Compare commits

..

29 Commits

Author SHA1 Message Date
d-robotics
03dd5ffbfd merge: complete DEPLOY-002 Kubernetes Helm delivery 2026-09-03 13:07:19 +08:00
d-robotics
5100d98e3c deploy: add Kubernetes Helm delivery 2026-09-03 12:21:54 +08:00
d-robotics
d727d0a118 merge: complete DEPLOY-001 production images 2026-09-03 11:01:58 +08:00
d-robotics
72dc248bbc build: add production process images 2026-09-02 21:16:33 +08:00
d-robotics
a1eae16fb1 merge: complete DEMO-016 batch async upload 2026-09-02 20:43:26 +08:00
d-robotics
0fad825eb0 feat: add batch async ingestion 2026-09-02 20:42:52 +08:00
d-robotics
a8dfbe0cfa merge: DEMO-015 durable async ingestion 2026-09-02 20:19:28 +08:00
d-robotics
7938c0b9cf feat: add durable async ingestion jobs 2026-09-02 20:18:58 +08:00
d-robotics
4d73465888 merge: PLAN-002 public cluster delivery plan 2026-09-02 19:43:15 +08:00
d-robotics
f47634e104 docs: plan public knowledge cluster delivery 2026-09-02 19:41:13 +08:00
d-robotics
6f16c971e4 merge: close DEMO-012 WorkBuddy acceptance 2026-09-02 17:11:29 +08:00
d-robotics
123864b80d docs: close WorkBuddy MCP acceptance 2026-09-02 17:10:56 +08:00
d-robotics
9d53bdbe4b merge: complete DEMO-012 MCP search gateway 2026-09-02 16:40:00 +08:00
d-robotics
77119f4a80 feat: add authenticated knowledge MCP gateway 2026-09-02 16:39:26 +08:00
d-robotics
6af4ab465d merge: PLAN-001 async ingestion and MCP priority 2026-09-02 16:01:20 +08:00
d-robotics
e3b5b4d50f docs: plan async ingestion and prioritize MCP 2026-09-02 16:00:42 +08:00
d-robotics
fe9d85b6da merge: complete DEMO-014 branded admin console 2026-09-02 15:28:59 +08:00
d-robotics
f16c738035 feat: align DEMO-014 console with D-Robotics brand 2026-09-02 15:27:40 +08:00
d-robotics
8fba051da4 merge: complete DEMO-010 admin console 2026-09-02 13:49:38 +08:00
d-robotics
c13ae39543 feat: complete DEMO-010 admin console 2026-09-02 13:48:31 +08:00
d-robotics
9931224b46 merge: ARCH-006 OceanBase live acceptance 2026-09-02 13:09:08 +08:00
d-robotics
2558264d21 feat: complete ARCH-006 OceanBase live acceptance 2026-09-02 13:08:20 +08:00
d-robotics
c6421cc9d4 merge: ARCH-005 runtime cutover 2026-09-02 11:29:01 +08:00
d-robotics
df4b0ae91e feat: complete ARCH-005 runtime cutover 2026-09-02 11:28:13 +08:00
d-robotics
4ce74604ba merge: ARCH-004 BCE reranker alignment 2026-09-02 11:14:03 +08:00
d-robotics
0d21fae173 feat: align ARCH-004 reranker with BCE 2026-09-02 11:13:27 +08:00
d-robotics
92d098cc25 merge: ARCH-003 OceanBase vector adapter 2026-09-02 11:10:05 +08:00
d-robotics
6efc33e868 feat: add ARCH-003 OceanBase vector adapter 2026-09-02 11:09:04 +08:00
d-robotics
b98b04af17 merge: ARCH-002 BOS object storage adapter 2026-09-01 20:49:04 +08:00
133 changed files with 11891 additions and 943 deletions

21
.dockerignore Normal file
View File

@ -0,0 +1,21 @@
# Keep credentials, local state, caches, and developer artifacts out of every image context.
.env
.env.*
!.env.example
.git
.github
.idea
.vscode
.DS_Store
.mypy_cache
.pytest_cache
.ruff_cache
.venv
__pycache__
*.py[cod]
*.sqlite*
dist
storage
tests
docs
docker-compose.dev.yml

View File

@ -4,12 +4,26 @@ LOG_LEVEL=INFO
API_HOST=0.0.0.0 API_HOST=0.0.0.0
API_PORT=8000 API_PORT=8000
API_PREFIX=/api/v1 API_PREFIX=/api/v1
# Optional outside source checkouts; production image fixes this to /app/apps/admin-web.
# ADMIN_WEB_ROOT=/absolute/path/to/apps/admin-web
DATABASE_URL=postgresql+psycopg://ekp:ekp@localhost:5432/ekp DATABASE_URL=mysql+pymysql://ekp:change-me@localhost:2881/ekp
REDIS_URL=redis://localhost:6379/0 DATABASE_POOL_SIZE=10
RABBITMQ_URL=amqp://guest:guest@localhost:5672// DATABASE_MAX_OVERFLOW=10
DATABASE_POOL_RECYCLE_SECONDS=600
DATABASE_POOL_TIMEOUT_SECONDS=30
DATABASE_CONNECT_TIMEOUT_SECONDS=30
# DEMO-015 queue. Inject the real credential at runtime; never commit its value.
RABBITMQ_URL=
INGESTION_MAX_ATTEMPTS=3
INGESTION_RETRY_BACKOFF_SECONDS=2
INGESTION_RETRY_BACKOFF_MAX_SECONDS=60
INGESTION_WORKER_LEASE_SECONDS=300
INGESTION_RECOVERY_INTERVAL_SECONDS=30
BATCH_MAX_FILES=20
BATCH_MAX_TOTAL_BYTES=209715200
OBJECT_STORAGE_PROVIDER=local OBJECT_STORAGE_PROVIDER=bos
OBJECT_STORAGE_PATH=storage OBJECT_STORAGE_PATH=storage
BOS_ENDPOINT= BOS_ENDPOINT=
BOS_ACCESS_KEY_ID= BOS_ACCESS_KEY_ID=
@ -22,31 +36,55 @@ CHUNK_MAX_TOKENS=450
CHUNK_OVERLAP_TOKENS=75 CHUNK_OVERLAP_TOKENS=75
CHUNK_MAX_CHARACTERS=2000 CHUNK_MAX_CHARACTERS=2000
VECTOR_STORE_PROVIDER=mochow VECTOR_STORE_PROVIDER=oceanbase
BAIDU_MOCHOW_ACCOUNT=root OCEANBASE_VECTOR_URI=
BAIDU_MOCHOW_API_KEY= OCEANBASE_VECTOR_USER=
BAIDU_MOCHOW_ENDPOINT= OCEANBASE_VECTOR_PASSWORD=
KNOWLEDGE_VDB_DB_NAME= OCEANBASE_VECTOR_DATABASE=
MOCHOW_TABLE=knowledge_chunks OCEANBASE_VECTOR_TABLE=knowledge_chunks_v1
MOCHOW_BATCH_SIZE=100 OCEANBASE_VECTOR_INDEX=vidx_knowledge_chunks_embedding
MOCHOW_SEARCH_EF=200 OCEANBASE_VECTOR_BATCH_SIZE=100
MOCHOW_TIMEOUT_SECONDS=30 OCEANBASE_VECTOR_TIMEOUT_SECONDS=30
EMBEDDING_PROVIDER=qianfan EMBEDDING_PROVIDER=qianfan
EMBEDDING_MODEL=bge-large-zh EMBEDDING_MODEL=bge-large-zh
EMBEDDING_ENDPOINT=https://qianfan.baidubce.com/v2/embeddings EMBEDDING_ENDPOINT=https://qianfan.baidubce.com/v2/embeddings
BAIDU_QIANFAN_API_KEY= BAIDU_QIANFAN_API_KEY=
EMBEDDING_TIMEOUT_SECONDS=30 EMBEDDING_TIMEOUT_SECONDS=30
INDEX_PROFILE_ID=demo-bge-large-zh-structure-v1 INDEX_PROFILE_ID=demo-bge-large-zh-structure-v1-oceanbase-v1
DENSE_TOP_N=20 DENSE_TOP_N=20
RERANKER_PROVIDER=qianfan RERANKER_PROVIDER=qianfan
RERANKER_MODEL=qwen3-reranker-0.6b RERANKER_MODEL=bce-reranker-base
RERANKER_ENDPOINT=https://qianfan.baidubce.com/v2/rerank RERANKER_ENDPOINT=https://qianfan.baidubce.com/v2/rerank
RERANKER_API_KEY= RERANKER_API_KEY=
RERANKER_TIMEOUT_SECONDS=30 RERANKER_TIMEOUT_SECONDS=30
SEARCH_TOP_K=5 SEARCH_TOP_K=5
# DEMO-012 MCP gateway. Inject MCP_AUTH_TOKEN at runtime; never commit its value.
MCP_HOST=127.0.0.1
MCP_PORT=8766
MCP_PATH=/mcp
MCP_PUBLIC_URL=http://127.0.0.1:8766/mcp
MCP_ISSUER_URL=http://127.0.0.1:8766
MCP_AUTH_TOKEN=
MCP_CLIENT_ID=workbuddy-enterprise
MCP_SERVICE_IDENTITY=workbuddy-demo
MCP_REQUIRED_SCOPE=knowledge:search
MCP_ALLOWED_HOSTS='["127.0.0.1:*","localhost:*","[::1]:*","testserver"]'
MCP_ALLOWED_ORIGINS='["http://127.0.0.1:*","http://localhost:*","http://[::1]:*"]'
MCP_MAX_REQUEST_BODY_BYTES=1048576
# Controlled ARCH-006 acceptance; keep disabled outside an authorized test run.
EKP_RUN_LIVE_TESTS=0
EKP_CREDENTIALS_AUTHORIZED=0
EKP_LIVE_RESOURCE_ACK=0
EKP_LIVE_MIN_RECALL_AT_1=0.90
EKP_LIVE_MIN_RECALL_AT_5=1.0
EKP_LIVE_MIN_MRR=0.90
EKP_LIVE_MIN_NDCG_AT_5=0.90
EKP_LIVE_MAX_P95_MS=10000
LLM_PROVIDER=qianfan LLM_PROVIDER=qianfan
LLM_MODEL= LLM_MODEL=
LLM_ENDPOINT= LLM_ENDPOINT=

View File

@ -19,8 +19,8 @@ Do not redesign accepted architecture unless the task explicitly asks for an arc
1. API First. 1. API First.
2. V1 uses a modular monolith plus independent async/GPU workers and an MCP gateway. 2. V1 uses a modular monolith plus independent async/GPU workers and an MCP gateway.
3. PostgreSQL + object storage hold canonical data. 3. OceanBase relational tables + object storage hold canonical data.
4. Vector databases are rebuildable indexes, not sources of truth. 4. OceanBase vector tables are rebuildable indexes, not sources of truth, even when co-located with canonical relational tables.
5. Business services depend on `VectorStore` abstractions, never directly on Mochow. 5. Business services depend on `VectorStore` abstractions, never directly on Mochow.
6. Embedding, reranking, parsing, OCR, object storage, and LLMs use provider interfaces. 6. Embedding, reranking, parsing, OCR, object storage, and LLMs use provider interfaces.
7. Original source documents are immutable. 7. Original source documents are immutable.

View File

@ -1,4 +1,4 @@
.PHONY: help install run format format-check lint typecheck test test-unit test-integration migration-check eval up down .PHONY: help install run run-mcp run-worker images images-print images-push deploy deploy-lint deploy-render format format-check lint typecheck test test-unit test-integration migration-check eval live-acceptance up down
UV_CACHE_DIR ?= /tmp/ekp-uv-cache UV_CACHE_DIR ?= /tmp/ekp-uv-cache
export UV_CACHE_DIR export UV_CACHE_DIR
@ -7,6 +7,14 @@ help:
@echo "Enterprise Knowledge Platform" @echo "Enterprise Knowledge Platform"
@echo " install Install development dependencies" @echo " install Install development dependencies"
@echo " run Start the development API" @echo " run Start the development API"
@echo " run-mcp Start the authenticated MCP gateway"
@echo " run-worker Start the ingestion worker and recovery scheduler"
@echo " images Build production images for the current builder"
@echo " images-print Render the multi-architecture image build plan"
@echo " images-push Build and push amd64/arm64 production images"
@echo " deploy Install or upgrade Kubernetes with VALUES=<file>"
@echo " deploy-lint Validate Helm values and templates"
@echo " deploy-render Render non-secret Kubernetes manifests"
@echo " format Format code" @echo " format Format code"
@echo " format-check Check formatting without modifying files" @echo " format-check Check formatting without modifying files"
@echo " lint Run linters" @echo " lint Run linters"
@ -16,6 +24,7 @@ help:
@echo " test-integration Run integration tests" @echo " test-integration Run integration tests"
@echo " migration-check Validate the forward migration path" @echo " migration-check Validate the forward migration path"
@echo " eval Run RAG evaluation" @echo " eval Run RAG evaluation"
@echo " live-acceptance Run controlled target-provider acceptance"
@echo " up Start local dependencies" @echo " up Start local dependencies"
@echo " down Stop local dependencies" @echo " down Stop local dependencies"
@ -23,6 +32,24 @@ install:
uv sync --all-groups uv sync --all-groups
run: run:
uv run uvicorn ekp_api.main:app --host "$${API_HOST:-0.0.0.0}" --port "$${API_PORT:-8000}" uv run uvicorn ekp_api.main:app --host "$${API_HOST:-0.0.0.0}" --port "$${API_PORT:-8000}"
run-mcp:
uv run ekp-mcp
run-worker:
uv run ekp-worker
images:
docker buildx bake -f deploy/docker/docker-bake.hcl
images-print:
docker buildx bake -f deploy/docker/docker-bake.hcl --print
images-push:
docker buildx bake -f deploy/docker/docker-bake.hcl --push
deploy:
./scripts/deploy.sh "$(VALUES)"
deploy-lint:
@test -n "$(VALUES)" || (echo "VALUES must point to an environment values file" >&2; exit 2)
helm lint deploy/helm/knowledge-platform --values "$(VALUES)"
deploy-render:
@test -n "$(VALUES)" || (echo "VALUES must point to an environment values file" >&2; exit 2)
helm template "$${RELEASE:-knowledge-platform}" deploy/helm/knowledge-platform --namespace "$${NAMESPACE:-knowledge-platform}" --values "$(VALUES)"
format: format:
uv run ruff format . uv run ruff format .
format-check: format-check:
@ -30,7 +57,7 @@ format-check:
lint: lint:
uv run ruff check . uv run ruff check .
typecheck: typecheck:
uv run mypy apps/api/src packages tests uv run mypy apps/api/src apps/mcp-server/src packages tests
test: test:
uv run pytest tests/unit tests/integration uv run pytest tests/unit tests/integration
test-unit: test-unit:
@ -42,6 +69,8 @@ migration-check:
uv run alembic upgrade head --sql >/dev/null uv run alembic upgrade head --sql >/dev/null
eval: eval:
uv run pytest tests/evaluation uv run pytest tests/evaluation
live-acceptance:
uv run ekp-live-acceptance
up: up:
docker compose -f docker-compose.dev.yml up -d docker compose -f docker-compose.dev.yml up -d
down: down:

View File

@ -19,8 +19,8 @@ Enterprise internal knowledge infrastructure.
- API First + Admin Console - API First + Admin Console
- Modular Monolith + async workers + MCP gateway - Modular Monolith + async workers + MCP gateway
- PostgreSQL + object storage as canonical source of truth - OceanBase MySQL relational tables + object storage as canonical source of truth
- OceanBase Vector as the target retrieval index, behind a replaceable adapter - Co-located OceanBase Vector tables as rebuildable retrieval indexes behind an adapter
- Baidu BOS as target object storage; local storage remains for tests/development - Baidu BOS as target object storage; local storage remains for tests/development
- Embedding / reranker / parser / LLM providers are replaceable - Embedding / reranker / parser / LLM providers are replaceable
- Raw source is immutable - Raw source is immutable
@ -46,19 +46,40 @@ The current implementation target is the retrieval demo defined in
Prerequisites: Prerequisites:
- `uv` - `uv`
- Docker (for PostgreSQL and later integration dependencies) - Docker (for optional local integration dependencies)
Install dependencies and run the API: Install dependencies and run the API:
```bash ```bash
cp .env.example .env cp .env.example .env
make install make install
make run-worker
make run make run
``` ```
Run the independent authenticated MCP Gateway after injecting a dedicated
runtime token of at least 32 characters:
```bash
export MCP_AUTH_TOKEN="<runtime-secret-with-at-least-32-characters>"
make run-mcp
```
The Streamable HTTP endpoint defaults to `http://127.0.0.1:8766/mcp` and exposes
only `search_knowledge`. See `docs/development/MCP_SETUP.md` for WorkBuddy configuration,
identity boundaries, remote allowlists, and protocol verification.
The health endpoint is available at `http://localhost:8000/healthz`, and the The health endpoint is available at `http://localhost:8000/healthz`, and the
versioned OpenAPI document is available at versioned OpenAPI document is available at
`http://localhost:8000/api/v1/openapi.json`. `http://localhost:8000/api/v1/openapi.json`.
The Demo Admin Console is available at `http://localhost:8000/admin` and contains
only document upload, document management, and Search Lab pages. See
`docs/development/ADMIN_CONSOLE.md` for the operation and scope boundaries.
DEPLOY-001 provides locked, non-root `api`, `mcp`, and `worker` production image targets with an
amd64/arm64 Buildx Bake plan. DEPLOY-002 adds the Kubernetes/Helm Chart, runtime Secret references,
serialized Migration, Ingress/TLS and the `make deploy VALUES=<file>` entrypoint. See
`docs/development/CONTAINER_IMAGES.md` and `docs/development/KUBERNETES_DEPLOYMENT.md`.
Run the development checks with: Run the development checks with:
@ -66,7 +87,12 @@ Run the development checks with:
make format-check lint typecheck test migration-check eval make format-check lint typecheck test migration-check eval
``` ```
Document metadata is canonical in PostgreSQL. Demo source files use the local Controlled target-provider acceptance uses 32 synthetic queries and is disabled by
default. It requires rotated secrets, dedicated test resources, and all safety
acknowledgements described in `docs/development/LIVE_ACCEPTANCE.md`; only then run
`make live-acceptance`.
Document metadata is canonical in OceanBase MySQL relational tables. Demo source files use the local
`storage/` directory through the provider-neutral `ObjectStorage` interface; `storage/` directory through the provider-neutral `ObjectStorage` interface;
existing objects cannot be overwritten. The upload service accepts PDF, DOCX, existing objects cannot be overwritten. The upload service accepts PDF, DOCX,
TXT, and MD sources, streams at most `MAX_UPLOAD_BYTES`, and compensates stored TXT, and MD sources, streams at most `MAX_UPLOAD_BYTES`, and compensates stored
@ -77,39 +103,48 @@ versioned parsed JSON artifact under `parsed/{document_id}/document.json`.
The `structure_v1` chunk strategy keeps heading/section boundaries, combines The `structure_v1` chunk strategy keeps heading/section boundaries, combines
paragraphs, and creates configurable 450-token windows with a 75-token overlap. paragraphs, and creates configurable 450-token windows with a 75-token overlap.
Chunk metadata, including deterministic IDs, filename, page range, section, Chunk metadata, including deterministic IDs, filename, page range, section,
index, and strategy version, is stored atomically in PostgreSQL. The lightweight index, and strategy version, is stored atomically in canonical relational tables. The lightweight
token counter is replaceable so DEMO-006 can inject model-specific behavior. token counter is replaceable so DEMO-006 can inject model-specific behavior.
Documents remain `processing` until embedding/indexing completes; pipeline Documents remain `processing` until embedding/indexing completes; pipeline
failures become `failed` without deleting the immutable original. DEMO-009 failures become `failed` without deleting the immutable original. DEMO-015 now
exposes this pipeline through the versioned upload endpoint. accepts a source plus durable Job with HTTP `202`, while a Celery/RabbitMQ worker
runs the pipeline. Bounded transient retries clean every derived artifact before
starting the next Attempt; broker redelivery and expired worker leases converge on
OceanBase Job state. See `docs/development/ASYNC_INGESTION_SETUP.md`.
DEMO-006 added a provider-neutral `EmbeddingProvider` and `VectorStore`, a DEMO-006 added a provider-neutral `EmbeddingProvider` and `VectorStore`, plus a
validated Qianfan `bge-large-zh` adapter, and the initial isolated `pymochow` adapter. validated Qianfan `bge-large-zh` adapter. ARCH-005 now wires that boundary to OceanBase.
Chunk vectors are upserted with source metadata under a versioned index profile; Chunk vectors are upserted with source metadata under a versioned index profile;
PostgreSQL records canonical index state while Mochow remains rebuildable. A OceanBase relational tables record canonical index state while vector rows remain rebuildable. A
document becomes `ready` only after every chunk is indexed. See document becomes `ready` only after every chunk is indexed. See
`docs/development/MOCHOW_SETUP.md` for the required environment variables and `docs/development/OCEANBASE_VECTOR_SETUP.md` for the required environment variables
pre-provisioned table schema. and pre-provisioned table schema.
The accepted provider migration is tracked by `ARCH-001` through `ARCH-006`: The accepted provider migration is tracked by `ARCH-001` through `ARCH-006`:
Baidu BOS, OceanBase Vector, and Qianfan `bce-reranker-base` replace the initial Baidu BOS, OceanBase Vector, and Qianfan `bce-reranker-base` replace the initial
demo providers without changing PostgreSQL canonical ownership. Until the runtime demo providers, and ARCH-006 aligns canonical ownership with the verified OceanBase target. BOS and OceanBase
cutover task is merged, the paragraphs above describe the current implementation. are now the runtime defaults; LocalObjectStorage and FakeVectorStore remain available
ARCH-002 adds the selectable BOS adapter and safe runtime configuration documented for explicit local/offline tests. ARCH-005 also adds the controlled shadow rebuild
in `docs/development/BOS_SETUP.md`; local storage remains the default until ARCH-005. workflow documented in `docs/development/INDEX_REBUILD.md`.
DEMO-007 adds query embedding and profile-scoped Dense Top-N retrieval. The DEMO-007 adds query embedding and profile-scoped Dense Top-N retrieval. The
default is Top 20 with a configurable Mochow HNSW `ef`; the in-memory fake uses default is Top 20; the in-memory fake uses
exact cosine search for deterministic offline regression. Returned objects are exact cosine search for deterministic offline regression. Returned objects are
raw dense candidates only. raw dense candidates only.
DEMO-008 adds a provider-neutral reranker, a validated Qianfan DEMO-008 adds a provider-neutral reranker, and ARCH-004 aligns its validated Qianfan
`qwen3-reranker-0.6b` adapter, and the final Top20-to-Top5 search pipeline. Final adapter with the `bce-reranker-base` baseline. The final Top20-to-Top5 pipeline and
source results are rehydrated from canonical PostgreSQL chunks and retain both source results are rehydrated from canonical relational chunks and retain both
Dense and reranker scores; stale vector content fails closed. See Dense and reranker scores; stale vector content fails closed. See
`docs/development/RERANKER_SETUP.md` for configuration and validation limits. `docs/development/RERANKER_SETUP.md` for configuration and validation limits.
DEMO-009 exposes the completed synchronous pipeline through versioned document DEMO-009 exposes document and search REST endpoints under `/api/v1`; DEMO-015
and search REST endpoints under `/api/v1`. Upload, list, detail, direct delete, changes upload to asynchronous acceptance and adds Job status. DEMO-016 adds bounded multi-file
submission, batch/per-file status, and automatic Admin Console polling. Upload, list, detail, direct delete,
and reranked source search are documented in and reranked source search are documented in
`docs/development/REST_API_SETUP.md`; provider and storage internals remain `docs/development/REST_API_SETUP.md`; provider and storage internals remain
outside the public response contract. outside the public response contract.
DEMO-012 adds the independent authenticated MCP Gateway. It uses the official
MCP Python SDK, maps a runtime-only Bearer credential to a least-privilege service
identity, and delegates `search_knowledge` to the same `SearchService` used by REST.
No provider, storage, database, model, or index-profile details are exposed.

View File

@ -0,0 +1,411 @@
// The Admin Console owns presentation state only. Batch acceptance, Job state,
// and retrieval remain behind the versioned public REST contract.
const API_BASE = "/api/v1";
const BATCH_POLL_INTERVAL_MS = 1500;
const terminalBatchStatuses = new Set(["completed", "partial_failure", "failed"]);
let batchPollTimer;
let activeBatchId;
const statusLabels = {
uploaded: "Uploaded",
processing: "Processing",
ready: "Ready",
failed: "Failed",
pending: "Pending",
accepted: "Accepted",
rejected: "Rejected",
queued: "Queued",
running: "Running",
retrying: "Retrying",
succeeded: "Succeeded",
completed: "Completed",
partial_failure: "Partial failure",
};
const stageLabels = {
parsing: "解析",
cleaning: "清理",
chunking: "切分",
embedding: "向量化",
indexing: "索引",
};
const pageTitles = {
upload: "上传文档",
documents: "文档列表",
search: "Search Lab",
};
function currentPage() {
const route = window.location.pathname.split("/").filter(Boolean).at(-1);
return Object.hasOwn(pageTitles, route) ? route : "upload";
}
function showPage(page) {
document.querySelectorAll("[data-page]").forEach((section) => {
section.hidden = section.dataset.page !== page;
});
document.querySelectorAll("[data-nav]").forEach((link) => {
const active = link.dataset.nav === page;
link.classList.toggle("active", active);
if (active) link.setAttribute("aria-current", "page");
else link.removeAttribute("aria-current");
});
document.querySelector("#page-title").textContent = pageTitles[page];
}
function showNotice(message, kind = "success") {
const notice = document.querySelector("#notice");
notice.textContent = message;
notice.className = `notice ${kind}`;
notice.hidden = false;
window.clearTimeout(showNotice.timeout);
showNotice.timeout = window.setTimeout(() => {
notice.hidden = true;
}, 6000);
}
async function requestJson(path, options = {}) {
const response = await fetch(`${API_BASE}${path}`, options);
if (!response.ok) {
let message = `请求失败(HTTP ${response.status})`;
try {
const body = await response.json();
if (body.detail) message = body.detail;
} catch {
// A non-JSON upstream error is represented by the stable HTTP message above.
}
throw new Error(message);
}
return response.status === 204 ? null : response.json();
}
function formatDate(value) {
if (!value) return "—";
return new Intl.DateTimeFormat("zh-CN", {
dateStyle: "medium",
timeStyle: "short",
}).format(new Date(value));
}
function formatSize(bytes) {
if (bytes < 1024) return `${bytes} B`;
if (bytes < 1024 ** 2) return `${(bytes / 1024).toFixed(1)} KB`;
return `${(bytes / 1024 ** 2).toFixed(1)} MB`;
}
function statusPill(status) {
const pill = document.createElement("span");
pill.className = `status-pill status-${status}`;
pill.textContent = statusLabels[status] ?? status;
return pill;
}
function setButtonBusy(button, busy, busyText) {
if (!button.dataset.label) button.dataset.label = button.textContent;
button.disabled = busy;
button.textContent = busy ? busyText : button.dataset.label;
}
function initializeUpload() {
const form = document.querySelector("#upload-form");
const input = document.querySelector("#file-input");
const selected = document.querySelector("#selected-file");
const button = document.querySelector("#upload-button");
const updateFilename = () => {
const files = Array.from(input.files);
if (!files.length) {
selected.textContent = "尚未选择文件";
} else if (files.length === 1) {
selected.textContent = files[0].name;
} else {
selected.textContent = `已选择 ${files.length} 个文件`;
}
};
input.addEventListener("change", updateFilename);
form.addEventListener("submit", async (event) => {
event.preventDefault();
if (!input.files.length) return;
window.clearTimeout(batchPollTimer);
activeBatchId = undefined;
setButtonBusy(button, true, "提交中…");
try {
const formData = new FormData();
Array.from(input.files).forEach((file) => formData.append("files", file));
const batch = await requestJson("/ingestion-batches", {
method: "POST",
body: formData,
});
activeBatchId = batch.batch_id;
renderBatchResult(batch);
scheduleBatchPoll(batch);
showNotice(
`批次已接收:${batch.accepted_files} 个进入后台队列,${batch.rejected_files} 个被拒绝。`,
batch.rejected_files ? "error" : "success",
);
form.reset();
updateFilename();
} catch (error) {
showNotice(`上传失败:${error.message}`, "error");
} finally {
setButtonBusy(button, false, "");
}
});
}
function scheduleBatchPoll(batch) {
window.clearTimeout(batchPollTimer);
if (batch.batch_id !== activeBatchId) return;
if (terminalBatchStatuses.has(batch.status)) return;
batchPollTimer = window.setTimeout(async () => {
try {
const latest = await requestJson(`/ingestion-batches/${batch.batch_id}`);
renderBatchResult(latest);
scheduleBatchPoll(latest);
} catch (error) {
showNotice(`批次状态刷新失败:${error.message}`, "error");
scheduleBatchPoll(batch);
}
}, BATCH_POLL_INTERVAL_MS);
}
function renderBatchResult(batch) {
const card = document.querySelector("#upload-result");
document.querySelector("#upload-batch-title").textContent =
`批次 ${batch.batch_id.slice(0, 8)} · ${formatDate(batch.created_at)}`;
document.querySelector("#upload-total").textContent = `${batch.total_files} 个`;
document.querySelector("#upload-completed").textContent =
`${batch.completed_files} / ${batch.total_files}`;
document.querySelector("#upload-accepted").textContent =
`${batch.accepted_files} / ${batch.rejected_files}`;
document.querySelector("#upload-status").replaceWith(statusPill(batch.status));
const status = card.querySelector(".status-pill");
status.id = "upload-status";
const items = document.querySelector("#batch-items");
items.replaceChildren(...batch.items.map(batchItem));
card.hidden = false;
}
function batchItem(item) {
const row = document.createElement("article");
row.className = "batch-item";
const order = document.createElement("span");
order.className = "batch-item-order";
order.textContent = String(item.item_index + 1).padStart(2, "0");
const body = document.createElement("div");
body.className = "batch-item-body";
const name = document.createElement("strong");
name.textContent = item.filename;
const detail = document.createElement("span");
const stage = item.stage ? ` · ${stageLabels[item.stage] ?? item.stage}` : "";
const attempts = item.max_attempts == null ? "" : ` · 尝试 ${item.attempt}/${item.max_attempts}`;
detail.textContent = item.error_message ?? `后台任务${stage}${attempts}`;
body.append(name, detail);
const state = item.job_status ?? item.acceptance_status;
row.append(order, body, statusPill(state));
return row;
}
async function loadDocuments() {
const refresh = document.querySelector("#refresh-documents");
setButtonBusy(refresh, true, "加载中…");
try {
const documents = await requestJson("/documents");
renderDocuments(documents);
} catch (error) {
showNotice(`文档列表加载失败:${error.message}`, "error");
} finally {
setButtonBusy(refresh, false, "");
}
}
function renderDocuments(documents) {
const rows = document.querySelector("#document-rows");
const empty = document.querySelector("#document-empty");
rows.replaceChildren();
empty.hidden = documents.length !== 0;
document.querySelector("#document-count").textContent = `${documents.length} 个文档`;
document.querySelector("#document-refresh-time").textContent =
`更新于 ${new Intl.DateTimeFormat("zh-CN", { timeStyle: "short" }).format(new Date())}`;
documents.forEach((documentData) => {
const row = document.createElement("tr");
row.append(
tableCell(documentData.filename, "filename-cell"),
tableCell(documentData.media_type),
tableCell(formatDate(documentData.created_at)),
);
const statusCell = document.createElement("td");
statusCell.append(statusPill(documentData.status));
row.append(statusCell);
const actions = document.createElement("td");
actions.className = "row-actions";
const view = actionButton("查看", () => showDocumentDetail(documentData));
const remove = actionButton("删除", () => deleteDocument(documentData), "danger-link");
actions.append(view, remove);
row.append(actions);
rows.append(row);
});
}
function tableCell(value, className = "") {
const cell = document.createElement("td");
cell.textContent = value;
cell.className = className;
return cell;
}
function actionButton(label, handler, className = "") {
const button = document.createElement("button");
button.type = "button";
button.className = `table-action ${className}`;
button.textContent = label;
button.addEventListener("click", handler);
return button;
}
function showDocumentDetail(documentData) {
const dialog = document.querySelector("#document-dialog");
document.querySelector("#dialog-filename").textContent = documentData.filename;
const metadata = document.querySelector("#dialog-metadata");
metadata.replaceChildren();
[
["文档 ID", documentData.id],
["文件类型", documentData.media_type],
["文件大小", formatSize(documentData.size_bytes)],
["当前状态", statusLabels[documentData.status] ?? documentData.status],
["上传时间", formatDate(documentData.created_at)],
["更新时间", formatDate(documentData.updated_at)],
["错误信息", documentData.error_message ?? "无"],
].forEach(([term, value]) => {
const wrapper = document.createElement("div");
const dt = document.createElement("dt");
const dd = document.createElement("dd");
dt.textContent = term;
dd.textContent = value;
wrapper.append(dt, dd);
metadata.append(wrapper);
});
dialog.showModal();
}
async function deleteDocument(documentData) {
if (!window.confirm(`确认删除“${documentData.filename}”及其索引数据?`)) return;
try {
await requestJson(`/documents/${documentData.id}`, { method: "DELETE" });
showNotice(`已删除 ${documentData.filename}`, "success");
await loadDocuments();
} catch (error) {
showNotice(`删除失败:${error.message}`, "error");
}
}
function initializeSearch() {
const form = document.querySelector("#search-form");
const button = document.querySelector("#search-button");
form.addEventListener("submit", async (event) => {
event.preventDefault();
const query = document.querySelector("#search-query").value.trim();
const topK = Number(document.querySelector("#search-top-k").value);
if (!query) return;
setButtonBusy(button, true, "检索中…");
try {
const startedAt = performance.now();
const response = await requestJson("/search", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ query, top_k: topK }),
});
renderSearchResults(response.results, performance.now() - startedAt);
} catch (error) {
showNotice(`检索失败:${error.message}`, "error");
} finally {
setButtonBusy(button, false, "");
}
});
}
function renderSearchResults(results, elapsedMs) {
const container = document.querySelector("#search-results");
container.replaceChildren();
document.querySelector("#search-summary").textContent =
`${results.length} 条结果 · ${(elapsedMs / 1000).toFixed(2)} 秒`;
if (!results.length) {
const empty = document.createElement("div");
empty.className = "empty-state search-empty";
const strong = document.createElement("strong");
strong.textContent = "没有找到匹配内容";
const detail = document.createElement("span");
detail.textContent = "可以换一种问法,或先确认文档状态为 Ready。";
empty.append(strong, detail);
container.append(empty);
return;
}
results.forEach((result, index) => {
const card = document.createElement("article");
card.className = "search-result-card";
const rank = document.createElement("span");
rank.className = "rank";
rank.textContent = String(index + 1).padStart(2, "0");
const body = document.createElement("div");
body.className = "result-body";
const heading = document.createElement("div");
heading.className = "result-title-row";
const title = document.createElement("h4");
title.textContent = result.filename;
const sourceMeta = document.createElement("div");
sourceMeta.className = "source-meta";
sourceMeta.append(
metadataItem("Page", result.page == null ? "—" : String(result.page)),
metadataItem("Section", result.section ?? "—"),
);
heading.append(title, sourceMeta);
const content = document.createElement("p");
content.className = "chunk-content";
content.textContent = result.content;
const scores = document.createElement("div");
scores.className = "score-row";
scores.append(
scoreItem("Dense Score", result.score),
scoreItem("Rerank Score", result.rerank_score),
);
body.append(heading, content, scores);
card.append(rank, body);
container.append(card);
});
}
function scoreItem(label, value) {
const item = document.createElement("span");
const labelNode = document.createElement("small");
const valueNode = document.createElement("strong");
labelNode.textContent = label;
valueNode.textContent = Number(value).toFixed(4);
item.append(labelNode, valueNode);
return item;
}
function metadataItem(label, value) {
const item = document.createElement("span");
const labelNode = document.createElement("small");
const valueNode = document.createElement("strong");
labelNode.textContent = label;
valueNode.textContent = value;
item.append(labelNode, valueNode);
return item;
}
const page = currentPage();
showPage(page);
initializeUpload();
initializeSearch();
document.querySelector("#refresh-documents").addEventListener("click", loadDocuments);
if (page === "documents") loadDocuments();
window.addEventListener("pagehide", () => window.clearTimeout(batchPollTimer));

File diff suppressed because it is too large Load Diff

201
apps/admin-web/index.html Normal file
View File

@ -0,0 +1,201 @@
<!doctype html>
<html lang="zh-CN">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1" />
<meta
name="description"
content="Enterprise Knowledge Platform Demo 管理控制台"
/>
<title>Enterprise Knowledge Platform</title>
<link rel="stylesheet" href="/admin/assets/styles.css" />
<script type="module" src="/admin/assets/app.js"></script>
</head>
<body>
<div class="app-shell">
<aside class="sidebar" aria-label="管理控制台导航">
<a class="brand" href="/admin/upload" aria-label="Enterprise Knowledge Platform 首页">
<svg class="brand-mark" viewBox="0 0 48 36" aria-hidden="true">
<path d="M15 3h17a15 15 0 0 1 0 30H15v-9h17a6 6 0 0 0 0-12H15z" />
<circle cx="11" cy="18" r="8" />
</svg>
<span>
<strong>地瓜机器人</strong>
<small>Knowledge Platform</small>
</span>
</a>
<div class="workspace-label">
<span>CONTROL DESK</span>
<strong>企业知识运营台</strong>
</div>
<nav class="primary-nav">
<a data-nav="upload" href="/admin/upload">
<span aria-hidden="true">01</span>上传文档
</a>
<a data-nav="documents" href="/admin/documents">
<span aria-hidden="true">02</span>文档列表
</a>
<a data-nav="search" href="/admin/search">
<span aria-hidden="true">03</span>Search Lab
</a>
</nav>
<div class="sidebar-foot">
<span class="environment-dot" aria-hidden="true"></span>
DEMO ENVIRONMENT
</div>
</aside>
<main class="workspace">
<header class="topbar">
<div>
<p class="eyebrow">KNOWLEDGE OPERATIONS</p>
<h1 id="page-title">上传文档</h1>
</div>
<div class="topbar-actions">
<span class="system-state"><i aria-hidden="true"></i>API FIRST · V1</span>
<a class="api-link" href="/api/v1/docs" target="_blank" rel="noreferrer">
API 文档 <span aria-hidden="true">↗</span>
</a>
</div>
</header>
<div class="signal-line" aria-hidden="true"><span></span><span></span><span></span></div>
<div id="notice" class="notice" role="status" aria-live="polite" hidden></div>
<section id="upload-page" class="page" data-page="upload" aria-labelledby="page-title">
<div class="page-heading">
<div>
<p class="section-label">01 / INGESTION</p>
<h2>让知识进入检索链路</h2>
<p>批量提交 PDF、DOCX、TXT 或 Markdown 文件,系统将逐个接收并在后台独立完成解析、切分与索引。</p>
</div>
</div>
<form id="upload-form" class="upload-panel">
<label id="drop-zone" class="drop-zone" for="file-input">
<input
id="file-input"
name="files"
type="file"
multiple
accept=".pdf,.docx,.txt,.md,application/pdf,application/vnd.openxmlformats-officedocument.wordprocessingml.document,text/plain,text/markdown"
required
/>
<span class="upload-icon" aria-hidden="true">↑</span>
<strong>选择一批知识文件</strong>
<span>支持 PDF、DOCX、TXT、Markdown · 每个文件独立处理</span>
<span id="selected-file" class="selected-file">尚未选择文件</span>
</label>
<button id="upload-button" class="primary-button" type="submit">上传并处理</button>
</form>
<article id="upload-result" class="result-card" hidden>
<div>
<p class="section-label">LATEST BATCH</p>
<h3 id="upload-batch-title"></h3>
</div>
<span id="upload-status" class="status-pill"></span>
<dl class="metadata-grid">
<div><dt>总文件</dt><dd id="upload-total"></dd></div>
<div><dt>已完成</dt><dd id="upload-completed"></dd></div>
<div><dt>接收 / 拒绝</dt><dd id="upload-accepted"></dd></div>
</dl>
<div id="batch-items" class="batch-items" aria-live="polite"></div>
<a class="text-link" href="/admin/search">前往 Search Lab 检索 <span>→</span></a>
</article>
</section>
<section id="documents-page" class="page" data-page="documents" aria-labelledby="page-title" hidden>
<div class="page-heading heading-row">
<div>
<p class="section-label">02 / DOCUMENTS</p>
<h2>文档列表</h2>
<p>查看知识文档的状态、元数据并管理 Demo 数据。</p>
</div>
<button id="refresh-documents" class="secondary-button" type="button">刷新列表</button>
</div>
<div class="table-card">
<div class="table-meta">
<span id="document-count">0 个文档</span>
<span id="document-refresh-time">尚未加载</span>
</div>
<div class="table-scroll">
<table>
<thead>
<tr>
<th>文件名</th>
<th>类型</th>
<th>上传时间</th>
<th>状态</th>
<th aria-label="操作"></th>
</tr>
</thead>
<tbody id="document-rows"></tbody>
</table>
</div>
<div id="document-empty" class="empty-state" hidden>
<strong>还没有文档</strong>
<span>上传第一个文件后,它会出现在这里。</span>
<a class="primary-button compact" href="/admin/upload">上传文档</a>
</div>
</div>
</section>
<section id="search-page" class="page" data-page="search" aria-labelledby="page-title" hidden>
<div class="page-heading">
<div>
<p class="section-label">03 / RETRIEVAL</p>
<h2>Search Lab</h2>
<p>输入问题,检查 Dense 召回与 Reranker 精排后的 Top-K 来源。</p>
</div>
</div>
<form id="search-form" class="search-panel">
<label for="search-query">检索问题</label>
<div class="search-controls">
<textarea
id="search-query"
name="query"
rows="3"
maxlength="1600"
placeholder="例如:产品 A 的退款周期是多少?"
required
></textarea>
<label class="top-k-control" for="search-top-k">
Top-K
<select id="search-top-k" name="top_k">
<option value="3">3</option>
<option value="5" selected>5</option>
<option value="10">10</option>
</select>
</label>
<button id="search-button" class="primary-button" type="submit">开始检索</button>
</div>
</form>
<div class="results-heading">
<div>
<p class="section-label">RESULTS</p>
<h3>来源结果</h3>
</div>
<span id="search-summary">等待检索</span>
</div>
<div id="search-results" class="search-results" aria-live="polite">
<div class="empty-state search-empty">
<strong>检索结果会显示在这里</strong>
<span>结果包含排名、文件、页码、章节、Chunk 内容和两阶段分数。</span>
</div>
</div>
</section>
</main>
</div>
<dialog id="document-dialog" class="document-dialog">
<form method="dialog"><button class="icon-button" aria-label="关闭详情">×</button></form>
<p class="section-label">DOCUMENT DETAIL</p>
<h2 id="dialog-filename"></h2>
<dl id="dialog-metadata" class="dialog-metadata"></dl>
</dialog>
</body>
</html>

View File

@ -0,0 +1,55 @@
"""Celery/RabbitMQ adapter behind the queue-neutral ingestion dispatcher."""
from uuid import UUID
from celery import Celery # type: ignore[import-untyped]
from ekp_core.config import Settings
INGESTION_TASK_NAME = "ekp.ingestion.run"
RECOVERY_TASK_NAME = "ekp.ingestion.recover"
def create_celery_app(settings: Settings) -> Celery:
"""Create a secret-safe broker client without opening a connection eagerly."""
if settings.rabbitmq_url is None or not settings.rabbitmq_url.get_secret_value().strip():
raise ValueError("RABBITMQ_URL is required for asynchronous ingestion")
app = Celery(
"enterprise-knowledge-platform",
broker=settings.rabbitmq_url.get_secret_value(),
)
app.conf.update(
task_ignore_result=True,
task_acks_late=True,
task_reject_on_worker_lost=True,
worker_prefetch_multiplier=1,
broker_connection_retry_on_startup=True,
timezone="UTC",
enable_utc=True,
task_routes={
INGESTION_TASK_NAME: {"queue": "ingestion"},
RECOVERY_TASK_NAME: {"queue": "ingestion"},
},
beat_schedule={
"recover-durable-ingestion-jobs": {
"task": RECOVERY_TASK_NAME,
"schedule": float(settings.ingestion_recovery_interval_seconds),
}
},
)
return app
class CeleryIngestionDispatcher:
"""Publish only a durable Job id; source bytes and state stay canonical."""
def __init__(self, app: Celery) -> None:
self._app = app
def enqueue(self, job_id: UUID) -> None:
"""Deliver one idempotent execution hint to the ingestion queue."""
self._app.send_task(
INGESTION_TASK_NAME,
args=[str(job_id)],
queue="ingestion",
)

View File

@ -0,0 +1,67 @@
"""Controlled CLI for the ARCH-005 shadow index cutover workflow."""
import argparse
import json
import sys
from collections.abc import Sequence
from dataclasses import asdict
from ekp_api.providers import (
create_embedding_provider,
create_index_profile,
create_vector_store,
)
from ekp_core.config import Settings
from ekp_core.database import create_database_engine, create_session_factory
from ekp_core.index_rebuild import IndexBuildSummary, IndexRebuildService
def _parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(
description="Build, verify, and activate a versioned shadow vector index."
)
parser.add_argument(
"command",
choices=("build", "verify", "activate"),
help="Lifecycle step to execute for INDEX_PROFILE_ID.",
)
return parser
def _create_service(settings: Settings) -> IndexRebuildService:
"""Compose the command from the same provider settings as the API runtime."""
engine = create_database_engine(settings)
session_factory = create_session_factory(engine)
return IndexRebuildService(
session_factory=session_factory,
embedding_provider=create_embedding_provider(settings),
vector_store=create_vector_store(settings),
target_profile=create_index_profile(settings),
)
def _execute(service: IndexRebuildService, command: str) -> IndexBuildSummary:
"""Run exactly one explicit lifecycle step."""
if command == "build":
return service.rebuild()
if command == "verify":
return service.verify()
if command == "activate":
return service.activate()
raise ValueError("unsupported index rebuild command")
def main(argv: Sequence[str] | None = None) -> int:
"""Return a shell-safe status without printing endpoints, SQL, or credentials."""
args = _parser().parse_args(argv)
try:
summary = _execute(_create_service(Settings()), args.command)
except Exception:
print("index profile command failed; inspect sanitized service logs", file=sys.stderr)
return 1
print(json.dumps(asdict(summary), sort_keys=True, separators=(",", ":")))
return 0
if __name__ == "__main__":
raise SystemExit(main())

View File

@ -0,0 +1,279 @@
"""Controlled end-to-end acceptance for the target provider chain."""
import json
import math
import os
import sys
from collections.abc import Sequence
from dataclasses import asdict, dataclass
from io import BytesIO
from pathlib import Path
from time import perf_counter
from typing import cast
from uuid import uuid4
from ekp_api.providers import ApiServices, create_api_services
from ekp_core.config import Settings
from ekp_core.search import SourceResult
MIN_GOLDEN_QUERIES = 30
MAX_GOLDEN_QUERIES = 50
DEFAULT_MIN_RECALL_AT_1 = 0.90
DEFAULT_MIN_RECALL_AT_5 = 1.0
DEFAULT_MIN_MRR = 0.90
DEFAULT_MIN_NDCG_AT_5 = 0.90
DEFAULT_MAX_P95_MS = 10_000.0
class LiveAcceptanceError(RuntimeError):
"""Raised for a safe precondition or acceptance failure."""
@dataclass(frozen=True, slots=True)
class GoldenQuery:
"""One anonymized expected source and section."""
query_id: str
query: str
expected_document: str
expected_sections: tuple[str, ...]
@dataclass(frozen=True, slots=True)
class AcceptanceReport:
"""Provider-neutral quality and latency metrics without retrieved content."""
query_count: int
recall_at_1: float
recall_at_3: float
recall_at_5: float
mrr: float
ndcg_at_5: float
latency_p50_ms: float
latency_p95_ms: float
passed: bool
@dataclass(frozen=True, slots=True)
class AcceptanceThresholds:
"""Explicit gates for controlled live acceptance."""
min_recall_at_1: float = DEFAULT_MIN_RECALL_AT_1
min_recall_at_5: float = DEFAULT_MIN_RECALL_AT_5
min_mrr: float = DEFAULT_MIN_MRR
min_ndcg_at_5: float = DEFAULT_MIN_NDCG_AT_5
max_p95_ms: float = DEFAULT_MAX_P95_MS
def __post_init__(self) -> None:
quality_values = (
self.min_recall_at_1,
self.min_recall_at_5,
self.min_mrr,
self.min_ndcg_at_5,
)
if any(not 0 <= value <= 1 for value in quality_values):
raise ValueError("live quality thresholds must be between zero and one")
if self.max_p95_ms <= 0:
raise ValueError("live latency threshold must be positive")
def _project_root() -> Path:
return Path(__file__).resolve().parents[4]
def _load_golden_queries(path: Path) -> tuple[GoldenQuery, ...]:
"""Load only the minimal fields needed by the live evaluator."""
records: list[GoldenQuery] = []
for line in path.read_text(encoding="utf-8").splitlines():
if not line:
continue
raw: object = json.loads(line)
if not isinstance(raw, dict):
raise LiveAcceptanceError("Golden Query record must be an object")
documents = raw.get("expected_documents")
sections = raw.get("expected_sections")
if (
not isinstance(raw.get("id"), str)
or not isinstance(raw.get("query"), str)
or not isinstance(documents, list)
or len(documents) != 1
or not isinstance(documents[0], str)
or not isinstance(sections, list)
or not sections
or any(not isinstance(section, str) for section in sections)
):
raise LiveAcceptanceError("Golden Query record violates the live contract")
records.append(
GoldenQuery(
query_id=cast(str, raw["id"]),
query=cast(str, raw["query"]),
expected_document=documents[0],
expected_sections=tuple(cast(list[str], sections)),
)
)
if not MIN_GOLDEN_QUERIES <= len(records) <= MAX_GOLDEN_QUERIES:
raise LiveAcceptanceError("live Golden Dataset must contain 30 to 50 queries")
if len({record.query_id for record in records}) != len(records):
raise LiveAcceptanceError("live Golden Query IDs must be unique")
return tuple(records)
def _matching_rank(
results: Sequence[SourceResult],
*,
expected_filename: str,
expected_sections: tuple[str, ...],
) -> int | None:
"""Return the one-based rank of the first matching canonical source."""
for rank, result in enumerate(results, start=1):
if result.filename != expected_filename or result.section is None:
continue
if any(section in result.section for section in expected_sections):
return rank
return None
def _percentile(values: Sequence[float], percentile: float) -> float:
"""Use nearest-rank percentiles for a small, deterministic acceptance set."""
if not values:
raise ValueError("latency sample must not be empty")
ordered = sorted(values)
index = max(0, math.ceil(percentile * len(ordered)) - 1)
return ordered[index]
def _build_report(
ranks: Sequence[int | None],
latencies_ms: Sequence[float],
thresholds: AcceptanceThresholds,
) -> AcceptanceReport:
"""Calculate binary-relevance metrics and apply explicit acceptance gates."""
if not ranks or len(ranks) != len(latencies_ms):
raise ValueError("live ranks and latencies must have equal non-zero length")
query_count = len(ranks)
def recall_at(limit: int) -> float:
return sum(rank is not None and rank <= limit for rank in ranks) / query_count
reciprocal_ranks = [0.0 if rank is None else 1.0 / rank for rank in ranks]
discounted_gains = [
0.0 if rank is None or rank > 5 else 1.0 / math.log2(rank + 1) for rank in ranks
]
recall_at_1 = recall_at(1)
recall_at_3 = recall_at(3)
recall_at_5 = recall_at(5)
mrr = sum(reciprocal_ranks) / query_count
ndcg_at_5 = sum(discounted_gains) / query_count
p50 = _percentile(latencies_ms, 0.50)
p95 = _percentile(latencies_ms, 0.95)
passed = (
recall_at_1 >= thresholds.min_recall_at_1
and recall_at_5 >= thresholds.min_recall_at_5
and mrr >= thresholds.min_mrr
and ndcg_at_5 >= thresholds.min_ndcg_at_5
and p95 <= thresholds.max_p95_ms
)
return AcceptanceReport(
query_count=query_count,
recall_at_1=recall_at_1,
recall_at_3=recall_at_3,
recall_at_5=recall_at_5,
mrr=mrr,
ndcg_at_5=ndcg_at_5,
latency_p50_ms=p50,
latency_p95_ms=p95,
passed=passed,
)
def _thresholds_from_environment() -> AcceptanceThresholds:
return AcceptanceThresholds(
min_recall_at_1=float(os.getenv("EKP_LIVE_MIN_RECALL_AT_1", str(DEFAULT_MIN_RECALL_AT_1))),
min_recall_at_5=float(os.getenv("EKP_LIVE_MIN_RECALL_AT_5", str(DEFAULT_MIN_RECALL_AT_5))),
min_mrr=float(os.getenv("EKP_LIVE_MIN_MRR", str(DEFAULT_MIN_MRR))),
min_ndcg_at_5=float(os.getenv("EKP_LIVE_MIN_NDCG_AT_5", str(DEFAULT_MIN_NDCG_AT_5))),
max_p95_ms=float(os.getenv("EKP_LIVE_MAX_P95_MS", str(DEFAULT_MAX_P95_MS))),
)
def _require_safe_live_settings(settings: Settings) -> None:
"""Refuse external writes unless rotation and resource authorization are explicit."""
required_flags = (
"EKP_RUN_LIVE_TESTS",
"EKP_CREDENTIALS_AUTHORIZED",
"EKP_LIVE_RESOURCE_ACK",
)
if any(os.getenv(name) != "1" for name in required_flags):
raise LiveAcceptanceError("live acceptance safety flags are not enabled")
if settings.app_env != "test":
raise LiveAcceptanceError("live acceptance requires APP_ENV=test")
if settings.object_storage_provider != "bos" or settings.vector_store_provider != "oceanbase":
raise LiveAcceptanceError("live acceptance requires the target provider baseline")
if not settings.bos_key_prefix.startswith("live-acceptance/"):
raise LiveAcceptanceError("BOS_KEY_PREFIX must use a dedicated live-acceptance prefix")
def _cleanup_run_documents(services: ApiServices, filename_prefix: str) -> None:
"""Delete only documents created under this random acceptance-run prefix."""
documents = tuple(
document
for document in services.documents.list_documents()
if document.filename.startswith(filename_prefix)
)
for document in documents:
services.documents.delete_document(document.id)
def _run_live(settings: Settings) -> AcceptanceReport:
"""Upload synthetic sources, execute every query, and clean run-owned data."""
_require_safe_live_settings(settings)
project_root = _project_root()
golden = _load_golden_queries(project_root / "fixtures/evaluation/golden.jsonl")
services = create_api_services(settings)
filename_prefix = f"live-{uuid4().hex[:12]}-"
uploaded_names: dict[str, str] = {}
cleanup_error: Exception | None = None
try:
source_names = sorted({record.expected_document for record in golden})
for source_name in source_names:
live_name = f"{filename_prefix}{source_name}"
payload = (project_root / "fixtures/documents" / source_name).read_bytes()
services.ingestion.ingest(live_name, "text/markdown", BytesIO(payload))
uploaded_names[source_name] = live_name
ranks: list[int | None] = []
latencies_ms: list[float] = []
for record in golden:
started = perf_counter()
result = services.search.search(record.query, top_k=5)
latencies_ms.append((perf_counter() - started) * 1000)
ranks.append(
_matching_rank(
result.results,
expected_filename=uploaded_names[record.expected_document],
expected_sections=record.expected_sections,
)
)
return _build_report(ranks, latencies_ms, _thresholds_from_environment())
finally:
try:
_cleanup_run_documents(services, filename_prefix)
except Exception as error:
cleanup_error = error
if cleanup_error is not None:
raise LiveAcceptanceError("live acceptance cleanup failed") from cleanup_error
def main() -> int:
"""Print only aggregate metrics and return non-zero for unsafe or failed runs."""
try:
report = _run_live(Settings())
except Exception:
print("live acceptance failed; no provider details were emitted", file=sys.stderr)
return 1
print(json.dumps(asdict(report), sort_keys=True, separators=(",", ":")))
return 0 if report.passed else 2
if __name__ == "__main__":
raise SystemExit(main())

View File

@ -1,12 +1,18 @@
"""FastAPI application entry point.""" """FastAPI application entry point."""
from pathlib import Path
from fastapi import FastAPI from fastapi import FastAPI
from fastapi.responses import FileResponse
from fastapi.staticfiles import StaticFiles
from pydantic import BaseModel from pydantic import BaseModel
from ekp_api.providers import ApiServices from ekp_api.providers import ApiServices
from ekp_api.routes import router from ekp_api.routes import router
from ekp_core.config import Settings, get_settings from ekp_core.config import Settings, get_settings
ADMIN_WEB_ROOT = Path(__file__).resolve().parents[3] / "admin-web"
class HealthResponse(BaseModel): class HealthResponse(BaseModel):
"""Health endpoint response.""" """Health endpoint response."""
@ -31,6 +37,23 @@ def create_app(
application.state.settings = resolved_settings application.state.settings = resolved_settings
application.state.api_services = api_services application.state.api_services = api_services
application.include_router(router, prefix=resolved_settings.api_prefix) application.include_router(router, prefix=resolved_settings.api_prefix)
admin_web_root = (resolved_settings.admin_web_root or ADMIN_WEB_ROOT).resolve()
# The Demo console is a deliberately thin, same-origin client. It contains
# no business logic and calls only the versioned public REST API.
application.mount(
"/admin/assets",
StaticFiles(directory=admin_web_root / "assets"),
name="admin-assets",
)
@application.get("/admin", include_in_schema=False)
@application.get("/admin/upload", include_in_schema=False)
@application.get("/admin/documents", include_in_schema=False)
@application.get("/admin/search", include_in_schema=False)
async def admin_console() -> FileResponse:
"""Serve the three-page Demo Admin Console shell."""
return FileResponse(admin_web_root / "index.html")
@application.get("/healthz", response_model=HealthResponse, tags=["system"]) @application.get("/healthz", response_model=HealthResponse, tags=["system"])
async def health() -> HealthResponse: async def health() -> HealthResponse:

View File

@ -4,6 +4,7 @@ from dataclasses import dataclass
from sqlalchemy.orm import Session, sessionmaker from sqlalchemy.orm import Session, sessionmaker
from ekp_api.celery_queue import CeleryIngestionDispatcher, create_celery_app
from ekp_core.chunking import StructureAwareChunker from ekp_core.chunking import StructureAwareChunker
from ekp_core.config import Settings from ekp_core.config import Settings
from ekp_core.database import create_database_engine, create_session_factory from ekp_core.database import create_database_engine, create_session_factory
@ -16,6 +17,14 @@ from ekp_core.document_indexing import (
from ekp_core.document_management import DocumentManagementService from ekp_core.document_management import DocumentManagementService
from ekp_core.document_parsing import DocumentParsingService from ekp_core.document_parsing import DocumentParsingService
from ekp_core.document_upload import DocumentUploadService from ekp_core.document_upload import DocumentUploadService
from ekp_core.index_rebuild import resolve_active_index_profile
from ekp_core.ingestion_batches import IngestionBatchService
from ekp_core.ingestion_jobs import (
AsyncIngestionSubmissionService,
DerivedDocumentCleanupService,
IngestionJobRunner,
IngestionJobService,
)
from ekp_core.search import SearchService from ekp_core.search import SearchService
from ekp_domain import IndexProfile from ekp_domain import IndexProfile
from ekp_embedding import EmbeddingProvider, QianfanEmbeddingProvider from ekp_embedding import EmbeddingProvider, QianfanEmbeddingProvider
@ -23,7 +32,10 @@ from ekp_object_storage import ObjectStorage, create_bos_object_storage
from ekp_object_storage.local import LocalObjectStorage from ekp_object_storage.local import LocalObjectStorage
from ekp_parser import ParserRouter from ekp_parser import ParserRouter
from ekp_reranker import QianfanRerankerProvider, RerankerProvider from ekp_reranker import QianfanRerankerProvider, RerankerProvider
from ekp_vector_store import MochowVectorStore, VectorStore, create_mochow_vector_store from ekp_vector_store import (
VectorStore,
create_oceanbase_vector_store,
)
BGE_LARGE_ZH_DIMENSIONS = 1024 BGE_LARGE_ZH_DIMENSIONS = 1024
DEMO_CHUNK_STRATEGY_VERSION = "structure_v1" DEMO_CHUNK_STRATEGY_VERSION = "structure_v1"
@ -38,18 +50,53 @@ class ApiServices:
"""Business services injected into REST routes without provider leakage.""" """Business services injected into REST routes without provider leakage."""
ingestion: DocumentIndexingPipelineService ingestion: DocumentIndexingPipelineService
submissions: AsyncIngestionSubmissionService
batches: IngestionBatchService
jobs: IngestionJobService
documents: DocumentManagementService documents: DocumentManagementService
search: SearchService search: SearchService
def create_index_profile(settings: Settings) -> IndexProfile: @dataclass(frozen=True, slots=True)
"""Bind the Demo's chunk and embedding versions without vendor SDK types.""" class WorkerServices:
return IndexProfile( """Provider graph required by Celery without REST search-only services."""
runner: IngestionJobRunner
jobs: IngestionJobService
dispatcher: CeleryIngestionDispatcher
@dataclass(frozen=True, slots=True)
class _IngestionComponents:
"""Shared composition result used by API and independent worker processes."""
session_factory: sessionmaker[Session]
storage: ObjectStorage
vector_store: VectorStore
embedding_provider: EmbeddingProvider
index_profile: IndexProfile
upload: DocumentUploadService
parsing: DocumentParsingService
chunking: DocumentChunkingService
indexing: DocumentIndexingService
jobs: IngestionJobService
dispatcher: CeleryIngestionDispatcher
def create_index_profile(
settings: Settings,
session_factory: sessionmaker[Session] | None = None,
) -> IndexProfile:
"""Resolve the canonical active profile or the fresh-deployment baseline."""
configured = IndexProfile(
profile_id=settings.index_profile_id, profile_id=settings.index_profile_id,
chunk_strategy_version=DEMO_CHUNK_STRATEGY_VERSION, chunk_strategy_version=DEMO_CHUNK_STRATEGY_VERSION,
embedding_model=settings.embedding_model, embedding_model=settings.embedding_model,
vector_dimension=BGE_LARGE_ZH_DIMENSIONS, vector_dimension=BGE_LARGE_ZH_DIMENSIONS,
) )
if session_factory is None:
return configured
return resolve_active_index_profile(session_factory, configured)
def create_embedding_provider(settings: Settings) -> QianfanEmbeddingProvider: def create_embedding_provider(settings: Settings) -> QianfanEmbeddingProvider:
@ -85,23 +132,26 @@ def create_object_storage(settings: Settings) -> ObjectStorage:
) )
def create_vector_store(settings: Settings) -> MochowVectorStore: def create_vector_store(settings: Settings) -> VectorStore:
"""Create the Mochow adapter for an explicitly configured database/table.""" """Create the OceanBase adapter from explicit, secret-safe settings."""
if settings.baidu_mochow_api_key is None: if settings.oceanbase_vector_uri is None:
raise ProviderConfigurationError("BAIDU_MOCHOW_API_KEY is required") raise ProviderConfigurationError("OCEANBASE_VECTOR_URI is required")
if not settings.baidu_mochow_endpoint: if settings.oceanbase_vector_user is None:
raise ProviderConfigurationError("BAIDU_MOCHOW_ENDPOINT is required") raise ProviderConfigurationError("OCEANBASE_VECTOR_USER is required")
if not settings.knowledge_vdb_db_name: if settings.oceanbase_vector_password is None:
raise ProviderConfigurationError("KNOWLEDGE_VDB_DB_NAME is required") raise ProviderConfigurationError("OCEANBASE_VECTOR_PASSWORD is required")
return create_mochow_vector_store( if settings.oceanbase_vector_database is None:
account=settings.baidu_mochow_account, raise ProviderConfigurationError("OCEANBASE_VECTOR_DATABASE is required")
api_key=settings.baidu_mochow_api_key.get_secret_value(), return create_oceanbase_vector_store(
endpoint=settings.baidu_mochow_endpoint, uri=settings.oceanbase_vector_uri,
database_name=settings.knowledge_vdb_db_name, user=settings.oceanbase_vector_user,
table_name=settings.mochow_table, password=settings.oceanbase_vector_password.get_secret_value(),
batch_size=settings.mochow_batch_size, database_name=settings.oceanbase_vector_database,
search_ef=settings.mochow_search_ef, table_name=settings.oceanbase_vector_table,
timeout_seconds=settings.mochow_timeout_seconds, index_name=settings.oceanbase_vector_index,
vector_dimension=BGE_LARGE_ZH_DIMENSIONS,
batch_size=settings.oceanbase_vector_batch_size,
timeout_seconds=settings.oceanbase_vector_timeout_seconds,
) )
@ -121,7 +171,7 @@ def create_dense_retrieval_service(
def create_reranker_provider(settings: Settings) -> QianfanRerankerProvider: def create_reranker_provider(settings: Settings) -> QianfanRerankerProvider:
"""Create the Demo Qwen3 reranker using an explicit or shared Qianfan key.""" """Create the Demo BCE reranker using an explicit or shared Qianfan key."""
secret = settings.reranker_api_key or settings.baidu_qianfan_api_key secret = settings.reranker_api_key or settings.baidu_qianfan_api_key
if secret is None: if secret is None:
raise ProviderConfigurationError("RERANKER_API_KEY or BAIDU_QIANFAN_API_KEY is required") raise ProviderConfigurationError("RERANKER_API_KEY or BAIDU_QIANFAN_API_KEY is required")
@ -148,12 +198,35 @@ def create_search_service(
) )
def create_api_services(settings: Settings) -> ApiServices: def create_runtime_search_service(settings: Settings) -> SearchService:
"""Compose the synchronous Demo runtime once for all business endpoints.""" """Compose only read-path providers for the independent MCP Gateway."""
engine = create_database_engine(settings)
session_factory = create_session_factory(engine)
index_profile = create_index_profile(settings, session_factory)
embedding_provider = create_embedding_provider(settings)
vector_store = create_vector_store(settings)
dense_retrieval = create_dense_retrieval_service(
settings,
embedding_provider,
vector_store,
index_profile,
)
return create_search_service(
settings,
session_factory,
dense_retrieval,
create_reranker_provider(settings),
)
def _create_ingestion_components(settings: Settings) -> _IngestionComponents:
"""Compose provider-neutral ingestion stages and the Celery delivery adapter."""
if settings.rabbitmq_url is None or not settings.rabbitmq_url.get_secret_value().strip():
raise ProviderConfigurationError("RABBITMQ_URL is required")
engine = create_database_engine(settings) engine = create_database_engine(settings)
session_factory = create_session_factory(engine) session_factory = create_session_factory(engine)
storage = create_object_storage(settings) storage = create_object_storage(settings)
index_profile = create_index_profile(settings) index_profile = create_index_profile(settings, session_factory)
embedding_provider = create_embedding_provider(settings) embedding_provider = create_embedding_provider(settings)
vector_store = create_vector_store(settings) vector_store = create_vector_store(settings)
@ -172,35 +245,103 @@ def create_api_services(settings: Settings) -> ApiServices:
max_characters=settings.chunk_max_characters, max_characters=settings.chunk_max_characters,
), ),
) )
document_pipeline = DocumentPipelineService(
upload_service,
parsing_service,
chunking_service,
)
indexing_service = DocumentIndexingService( indexing_service = DocumentIndexingService(
session_factory, session_factory,
embedding_provider, embedding_provider,
vector_store, vector_store,
index_profile, index_profile,
) )
ingestion = DocumentIndexingPipelineService(document_pipeline, indexing_service) jobs = IngestionJobService(session_factory, settings.ingestion_max_attempts)
dispatcher = CeleryIngestionDispatcher(create_celery_app(settings))
return _IngestionComponents(
session_factory=session_factory,
storage=storage,
vector_store=vector_store,
embedding_provider=embedding_provider,
index_profile=index_profile,
upload=upload_service,
parsing=parsing_service,
chunking=chunking_service,
indexing=indexing_service,
jobs=jobs,
dispatcher=dispatcher,
)
def _create_runner(settings: Settings, components: _IngestionComponents) -> IngestionJobRunner:
"""Compose retry cleanup and execution policy from non-sensitive settings."""
cleanup = DerivedDocumentCleanupService(
components.session_factory,
components.storage,
components.vector_store,
components.index_profile.profile_id,
)
return IngestionJobRunner(
components.session_factory,
components.parsing,
components.chunking,
components.indexing,
cleanup,
retry_backoff_seconds=settings.ingestion_retry_backoff_seconds,
retry_backoff_max_seconds=settings.ingestion_retry_backoff_max_seconds,
lease_seconds=settings.ingestion_worker_lease_seconds,
)
def create_worker_services(settings: Settings) -> WorkerServices:
"""Build the independent worker runtime from the same application boundaries."""
components = _create_ingestion_components(settings)
return WorkerServices(
runner=_create_runner(settings, components),
jobs=components.jobs,
dispatcher=components.dispatcher,
)
def create_api_services(settings: Settings) -> ApiServices:
"""Compose REST services while ingestion execution remains in workers."""
components = _create_ingestion_components(settings)
document_pipeline = DocumentPipelineService(
components.upload,
components.parsing,
components.chunking,
)
ingestion = DocumentIndexingPipelineService(document_pipeline, components.indexing)
submissions = AsyncIngestionSubmissionService(
components.upload,
components.jobs,
components.dispatcher,
)
batches = IngestionBatchService(
components.session_factory,
submissions,
max_files=settings.batch_max_files,
max_total_bytes=settings.batch_max_total_bytes,
)
dense_retrieval = create_dense_retrieval_service( dense_retrieval = create_dense_retrieval_service(
settings, settings,
embedding_provider, components.embedding_provider,
vector_store, components.vector_store,
index_profile, components.index_profile,
) )
reranker_provider = create_reranker_provider(settings) reranker_provider = create_reranker_provider(settings)
search = create_search_service( search = create_search_service(
settings, settings,
session_factory, components.session_factory,
dense_retrieval, dense_retrieval,
reranker_provider, reranker_provider,
) )
documents = DocumentManagementService( documents = DocumentManagementService(
session_factory, components.session_factory,
storage, components.storage,
vector_store, components.vector_store,
active_index_profile_id=index_profile.profile_id, active_index_profile_id=components.index_profile.profile_id,
)
return ApiServices(
ingestion=ingestion,
submissions=submissions,
batches=batches,
jobs=components.jobs,
documents=documents,
search=search,
) )
return ApiServices(ingestion=ingestion, documents=documents, search=search)

View File

@ -6,10 +6,18 @@ from uuid import UUID
from fastapi import APIRouter, Depends, File, HTTPException, Request, Response, UploadFile, status from fastapi import APIRouter, Depends, File, HTTPException, Request, Response, UploadFile, status
from ekp_api.providers import ApiServices, ProviderConfigurationError, create_api_services from ekp_api.providers import ApiServices, ProviderConfigurationError, create_api_services
from ekp_api.schemas import DocumentResponse, SearchRequest, SearchResponse from ekp_api.schemas import (
DocumentResponse,
IngestionAcceptedResponse,
IngestionBatchResponse,
IngestionJobResponse,
SearchRequest,
SearchResponse,
)
from ekp_core.dense_retrieval import InvalidRetrievalQueryError from ekp_core.dense_retrieval import InvalidRetrievalQueryError
from ekp_core.document_management import ( from ekp_core.document_management import (
DocumentArtifactCleanupError, DocumentArtifactCleanupError,
DocumentIngestionActiveError,
DocumentNotFoundError, DocumentNotFoundError,
) )
from ekp_core.document_upload import ( from ekp_core.document_upload import (
@ -18,7 +26,8 @@ from ekp_core.document_upload import (
MediaTypeMismatchError, MediaTypeMismatchError,
UnsupportedDocumentTypeError, UnsupportedDocumentTypeError,
) )
from ekp_parser import DocumentParseError from ekp_core.ingestion_batches import BatchFileCountLimitError, BatchNotFoundError, BatchSource
from ekp_core.ingestion_jobs import IngestionJobNotFoundError
router = APIRouter() router = APIRouter()
@ -44,17 +53,17 @@ Services = Annotated[ApiServices, Depends(get_api_services)]
@router.post( @router.post(
"/documents", "/documents",
response_model=DocumentResponse, response_model=IngestionAcceptedResponse,
status_code=status.HTTP_201_CREATED, status_code=status.HTTP_202_ACCEPTED,
tags=["documents"], tags=["documents"],
) )
def upload_document( def upload_document(
services: Services, services: Services,
file: Annotated[UploadFile, File(description="One PDF, DOCX, TXT, or Markdown file")], file: Annotated[UploadFile, File(description="One PDF, DOCX, TXT, or Markdown file")],
) -> DocumentResponse: ) -> IngestionAcceptedResponse:
"""Upload one file and synchronously run the complete Demo ingestion pipeline.""" """Persist one immutable source and durable Job, then return immediately."""
try: try:
result = services.ingestion.ingest(file.filename or "", file.content_type, file.file) submission = services.submissions.submit(file.filename or "", file.content_type, file.file)
except DocumentTooLargeError as error: except DocumentTooLargeError as error:
raise HTTPException( raise HTTPException(
status_code=status.HTTP_413_CONTENT_TOO_LARGE, status_code=status.HTTP_413_CONTENT_TOO_LARGE,
@ -65,21 +74,85 @@ def upload_document(
status_code=status.HTTP_415_UNSUPPORTED_MEDIA_TYPE, status_code=status.HTTP_415_UNSUPPORTED_MEDIA_TYPE,
detail="unsupported document type or media type", detail="unsupported document type or media type",
) from error ) from error
except (DocumentUploadError, DocumentParseError) as error: except DocumentUploadError as error:
raise HTTPException( raise HTTPException(
status_code=status.HTTP_422_UNPROCESSABLE_CONTENT, status_code=status.HTTP_422_UNPROCESSABLE_CONTENT,
detail="document could not be accepted or parsed", detail="document could not be accepted",
) from error ) from error
except Exception as error: except Exception as error:
# Provider and persistence exceptions are intentionally not reflected to clients. # Provider and persistence exceptions are intentionally not reflected to clients.
raise HTTPException( raise HTTPException(
status_code=status.HTTP_502_BAD_GATEWAY, status_code=status.HTTP_502_BAD_GATEWAY,
detail="document processing failed", detail="document acceptance failed",
) from error ) from error
# Reload after every pipeline stage so server-generated timestamps and the return IngestionAcceptedResponse.from_submission(submission)
# final persisted status, not a detached intermediate ORM object, are returned.
document = services.documents.get_document(result.ingestion.document.id)
return DocumentResponse.from_document(document) @router.get("/jobs/{job_id}", response_model=IngestionJobResponse, tags=["jobs"])
def get_ingestion_job(job_id: UUID, services: Services) -> IngestionJobResponse:
"""Return durable stage, attempt and safe failure state for one Job."""
try:
job = services.jobs.get(job_id)
except IngestionJobNotFoundError as error:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND, detail="job not found"
) from error
return IngestionJobResponse.from_job(job)
@router.post(
"/ingestion-batches",
response_model=IngestionBatchResponse,
status_code=status.HTTP_202_ACCEPTED,
tags=["jobs"],
)
def upload_ingestion_batch(
services: Services,
files: Annotated[
list[UploadFile],
File(description="One or more PDF, DOCX, TXT, or Markdown files"),
],
) -> IngestionBatchResponse:
"""Accept a bounded batch while isolating every file's result."""
try:
view = services.batches.submit(
tuple(
BatchSource(
filename=file.filename or "",
media_type=file.content_type,
source=file.file,
)
for file in files
)
)
except BatchFileCountLimitError as error:
raise HTTPException(
status_code=status.HTTP_413_CONTENT_TOO_LARGE,
detail="batch file count exceeds the configured limit",
) from error
except Exception as error:
raise HTTPException(
status_code=status.HTTP_502_BAD_GATEWAY,
detail="batch could not be accepted",
) from error
return IngestionBatchResponse.from_view(view)
@router.get(
"/ingestion-batches/{batch_id}",
response_model=IngestionBatchResponse,
tags=["jobs"],
)
def get_ingestion_batch(batch_id: UUID, services: Services) -> IngestionBatchResponse:
"""Return aggregate and per-file status for Admin Console polling."""
try:
view = services.batches.get(batch_id)
except BatchNotFoundError as error:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail="ingestion batch not found",
) from error
return IngestionBatchResponse.from_view(view)
@router.get("/documents", response_model=list[DocumentResponse], tags=["documents"]) @router.get("/documents", response_model=list[DocumentResponse], tags=["documents"])
@ -115,6 +188,11 @@ def delete_document(document_id: UUID, services: Services) -> Response:
raise HTTPException( raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND, detail="document not found" status_code=status.HTTP_404_NOT_FOUND, detail="document not found"
) from error ) from error
except DocumentIngestionActiveError as error:
raise HTTPException(
status_code=status.HTTP_409_CONFLICT,
detail="document ingestion is still active",
) from error
except DocumentArtifactCleanupError as error: except DocumentArtifactCleanupError as error:
raise HTTPException( raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,

View File

@ -5,8 +5,17 @@ from uuid import UUID
from pydantic import BaseModel, ConfigDict, Field from pydantic import BaseModel, ConfigDict, Field
from ekp_core.ingestion_batches import BatchAggregateStatus, BatchItemView, BatchView
from ekp_core.ingestion_jobs import IngestionSubmission
from ekp_core.search import SearchResult from ekp_core.search import SearchResult
from ekp_domain import Document, DocumentStatus from ekp_domain import (
BatchItemAcceptanceStatus,
Document,
DocumentStatus,
IngestionJob,
IngestionJobStatus,
IngestionStage,
)
class DocumentResponse(BaseModel): class DocumentResponse(BaseModel):
@ -29,6 +38,138 @@ class DocumentResponse(BaseModel):
return cls.model_validate(document) return cls.model_validate(document)
class IngestionAcceptedResponse(DocumentResponse):
"""Single-file acceptance with the durable Job needed for status polling."""
job_id: UUID
job_status: IngestionJobStatus
max_attempts: int
@classmethod
def from_submission(cls, submission: IngestionSubmission) -> "IngestionAcceptedResponse":
"""Flatten identifiers while retaining the existing document fields."""
document = submission.document
job = submission.job
return cls(
id=document.id,
filename=document.filename,
media_type=document.media_type,
size_bytes=document.size_bytes,
status=document.status,
error_message=document.error_message,
created_at=document.created_at,
updated_at=document.updated_at,
job_id=job.id,
job_status=job.status,
max_attempts=job.max_attempts,
)
class IngestionJobResponse(BaseModel):
"""Public queue-independent Job status without provider or broker details."""
id: UUID
document_id: UUID
status: IngestionJobStatus
stage: IngestionStage | None
attempt: int
max_attempts: int
error_message: str | None
created_at: datetime
updated_at: datetime
completed_at: datetime | None
@classmethod
def from_job(cls, job: IngestionJob) -> "IngestionJobResponse":
"""Serialize only client-actionable state and safe diagnostics."""
return cls(
id=job.id,
document_id=job.document_id,
status=job.status,
stage=job.stage,
attempt=job.attempt_count,
max_attempts=job.max_attempts,
error_message=job.error_message,
created_at=job.created_at,
updated_at=job.updated_at,
completed_at=job.completed_at,
)
class IngestionBatchItemResponse(BaseModel):
"""Per-file acceptance and latest asynchronous processing state."""
item_index: int
filename: str
acceptance_status: BatchItemAcceptanceStatus
document_id: UUID | None
job_id: UUID | None
job_status: IngestionJobStatus | None
stage: IngestionStage | None
attempt: int
max_attempts: int | None
error_message: str | None
@classmethod
def from_view(cls, view: BatchItemView) -> "IngestionBatchItemResponse":
job = view.job
return cls(
item_index=view.item.item_index,
filename=view.item.filename,
acceptance_status=view.item.acceptance_status,
document_id=view.item.document_id,
job_id=view.item.job_id,
job_status=job.status if job is not None else None,
stage=job.stage if job is not None else None,
attempt=job.attempt_count if job is not None else 0,
max_attempts=job.max_attempts if job is not None else None,
error_message=(job.error_message if job is not None else view.item.error_message),
)
class IngestionBatchResponse(BaseModel):
"""Batch aggregate with ordered independently processed files."""
batch_id: UUID
status: BatchAggregateStatus
total_files: int
accepted_files: int
rejected_files: int
completed_files: int
created_at: datetime
items: list[IngestionBatchItemResponse]
@classmethod
def from_view(cls, view: BatchView) -> "IngestionBatchResponse":
items = [IngestionBatchItemResponse.from_view(item) for item in view.items]
accepted = sum(
item.acceptance_status == BatchItemAcceptanceStatus.ACCEPTED for item in items
)
rejected = sum(
item.acceptance_status == BatchItemAcceptanceStatus.REJECTED for item in items
)
completed = sum(
item.acceptance_status == BatchItemAcceptanceStatus.REJECTED
or item.job_status
in {
IngestionJobStatus.SUCCEEDED,
IngestionJobStatus.FAILED,
IngestionJobStatus.CANCELLED,
}
for item in items
)
return cls(
batch_id=view.batch.id,
status=view.status,
total_files=view.batch.total_files,
accepted_files=accepted,
rejected_files=rejected,
completed_files=completed,
created_at=view.batch.created_at,
items=items,
)
class SearchRequest(BaseModel): class SearchRequest(BaseModel):
"""Validated public search input; normalization remains in the core service.""" """Validated public search input; normalization remains in the core service."""

View File

@ -0,0 +1,51 @@
"""Celery worker entry point for durable ingestion Jobs."""
from functools import lru_cache
from typing import Any
from uuid import UUID
from ekp_api.celery_queue import (
INGESTION_TASK_NAME,
RECOVERY_TASK_NAME,
create_celery_app,
)
from ekp_api.providers import WorkerServices, create_worker_services
from ekp_core.config import get_settings
from ekp_core.ingestion_jobs import JobRunAction, dispatch_pending_jobs
_settings = get_settings()
celery_app = create_celery_app(_settings)
@lru_cache
def _worker_services() -> WorkerServices:
"""Build one provider/service graph per worker process."""
return create_worker_services(_settings)
@celery_app.task( # type: ignore[untyped-decorator]
bind=True,
name=INGESTION_TASK_NAME,
max_retries=None,
)
def run_ingestion(self: Any, job_id: str) -> None:
"""Execute one delivery and ask Celery to redeliver only when required.
Celery delivery retries are unbounded because valid-lease deferrals are not
business Attempts; the durable runner independently enforces max_attempts.
"""
result = _worker_services().runner.run(UUID(job_id))
if result.action == JobRunAction.RETRY:
raise self.retry(countdown=result.retry_delay_seconds or 1, max_retries=None)
@celery_app.task(name=RECOVERY_TASK_NAME) # type: ignore[untyped-decorator]
def recover_ingestion_jobs() -> int:
"""Redeliver due durable Jobs after API/broker or worker interruptions."""
services = _worker_services()
return dispatch_pending_jobs(services.jobs, services.dispatcher)
def main() -> None:
"""Run one worker with the embedded Demo recovery scheduler."""
celery_app.worker_main(["worker", "--loglevel=INFO", "--queues=ingestion", "--beat"])

View File

@ -0,0 +1,5 @@
"""Enterprise Knowledge MCP gateway."""
from ekp_mcp.server import create_mcp_http_app, create_mcp_server
__all__ = ["create_mcp_http_app", "create_mcp_server"]

View File

@ -0,0 +1,76 @@
"""Request-scoped MCP service identity and bearer-token verification."""
from dataclasses import dataclass
from secrets import compare_digest
from mcp.server.auth.middleware.auth_context import get_access_token
from mcp.server.auth.provider import AccessToken
from mcp.server.mcpserver.exceptions import ToolError
from pydantic import SecretStr
@dataclass(frozen=True, slots=True)
class McpIdentity:
"""Authenticated client/service identity passed across the application boundary."""
client_id: str
subject: str
scopes: frozenset[str]
class StaticServiceTokenVerifier:
"""Validate one runtime-injected, least-privilege Demo service credential.
The verifier is intentionally small for DEMO-012. It establishes the MCP
identity boundary without pretending to be the V1 OAuth or ACL solution.
"""
__slots__ = (
"_client_id",
"_expected_token",
"_issuer",
"_scope",
"_subject",
)
def __init__(
self,
expected_token: SecretStr,
*,
client_id: str,
subject: str,
scope: str,
issuer: str,
) -> None:
self._expected_token = expected_token
self._client_id = client_id
self._subject = subject
self._scope = scope
self._issuer = issuer
async def verify_token(self, token: str) -> AccessToken | None:
"""Return scoped identity metadata only for a constant-time token match."""
if not compare_digest(token, self._expected_token.get_secret_value()):
return None
return AccessToken(
token=token,
client_id=self._client_id,
subject=self._subject,
scopes=[self._scope],
claims={"iss": self._issuer},
)
def current_mcp_identity() -> McpIdentity:
"""Resolve the SDK-authenticated principal for the active tool request."""
access_token = get_access_token()
if access_token is None:
# HTTP middleware normally rejects this earlier; retaining the guard
# prevents accidental unauthenticated in-process tool invocation.
raise ToolError("authentication required")
subject = access_token.subject or access_token.client_id
return McpIdentity(
client_id=access_token.client_id,
subject=subject,
scopes=frozenset(access_token.scopes),
)

View File

@ -0,0 +1,30 @@
"""Command-line entry point for the independent MCP gateway process."""
from mcp.server.transport_security import TransportSecuritySettings
from ekp_core.config import get_settings
from ekp_mcp.server import create_mcp_server
def main() -> None:
"""Run the authenticated Streamable HTTP server from environment settings."""
settings = get_settings()
server = create_mcp_server(settings)
server.run(
transport="streamable-http",
host=settings.mcp_host,
port=settings.mcp_port,
streamable_http_path=settings.mcp_path,
json_response=True,
stateless_http=True,
max_request_body_size=settings.mcp_max_request_body_bytes,
transport_security=TransportSecuritySettings(
enable_dns_rebinding_protection=True,
allowed_hosts=settings.mcp_allowed_hosts,
allowed_origins=settings.mcp_allowed_origins,
),
)
if __name__ == "__main__": # pragma: no cover - exercised through the console script.
main()

View File

@ -0,0 +1,77 @@
"""MCP-specific application boundary over the provider-neutral SearchService."""
from dataclasses import dataclass
from typing import Protocol
from uuid import UUID
from pydantic import BaseModel
from ekp_core.search import SearchResult
from ekp_mcp.auth import McpIdentity
class KnowledgeSearcher(Protocol):
"""Subset of SearchService required by the MCP gateway."""
def search(self, query: str, top_k: int | None = None) -> SearchResult:
"""Return reranked canonical sources for a normalized query."""
class McpSourceResponse(BaseModel):
"""Provider-neutral source shape exposed by `search_knowledge`."""
document_id: UUID
filename: str
page: int | None
section: str | None
chunk_id: UUID
content: str
score: float
rerank_score: float
class McpSearchResponse(BaseModel):
"""Structured MCP result aligned with the REST search source fields."""
results: list[McpSourceResponse]
class McpKnowledgeSearch(Protocol):
"""Identity-aware search boundary consumed by the protocol server."""
def search(self, identity: McpIdentity, query: str, top_k: int) -> McpSearchResponse:
"""Return authorized, structured knowledge sources."""
@dataclass(frozen=True, slots=True)
class McpSearchApplication:
"""Require an identity before delegating to the shared SearchService.
Document ACL is outside the current Demo scope. Keeping identity explicit
here prevents the gateway from becoming an anonymous bypass when the
permission resolver is added in the V1 tasks.
"""
searcher: KnowledgeSearcher
required_scope: str
def search(self, identity: McpIdentity, query: str, top_k: int) -> McpSearchResponse:
"""Authorize the Demo scope and serialize canonical search results."""
if self.required_scope not in identity.scopes:
raise PermissionError("MCP identity lacks the knowledge search scope")
result = self.searcher.search(query, top_k)
return McpSearchResponse(
results=[
McpSourceResponse(
document_id=source.document_id,
filename=source.filename,
page=source.page,
section=source.section,
chunk_id=source.chunk_id,
content=source.content,
score=source.score,
rerank_score=source.rerank_score,
)
for source in result.results
]
)

View File

@ -0,0 +1,133 @@
"""Authenticated Streamable HTTP MCP server exposing Demo business tools."""
import logging
from functools import partial
from typing import Annotated
import anyio
from mcp.server import MCPServer
from mcp.server.auth.settings import AuthSettings
from mcp.server.mcpserver.exceptions import ToolError
from mcp.server.transport_security import TransportSecuritySettings
from pydantic import Field
from starlette.applications import Starlette
from ekp_api.providers import ApiServices, create_runtime_search_service
from ekp_core.config import Settings, get_settings
from ekp_core.dense_retrieval import InvalidRetrievalQueryError
from ekp_mcp.auth import StaticServiceTokenVerifier, current_mcp_identity
from ekp_mcp.search import McpKnowledgeSearch, McpSearchApplication, McpSearchResponse
logger = logging.getLogger(__name__)
class McpConfigurationError(ValueError):
"""Raised when the gateway lacks its dedicated runtime credential."""
def _create_search_application(
settings: Settings,
api_services: ApiServices | None,
) -> McpSearchApplication:
searcher = (
api_services.search if api_services is not None else create_runtime_search_service(settings)
)
return McpSearchApplication(searcher, settings.mcp_required_scope)
def create_mcp_server(
settings: Settings | None = None,
*,
api_services: ApiServices | None = None,
search_application: McpKnowledgeSearch | None = None,
) -> MCPServer[dict[str, object]]:
"""Create the authenticated MCP gateway with only `search_knowledge`."""
resolved_settings = settings or get_settings()
if (
resolved_settings.mcp_auth_token is None
or len(resolved_settings.mcp_auth_token.get_secret_value()) < 32
):
raise McpConfigurationError("MCP_AUTH_TOKEN must contain at least 32 characters")
application = search_application or _create_search_application(
resolved_settings,
api_services,
)
token_verifier = StaticServiceTokenVerifier(
resolved_settings.mcp_auth_token,
client_id=resolved_settings.mcp_client_id,
subject=resolved_settings.mcp_service_identity,
scope=resolved_settings.mcp_required_scope,
issuer=str(resolved_settings.mcp_issuer_url),
)
server: MCPServer[dict[str, object]] = MCPServer(
name="enterprise-knowledge",
title="Enterprise Knowledge MCP Server",
description="Authenticated enterprise knowledge search backed by canonical sources.",
version="0.1.0",
token_verifier=token_verifier,
auth=AuthSettings(
issuer_url=resolved_settings.mcp_issuer_url,
resource_server_url=resolved_settings.mcp_public_url,
required_scopes=[resolved_settings.mcp_required_scope],
),
log_level=resolved_settings.log_level,
)
@server.tool(
name="search_knowledge",
description=(
"Search authorized enterprise knowledge and return traceable source passages. "
"Use for factual questions that should be grounded in indexed company documents."
),
structured_output=True,
)
async def search_knowledge(
query: Annotated[str, Field(min_length=1, max_length=1600)],
top_k: Annotated[int, Field(ge=1, le=64, strict=True)] = 5,
) -> McpSearchResponse:
"""Run the shared synchronous search pipeline outside the ASGI event loop."""
identity = current_mcp_identity()
try:
return await anyio.to_thread.run_sync(
partial(application.search, identity, query, top_k)
)
except InvalidRetrievalQueryError as error:
raise ToolError("search query must contain non-whitespace text") from error
except PermissionError as error:
raise ToolError("knowledge search is not authorized") from error
except Exception:
# Do not reflect provider, database, endpoint, or credential details
# into MCP content or logs. Operations can correlate this generic
# event with upstream sanitized service telemetry.
logger.error("MCP knowledge search failed")
raise ToolError("knowledge search failed") from None
return server
def create_mcp_http_app(
settings: Settings | None = None,
*,
api_services: ApiServices | None = None,
search_application: McpKnowledgeSearch | None = None,
) -> Starlette:
"""Build the stateless Streamable HTTP app used by WorkBuddy and tests."""
resolved_settings = settings or get_settings()
server = create_mcp_server(
resolved_settings,
api_services=api_services,
search_application=search_application,
)
return server.streamable_http_app(
streamable_http_path=resolved_settings.mcp_path,
json_response=True,
stateless_http=True,
max_request_body_size=resolved_settings.mcp_max_request_body_bytes,
host=resolved_settings.mcp_host,
transport_security=TransportSecuritySettings(
enable_dns_rebinding_protection=True,
allowed_hosts=resolved_settings.mcp_allowed_hosts,
allowed_origins=resolved_settings.mcp_allowed_origins,
),
)

61
deploy/docker/Dockerfile Normal file
View File

@ -0,0 +1,61 @@
# syntax=docker/dockerfile:1.7
# One locked runtime produces three independently deployable process images.
ARG PYTHON_IMAGE=python:3.12.14-slim-bookworm@sha256:782412e85d0f0984994c290652577d4018aff08145c85b262bb63dc0c7522254
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.12.5@sha256:e85be844203885286c60ffad8a858d48afb6c5a5c237ca0e67f12e74b8f174b1
FROM ${UV_IMAGE} AS uv-bin
FROM ${PYTHON_IMAGE} AS runtime
ENV PYTHONUNBUFFERED=1 \
PYTHONDONTWRITEBYTECODE=1 \
UV_COMPILE_BYTECODE=1 \
UV_LINK_MODE=copy \
VIRTUAL_ENV=/app/.venv \
PATH=/app/.venv/bin:${PATH} \
HOME=/tmp/ekp \
XDG_CACHE_HOME=/tmp/ekp/cache \
APP_ENV=production \
API_HOST=0.0.0.0 \
MCP_HOST=0.0.0.0 \
ADMIN_WEB_ROOT=/app/apps/admin-web
WORKDIR /app
COPY --from=uv-bin /uv /uvx /usr/local/bin/
COPY pyproject.toml uv.lock README.md alembic.ini ./
COPY apps ./apps
COPY packages ./packages
COPY migrations ./migrations
# --frozen prevents lock changes; --no-dev and --no-editable keep the runtime
# independent from source-tree import hooks while retaining migration sources.
RUN uv sync --frozen --no-dev --no-editable \
&& test -f /app/apps/admin-web/index.html \
&& groupadd --gid 10001 ekp \
&& useradd --uid 10001 --gid 10001 --no-create-home --shell /usr/sbin/nologin ekp \
&& mkdir -p /tmp/ekp/cache \
&& chown -R 10001:10001 /tmp/ekp \
&& rm -rf /root/.cache/uv
USER 10001:10001
FROM runtime AS api
EXPOSE 8000
# This endpoint is intentionally credential-free and suitable for container liveness.
HEALTHCHECK --interval=30s --timeout=3s --start-period=10s --retries=3 \
CMD ["python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/healthz', timeout=2)"]
CMD ["uvicorn", "ekp_api.main:app", "--host", "0.0.0.0", "--port", "8000", "--no-access-log"]
FROM runtime AS mcp
EXPOSE 8766
# MCP requests require authentication, so the image-level check only verifies PID 1.
HEALTHCHECK --interval=30s --timeout=3s --start-period=10s --retries=3 \
CMD ["python", "-c", "import os; os.kill(1, 0)"]
CMD ["ekp-mcp"]
FROM runtime AS worker
# Worker health is process-level here; durable Job recovery is validated by the service tests.
HEALTHCHECK --interval=30s --timeout=3s --start-period=10s --retries=3 \
CMD ["python", "-c", "import os; os.kill(1, 0)"]
CMD ["ekp-worker"]

View File

@ -0,0 +1,36 @@
# Reproducible target names consumed by local and CI buildx invocations.
variable "IMAGE_REGISTRY" {
default = "knowledge-platform"
}
variable "IMAGE_TAG" {
default = "dev"
}
group "default" {
targets = ["api", "mcp", "worker"]
}
target "common" {
context = "."
dockerfile = "deploy/docker/Dockerfile"
platforms = ["linux/amd64", "linux/arm64"]
}
target "api" {
inherits = ["common"]
target = "api"
tags = ["${IMAGE_REGISTRY}/enterprise-knowledge-api:${IMAGE_TAG}"]
}
target "mcp" {
inherits = ["common"]
target = "mcp"
tags = ["${IMAGE_REGISTRY}/enterprise-knowledge-mcp:${IMAGE_TAG}"]
}
target "worker" {
inherits = ["common"]
target = "worker"
tags = ["${IMAGE_REGISTRY}/enterprise-knowledge-worker:${IMAGE_TAG}"]
}

View File

@ -0,0 +1,50 @@
# Copy this file outside the repository for one environment. It contains only
# non-sensitive settings and Kubernetes Secret references; never add values for
# DATABASE_URL, RABBITMQ_URL, provider keys, passwords or MCP_AUTH_TOKEN here.
existingSecret: knowledge-platform-runtime
images:
api:
repository: registry.example.internal/knowledge/enterprise-knowledge-api
tag: "0.1.0"
digest: ""
mcp:
repository: registry.example.internal/knowledge/enterprise-knowledge-mcp
tag: "0.1.0"
digest: ""
worker:
repository: registry.example.internal/knowledge/enterprise-knowledge-worker
tag: "0.1.0"
digest: ""
ingress:
enabled: true
className: nginx
api:
host: knowledge.example.internal
tlsSecretName: knowledge-platform-api-tls
mcp:
host: mcp.example.internal
tlsSecretName: knowledge-platform-mcp-tls
config:
objectStorage:
endpoint: https://s3.example.internal
bucket: knowledge-platform
vectorStore:
uri: oceanbase.example.internal:2881
user: knowledge-platform
database: knowledge_platform
mcp:
publicUrl: https://mcp.example.internal/mcp
issuerUrl: https://mcp.example.internal
allowedHosts:
- mcp.example.internal
allowedOrigins:
- https://mcp.example.internal
# External RabbitMQ is the production default. If this local dependency is
# explicitly enabled for a demo cluster, create its Secret before deployment.
rabbitmq:
enabled: false
existingSecret: knowledge-platform-rabbitmq

View File

@ -0,0 +1,15 @@
apiVersion: v2
name: knowledge-platform
description: Enterprise Knowledge Platform public-knowledge deployment
type: application
version: 0.1.0
appVersion: "0.1.0"
# This local dependency is disabled by default. Production deployments should
# use an externally managed RabbitMQ service with an independently owned data
# lifecycle; the subchart exists only for explicit development/demo use.
dependencies:
- name: rabbitmq
version: 0.1.0
repository: file://charts/rabbitmq
condition: rabbitmq.enabled

View File

@ -0,0 +1,6 @@
apiVersion: v2
name: rabbitmq
description: Minimal single-node RabbitMQ dependency for local demo clusters only
type: application
version: 0.1.0
appVersion: "4.1.4"

View File

@ -0,0 +1,11 @@
{{- define "rabbitmq.fullname" -}}
{{- printf "%s-rabbitmq" .Release.Name | trunc 63 | trimSuffix "-" -}}
{{- end -}}
{{- define "rabbitmq.image" -}}
{{- if .Values.image.digest -}}
{{- printf "%s@%s" .Values.image.repository .Values.image.digest -}}
{{- else -}}
{{- printf "%s:%s" .Values.image.repository .Values.image.tag -}}
{{- end -}}
{{- end -}}

View File

@ -0,0 +1,16 @@
apiVersion: v1
kind: Service
metadata:
name: {{ include "rabbitmq.fullname" . }}
labels:
app.kubernetes.io/name: rabbitmq
app.kubernetes.io/instance: {{ .Release.Name }}
spec:
clusterIP: None
ports:
- name: amqp
port: {{ .Values.service.port }}
targetPort: amqp
selector:
app.kubernetes.io/name: rabbitmq
app.kubernetes.io/instance: {{ .Release.Name }}

View File

@ -0,0 +1,8 @@
apiVersion: v1
kind: ServiceAccount
metadata:
name: {{ include "rabbitmq.fullname" . }}
labels:
app.kubernetes.io/name: rabbitmq
app.kubernetes.io/instance: {{ .Release.Name }}
automountServiceAccountToken: false

View File

@ -0,0 +1,109 @@
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: {{ include "rabbitmq.fullname" . }}
labels:
app.kubernetes.io/name: rabbitmq
app.kubernetes.io/instance: {{ .Release.Name }}
spec:
serviceName: {{ include "rabbitmq.fullname" . }}
replicas: 1
selector:
matchLabels:
app.kubernetes.io/name: rabbitmq
app.kubernetes.io/instance: {{ .Release.Name }}
template:
metadata:
labels:
app.kubernetes.io/name: rabbitmq
app.kubernetes.io/instance: {{ .Release.Name }}
spec:
serviceAccountName: {{ include "rabbitmq.fullname" . }}
automountServiceAccountToken: false
securityContext:
runAsNonRoot: true
runAsUser: 999
runAsGroup: 999
fsGroup: 999
seccompProfile:
type: RuntimeDefault
containers:
- name: rabbitmq
image: {{ include "rabbitmq.image" . | quote }}
imagePullPolicy: {{ .Values.image.pullPolicy }}
securityContext:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
capabilities:
drop: ["ALL"]
ports:
- name: amqp
containerPort: 5672
env:
- name: RABBITMQ_DEFAULT_USER
valueFrom:
secretKeyRef:
name: {{ .Values.existingSecret }}
key: {{ .Values.secretKeys.username }}
- name: RABBITMQ_DEFAULT_PASS
valueFrom:
secretKeyRef:
name: {{ .Values.existingSecret }}
key: {{ .Values.secretKeys.password }}
- name: RABBITMQ_ERLANG_COOKIE
valueFrom:
secretKeyRef:
name: {{ .Values.existingSecret }}
key: {{ .Values.secretKeys.erlangCookie }}
startupProbe:
exec:
command: ["rabbitmq-diagnostics", "-q", "ping"]
periodSeconds: 5
failureThreshold: 24
readinessProbe:
exec:
command: ["rabbitmq-diagnostics", "-q", "check_running"]
periodSeconds: 10
timeoutSeconds: 5
failureThreshold: 3
livenessProbe:
exec:
command: ["rabbitmq-diagnostics", "-q", "ping"]
periodSeconds: 20
timeoutSeconds: 5
failureThreshold: 3
resources:
{{- toYaml .Values.resources | nindent 12 }}
volumeMounts:
- name: data
mountPath: /var/lib/rabbitmq
- name: config
mountPath: /etc/rabbitmq/conf.d
- name: log
mountPath: /var/log/rabbitmq
- name: tmp
mountPath: /tmp
volumes:
- name: config
emptyDir: {}
- name: log
emptyDir: {}
- name: tmp
emptyDir: {}
{{- if not .Values.persistence.enabled }}
- name: data
emptyDir: {}
{{- end }}
{{- if .Values.persistence.enabled }}
volumeClaimTemplates:
- metadata:
name: data
spec:
accessModes: [{{ .Values.persistence.accessMode | quote }}]
{{- if .Values.persistence.storageClass }}
storageClassName: {{ .Values.persistence.storageClass | quote }}
{{- end }}
resources:
requests:
storage: {{ .Values.persistence.size }}
{{- end }}

View File

@ -0,0 +1,69 @@
{
"$schema": "https://json-schema.org/draft-07/schema#",
"type": "object",
"additionalProperties": false,
"required": ["enabled", "existingSecret", "secretKeys", "image", "service", "persistence", "resources"],
"properties": {
"global": {"type": "object"},
"enabled": {"type": "boolean"},
"existingSecret": {"type": "string", "minLength": 1},
"secretKeys": {
"type": "object",
"additionalProperties": false,
"required": ["username", "password", "erlangCookie"],
"properties": {
"username": {"type": "string", "minLength": 1},
"password": {"type": "string", "minLength": 1},
"erlangCookie": {"type": "string", "minLength": 1}
}
},
"image": {
"type": "object",
"additionalProperties": false,
"required": ["repository", "tag", "digest", "pullPolicy"],
"properties": {
"repository": {"type": "string", "minLength": 1},
"tag": {"type": "string", "minLength": 1, "not": {"enum": ["latest"]}},
"digest": {"type": "string", "pattern": "^$|^sha256:[a-f0-9]{64}$"},
"pullPolicy": {"enum": ["Always", "IfNotPresent", "Never"]}
}
},
"service": {
"type": "object",
"additionalProperties": false,
"required": ["port"],
"properties": {"port": {"type": "integer", "minimum": 1, "maximum": 65535}}
},
"persistence": {
"type": "object",
"additionalProperties": false,
"required": ["enabled", "storageClass", "accessMode", "size"],
"properties": {
"enabled": {"type": "boolean"},
"storageClass": {"type": "string"},
"accessMode": {"enum": ["ReadWriteOnce", "ReadWriteMany"]},
"size": {"type": "string", "minLength": 2}
}
},
"resources": {
"type": "object",
"additionalProperties": false,
"required": ["requests", "limits"],
"properties": {
"requests": {"$ref": "#/definitions/resourcePair"},
"limits": {"$ref": "#/definitions/resourcePair"}
}
}
},
"definitions": {
"resourcePair": {
"type": "object",
"additionalProperties": false,
"required": ["cpu", "memory"],
"properties": {
"cpu": {"type": ["string", "number"]},
"memory": {"type": "string", "minLength": 2}
}
}
}
}

View File

@ -0,0 +1,25 @@
enabled: false
existingSecret: knowledge-platform-rabbitmq
secretKeys:
username: username
password: password
erlangCookie: erlang-cookie
image:
repository: rabbitmq
tag: 4.1.4-alpine
digest: sha256:b736d649308e1b3e1a116c3f36986b605ee3d03e88f10166be2900083d2e63f2
pullPolicy: IfNotPresent
service:
port: 5672
persistence:
enabled: true
storageClass: ""
accessMode: ReadWriteOnce
size: 8Gi
resources:
requests:
cpu: 250m
memory: 512Mi
limits:
cpu: "1"
memory: 1Gi

View File

@ -0,0 +1,12 @@
Enterprise Knowledge Platform {{ .Chart.AppVersion }} is managed by Helm release {{ .Release.Name }}.
Runtime credentials are read only from Secret {{ .Values.existingSecret }}.
{{- if .Values.ingress.enabled }}
Admin/API: https://{{ .Values.ingress.api.host }}{{ .Values.ingress.api.path }}
MCP: https://{{ .Values.ingress.mcp.host }}{{ .Values.ingress.mcp.path }}
{{- else }}
Ingress is disabled. Enable it in the environment values file before remote use.
{{- end }}
Inspect rollout state without printing environment variables:
kubectl -n {{ .Release.Namespace }} get deploy,job,pod

View File

@ -0,0 +1,106 @@
{{/* Stable resource name helpers keep every workload under one release identity. */}}
{{- define "knowledge-platform.name" -}}
{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
{{- end -}}
{{- define "knowledge-platform.fullname" -}}
{{- if .Values.fullnameOverride -}}
{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
{{- else -}}
{{- $name := include "knowledge-platform.name" . -}}
{{- if contains $name .Release.Name -}}
{{- .Release.Name | trunc 63 | trimSuffix "-" -}}
{{- else -}}
{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
{{- end -}}
{{- end -}}
{{- end -}}
{{- define "knowledge-platform.labels" -}}
helm.sh/chart: {{ printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | quote }}
app.kubernetes.io/name: {{ include "knowledge-platform.name" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
{{- end -}}
{{- define "knowledge-platform.selectorLabels" -}}
app.kubernetes.io/name: {{ include "knowledge-platform.name" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
{{- end -}}
{{- define "knowledge-platform.serviceAccountName" -}}
{{- if .Values.serviceAccount.create -}}
{{- default (include "knowledge-platform.fullname" .) .Values.serviceAccount.name -}}
{{- else -}}
{{- required "serviceAccount.name is required when serviceAccount.create=false" .Values.serviceAccount.name -}}
{{- end -}}
{{- end -}}
{{/* A supplied digest takes precedence over the immutable release tag. */}}
{{- define "knowledge-platform.image" -}}
{{- if .digest -}}
{{- printf "%s@%s" .repository .digest -}}
{{- else -}}
{{- printf "%s:%s" .repository .tag -}}
{{- end -}}
{{- end -}}
{{- define "knowledge-platform.configMapName" -}}
{{- printf "%s-runtime" (include "knowledge-platform.fullname" .) -}}
{{- end -}}
{{/* Cross-field rules complement values.schema.json's structural checks. */}}
{{- define "knowledge-platform.validateValues" -}}
{{- if lt (float64 .Values.config.ingestion.retryBackoffMaxSeconds) (float64 .Values.config.ingestion.retryBackoffSeconds) -}}
{{- fail "config.ingestion.retryBackoffMaxSeconds must be >= retryBackoffSeconds" -}}
{{- end -}}
{{- if lt (float64 .Values.config.ingestion.batchMaxTotalBytes) (float64 .Values.config.ingestion.maxUploadBytes) -}}
{{- fail "config.ingestion.batchMaxTotalBytes must be >= maxUploadBytes" -}}
{{- end -}}
{{- if ge (float64 .Values.config.chunking.overlapTokens) (float64 .Values.config.chunking.maxTokens) -}}
{{- fail "config.chunking.overlapTokens must be < maxTokens" -}}
{{- end -}}
{{- if gt (float64 .Values.config.retrieval.searchTopK) (float64 .Values.config.retrieval.denseTopN) -}}
{{- fail "config.retrieval.searchTopK must be <= denseTopN" -}}
{{- end -}}
{{- end -}}
{{- define "knowledge-platform.podSecurityContext" -}}
runAsNonRoot: true
runAsUser: 10001
runAsGroup: 10001
fsGroup: 10001
fsGroupChangePolicy: OnRootMismatch
seccompProfile:
type: RuntimeDefault
{{- end -}}
{{- define "knowledge-platform.containerSecurityContext" -}}
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
runAsNonRoot: true
runAsUser: 10001
runAsGroup: 10001
capabilities:
drop:
- ALL
{{- end -}}
{{- define "knowledge-platform.envFrom" -}}
- configMapRef:
name: {{ include "knowledge-platform.configMapName" . }}
- secretRef:
name: {{ .Values.existingSecret }}
{{- end -}}
{{- define "knowledge-platform.tmpVolume" -}}
- name: tmp
emptyDir:
sizeLimit: 1Gi
{{- end -}}
{{- define "knowledge-platform.tmpMount" -}}
- name: tmp
mountPath: /tmp
{{- end -}}

View File

@ -0,0 +1,75 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ include "knowledge-platform.fullname" . }}-api
labels:
{{- include "knowledge-platform.labels" . | nindent 4 }}
app.kubernetes.io/component: api
spec:
replicas: {{ .Values.api.replicaCount }}
minReadySeconds: 5
strategy:
type: RollingUpdate
rollingUpdate:
maxUnavailable: 0
maxSurge: 1
selector:
matchLabels:
{{- include "knowledge-platform.selectorLabels" . | nindent 6 }}
app.kubernetes.io/component: api
template:
metadata:
labels:
{{- include "knowledge-platform.selectorLabels" . | nindent 8 }}
app.kubernetes.io/component: api
{{- with .Values.podLabels }}
{{- toYaml . | nindent 8 }}
{{- end }}
annotations:
checksum/runtime-config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }}
checksum/runtime-secret-reference: {{ .Values.existingSecret | sha256sum }}
{{- with .Values.podAnnotations }}
{{- toYaml . | nindent 8 }}
{{- end }}
spec:
serviceAccountName: {{ include "knowledge-platform.serviceAccountName" . }}
automountServiceAccountToken: false
securityContext:
{{- include "knowledge-platform.podSecurityContext" . | nindent 8 }}
{{- with .Values.imagePullSecrets }}
imagePullSecrets:
{{- toYaml . | nindent 8 }}
{{- end }}
containers:
- name: api
image: {{ include "knowledge-platform.image" .Values.images.api | quote }}
imagePullPolicy: {{ .Values.images.api.pullPolicy }}
securityContext:
{{- include "knowledge-platform.containerSecurityContext" . | nindent 12 }}
ports:
- name: http
containerPort: 8000
protocol: TCP
envFrom:
{{- include "knowledge-platform.envFrom" . | nindent 12 }}
startupProbe:
httpGet:
path: /healthz
port: http
{{- toYaml .Values.api.probes.startup | nindent 12 }}
readinessProbe:
httpGet:
path: /healthz
port: http
{{- toYaml .Values.api.probes.readiness | nindent 12 }}
livenessProbe:
httpGet:
path: /healthz
port: http
{{- toYaml .Values.api.probes.liveness | nindent 12 }}
resources:
{{- toYaml .Values.api.resources | nindent 12 }}
volumeMounts:
{{- include "knowledge-platform.tmpMount" . | nindent 12 }}
volumes:
{{- include "knowledge-platform.tmpVolume" . | nindent 8 }}

View File

@ -0,0 +1,17 @@
apiVersion: v1
kind: Service
metadata:
name: {{ include "knowledge-platform.fullname" . }}-api
labels:
{{- include "knowledge-platform.labels" . | nindent 4 }}
app.kubernetes.io/component: api
spec:
type: {{ .Values.api.service.type }}
ports:
- name: http
port: {{ .Values.api.service.port }}
targetPort: http
protocol: TCP
selector:
{{- include "knowledge-platform.selectorLabels" . | nindent 4 }}
app.kubernetes.io/component: api

View File

@ -0,0 +1,66 @@
{{- include "knowledge-platform.validateValues" . }}
apiVersion: v1
kind: ConfigMap
metadata:
name: {{ include "knowledge-platform.configMapName" . }}
labels:
{{- include "knowledge-platform.labels" . | nindent 4 }}
data:
# Only the explicitly enumerated non-sensitive settings are rendered here.
APP_ENV: "production"
APP_NAME: {{ .Values.config.appName | quote }}
LOG_LEVEL: {{ .Values.config.logLevel | quote }}
API_HOST: "0.0.0.0"
API_PORT: "8000"
API_PREFIX: "/api/v1"
DATABASE_POOL_SIZE: {{ .Values.config.database.poolSize | quote }}
DATABASE_MAX_OVERFLOW: {{ .Values.config.database.maxOverflow | quote }}
DATABASE_POOL_RECYCLE_SECONDS: {{ .Values.config.database.poolRecycleSeconds | quote }}
DATABASE_POOL_TIMEOUT_SECONDS: {{ .Values.config.database.poolTimeoutSeconds | quote }}
DATABASE_CONNECT_TIMEOUT_SECONDS: {{ .Values.config.database.connectTimeoutSeconds | quote }}
INGESTION_MAX_ATTEMPTS: {{ .Values.config.ingestion.maxAttempts | quote }}
INGESTION_RETRY_BACKOFF_SECONDS: {{ .Values.config.ingestion.retryBackoffSeconds | quote }}
INGESTION_RETRY_BACKOFF_MAX_SECONDS: {{ .Values.config.ingestion.retryBackoffMaxSeconds | quote }}
INGESTION_WORKER_LEASE_SECONDS: {{ .Values.config.ingestion.workerLeaseSeconds | quote }}
INGESTION_RECOVERY_INTERVAL_SECONDS: {{ .Values.config.ingestion.recoveryIntervalSeconds | quote }}
BATCH_MAX_FILES: {{ .Values.config.ingestion.batchMaxFiles | quote }}
BATCH_MAX_TOTAL_BYTES: {{ printf "%.0f" .Values.config.ingestion.batchMaxTotalBytes | quote }}
MAX_UPLOAD_BYTES: {{ printf "%.0f" .Values.config.ingestion.maxUploadBytes | quote }}
OBJECT_STORAGE_PROVIDER: {{ .Values.config.objectStorage.provider | quote }}
BOS_ENDPOINT: {{ .Values.config.objectStorage.endpoint | quote }}
BOS_BUCKET: {{ .Values.config.objectStorage.bucket | quote }}
BOS_KEY_PREFIX: {{ .Values.config.objectStorage.keyPrefix | quote }}
BOS_TIMEOUT_SECONDS: {{ .Values.config.objectStorage.timeoutSeconds | quote }}
CHUNK_MAX_TOKENS: {{ .Values.config.chunking.maxTokens | quote }}
CHUNK_OVERLAP_TOKENS: {{ .Values.config.chunking.overlapTokens | quote }}
CHUNK_MAX_CHARACTERS: {{ .Values.config.chunking.maxCharacters | quote }}
EMBEDDING_PROVIDER: {{ .Values.config.embedding.provider | quote }}
EMBEDDING_MODEL: {{ .Values.config.embedding.model | quote }}
EMBEDDING_ENDPOINT: {{ .Values.config.embedding.endpoint | quote }}
EMBEDDING_TIMEOUT_SECONDS: {{ .Values.config.embedding.timeoutSeconds | quote }}
VECTOR_STORE_PROVIDER: {{ .Values.config.vectorStore.provider | quote }}
OCEANBASE_VECTOR_URI: {{ .Values.config.vectorStore.uri | quote }}
OCEANBASE_VECTOR_USER: {{ .Values.config.vectorStore.user | quote }}
OCEANBASE_VECTOR_DATABASE: {{ .Values.config.vectorStore.database | quote }}
OCEANBASE_VECTOR_TABLE: {{ .Values.config.vectorStore.table | quote }}
OCEANBASE_VECTOR_INDEX: {{ .Values.config.vectorStore.index | quote }}
OCEANBASE_VECTOR_BATCH_SIZE: {{ .Values.config.vectorStore.batchSize | quote }}
OCEANBASE_VECTOR_TIMEOUT_SECONDS: {{ .Values.config.vectorStore.timeoutSeconds | quote }}
INDEX_PROFILE_ID: {{ .Values.config.retrieval.indexProfileId | quote }}
DENSE_TOP_N: {{ .Values.config.retrieval.denseTopN | quote }}
SEARCH_TOP_K: {{ .Values.config.retrieval.searchTopK | quote }}
RERANKER_PROVIDER: {{ .Values.config.reranker.provider | quote }}
RERANKER_MODEL: {{ .Values.config.reranker.model | quote }}
RERANKER_ENDPOINT: {{ .Values.config.reranker.endpoint | quote }}
RERANKER_TIMEOUT_SECONDS: {{ .Values.config.reranker.timeoutSeconds | quote }}
MCP_HOST: "0.0.0.0"
MCP_PORT: "8766"
MCP_PATH: "/mcp"
MCP_PUBLIC_URL: {{ .Values.config.mcp.publicUrl | quote }}
MCP_ISSUER_URL: {{ .Values.config.mcp.issuerUrl | quote }}
MCP_CLIENT_ID: {{ .Values.config.mcp.clientId | quote }}
MCP_SERVICE_IDENTITY: {{ .Values.config.mcp.serviceIdentity | quote }}
MCP_REQUIRED_SCOPE: {{ .Values.config.mcp.requiredScope | quote }}
MCP_ALLOWED_HOSTS: {{ .Values.config.mcp.allowedHosts | toJson | quote }}
MCP_ALLOWED_ORIGINS: {{ .Values.config.mcp.allowedOrigins | toJson | quote }}
MCP_MAX_REQUEST_BODY_BYTES: {{ printf "%.0f" .Values.config.mcp.maxRequestBodyBytes | quote }}

View File

@ -0,0 +1,42 @@
{{- if .Values.ingress.enabled }}
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: {{ include "knowledge-platform.fullname" . }}
labels:
{{- include "knowledge-platform.labels" . | nindent 4 }}
{{- with .Values.ingress.annotations }}
annotations:
{{- toYaml . | nindent 4 }}
{{- end }}
spec:
ingressClassName: {{ .Values.ingress.className }}
tls:
- hosts:
- {{ .Values.ingress.api.host | quote }}
secretName: {{ .Values.ingress.api.tlsSecretName }}
- hosts:
- {{ .Values.ingress.mcp.host | quote }}
secretName: {{ .Values.ingress.mcp.tlsSecretName }}
rules:
- host: {{ .Values.ingress.api.host | quote }}
http:
paths:
- path: {{ .Values.ingress.api.path }}
pathType: Prefix
backend:
service:
name: {{ include "knowledge-platform.fullname" . }}-api
port:
number: {{ .Values.api.service.port }}
- host: {{ .Values.ingress.mcp.host | quote }}
http:
paths:
- path: {{ .Values.ingress.mcp.path }}
pathType: Prefix
backend:
service:
name: {{ include "knowledge-platform.fullname" . }}-mcp
port:
number: {{ .Values.mcp.service.port }}
{{- end }}

View File

@ -0,0 +1,72 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ include "knowledge-platform.fullname" . }}-mcp
labels:
{{- include "knowledge-platform.labels" . | nindent 4 }}
app.kubernetes.io/component: mcp
spec:
replicas: {{ .Values.mcp.replicaCount }}
minReadySeconds: 5
strategy:
type: RollingUpdate
rollingUpdate:
maxUnavailable: 0
maxSurge: 1
selector:
matchLabels:
{{- include "knowledge-platform.selectorLabels" . | nindent 6 }}
app.kubernetes.io/component: mcp
template:
metadata:
labels:
{{- include "knowledge-platform.selectorLabels" . | nindent 8 }}
app.kubernetes.io/component: mcp
{{- with .Values.podLabels }}
{{- toYaml . | nindent 8 }}
{{- end }}
annotations:
checksum/runtime-config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }}
checksum/runtime-secret-reference: {{ .Values.existingSecret | sha256sum }}
{{- with .Values.podAnnotations }}
{{- toYaml . | nindent 8 }}
{{- end }}
spec:
serviceAccountName: {{ include "knowledge-platform.serviceAccountName" . }}
automountServiceAccountToken: false
securityContext:
{{- include "knowledge-platform.podSecurityContext" . | nindent 8 }}
{{- with .Values.imagePullSecrets }}
imagePullSecrets:
{{- toYaml . | nindent 8 }}
{{- end }}
containers:
- name: mcp
image: {{ include "knowledge-platform.image" .Values.images.mcp | quote }}
imagePullPolicy: {{ .Values.images.mcp.pullPolicy }}
securityContext:
{{- include "knowledge-platform.containerSecurityContext" . | nindent 12 }}
ports:
- name: mcp
containerPort: 8766
protocol: TCP
envFrom:
{{- include "knowledge-platform.envFrom" . | nindent 12 }}
startupProbe:
tcpSocket:
port: mcp
{{- toYaml .Values.mcp.probes.startup | nindent 12 }}
readinessProbe:
tcpSocket:
port: mcp
{{- toYaml .Values.mcp.probes.readiness | nindent 12 }}
livenessProbe:
tcpSocket:
port: mcp
{{- toYaml .Values.mcp.probes.liveness | nindent 12 }}
resources:
{{- toYaml .Values.mcp.resources | nindent 12 }}
volumeMounts:
{{- include "knowledge-platform.tmpMount" . | nindent 12 }}
volumes:
{{- include "knowledge-platform.tmpVolume" . | nindent 8 }}

View File

@ -0,0 +1,17 @@
apiVersion: v1
kind: Service
metadata:
name: {{ include "knowledge-platform.fullname" . }}-mcp
labels:
{{- include "knowledge-platform.labels" . | nindent 4 }}
app.kubernetes.io/component: mcp
spec:
type: {{ .Values.mcp.service.type }}
ports:
- name: mcp
port: {{ .Values.mcp.service.port }}
targetPort: mcp
protocol: TCP
selector:
{{- include "knowledge-platform.selectorLabels" . | nindent 4 }}
app.kubernetes.io/component: mcp

View File

@ -0,0 +1,59 @@
{{- if .Values.migration.enabled }}
apiVersion: batch/v1
kind: Job
metadata:
name: {{ include "knowledge-platform.fullname" . }}-migration
labels:
{{- include "knowledge-platform.labels" . | nindent 4 }}
app.kubernetes.io/component: migration
annotations:
# Helm waits for pre-install/pre-upgrade hooks. A failed migration blocks
# the application rollout, and the previous Job remains until the next run.
helm.sh/hook: pre-install,pre-upgrade
helm.sh/hook-weight: "-10"
helm.sh/hook-delete-policy: before-hook-creation
spec:
backoffLimit: {{ .Values.migration.backoffLimit }}
activeDeadlineSeconds: {{ .Values.migration.activeDeadlineSeconds }}
ttlSecondsAfterFinished: {{ .Values.migration.ttlSecondsAfterFinished }}
template:
metadata:
labels:
{{- include "knowledge-platform.selectorLabels" . | nindent 8 }}
app.kubernetes.io/component: migration
spec:
restartPolicy: Never
# Pre-install hooks run before the release ServiceAccount exists. The
# namespace default account is used with token mounting disabled; the
# migration needs no Kubernetes API permissions.
serviceAccountName: default
automountServiceAccountToken: false
securityContext:
{{- include "knowledge-platform.podSecurityContext" . | nindent 8 }}
{{- with .Values.imagePullSecrets }}
imagePullSecrets:
{{- toYaml . | nindent 8 }}
{{- end }}
containers:
- name: migration
image: {{ include "knowledge-platform.image" .Values.images.api | quote }}
imagePullPolicy: {{ .Values.images.api.pullPolicy }}
command: ["alembic"]
args: ["upgrade", "head"]
securityContext:
{{- include "knowledge-platform.containerSecurityContext" . | nindent 12 }}
env:
- name: APP_ENV
value: production
- name: DATABASE_URL
valueFrom:
secretKeyRef:
name: {{ .Values.existingSecret }}
key: DATABASE_URL
resources:
{{- toYaml .Values.migration.resources | nindent 12 }}
volumeMounts:
{{- include "knowledge-platform.tmpMount" . | nindent 12 }}
volumes:
{{- include "knowledge-platform.tmpVolume" . | nindent 8 }}
{{- end }}

View File

@ -0,0 +1,71 @@
{{- if .Values.scheduler.enabled }}
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ include "knowledge-platform.fullname" . }}-scheduler
labels:
{{- include "knowledge-platform.labels" . | nindent 4 }}
app.kubernetes.io/component: scheduler
spec:
# A single Celery Beat owns periodic durable-Job recovery. Recreate prevents
# two schedulers from overlapping during upgrades.
replicas: 1
strategy:
type: Recreate
selector:
matchLabels:
{{- include "knowledge-platform.selectorLabels" . | nindent 6 }}
app.kubernetes.io/component: scheduler
template:
metadata:
labels:
{{- include "knowledge-platform.selectorLabels" . | nindent 8 }}
app.kubernetes.io/component: scheduler
{{- with .Values.podLabels }}
{{- toYaml . | nindent 8 }}
{{- end }}
annotations:
checksum/runtime-config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }}
checksum/runtime-secret-reference: {{ .Values.existingSecret | sha256sum }}
{{- with .Values.podAnnotations }}
{{- toYaml . | nindent 8 }}
{{- end }}
spec:
serviceAccountName: {{ include "knowledge-platform.serviceAccountName" . }}
automountServiceAccountToken: false
securityContext:
{{- include "knowledge-platform.podSecurityContext" . | nindent 8 }}
{{- with .Values.imagePullSecrets }}
imagePullSecrets:
{{- toYaml . | nindent 8 }}
{{- end }}
containers:
- name: scheduler
image: {{ include "knowledge-platform.image" .Values.images.worker | quote }}
imagePullPolicy: {{ .Values.images.worker.pullPolicy }}
workingDir: /tmp
command: ["celery"]
args:
- --app=ekp_api.worker:celery_app
- beat
- --loglevel=INFO
- --schedule=/tmp/celerybeat-schedule
securityContext:
{{- include "knowledge-platform.containerSecurityContext" . | nindent 12 }}
envFrom:
{{- include "knowledge-platform.envFrom" . | nindent 12 }}
startupProbe:
exec:
command: ["python", "-c", "import os; os.kill(1, 0)"]
{{- toYaml .Values.scheduler.probes.startup | nindent 12 }}
livenessProbe:
exec:
command: ["python", "-c", "import os; os.kill(1, 0)"]
{{- toYaml .Values.scheduler.probes.liveness | nindent 12 }}
resources:
{{- toYaml .Values.scheduler.resources | nindent 12 }}
volumeMounts:
{{- include "knowledge-platform.tmpMount" . | nindent 12 }}
volumes:
{{- include "knowledge-platform.tmpVolume" . | nindent 8 }}
{{- end }}

View File

@ -0,0 +1,13 @@
{{- if .Values.serviceAccount.create }}
apiVersion: v1
kind: ServiceAccount
metadata:
name: {{ include "knowledge-platform.serviceAccountName" . }}
labels:
{{- include "knowledge-platform.labels" . | nindent 4 }}
{{- with .Values.serviceAccount.annotations }}
annotations:
{{- toYaml . | nindent 4 }}
{{- end }}
automountServiceAccountToken: false
{{- end }}

View File

@ -0,0 +1,17 @@
apiVersion: v1
kind: ConfigMap
metadata:
name: {{ include "knowledge-platform.fullname" . }}-preflight-references
labels:
{{- include "knowledge-platform.labels" . | nindent 4 }}
annotations:
# The Make preflight renders this test hook to discover only resource
# names. It never renders or reads Secret data.
helm.sh/hook: test
data:
runtimeSecret: {{ .Values.existingSecret | quote }}
ingressEnabled: {{ .Values.ingress.enabled | quote }}
apiTlsSecret: {{ .Values.ingress.api.tlsSecretName | quote }}
mcpTlsSecret: {{ .Values.ingress.mcp.tlsSecretName | quote }}
rabbitmqEnabled: {{ .Values.rabbitmq.enabled | quote }}
rabbitmqSecret: {{ .Values.rabbitmq.existingSecret | quote }}

View File

@ -0,0 +1,71 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ include "knowledge-platform.fullname" . }}-worker
labels:
{{- include "knowledge-platform.labels" . | nindent 4 }}
app.kubernetes.io/component: worker
spec:
replicas: {{ .Values.worker.replicaCount }}
strategy:
type: RollingUpdate
rollingUpdate:
maxUnavailable: 1
maxSurge: 1
selector:
matchLabels:
{{- include "knowledge-platform.selectorLabels" . | nindent 6 }}
app.kubernetes.io/component: worker
template:
metadata:
labels:
{{- include "knowledge-platform.selectorLabels" . | nindent 8 }}
app.kubernetes.io/component: worker
{{- with .Values.podLabels }}
{{- toYaml . | nindent 8 }}
{{- end }}
annotations:
checksum/runtime-config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }}
checksum/runtime-secret-reference: {{ .Values.existingSecret | sha256sum }}
{{- with .Values.podAnnotations }}
{{- toYaml . | nindent 8 }}
{{- end }}
spec:
serviceAccountName: {{ include "knowledge-platform.serviceAccountName" . }}
automountServiceAccountToken: false
terminationGracePeriodSeconds: {{ .Values.worker.terminationGracePeriodSeconds }}
securityContext:
{{- include "knowledge-platform.podSecurityContext" . | nindent 8 }}
{{- with .Values.imagePullSecrets }}
imagePullSecrets:
{{- toYaml . | nindent 8 }}
{{- end }}
containers:
- name: worker
image: {{ include "knowledge-platform.image" .Values.images.worker | quote }}
imagePullPolicy: {{ .Values.images.worker.pullPolicy }}
workingDir: /tmp
command: ["celery"]
args:
- --app=ekp_api.worker:celery_app
- worker
- --loglevel=INFO
- --queues=ingestion
securityContext:
{{- include "knowledge-platform.containerSecurityContext" . | nindent 12 }}
envFrom:
{{- include "knowledge-platform.envFrom" . | nindent 12 }}
startupProbe:
exec:
command: ["python", "-c", "import os; os.kill(1, 0)"]
{{- toYaml .Values.worker.probes.startup | nindent 12 }}
livenessProbe:
exec:
command: ["python", "-c", "import os; os.kill(1, 0)"]
{{- toYaml .Values.worker.probes.liveness | nindent 12 }}
resources:
{{- toYaml .Values.worker.resources | nindent 12 }}
volumeMounts:
{{- include "knowledge-platform.tmpMount" . | nindent 12 }}
volumes:
{{- include "knowledge-platform.tmpVolume" . | nindent 8 }}

View File

@ -0,0 +1,338 @@
{
"$schema": "https://json-schema.org/draft-07/schema#",
"title": "Enterprise Knowledge Platform Helm values",
"type": "object",
"additionalProperties": false,
"required": [
"existingSecret",
"images",
"serviceAccount",
"api",
"mcp",
"worker",
"scheduler",
"migration",
"ingress",
"config",
"rabbitmq"
],
"properties": {
"nameOverride": {"type": "string"},
"fullnameOverride": {"type": "string"},
"existingSecret": {"$ref": "#/definitions/kubernetesName"},
"imagePullSecrets": {
"type": "array",
"items": {
"type": "object",
"additionalProperties": false,
"required": ["name"],
"properties": {"name": {"$ref": "#/definitions/kubernetesName"}}
}
},
"images": {
"type": "object",
"additionalProperties": false,
"required": ["api", "mcp", "worker"],
"properties": {
"api": {"$ref": "#/definitions/image"},
"mcp": {"$ref": "#/definitions/image"},
"worker": {"$ref": "#/definitions/image"}
}
},
"serviceAccount": {
"type": "object",
"additionalProperties": false,
"required": ["create", "name", "annotations"],
"properties": {
"create": {"type": "boolean"},
"name": {"type": "string"},
"annotations": {"$ref": "#/definitions/stringMap"}
}
},
"podAnnotations": {"$ref": "#/definitions/stringMap"},
"podLabels": {"$ref": "#/definitions/stringMap"},
"api": {"$ref": "#/definitions/httpWorkload"},
"mcp": {"$ref": "#/definitions/httpWorkload"},
"worker": {
"type": "object",
"additionalProperties": false,
"required": ["replicaCount", "terminationGracePeriodSeconds", "resources", "probes"],
"properties": {
"replicaCount": {"type": "integer", "minimum": 1, "maximum": 100},
"terminationGracePeriodSeconds": {"type": "integer", "minimum": 30, "maximum": 3600},
"resources": {"$ref": "#/definitions/resources"},
"probes": {"$ref": "#/definitions/processProbes"}
}
},
"scheduler": {
"type": "object",
"additionalProperties": false,
"required": ["enabled", "resources", "probes"],
"properties": {
"enabled": {"type": "boolean"},
"resources": {"$ref": "#/definitions/resources"},
"probes": {"$ref": "#/definitions/processProbes"}
}
},
"migration": {
"type": "object",
"additionalProperties": false,
"required": ["enabled", "activeDeadlineSeconds", "backoffLimit", "ttlSecondsAfterFinished", "resources"],
"properties": {
"enabled": {"type": "boolean"},
"activeDeadlineSeconds": {"type": "integer", "minimum": 60, "maximum": 3600},
"backoffLimit": {"type": "integer", "minimum": 0, "maximum": 3},
"ttlSecondsAfterFinished": {"type": "integer", "minimum": 0},
"resources": {"$ref": "#/definitions/resources"}
}
},
"ingress": {
"type": "object",
"additionalProperties": false,
"required": ["enabled", "className", "annotations", "api", "mcp"],
"properties": {
"enabled": {"type": "boolean"},
"className": {"type": "string", "minLength": 1},
"annotations": {"$ref": "#/definitions/stringMap"},
"api": {"$ref": "#/definitions/ingressRoute"},
"mcp": {"$ref": "#/definitions/ingressRoute"}
}
},
"config": {
"type": "object",
"additionalProperties": false,
"required": ["appName", "logLevel", "database", "ingestion", "objectStorage", "chunking", "embedding", "vectorStore", "retrieval", "reranker", "mcp"],
"properties": {
"appName": {"type": "string", "minLength": 1},
"logLevel": {"enum": ["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"]},
"database": {"$ref": "#/definitions/databaseConfig"},
"ingestion": {"$ref": "#/definitions/ingestionConfig"},
"objectStorage": {"$ref": "#/definitions/objectStorageConfig"},
"chunking": {"$ref": "#/definitions/chunkingConfig"},
"embedding": {"$ref": "#/definitions/embeddingConfig"},
"vectorStore": {"$ref": "#/definitions/vectorStoreConfig"},
"retrieval": {"$ref": "#/definitions/retrievalConfig"},
"reranker": {"$ref": "#/definitions/rerankerConfig"},
"mcp": {"$ref": "#/definitions/mcpConfig"}
}
},
"rabbitmq": {
"type": "object",
"required": ["enabled", "existingSecret"],
"properties": {
"enabled": {"type": "boolean"},
"existingSecret": {"$ref": "#/definitions/kubernetesName"}
}
}
},
"definitions": {
"kubernetesName": {
"type": "string",
"minLength": 1,
"maxLength": 253,
"pattern": "^[a-z0-9]([-a-z0-9.]*[a-z0-9])?$"
},
"stringMap": {
"type": "object",
"additionalProperties": {"type": "string"}
},
"image": {
"type": "object",
"additionalProperties": false,
"required": ["repository", "tag", "digest", "pullPolicy"],
"properties": {
"repository": {"type": "string", "minLength": 1},
"tag": {"type": "string", "minLength": 1, "not": {"enum": ["latest"]}},
"digest": {"type": "string", "pattern": "^$|^sha256:[a-f0-9]{64}$"},
"pullPolicy": {"enum": ["Always", "IfNotPresent", "Never"]}
}
},
"probeTiming": {
"type": "object",
"additionalProperties": false,
"required": ["failureThreshold", "periodSeconds"],
"properties": {
"failureThreshold": {"type": "integer", "minimum": 1},
"periodSeconds": {"type": "integer", "minimum": 1},
"timeoutSeconds": {"type": "integer", "minimum": 1}
}
},
"httpProbes": {
"type": "object",
"additionalProperties": false,
"required": ["startup", "readiness", "liveness"],
"properties": {
"startup": {"$ref": "#/definitions/probeTiming"},
"readiness": {"$ref": "#/definitions/probeTiming"},
"liveness": {"$ref": "#/definitions/probeTiming"}
}
},
"processProbes": {
"type": "object",
"additionalProperties": false,
"required": ["startup", "liveness"],
"properties": {
"startup": {"$ref": "#/definitions/probeTiming"},
"liveness": {"$ref": "#/definitions/probeTiming"}
}
},
"resources": {
"type": "object",
"additionalProperties": false,
"required": ["requests", "limits"],
"properties": {
"requests": {"$ref": "#/definitions/resourcePair"},
"limits": {"$ref": "#/definitions/resourcePair"}
}
},
"resourcePair": {
"type": "object",
"additionalProperties": false,
"required": ["cpu", "memory"],
"properties": {
"cpu": {"type": ["string", "number"]},
"memory": {"type": "string", "minLength": 2}
}
},
"httpWorkload": {
"type": "object",
"additionalProperties": false,
"required": ["replicaCount", "service", "resources", "probes"],
"properties": {
"replicaCount": {"type": "integer", "minimum": 2, "maximum": 100},
"service": {
"type": "object",
"additionalProperties": false,
"required": ["type", "port"],
"properties": {
"type": {"enum": ["ClusterIP"]},
"port": {"type": "integer", "minimum": 1, "maximum": 65535}
}
},
"resources": {"$ref": "#/definitions/resources"},
"probes": {"$ref": "#/definitions/httpProbes"}
}
},
"ingressRoute": {
"type": "object",
"additionalProperties": false,
"required": ["host", "path", "tlsSecretName"],
"properties": {
"host": {"type": "string", "minLength": 1},
"path": {"type": "string", "pattern": "^/"},
"tlsSecretName": {"$ref": "#/definitions/kubernetesName"}
}
},
"databaseConfig": {
"type": "object",
"additionalProperties": false,
"required": ["poolSize", "maxOverflow", "poolRecycleSeconds", "poolTimeoutSeconds", "connectTimeoutSeconds"],
"properties": {
"poolSize": {"type": "integer", "minimum": 1, "maximum": 100},
"maxOverflow": {"type": "integer", "minimum": 0, "maximum": 100},
"poolRecycleSeconds": {"type": "integer", "minimum": 1},
"poolTimeoutSeconds": {"type": "number", "exclusiveMinimum": 0},
"connectTimeoutSeconds": {"type": "integer", "minimum": 1}
}
},
"ingestionConfig": {
"type": "object",
"additionalProperties": false,
"required": ["maxAttempts", "retryBackoffSeconds", "retryBackoffMaxSeconds", "workerLeaseSeconds", "recoveryIntervalSeconds", "batchMaxFiles", "batchMaxTotalBytes", "maxUploadBytes"],
"properties": {
"maxAttempts": {"type": "integer", "minimum": 1, "maximum": 10},
"retryBackoffSeconds": {"type": "integer", "minimum": 1},
"retryBackoffMaxSeconds": {"type": "integer", "minimum": 1},
"workerLeaseSeconds": {"type": "integer", "minimum": 30},
"recoveryIntervalSeconds": {"type": "integer", "minimum": 5},
"batchMaxFiles": {"type": "integer", "minimum": 1, "maximum": 100},
"batchMaxTotalBytes": {"type": "integer", "minimum": 1},
"maxUploadBytes": {"type": "integer", "minimum": 1}
}
},
"objectStorageConfig": {
"type": "object",
"additionalProperties": false,
"required": ["provider", "endpoint", "bucket", "keyPrefix", "timeoutSeconds"],
"properties": {
"provider": {"const": "bos"},
"endpoint": {"type": "string", "minLength": 1},
"bucket": {"type": "string", "minLength": 1},
"keyPrefix": {"type": "string", "minLength": 1},
"timeoutSeconds": {"type": "number", "exclusiveMinimum": 0}
}
},
"chunkingConfig": {
"type": "object",
"additionalProperties": false,
"required": ["maxTokens", "overlapTokens", "maxCharacters"],
"properties": {
"maxTokens": {"type": "integer", "minimum": 1},
"overlapTokens": {"type": "integer", "minimum": 0},
"maxCharacters": {"type": "integer", "minimum": 4}
}
},
"embeddingConfig": {
"type": "object",
"additionalProperties": false,
"required": ["provider", "model", "endpoint", "timeoutSeconds"],
"properties": {
"provider": {"const": "qianfan"},
"model": {"const": "bge-large-zh"},
"endpoint": {"type": "string", "minLength": 1},
"timeoutSeconds": {"type": "number", "exclusiveMinimum": 0}
}
},
"vectorStoreConfig": {
"type": "object",
"additionalProperties": false,
"required": ["provider", "uri", "user", "database", "table", "index", "batchSize", "timeoutSeconds"],
"properties": {
"provider": {"const": "oceanbase"},
"uri": {"type": "string", "minLength": 1},
"user": {"type": "string", "minLength": 1},
"database": {"type": "string", "minLength": 1},
"table": {"type": "string", "minLength": 1},
"index": {"type": "string", "minLength": 1},
"batchSize": {"type": "integer", "minimum": 1, "maximum": 1000},
"timeoutSeconds": {"type": "number", "exclusiveMinimum": 0}
}
},
"retrievalConfig": {
"type": "object",
"additionalProperties": false,
"required": ["indexProfileId", "denseTopN", "searchTopK"],
"properties": {
"indexProfileId": {"type": "string", "minLength": 1},
"denseTopN": {"type": "integer", "minimum": 1, "maximum": 64},
"searchTopK": {"type": "integer", "minimum": 1, "maximum": 64}
}
},
"rerankerConfig": {
"type": "object",
"additionalProperties": false,
"required": ["provider", "model", "endpoint", "timeoutSeconds"],
"properties": {
"provider": {"const": "qianfan"},
"model": {"const": "bce-reranker-base"},
"endpoint": {"type": "string", "minLength": 1},
"timeoutSeconds": {"type": "number", "exclusiveMinimum": 0}
}
},
"mcpConfig": {
"type": "object",
"additionalProperties": false,
"required": ["publicUrl", "issuerUrl", "clientId", "serviceIdentity", "requiredScope", "allowedHosts", "allowedOrigins", "maxRequestBodyBytes"],
"properties": {
"publicUrl": {"type": "string", "pattern": "^https://"},
"issuerUrl": {"type": "string", "pattern": "^https://"},
"clientId": {"type": "string", "minLength": 1},
"serviceIdentity": {"type": "string", "minLength": 1},
"requiredScope": {"const": "knowledge:search"},
"allowedHosts": {"type": "array", "minItems": 1, "items": {"type": "string", "minLength": 1}},
"allowedOrigins": {"type": "array", "minItems": 1, "items": {"type": "string", "pattern": "^https://"}},
"maxRequestBodyBytes": {"type": "integer", "minimum": 1024, "maximum": 4194304}
}
}
}
}

View File

@ -0,0 +1,218 @@
# DEPLOY-002 safe defaults. Real endpoints, immutable image coordinates and
# Secret names are supplied in an environment-specific values file.
nameOverride: ""
fullnameOverride: ""
existingSecret: knowledge-platform-runtime
imagePullSecrets: []
images:
api:
repository: knowledge-platform/enterprise-knowledge-api
tag: "0.1.0"
digest: ""
pullPolicy: IfNotPresent
mcp:
repository: knowledge-platform/enterprise-knowledge-mcp
tag: "0.1.0"
digest: ""
pullPolicy: IfNotPresent
worker:
repository: knowledge-platform/enterprise-knowledge-worker
tag: "0.1.0"
digest: ""
pullPolicy: IfNotPresent
serviceAccount:
create: true
name: ""
annotations: {}
podAnnotations: {}
podLabels: {}
api:
replicaCount: 2
service:
type: ClusterIP
port: 80
resources:
requests:
cpu: 250m
memory: 512Mi
limits:
cpu: "1"
memory: 1Gi
probes:
startup:
failureThreshold: 30
periodSeconds: 2
readiness:
periodSeconds: 10
timeoutSeconds: 2
failureThreshold: 3
liveness:
periodSeconds: 20
timeoutSeconds: 2
failureThreshold: 3
mcp:
replicaCount: 2
service:
type: ClusterIP
port: 80
resources:
requests:
cpu: 250m
memory: 512Mi
limits:
cpu: "1"
memory: 1Gi
probes:
startup:
failureThreshold: 30
periodSeconds: 2
readiness:
periodSeconds: 10
timeoutSeconds: 2
failureThreshold: 3
liveness:
periodSeconds: 20
timeoutSeconds: 2
failureThreshold: 3
worker:
replicaCount: 2
terminationGracePeriodSeconds: 60
resources:
requests:
cpu: 500m
memory: 1Gi
limits:
cpu: "2"
memory: 2Gi
probes:
startup:
failureThreshold: 30
periodSeconds: 2
liveness:
periodSeconds: 20
timeoutSeconds: 2
failureThreshold: 3
# Celery Beat is separated from scalable workers so only one recovery
# scheduler publishes due durable Jobs at a time.
scheduler:
enabled: true
resources:
requests:
cpu: 100m
memory: 256Mi
limits:
cpu: 500m
memory: 512Mi
probes:
startup:
failureThreshold: 30
periodSeconds: 2
liveness:
periodSeconds: 20
timeoutSeconds: 2
failureThreshold: 3
migration:
enabled: true
activeDeadlineSeconds: 600
backoffLimit: 0
ttlSecondsAfterFinished: 86400
resources:
requests:
cpu: 100m
memory: 256Mi
limits:
cpu: 500m
memory: 512Mi
ingress:
enabled: false
className: nginx
annotations: {}
api:
host: knowledge.example.invalid
path: /
tlsSecretName: knowledge-platform-api-tls
mcp:
host: mcp.example.invalid
path: /mcp
tlsSecretName: knowledge-platform-mcp-tls
# Only non-sensitive runtime settings belong here. The templates enumerate
# keys explicitly so credentials cannot be added to this ConfigMap by mistake.
config:
appName: enterprise-knowledge-platform
logLevel: INFO
database:
poolSize: 10
maxOverflow: 10
poolRecycleSeconds: 600
poolTimeoutSeconds: 30
connectTimeoutSeconds: 30
ingestion:
maxAttempts: 3
retryBackoffSeconds: 2
retryBackoffMaxSeconds: 60
workerLeaseSeconds: 300
recoveryIntervalSeconds: 30
batchMaxFiles: 20
batchMaxTotalBytes: 209715200
maxUploadBytes: 52428800
objectStorage:
provider: bos
endpoint: https://s3.example.invalid
bucket: knowledge-platform
keyPrefix: knowledge-platform
timeoutSeconds: 30
chunking:
maxTokens: 450
overlapTokens: 75
maxCharacters: 2000
embedding:
provider: qianfan
model: bge-large-zh
endpoint: https://qianfan.baidubce.com/v2/embeddings
timeoutSeconds: 30
vectorStore:
provider: oceanbase
uri: oceanbase.example.invalid:2881
user: knowledge-platform
database: knowledge_platform
table: knowledge_chunks_v1
index: vidx_knowledge_chunks_embedding
batchSize: 100
timeoutSeconds: 30
retrieval:
indexProfileId: demo-bge-large-zh-structure-v1-oceanbase-v1
denseTopN: 20
searchTopK: 5
reranker:
provider: qianfan
model: bce-reranker-base
endpoint: https://qianfan.baidubce.com/v2/rerank
timeoutSeconds: 30
mcp:
publicUrl: https://mcp.example.invalid/mcp
issuerUrl: https://mcp.example.invalid
clientId: workbuddy-enterprise
serviceIdentity: workbuddy-public-knowledge
requiredScope: knowledge:search
allowedHosts:
- mcp.example.invalid
allowedOrigins:
- https://mcp.example.invalid
maxRequestBodyBytes: 1048576
# Optional local dependency. Keep disabled for production and provide
# RABBITMQ_URL through existingSecret for an externally managed broker.
rabbitmq:
enabled: false
existingSecret: knowledge-platform-rabbitmq

View File

@ -0,0 +1,31 @@
-- ARCH-003 rebuildable OceanBase Vector index for bge-large-zh (1024 dimensions).
-- Canonical metadata is co-located in OceanBase relational tables; this vector
-- table remains a derived index and may be rebuilt from canonical chunks.
-- Requires OceanBase 4.3.5.4+; ARCH-006 verifies PRE-FILTER in the target test cluster.
CREATE TABLE IF NOT EXISTS knowledge_chunks_v1 (
index_record_id VARCHAR(36) NOT NULL,
chunk_id VARCHAR(36) NOT NULL,
document_id VARCHAR(36) NOT NULL,
filename VARCHAR(255) NOT NULL,
page INT NULL,
page_end INT NULL,
section VARCHAR(1024) NULL,
chunk_index INT NOT NULL,
content MEDIUMTEXT NOT NULL,
embedding VECTOR(1024) NOT NULL,
embedding_model VARCHAR(128) NOT NULL,
index_profile_id VARCHAR(128) NOT NULL,
PRIMARY KEY (index_record_id),
KEY idx_knowledge_chunks_document_profile (document_id, index_profile_id),
KEY idx_knowledge_chunks_profile (index_profile_id),
VECTOR KEY vidx_knowledge_chunks_embedding (embedding)
WITH (
DISTANCE = COSINE,
TYPE = HNSW,
LIB = VSAG,
M = 16,
EF_CONSTRUCTION = 256,
EF_SEARCH = 200
) BLOCK_SIZE 16384
) ORGANIZATION = HEAP;

View File

@ -0,0 +1,12 @@
{
"mcpServers": {
"enterprise-knowledge": {
"type": "http",
"url": "${EKP_MCP_URL:-http://127.0.0.1:8766/mcp}",
"headers": {
"Authorization": "Bearer ${EKP_MCP_TOKEN}"
},
"description": "企业知识检索"
}
}
}

View File

@ -11,17 +11,22 @@ Use OpenAPI as executable source of truth once implementation starts. Provider-s
DEMO-009 公开以下最小接口。`docs/demo/DEMO_SCOPE.md` 中的 `/api/*` 逻辑路径按本契约统一 DEMO-009 公开以下最小接口。`docs/demo/DEMO_SCOPE.md` 中的 `/api/*` 逻辑路径按本契约统一
版本化为 `/api/v1/*`。 版本化为 `/api/v1/*`。
`DEMO-015` 已将单文件上传切换为持久化异步 Job;`DEMO-016` 已增加批量提交和聚合查询。
| Method | Path | 行为 | | Method | Path | 行为 |
|---|---|---| |---|---|---|
| `POST` | `/api/v1/documents` | multipart 单文件上传并同步执行完整入库流水线 | | `POST` | `/api/v1/documents` | multipart 单文件接收;持久化 Source、Document、Job 后返回 `202` |
| `POST` | `/api/v1/ingestion-batches` | multipart `files` 多文件接收;返回批次与逐文件状态 |
| `GET` | `/api/v1/ingestion-batches/{batch_id}` | 返回批次汇总和逐文件最新 Job 状态 |
| `GET` | `/api/v1/documents` | 按创建时间倒序返回文档列表 | | `GET` | `/api/v1/documents` | 按创建时间倒序返回文档列表 |
| `GET` | `/api/v1/documents/{document_id}` | 返回单个文档状态与安全错误信息 | | `GET` | `/api/v1/documents/{document_id}` | 返回单个文档状态与安全错误信息 |
| `DELETE` | `/api/v1/documents/{document_id}` | 直接删除向量、Metadata、解析产物和原始文件 | | `DELETE` | `/api/v1/documents/{document_id}` | 直接删除向量、Metadata、解析产物和原始文件 |
| `GET` | `/api/v1/jobs/{job_id}` | 返回阶段、状态、attempt、最大尝试次数和安全错误信息 |
| `POST` | `/api/v1/search` | Dense Top20、Reranker、TopK、来源补全 | | `POST` | `/api/v1/search` | Dense Top20、Reranker、TopK、来源补全 |
上传字段名为 `file`,支持 PDF、DOCX、TXT 和 Markdown。成功返回 `201` 和 Document; 上传字段名为 `file`,支持 PDF、DOCX、TXT 和 Markdown。成功返回 `202`、Document 字段及
Demo 同步流水线完成后状态为 `ready`。Document 响应不暴露 `storage_key`、Provider 配置或 `job_id`、`job_status=queued`、`max_attempts`;响应不等待 Parse、Clean、Chunk、Embedding 或
索引实现。 Index。Document 响应不暴露 `storage_key`、Provider 配置或索引实现。
Search 请求: Search 请求:
@ -52,5 +57,87 @@ Search 响应:
``` ```
公开错误语义:请求校验失败返回 `422`,不支持的上传类型返回 `415`,文件过大返回 `413`, 公开错误语义:请求校验失败返回 `422`,不支持的上传类型返回 `415`,文件过大返回 `413`,
文档不存在返回 `404`,运行时 Provider 未配置返回 `503`。上游 Provider 或持久化异常只返回 文档、Job 或批次不存在返回 `404`,活动 Job 的文档删除返回 `409`,运行时 Provider 未配置返回
经过收敛的 `5xx` 信息,不回显凭证、请求正文或厂商响应。 `503`。上游 Provider 或持久化异常只返回经过收敛的 `5xx` 信息,不回显凭证、请求正文或
厂商响应。
## Demo Admin 客户端路由
DEMO-010 提供 `/admin/upload`、`/admin/documents` 和 `/admin/search` 三个客户端路由,
`/admin` 默认展示上传页。它们是同源静态页面,不是新增业务 API,也不改变上述公开 REST
契约;所有业务操作仍调用 `/api/v1`。
## 当前公共知识库部署边界(PLAN-002)
当前阶段所有 Document 属于同一个公共知识可见域,不新增用户、知识库或 ACL 字段,也不把
这些完整未实现能力伪装为默认放行。Admin 写接口只允许部署在企业受控网络或统一接入网关后;
Ingress 是否进一步认证属于部署配置,但前端隐藏按钮或隐藏路由不构成安全控制。
集群重启、扩缩容和滚动部署不得改变本契约或要求浏览器重新注入 Provider 配置。数据库、BOS、
模型与队列凭据由服务端 Secret 提供,任何 API 响应都不得返回这些值。
## 异步入库契约(DEMO-015~016 已实现)
| Method | Path | 行为 |
|---|---|---|
| `POST` | `/api/v1/documents` | 已实现:单文件接收并返回 Document/Job 标识 |
| `POST` | `/api/v1/ingestion-batches` | multipart 多文件接收,返回批次及逐文件 Document/Job 标识 |
| `GET` | `/api/v1/ingestion-batches/{batch_id}` | 返回批次汇总和逐文件最新状态 |
| `GET` | `/api/v1/jobs/{job_id}` | 已实现:返回持久化 Job 状态 |
单文件响应示意:
```json
{
"id": "6e5ec370-5a45-4eb7-95d7-e84267ad8752",
"filename": "退款政策.md",
"media_type": "text/markdown",
"size_bytes": 1024,
"status": "uploaded",
"error_message": null,
"created_at": "2026-09-02T12:00:00Z",
"updated_at": "2026-09-02T12:00:00Z",
"job_id": "7adc4278-1157-4c3d-95a4-6926ad77a04e",
"job_status": "queued",
"max_attempts": 3
}
```
批量字段名为 `files`,成功返回 `202`。响应示意:
```json
{
"batch_id": "d21f8012-4c4e-4bb4-89af-e11c0f9f470e",
"status": "processing",
"total_files": 2,
"accepted_files": 1,
"rejected_files": 1,
"completed_files": 1,
"created_at": "2026-09-02T12:00:00Z",
"items": [
{
"item_index": 0,
"filename": "退款政策.md",
"acceptance_status": "accepted",
"document_id": "6e5ec370-5a45-4eb7-95d7-e84267ad8752",
"job_id": "7adc4278-1157-4c3d-95a4-6926ad77a04e",
"job_status": "queued",
"stage": null,
"attempt": 0,
"max_attempts": 3,
"error_message": null
}
]
}
```
Job 公开状态为 `queued | running | retrying | succeeded | failed | cancelled`,并通过 `stage`
表示 `parsing | cleaning | chunking | embedding | indexing`。错误字段只返回有界、安全摘要。
批次状态为 `processing | completed | partial_failure | failed`;逐文件接收状态为
`pending | accepted | rejected`。批次允许逐文件部分接收/失败,不以一个文件的处理结果回滚
其他文件。文件数量、单文件大小和请求总大小分别由 `BATCH_MAX_FILES`、`MAX_UPLOAD_BYTES`
和 `BATCH_MAX_TOTAL_BYTES` 限制。数量超限返回 `413`;类型、大小或内容不合规作为该文件的
`rejected` 安全结果返回,其他合规文件继续接收。
自动重试是服务端 Job 行为,不要求客户端再次上传,也不新增公开手动重试接口。每次重试前
清理派生数据但保留原始文件和稳定的 Document ID;达到次数上限或清理失败后进入 `failed`。

View File

@ -9,3 +9,43 @@ P0 business tools:
- `list_knowledge_bases` - `list_knowledge_bases`
Every request maps to a user/service identity and uses the same authorization/application services as REST. Do not expose OceanBase Vector, BOS, model names, parsers or other infrastructure choices. Every request maps to a user/service identity and uses the same authorization/application services as REST. Do not expose OceanBase Vector, BOS, model names, parsers or other infrastructure choices.
## Demo priority slice(DEMO-012,已实现)
`DEMO-012` 已提升为当前最高实现优先级。首个可运行 Gateway 只注册
`search_knowledge`,其输入与 `/api/v1/search` 对齐:
```json
{
"query": "产品A退款周期是多少?",
"top_k": 5
}
```
输出保留业务来源字段:`document_id`、`filename`、`page`、`section`、`chunk_id`、`content`
和排序分数。Gateway 必须复用现有 Search Service 或正式 REST API,不得复制检索、来源补全
或排序逻辑。
Demo 验证也必须建立 user/service identity 映射边界,禁止使用共享超级管理员 Token。由于
Document ACL 尚未进入 Demo Scope,`DEMO-012` 只能证明身份传递、认证拒绝和业务服务复用,
不能声称完成 V1 ACL 授权闭环。错误响应和日志不得包含 Provider Endpoint、凭据、数据库表名
或模型名。
验收至少覆盖:MCP 工具发现、参数校验、成功检索、无结果、认证失败、安全错误收敛,以及
WorkBuddy Enterprise 的一次端到端调用。其余 P0 tools 继续留在完整 V1 `EKP-071`。
实现使用独立 Streamable HTTP Gateway,默认端点为 `http://127.0.0.1:8766/mcp`。请求必须
携带 `Authorization: Bearer <token>`;Token 只从运行环境读取,并映射为显式 service identity
和 `knowledge:search` scope。当前静态 service credential 是 Demo 兼容层,不替代 V1 OAuth、
ACL 或用户委托授权。
公开错误边界:缺失/无效 Token 为 HTTP `401`,scope 不足为 `403`;Tool 参数不合法返回 MCP
参数错误;检索运行失败只返回 `knowledge search failed`。Host/Origin allowlist 默认只允许
本机,远程部署必须显式注入公开域名。
## 当前公共知识库部署边界(PLAN-002)
集群阶段仍只暴露只读 `search_knowledge`,并为 WorkBuddy 配置稳定的 HTTPS MCP 地址。
Credential 由 Kubernetes Secret 在服务端运行时注入,客户端只保存管理员分发的最小
`knowledge:search` Token;Pod 重启或滚动升级不得要求重新生成配置。公共知识域不等于匿名
访问或超级管理员身份,未来引入非公共知识前必须升级为用户委托身份与 ACL-aware Search。

View File

@ -2,22 +2,39 @@
- ADR-001: API First + Admin Console — Accepted - ADR-001: API First + Admin Console — Accepted
- ADR-002: Modular monolith + workers + MCP gateway for V1 — Accepted - ADR-002: Modular monolith + workers + MCP gateway for V1 — Accepted
- ADR-003: PostgreSQL + object storage are canonical — Accepted - ADR-003: PostgreSQL + object storage are canonical — Superseded by ADR-012
- ADR-004: Mochow is initial retrieval backend but replaceable — Superseded by ADR-010 - ADR-004: Mochow is initial retrieval backend but replaceable — Superseded by ADR-010
- ADR-005: Provider abstraction for embedding/reranker/parser/LLM — Accepted - ADR-005: Provider abstraction for embedding/reranker/parser/LLM — Accepted
- ADR-006: RAG + Knowledge Compilation — Accepted - ADR-006: RAG + Knowledge Compilation — Accepted
- ADR-007: ACL before/during retrieval — Accepted - ADR-007: ACL before/during retrieval — Accepted
- ADR-008: MCP hides infrastructure details — Accepted - ADR-008: MCP hides infrastructure details — Accepted
- ADR-009: Baidu BOS is the primary object-storage provider; local storage remains for tests and local development — Accepted - ADR-009: Baidu BOS is the primary object-storage provider; local storage remains for tests and local development — Accepted
- ADR-010: OceanBase Vector is the primary rebuildable retrieval index behind `VectorStore`; PostgreSQL remains canonical — Accepted - ADR-010: OceanBase Vector is the primary rebuildable retrieval index behind `VectorStore`; PostgreSQL remains canonical — Superseded by ADR-012
- ADR-011: Qianfan `bge-large-zh` plus `bce-reranker-base` is the initial Demo model baseline — Accepted - ADR-011: Qianfan `bge-large-zh` plus `bce-reranker-base` is the initial Demo model baseline — Accepted
- ADR-012: OceanBase MySQL relational tables + BOS are canonical; OceanBase vector tables remain rebuildable indexes — Accepted
## Architecture migration boundary ## Architecture migration boundary
ADR-009~011 adopt the verified internal platform provider baseline without copying its ARCH-006 的只读环境核验确认,目标平台没有独立 PostgreSQL,OceanBase MySQL 模式同时承载
canonical-database design. ADR-003 remains binding: OceanBase Vector stores only rebuildable 关系数据和向量数据。因此 ADR-012 取代 ADR-003/010 中的 PostgreSQL 部署假设:OceanBase
retrieval rows, while PostgreSQL and object storage retain canonical metadata and immutable 关系表保存 canonical metadata,BOS 保存不可变源文件,OceanBase 专用向量表仍只保存可由
source data. canonical Chunk 重建的检索行。共用数据库实例不改变两类数据的所有权边界,也不允许业务层
绕过 `VectorStore` 直接依赖 `pyobvector`。
目标 test 集群为 OceanBase 4.3.5.4 MySQL 模式。ARCH-006 已用只读 `EXPLAIN` 验证
`VECTOR_INDEX(... pre_filter)` 产生 `VECTOR INDEX ADAPTIVE SCAN (PRE-FILTER)`;版本下限与
运行检查据此调整为 4.3.5.4。生产升级和重新验收仍由运维流程控制。
## Public knowledge delivery boundary
PLAN-002 将当前交付阶段限定为企业受控网络内的单一公共知识可见域,并加入 Kubernetes/Helm
集群交付任务。这是部署范围和实施顺序调整,不是新的架构选型:继续采用 ADR-002 的模块化
单体 API、独立 Worker 和 MCP Gateway,以及架构既定的 Celery/RabbitMQ;OceanBase/BOS
边界继续遵循 ADR-012。
ADR-007 仍为 Accepted。当前阶段只是延后用户/组/角色和 Document ACL;开始接收任何非公共
知识前,必须先实现 Permission Resolver 和检索前/检索中 ACL 下推。公共知识库也不等于匿名
公网开放,Admin 写操作只允许经受控网络或统一接入网关访问,MCP 使用最小只读服务身份。
The migration retains the Demo's Dense Top20 → Reranker Top5 behavior and current lightweight The migration retains the Demo's Dense Top20 → Reranker Top5 behavior and current lightweight
Parser scope. Top50 retrieval, distance thresholds, dual-query RRF, Marker/Pandoc and LLM-derived Parser scope. Top50 retrieval, distance thresholds, dual-query RRF, Marker/Pandoc and LLM-derived

View File

@ -63,10 +63,11 @@ Wiki / FAQ / SOP / Summary 等派生知识
┌────────────────────────┼────────────────────────┐ ┌────────────────────────┼────────────────────────┐
│ │ │ │ │ │
▼ ▼ ▼ ▼ ▼ ▼
┌──────────────┐ ┌──────────────┐ ┌──────────────┐ ┌────────────────────────┐ ┌──────────────┐
│ PostgreSQL │ │ Object Store │ │ OceanBase │ │ OceanBase MySQL │ │ Object Store │
│ Canonical DB │ │ BOS │ │ Vector Index │ │ Canonical + Vector │ │ BOS │
└──────────────┘ └──────────────┘ └──────────────┘ │ (logical separation) │ └──────────────┘
└────────────────────────┘
▲ ▲ ▲ ▲
│ │ │ │
┌───────┴────────────────────────┘ ┌───────┴────────────────────────┘
@ -88,7 +89,7 @@ Wiki / FAQ / SOP / Summary 等派生知识
1. **API First**:后台、MCP、其他业务系统均调用统一 API/Service。 1. **API First**:后台、MCP、其他业务系统均调用统一 API/Service。
2. **模块化单体优先**:V1 不拆大量微服务,API 服务 + 异步 Worker + MCP Gateway 足够。 2. **模块化单体优先**:V1 不拆大量微服务,API 服务 + 异步 Worker + MCP Gateway 足够。
3. **Canonical Data 独立**:PostgreSQL + 对象存储保存事实数据,Vector DB 仅保存可重建索引。 3. **Canonical Data 逻辑独立**:OceanBase 关系表 + BOS 保存事实数据,同库向量表仅保存可重建索引。
4. **Provider 可替换**:Embedding、Reranker、Vector DB、Parser、LLM 均通过 Adapter 抽象。 4. **Provider 可替换**:Embedding、Reranker、Vector DB、Parser、LLM 均通过 Adapter 抽象。
5. **MCP 不暴露底层实现**:客户端只看到 `search_knowledge` 等业务工具。 5. **MCP 不暴露底层实现**:客户端只看到 `search_knowledge` 等业务工具。
@ -99,12 +100,12 @@ Wiki / FAQ / SOP / Summary 等派生知识
| 模块 | V1 当前选型 | 说明 | | 模块 | V1 当前选型 | 说明 |
|---|---|---| |---|---|---|
| Backend | Python + FastAPI | AI / 数据处理生态成熟,适合 MCP、异步任务和模型调用 | | Backend | Python + FastAPI | AI / 数据处理生态成熟,适合 MCP、异步任务和模型调用 |
| Metadata DB | PostgreSQL | 保存用户、权限、文档、版本、任务、Compiled Knowledge 等 Canonical Data | | Metadata DB | OceanBase MySQL | 保存用户、权限、文档、版本、任务、Compiled Knowledge 等 Canonical Data |
| Object Storage | Baidu BOS | 保存不可变原始文件、解析产物和中间文件;Local Adapter 仅用于测试/本地开发 | | Object Storage | Baidu BOS | 保存不可变原始文件、解析产物和中间文件;Local Adapter 仅用于测试/本地开发 |
| Task Queue | Celery + RabbitMQ | 文件解析、Embedding、索引、Compilation 等异步任务 | | Task Queue | Celery + RabbitMQ | 文件解析、Embedding、索引、Compilation 等异步任务 |
| PDF / OCR Parser | MinerU 优先 | 中文复杂 PDF、扫描件、表格、版面解析能力较强 | | PDF / OCR Parser | MinerU 优先 | 中文复杂 PDF、扫描件、表格、版面解析能力较强 |
| Office Parser | Docling 补充 | DOCX/PPTX/XLSX/HTML 等格式统一解析 | | Office Parser | Docling 补充 | DOCX/PPTX/XLSX/HTML 等格式统一解析 |
| Vector/Search DB | OceanBase Vector | 当前主检索索引,必须通过 `VectorStore` 隔离并可从 PostgreSQL 重建 | | Vector/Search DB | OceanBase Vector | 与关系表同库部署但逻辑隔离,必须通过 `VectorStore` 并可从 canonical Chunk 重建 |
| Embedding | Qianfan bge-large-zh | 当前 1024 维基线,后续模型变更使用版本化 Shadow Index | | Embedding | Qianfan bge-large-zh | 当前 1024 维基线,后续模型变更使用版本化 Shadow Index |
| Reranker | Qianfan bce-reranker-base | 当前 Dense Top20 → Reranker Top5 基线 | | Reranker | Qianfan bce-reranker-base | 当前 Dense Top20 → Reranker Top5 基线 |
| LLM | Provider 抽象,云端优先 | 用于 Knowledge Compilation、分类、摘要等 | | LLM | Provider 抽象,云端优先 | 用于 Knowledge Compilation、分类、摘要等 |
@ -173,6 +174,46 @@ Queued / Parsing / Cleaning / Chunking / Embedding / Indexing / Completed / Fail
- 软删除; - 软删除;
- 来源页码和章节追踪。 - 来源页码和章节追踪。
### 4.2.1 批量异步提交与失败恢复
API 只负责校验并持久化不可变源文件、Document 和 Ingestion Job,然后返回 `202`。批次是
提交与聚合查询边界,不是跨文件数据库事务;每个文件独立排队、执行和失败。
DEMO-016 以 canonical `ingestion_batches` / `ingestion_batch_items` 保存输入顺序和接收引用;
处理阶段、Attempt 与最终结果继续以 `ingestion_jobs` / `ingestion_attempts` 为事实源,批次表
不复制 Worker 状态。
```text
Batch Request
├─ Source A + Document A + Job A ─→ Worker
├─ Source B + Document B + Job B ─→ Worker
└─ Source C + Document C + Job C ─→ Worker
```
Worker 对可重试错误执行有界退避。重新执行前必须通过独立的派生数据清理服务按以下顺序
恢复干净状态:
```text
Vector/Sparse rows
→ ChunkIndexState
→ Chunks
→ Parsed artifact + parsed_storage_key
→ reset processing state
→ next attempt from immutable source
```
清理服务与 Document 删除语义分离:重试清理保留原始 BOS 对象、Document 和 Job/Attempt
历史。清理失败时 Job 终止,不能在未知状态继续执行。Job 通过数据库状态、唯一有效 Attempt
和幂等键避免重复投递产生并发执行;具体队列实现保持可替换,不进入业务服务接口。
公共知识库集群交付阶段由 Celery/RabbitMQ 作为首个生产执行器。API 只调用队列执行器抽象,
Celery task 只负责恢复应用上下文并调用 Ingestion Application Service;重试分类、Attempt 状态、
清理顺序和业务幂等仍由应用层持有。Broker 消息不是 Job 的 source of truth,Worker 重启或消息
重复投递必须以 OceanBase 中的持久化 Job/Attempt 状态收敛。
DEPLOY-001 将 API/Admin、MCP Gateway 与 Async Worker 构建为三个共享锁定 Python 运行时的
非 root 镜像 target;进程边界与上述模块边界一致,不新增服务或把 Provider 配置烘焙进镜像。
--- ---
### 4.3 文档结构与 Chunk ### 4.3 文档结构与 Chunk
@ -326,6 +367,11 @@ profile_v2
避免直接原地覆盖现有生产索引。 避免直接原地覆盖现有生产索引。
ARCH-005/006 将 Profile 生命周期以 `shadow / active / retired` 记录在 OceanBase 关系表;任一时刻至多
一个 Active Profile。重建从 canonical Chunk 生成确定性 `chunk_id + profile_id` 索引行,
重复执行跳过已完整文档,激活前重新核对全部 ChunkIndexState,并在同一事务内锁定 Chunk
写入、退役旧 Profile、激活新 Profile。OceanBase 表本身仍不是 canonical source。
--- ---
## 8. Vector Store 设计 ## 8. Vector Store 设计
@ -359,8 +405,9 @@ VectorStoreProvider
业务 Domain Model 不依赖 OceanBase/`pyobvector` SDK 数据结构。 业务 Domain Model 不依赖 OceanBase/`pyobvector` SDK 数据结构。
OceanBase Vector 中的数据全部视为 **可重建索引**。PostgreSQL 保存文档、Chunk、Profile OceanBase Vector 表中的数据全部视为 **可重建索引**。同一数据库的关系表保存文档、Chunk、
和索引状态等 canonical 数据,BOS 保存不可变原始文件和派生产物。 Profile 和索引状态等 canonical 数据,BOS 保存不可变原始文件和派生产物。物理共库不允许
向量表反向成为业务事实来源。
--- ---
@ -497,6 +544,17 @@ MinerU
首要验证客户端:**WorkBuddy Enterprise**。 首要验证客户端:**WorkBuddy Enterprise**。
### 11.1 DEMO-012 实现边界
当前已落地独立 Streamable HTTP Gateway,默认只绑定 `127.0.0.1:8766/mcp`,只注册
`search_knowledge`。Gateway 与 REST 共享 `SearchService`,同步检索通过工作线程执行,避免
阻塞 MCP ASGI 事件循环。
Demo 使用运行时专用 Bearer Token 映射到 client/service identity 和最小
`knowledge:search` scope,并启用 Host/Origin allowlist。该边界用于 WorkBuddy 接入验证,
后续 `EKP-070` 仍需用正式 OAuth/企业身份系统和 Permission Resolver 替换静态 Token;
`DEMO-012` 不改变 ADR-007 的 ACL 要求。
--- ---
## 12. REST API ## 12. REST API
@ -597,6 +655,11 @@ ReindexJob
EvaluationJob EvaluationJob
``` ```
当前增量实现先提供单个 `IngestionJob` 编排既有 Parse/Clean/Chunk/Embedding/Indexing 阶段,
保留阶段级状态和 Attempt 历史;以后拆分为独立阶段 Job 时不得改变公开批次/Job 查询契约。
Demo 默认最多 3 次总尝试,只有瞬时 Provider、网络、存储或数据库错误进入自动重试;输入
校验和确定性内容错误直接终止。
GPU 节点优先承担: GPU 节点优先承担:
- MinerU / OCR; - MinerU / OCR;
@ -609,7 +672,7 @@ LLM V1 可继续使用云端 Provider,不要求部署本地大模型。
## 15. 数据存储原则 ## 15. 数据存储原则
### PostgreSQL ### OceanBase MySQL 关系表
保存: 保存:
@ -658,15 +721,13 @@ Reverse Proxy / Ingress
├─ Knowledge API ├─ Knowledge API
└─ MCP Gateway └─ MCP Gateway
PostgreSQL OceanBase MySQL(Canonical + Vector,逻辑隔离)
RabbitMQ RabbitMQ
Baidu BOS Baidu BOS
CPU Worker CPU Worker
GPU Worker GPU Worker
OceanBase Vector
Observability Observability
``` ```
@ -683,6 +744,30 @@ Observability
未来根据负载再拆服务。 未来根据负载再拆服务。
### 16.1 当前公共知识库集群交付基线
PLAN-002 将当前交付范围收敛为企业受控网络内的单一公共知识可见域。该范围不重开 ADR-007:
所有现有文档共享同一可见性,因而当前检索不执行事后 ACL 过滤;接收任何非公共知识前,必须
先实现 Permission Resolver 和数据库/向量检索前置过滤。Admin 写接口不得直接暴露到匿名公网,
MCP 使用运行时 Secret 注入的最小 `knowledge:search` 服务身份。
集群中 API/Admin、MCP Gateway 和 Async Worker 分别部署并独立扩缩容,业务核心继续保持模块化
单体。当前 Parser 及云端 Embedding/Reranker 不需要 GPU Worker,因此它不进入最小交付,但角色
边界保留。OceanBase、BOS 和 AI Provider 使用外部服务;RabbitMQ 默认使用外部地址,可为开发/
演示显式启用 Chart 依赖。
Kubernetes/Helm 必须提供:生产镜像、`existingSecret`、串行 Migration Job、Ingress/TLS、
readiness/liveness、资源限制、非 root 与只读根文件系统、多副本和滚动更新。仓库的一键入口只
编排已有集群和外部服务,不创建云账号或把凭据写入 Values、镜像、命令行及日志。现有
`docker-compose.dev.yml` 包含已被 ADR-012 取代的 PostgreSQL/MinIO 假设,只能视为待清理的
本地开发资产,不能作为集群模板。详细依赖和验收见
[`PUBLIC_KNOWLEDGE_CLUSTER_PLAN.md`](../plans/PUBLIC_KNOWLEDGE_CLUSTER_PLAN.md)。
DEPLOY-002 的运行拓扑将 Celery Worker 与 Celery Beat 恢复调度器拆成独立 Pod:Worker 可横向
扩缩,Beat 固定单副本并使用 `Recreate`,避免每个 Worker 都发布周期恢复消息。二者仍调用同一
应用服务与持久化 Job/Attempt 状态,不形成新业务服务,也不改变 ADR-002。Migration 使用 Helm
`pre-install,pre-upgrade` Hook;失败会在业务 Deployment 更新前终止原子升级。
--- ---
## 17. V1 验收重点 ## 17. V1 验收重点
@ -737,7 +822,7 @@ WorkBuddy 可以通过 MCP 完成知识搜索和 Source 获取。
```text ```text
Backend Python + FastAPI Backend Python + FastAPI
Metadata PostgreSQL Metadata OceanBase MySQL relational tables
Object Storage Baidu BOS(Local Adapter 仅测试/本地开发) Object Storage Baidu BOS(Local Adapter 仅测试/本地开发)
Queue RabbitMQ + Celery Queue RabbitMQ + Celery
Parser MinerU 主力 + Docling 补充 Parser MinerU 主力 + Docling 补充

View File

@ -67,8 +67,8 @@ FastAPI
辅助存储: 辅助存储:
```text ```text
SQLite / PostgreSQL OceanBase MySQL
└── document / chunk metadata └── canonical document / chunk metadata
Local Storage / BOS Local Storage / BOS
└── original files └── original files
@ -83,7 +83,7 @@ Demo 阶段优先简单实现,不引入不必要的基础设施。
| 模块 | Demo 推荐选型 | 说明 | | 模块 | Demo 推荐选型 | 说明 |
|---|---|---| |---|---|---|
| Backend | FastAPI | 简单、开发效率高 | | Backend | FastAPI | 简单、开发效率高 |
| Metadata DB | SQLite 或 PostgreSQL | 单机 Demo 可 SQLite;已有环境可直接 PostgreSQL | | Metadata DB | OceanBase MySQL | 目标环境使用 OceanBase;SQLite 仅用于离线测试 |
| File Storage | 百度 BOS | Local Adapter 仅用于测试和本地开发,目标环境使用 BOS | | File Storage | 百度 BOS | Local Adapter 仅用于测试和本地开发,目标环境使用 BOS |
| Parser | PyMuPDF + python-docx 或 MinerU | 优先快速跑通;复杂 PDF 后续增强 | | Parser | PyMuPDF + python-docx 或 MinerU | 优先快速跑通;复杂 PDF 后续增强 |
| Cleaner | 自定义基础清洗 | 处理空行、空白、异常换行等 | | Cleaner | 自定义基础清洗 | 处理空行、空白、异常换行等 |
@ -92,7 +92,7 @@ Demo 阶段优先简单实现,不引入不必要的基础设施。
| Vector DB | OceanBase Vector | 当前默认,通过 `VectorStore` 隔离 SDK,不做多数据库运行时切换 | | Vector DB | OceanBase Vector | 当前默认,通过 `VectorStore` 隔离 SDK,不做多数据库运行时切换 |
| Reranker | `bce-reranker-base` | 延续当前可调用的千帆 Reranker,Demo 中保留 | | Reranker | `bce-reranker-base` | 延续当前可调用的千帆 Reranker,Demo 中保留 |
| Frontend | 简单 React/Vue 或 FastAPI HTML | 以最小可用为目标 | | Frontend | 简单 React/Vue 或 FastAPI HTML | 以最小可用为目标 |
| Deployment | 本地 / 单机 Docker | 暂不考虑 K8s | | Deployment | Kubernetes/Helm | PLAN-002 起面向企业受控网络交付;本地模式仅用于开发测试 |
--- ---
@ -189,7 +189,7 @@ embedding
Demo 阶段不实现 Qdrant/Milvus/pgvector,也不实现多 Vector Store 动态切换,但避免 Demo 阶段不实现 Qdrant/Milvus/pgvector,也不实现多 Vector Store 动态切换,但避免
OceanBase/pyobvector 调用散落在整个项目中。OceanBase Vector 行仍是可重建索引,不能替代 OceanBase/pyobvector 调用散落在整个项目中。OceanBase Vector 行仍是可重建索引,不能替代
PostgreSQL canonical Chunk。 同库关系表中的 canonical Chunk。
### 5.7 Dense Retrieval ### 5.7 Dense Retrieval
@ -305,7 +305,7 @@ Search Response 示例:
--- ---
## 8. P1 可选增强 ## 8. Demo 后续增强
### DEMO-011 Hybrid Search ### DEMO-011 Hybrid Search
@ -323,12 +323,26 @@ Reranker
### DEMO-012 MCP ### DEMO-012 MCP
只实现一个最小 MCP Tool:`search_knowledge`,内部调用 `/api/search`,用于验证 WorkBuddy 接入。 **已完成。** 只实现一个最小 MCP Tool:`search_knowledge`,复用现有
Search Service 或 `/api/v1/search`,用于验证标准 MCP、identity 边界和 WorkBuddy 接入。
不在本 Task 扩展其他 P0 tools 或声称完成尚未实施的 Document ACL。
### DEMO-013 OCR / MinerU ### DEMO-013 OCR / MinerU
增加扫描 PDF、图片、OCR 和复杂中文 PDF 处理能力。 增加扫描 PDF、图片、OCR 和复杂中文 PDF 处理能力。
### DEMO-015 异步入库 Job 与自动重试清理
**已完成。** 将现有同步入库流水线放入持久化 Ingestion Job。可重试失败自动执行有界重试;每次重试前
删除旧解析产物、Chunk、索引状态和向量数据,但保留不可变原始文件、Document 和 Attempt
历史。清理失败时终止,不在未知脏状态继续执行。通过执行器抽象接入 Celery/RabbitMQ,使
API 与 Worker 可独立部署,并以持久化状态处理 Worker 重启和重复消息。
### DEMO-016 批量异步上传
**已完成。** 在 DEMO-015 基础上支持一次提交多个文件、逐文件 Document/Job 状态、批次聚合查询和
Admin Console 状态展示。批次内文件相互隔离,不包含目录上传、压缩包解包或断点续传。
--- ---
## 9. 明确后置能力 ## 9. 明确后置能力
@ -348,12 +362,13 @@ Reranker
- 多 Embedding Benchmark Platform; - 多 Embedding Benchmark Platform;
- 多 Vector DB; - 多 Vector DB;
- 动态 Provider; - 动态 Provider;
- RabbitMQ / Celery;
- Kubernetes;
- 高可用;
- 灾备; - 灾备;
- 完整 Observability。 - 完整 Observability。
用户/权限隔离属于完整 V1 后置能力,不因“公共知识库”而取消;本阶段所有已接收知识共享一个
可见域,且仅部署在企业受控网络或统一接入网关后。RabbitMQ/Celery 与最小 Kubernetes 可用性
已由 PLAN-002 纳入当前交付,跨区域容灾和完整平台级高可用仍不在范围内。
--- ---
## 10. Demo Task 清单 ## 10. Demo Task 清单
@ -364,22 +379,50 @@ DEMO-002 Document 数据模型 + 本地文件存储
DEMO-003 PDF/DOCX/TXT/MD 上传 DEMO-003 PDF/DOCX/TXT/MD 上传
DEMO-004 Parser + Cleaner DEMO-004 Parser + Cleaner
DEMO-005 Chunk Pipeline DEMO-005 Chunk Pipeline
DEMO-006 bge-large-zh Embedding + 初始 Mochow 入库(将由 ARCH-003 迁移) DEMO-006 bge-large-zh Embedding + 初始 Mochow 入库(ARCH-003~005 已迁移为 OceanBase)
DEMO-007 Dense Retrieval DEMO-007 Dense Retrieval
DEMO-008 Reranker + Source Result DEMO-008 Reranker + Source Result
DEMO-009 Document/Search REST API DEMO-009 Document/Search REST API
DEMO-010 简单 Admin Console DEMO-010 简单 Admin Console
DEMO-012 MCP search_knowledge(已提升优先级)
DEMO-014 地瓜机器人品牌化前端优化(用户新增)
DEMO-015 异步入库 Job + 自动重试前清理(用户新增)
DEMO-016 批量异步上传 + 状态展示(用户新增)
ARCH-001~006 Provider 架构迁移与真实验收 ARCH-001~006 Provider 架构迁移与真实验收
PLAN-002 公共知识库集群交付规划(已完成)
DEPLOY-001 API/Admin、MCP Gateway、Async Worker 生产镜像
DEPLOY-002 Kubernetes/Helm 一键部署
DEPLOY-003 可用性、滚动升级和远程 WorkBuddy 验收
``` ```
实施状态(2026-09-02):`DEMO-001~DEMO-010`、用户新增的 `DEMO-014` 与
`ARCH-001~ARCH-006` 均已完成并通过当前 Scope 的闭环验收。`PLAN-001` 已完成现状审计和
任务拆分;`DEMO-012`、`DEMO-015~016` 已完成。`PLAN-002` 已将公共知识库集群交付纳入范围,
后续严格按 `DEPLOY-001~003` 顺序实施。`DEMO-011` 和 `DEMO-013` 仍为
可选项,不因本状态记录自动进入范围。
可选: 可选:
```text ```text
DEMO-011 Hybrid Search DEMO-011 Hybrid Search
DEMO-012 MCP search_knowledge
DEMO-013 OCR / MinerU DEMO-013 OCR / MinerU
``` ```
已批准顺序:
```text
PLAN-002 Public Knowledge Cluster Plan
→ DEMO-015 Celery/RabbitMQ Async Job + Retry Cleanup
→ DEMO-016 Batch Async Upload + UI
→ DEPLOY-001 Production Images
→ DEPLOY-002 Kubernetes/Helm One-click Deploy
→ DEPLOY-003 Availability + Remote Acceptance
```
详细 Task 映射、依赖与验收边界见
[`docs/plans/ASYNC_INGESTION_MCP_PLAN.md`](../plans/ASYNC_INGESTION_MCP_PLAN.md) 和
[`docs/plans/PUBLIC_KNOWLEDGE_CLUSTER_PLAN.md`](../plans/PUBLIC_KNOWLEDGE_CLUSTER_PLAN.md)。
--- ---
## 11. Demo 测试数据 ## 11. Demo 测试数据
@ -468,10 +511,13 @@ Demo 阶段不设严格生产指标。
## 14. Codex 开发约束 ## 14. Codex 开发约束
Codex 当前只能实现 `DEMO-001 ~ DEMO-010` 和已批准的 `ARCH-001 ~ ARCH-006`。 Codex 当前只能实现 `DEMO-001 ~ DEMO-010`、`DEMO-012`、`DEMO-014 ~ DEMO-016`、
ARCH 迁移任务不授权实现下列被排除的完整 V1 能力。 `ARCH-001 ~ ARCH-006`、`PLAN-001 ~ PLAN-002` 和 `DEPLOY-001 ~ DEPLOY-003`。
PLAN-002 只授权实现 Celery/RabbitMQ 异步执行器与最小 Kubernetes/Helm 集群交付,不授权
拆分额外微服务或扩大业务能力。
除非明确要求,否则禁止提前实现权限、审计、Knowledge Compilation、多数据库、多模型平台、微服务、Kubernetes 和企业级 Job Queue。 除非明确要求,否则禁止提前实现权限、审计、Knowledge Compilation、多数据库、多模型平台、
额外微服务、跨区域容灾和完整 Observability 平台。
当本文件与完整 PRD 的实现优先级冲突时: 当本文件与完整 PRD 的实现优先级冲突时:
@ -500,10 +546,10 @@ docs/demo/DEMO_SCOPE.md
> >
完整 V1 PRD 完整 V1 PRD
只允许实现 DEMO-001 ~ DEMO-010 以及本文件明确列出的 ARCH-001 ~ ARCH-006。 只允许实现本文件明确列出的 Demo、Architecture、Plan 与 Deploy Task,并且一次只执行一个
Provider 架构迁移任务按顺序完成后,再继续 DEMO-010。 Task。当前严格顺序为 DEMO-016 → DEPLOY-001 → DEPLOY-002 → DEPLOY-003。
当前 Demo 的目标只有: 当前公共知识库阶段的业务目标保持为:
Upload Upload
→ Parse/Clean → Parse/Clean
@ -515,6 +561,14 @@ Upload
→ Source Result → Source Result
→ Search UI → Search UI
并补齐以下交付能力:
Async Job / Retry Cleanup
→ Batch Async Upload
→ Production Images
→ Kubernetes/Helm One-click Deploy
→ Availability + Remote WorkBuddy Acceptance
不要提前实现: 不要提前实现:
- ACL - ACL
@ -524,9 +578,9 @@ Upload
- Knowledge Compilation - Knowledge Compilation
- Multi Vector Store - Multi Vector Store
- Embedding Benchmark Platform - Embedding Benchmark Platform
- Celery/RabbitMQ - 额外微服务
- Kubernetes - 跨区域容灾
- 企业级高可用 - 完整 Observability 平台
先不要修改代码。 先不要修改代码。
@ -535,7 +589,7 @@ Upload
1. 当前仓库状态; 1. 当前仓库状态;
2. 为完成 Demo 需要保留哪些现有结构; 2. 为完成 Demo 需要保留哪些现有结构;
3. 哪些完整 V1 设计可以暂时不实现; 3. 哪些完整 V1 设计可以暂时不实现;
4. DEMO-001 ~ DEMO-010 与 ARCH-001 ~ ARCH-006 的建议执行顺序; 4. 当前已批准 Task 的执行顺序;
5. 下一步应该实施的第一个 Task。 5. 下一步应该实施的第一个 Task。
``` ```
@ -546,10 +600,20 @@ Upload
```text ```text
Demo Demo
↓ ↓
Hybrid Search
↓
MCP MCP
↓ ↓
Async Job + Retry Cleanup
↓
Batch Async Upload
↓
Production Images
↓
Kubernetes/Helm One-click Deploy
↓
Availability + Remote WorkBuddy Acceptance
↓
Hybrid Search
↓
OCR / Complex Parsing OCR / Complex Parsing
↓ ↓
Document Version Document Version

View File

@ -0,0 +1,59 @@
# Demo Admin Console 使用说明
DEMO-010 在 FastAPI 服务中提供一个轻量、同源的管理客户端。页面只负责展示和交互,上传、
文档管理与检索仍全部通过 `/api/v1` 公开业务接口完成,不复制服务端业务逻辑。
## 启动与访问
完成运行时配置后启动 Worker 与 API:
```bash
make run-worker
make run
```
页面地址:
| 页面 | 地址 | 对应业务接口 |
|---|---|---|
| 文档上传 | `/admin/upload` | `POST/GET /api/v1/ingestion-batches*` |
| 文档列表与详情 | `/admin/documents` | `GET/DELETE /api/v1/documents*` |
| Search Lab | `/admin/search` | `POST /api/v1/search` |
`/admin` 默认展示上传页。OpenAPI 文档仍位于 `/api/v1/docs`。
## 品牌视觉基线
DEMO-014 参考地瓜机器人公开官网的设计语言,使用品牌橙 `#FF3C00`、黑白高对比、中文
无衬线标题、紧凑数据字体、低圆角容器和宽松留白。贯穿页面标题的“计算信号轨”是唯一的
强调性图形,序号对应上传、文档、检索三步真实操作顺序。
品牌图形使用仓库内的代码原生 SVG/CSS 绘制,不复制或热链官网图片、字体和脚本;页面
运行时不依赖品牌官网。前端设计方法参考 GitHub `anthropics/skills` 的 Apache-2.0
`frontend-design` Skill,并以现有 Demo 信息架构、可访问性和业务正确性为优先约束。
## Demo 操作闭环
1. 在上传页选择一个或多个 PDF、DOCX、TXT 或 Markdown 文件并提交;
2. 接口返回 `202` 后,页面自动轮询批次,逐文件显示接收状态、处理阶段、尝试次数和安全错误;
3. 批次进入 `Completed`、`Partial failure` 或 `Failed` 后停止轮询,再在文档列表查看元数据;
4. 在 Search Lab 输入问题并查看 Top-K 的文件名、页码、章节、Chunk 内容、Dense Score 和
Rerank Score;
5. 如需清理 Demo 数据,在文档列表确认后直接删除。
## 范围与安全边界
- 页面静态文件位于 `apps/admin-web`,由同一个 FastAPI 进程提供,避免额外前端运行时;
- API 返回内容仅通过 DOM `textContent` 渲染,不作为 HTML 注入;
- 页面不保存或展示 Provider 凭据、内部 Endpoint、storage key 或 Index Profile;
- Console 只轮询当前批次的聚合接口,不读取 Provider/队列细节,也不提供人工重试按钮;
- 批量上传不包含目录上传、压缩包解包、断点续传或跨批次去重;
- 本任务不包含 ACL/RBAC、审计、Document Version、Compiled Knowledge、MCP、Hybrid/BM25
或完整 V1 Admin Console 导航;
- 删除是 Demo 的直接删除语义,确认后会清理 canonical Metadata、派生产物、源文件与向量行。
## 验证
静态路由和范围契约由 `tests/integration/test_admin_console.py` 覆盖;现有 REST 全链路测试覆盖
上传、列表、详情、搜索与删除。浏览器验收使用确定性本地 Provider,不读取真实企业文档,
并覆盖桌面与移动布局。

View File

@ -0,0 +1,97 @@
# 异步入库与失败恢复说明
## 范围
DEMO-015 将单文件上传从 API 请求线程迁移到独立 Celery Worker,DEMO-016 在其上增加逐文件
隔离的批量接收和状态聚合。OceanBase 中的
`ingestion_jobs` 和 `ingestion_attempts` 是执行状态事实源;RabbitMQ 消息只携带 Job ID,
不携带文件内容、Provider 凭据或业务状态。
批次只聚合接收与 Job 状态,不改变每个文件的独立执行和重试边界。
## 运行时配置
以下值必须通过环境变量或部署 Secret 注入,不得提交真实值:
```text
RABBITMQ_URL=amqp://<user>:<password>@<host>:5672/<vhost>
INGESTION_MAX_ATTEMPTS=3
INGESTION_RETRY_BACKOFF_SECONDS=2
INGESTION_RETRY_BACKOFF_MAX_SECONDS=60
INGESTION_WORKER_LEASE_SECONDS=300
INGESTION_RECOVERY_INTERVAL_SECONDS=30
BATCH_MAX_FILES=20
BATCH_MAX_TOTAL_BYTES=209715200
```
Worker 与 API 还需要相同的 OceanBase、BOS、Embedding、VectorStore 和 Index Profile 配置。
RabbitMQ 用户应只具备指定 vhost/queue 的最小权限。
## 启动顺序
```bash
uv run alembic upgrade head
make run-worker
make run
```
`make run-worker` 在当前 Demo 中同时启动 Worker 和恢复调度器。恢复调度器周期扫描:
- 尚未成功投递的 `queued` Job;
- 已到重试时间的 `retrying` Job;
- Worker 丢失后租约已过期的 `running` Job。
任务投递使用 late acknowledgement、worker-lost reject 和单任务 prefetch。重复消息会先锁定
Job 行;有效租约内不会创建第二个 Attempt。租约过期后,旧 Attempt 以安全错误关闭,再按重试
规则恢复。
## 重试与清理
确定性的内容错误(无法解析、清洗后为空、无 Chunk、派生数据身份不一致)直接进入 `failed`。
其余运行时/Provider 错误按指数退避重试,默认最多三次总 Attempt。
第二次及后续 Attempt 创建前严格执行:
```text
OceanBase Vector rows
→ ChunkIndexState
→ Chunk
→ parsed JSON object
→ reset Document processing state
```
不可变源文件、Document、Job 和已结束 Attempt 始终保留。任一清理步骤失败时不创建新 Attempt,
Job 直接以 `retry cleanup failed` 终止,禁止在未知脏状态继续执行。
## API 状态
`POST /api/v1/documents` 成功返回 `202` 和 `job_id`。查询:
```text
GET /api/v1/jobs/{job_id}
```
Job 状态为 `queued | running | retrying | succeeded | failed | cancelled`;阶段为
`parsing | cleaning | chunking | embedding | indexing`。公开错误只返回固定安全摘要,不保存或
返回 Broker/Provider 异常正文。活动 Job 的 Document 暂不允许删除,API 返回 `409`。
批量提交与查询:
```text
POST /api/v1/ingestion-batches # multipart 字段 files,可重复
GET /api/v1/ingestion-batches/{batch_id}
```
批次创建后逐文件返回 `accepted` 或 `rejected`;已接收文件继续通过上述 Job 状态异步推进。
批次聚合状态为 `processing | completed | partial_failure | failed`。请求文件数超过
`BATCH_MAX_FILES` 时整体返回 `413`;单文件或共享总字节预算超限只拒绝当前文件,不删除此前
已接收文件。默认总预算 200 MiB,且配置不得小于单文件 `MAX_UPLOAD_BYTES`。
## 故障检查
1. Job 长期为 `queued`:检查 RabbitMQ 连通性、恢复调度器以及 `ingestion` queue;
2. Job 长期为 `running`:检查 Worker,租约到期后恢复调度器会重新投递;
3. Job 为 `retrying`:等待 `next_run_at`,客户端不应重新上传同一文件;
4. Job 为 `failed` 且错误为 `retry cleanup failed`:停止人工重试,先核对向量、关系数据与 BOS
派生对象的一致性;
5. 不得通过日志打印 `RABBITMQ_URL`、数据库 URL、BOS/模型凭据或原始 Provider 响应。

View File

@ -1,8 +1,8 @@
# ARCH-002 BOS 配置与对象契约 # ARCH-002 BOS 配置与对象契约
ARCH-002 在既有 `ObjectStorage` 契约后新增百度 BOS Adapter。业务服务仍只使用逻辑 key, ARCH-002 在既有 `ObjectStorage` 契约后新增百度 BOS Adapter。业务服务仍只使用逻辑 key,
不知道 Bucket、Prefix、SDK 类型或凭据。Local Adapter 继续用于单元测试和本地开发;正式 不知道 Bucket、Prefix、SDK 类型或凭据。ARCH-005 已将 BOS 设为正式运行默认值;Local
默认切换留在 ARCH-005,避免在 Vector/Reranker 迁移完成前形成半套运行配置。 Adapter 继续用于单元测试和显式配置的本地开发。
## 运行配置 ## 运行配置
@ -11,7 +11,7 @@ ARCH-002 在既有 `ObjectStorage` 契约后新增百度 BOS Adapter。业务服
```text ```text
OBJECT_STORAGE_PROVIDER=bos OBJECT_STORAGE_PROVIDER=bos
BOS_ENDPOINT=<已授权的 HTTP(S) BOS Origin> BOS_ENDPOINT=<已授权的 BOS Host 或 HTTP(S) Origin>
BOS_ACCESS_KEY_ID=<运行时注入> BOS_ACCESS_KEY_ID=<运行时注入>
BOS_SECRET_ACCESS_KEY=<运行时注入> BOS_SECRET_ACCESS_KEY=<运行时注入>
BOS_BUCKET=<预创建的 Bucket> BOS_BUCKET=<预创建的 Bucket>
@ -19,7 +19,8 @@ BOS_KEY_PREFIX=knowledge-platform
BOS_TIMEOUT_SECONDS=30 BOS_TIMEOUT_SECONDS=30
``` ```
- Endpoint 必须是无用户名、密码、Path、Query 和 Fragment 的显式 HTTP(S) Origin; - Endpoint 必须是无用户名、密码、Path、Query 和 Fragment 的 Host 或 HTTP(S) Origin;旧配置
的纯 Host 会在 Provider 边界强制规范化为 HTTPS,禁止沿用 SDK 的默认 HTTP;
- Bucket 名必须符合 BOS 的小写字母、数字和连字符规则; - Bucket 名必须符合 BOS 的小写字母、数字和连字符规则;
- Prefix 可为空;非空时使用安全的 POSIX 相对 key,不允许空段、`.`、`..` 或反斜杠; - Prefix 可为空;非空时使用安全的 POSIX 相对 key,不允许空段、`.`、`..` 或反斜杠;
- 应用不会创建、删除或修改 Bucket 配置。 - 应用不会创建、删除或修改 Bucket 配置。
@ -52,8 +53,8 @@ SDK Response Body 或凭据。
全局 List 权限。若生产策略要求来源不可物理删除,应在后续 Document Version/Retention 全局 List 权限。若生产策略要求来源不可物理删除,应在后续 Document Version/Retention
设计中单独处理;当前 Demo 的直接删除行为保持不变。 设计中单独处理;当前 Demo 的直接删除行为保持不变。
默认测试使用 Fake Client,不连接 BOS。真实连接、权限和临时对象清理统一在 ARCH-006 的 默认测试使用 Fake Client,不连接 BOS。真实连接、权限和临时对象清理已在 ARCH-006 的
受控测试资源中验证,并且只允许使用轮换后的凭据。 受控 test 资源中验证;凭据只允许运行时注入,不得写入仓库、日志或测试快照。
官方参考: 官方参考:

View File

@ -0,0 +1,91 @@
# 生产容器镜像说明
## 范围
DEPLOY-001 使用同一个锁定运行时构建三个独立镜像目标:
| Target | 默认镜像名 | 入口 | 端口 | 镜像健康检查 |
|---|---|---|---|---|
| `api` | `enterprise-knowledge-api` | Uvicorn + API/Admin | `8000` | `GET /healthz` |
| `mcp` | `enterprise-knowledge-mcp` | `ekp-mcp` | `8766` | PID 1 存活 |
| `worker` | `enterprise-knowledge-worker` | `ekp-worker` | 无 | PID 1 存活 |
镜像没有内置数据库迁移、Helm、Ingress 或 Secret 对象;DEPLOY-002 已在独立 Chart 中提供这些
编排能力。API 镜像包含 Alembic 和 migrations,由一次性 Migration Job 覆盖命令后复用。详见
[`KUBERNETES_DEPLOYMENT.md`](KUBERNETES_DEPLOYMENT.md)。
## 可重复构建基线
- Python 基础镜像固定为 `3.12.14-slim-bookworm`,uv 固定为 `0.12.5`,两者同时锁定 manifest
digest;
- Python 依赖只使用提交的 `uv.lock`,以 `uv sync --frozen --no-dev --no-editable` 安装;
- 三个目标共享相同依赖层,不复制不同版本的业务运行时;
- 进程均以固定 UID/GID `10001:10001` 运行,不具备 root 权限;
- `.dockerignore` 排除 `.env`、Git、虚拟环境、缓存、本地存储、测试和开发产物;
- Bake 计划同时声明 `linux/amd64` 与 `linux/arm64`,不下载或打包平台专属模型。
基础镜像 tag/digest 与应用依赖均已固定;镜像发布系统仍应保存本项目镜像的构建 digest,并让
Helm 按 digest 部署。本项目镜像 digest 由实际 Registry 发布产生,不把某个环境的 Registry
地址写入仓库。
## 构建命令
先确认 Docker Buildx builder 支持两个目标平台:
```bash
make images-print
```
构建全部目标:
```bash
IMAGE_REGISTRY=registry.example.internal/knowledge \
IMAGE_TAG=<immutable-release-tag> \
make images
```
只有发布流水线才执行推送:
```bash
IMAGE_REGISTRY=registry.example.internal/knowledge \
IMAGE_TAG=<immutable-release-tag> \
make images-push
```
不要把 Registry 密码放入变量或命令行;构建节点应提前通过其凭据助手登录。禁止使用
`latest` 作为集群部署版本。
## 运行时配置
镜像只内置以下非敏感进程默认值:`APP_ENV=production`、容器监听地址、静态页面绝对路径和
临时缓存目录。OceanBase、BOS、RabbitMQ、Embedding、Reranker 与 MCP Token 必须在容器启动
时由 Secret 注入。镜像构建参数、Layer、标签、健康检查和启动命令都不得包含这些值。
API 的 `/healthz` 只证明进程存活,不读取云端凭据或探测依赖。Kubernetes readiness、资源限制、
只读根文件系统所需临时卷及依赖就绪策略由 DEPLOY-002 Chart 配置。MCP endpoint 需要认证,Worker
没有 HTTP server,因此这两个镜像的 Docker 健康检查只验证主进程;业务可恢复性仍以持久化
Job/Attempt 和 DEPLOY-003 故障验收为准。
## 本地单目标检查
需要加载到本地 Docker 时,可直接构建当前平台的一个 target:
```bash
docker build --target api -f deploy/docker/Dockerfile -t ekp-api:test .
docker build --target mcp -f deploy/docker/Dockerfile -t ekp-mcp:test .
docker build --target worker -f deploy/docker/Dockerfile -t ekp-worker:test .
```
启动时使用 `--env-file` 或编排系统的 Secret 引用,不在命令中展开敏感值。API/Admin smoke test
至少检查 `/healthz`、`/admin/upload` 和 `/api/v1/openapi.json`;MCP 与 Worker 的真实依赖启动
验收在具备 RabbitMQ 和目标 Provider 的受控环境执行。
## DEPLOY-001 构建验收
2026-09-02 已使用 Buildx 分别构建并加载三个 `linux/arm64` target 和三个 `linux/amd64`
target。API/Admin 以非 root 用户完成健康、静态页面和 OpenAPI smoke test;MCP/Worker 完成非
root 安装入口检查。构建历史与容器文件系统检查未发现 `.env`、Git、测试文件或敏感配置。
这次验收只生成本地 `deploy001-*` 测试 tag,没有登录或推送 Registry,因此不产生可供集群
引用的项目镜像 digest。DEPLOY-002 必须使用操作者指定的不可变 Registry tag/digest,不得把
本地测试 tag 当作发布版本。

View File

@ -1,3 +1,19 @@
# Definition of Done # Definition of Done
A task is done only when scope is implemented, architecture invariants hold, format/lint/typecheck pass, relevant tests pass, migrations/contracts/docs are updated as needed, no secrets are committed, retrieval changes include RAG regression results, and known limitations are documented. A task is done only when scope is implemented, architecture invariants hold, format/lint/typecheck pass, relevant tests pass, migrations/contracts/docs are updated as needed, no secrets are committed, retrieval changes include RAG regression results, and known limitations are documented.
For asynchronous ingestion or retry tasks, relevant tests must also prove durable Job/Attempt state,
bounded retry classification, idempotent duplicate delivery, cleanup-before-retry, preservation of the
immutable source, and terminal behavior when cleanup fails. Batch ingestion must cover per-file partial
failure and configured limits.
For MCP tasks, contract tests must cover tool discovery, identity propagation, authorization/error
boundaries, REST/application-service parity, infrastructure redaction, and the explicitly targeted MCP
client compatibility check.
For cluster delivery tasks, completion also requires reproducible production images, Secret-only
sensitive configuration, serialized migration, readiness/liveness probes, resource and pod-security
defaults, idempotent Helm install/upgrade, rolling-restart verification, durable worker recovery, stable
HTTPS Admin/MCP endpoints, and remote WorkBuddy acceptance. Manifests, rendered output, logs and images
must be checked for credential leakage. Public-knowledge delivery does not complete or waive V1
user/ACL tasks.

View File

@ -0,0 +1,53 @@
# ARCH-005 Shadow Index 重建与切换
ARCH-005 将正式运行默认值切换为 BOS、OceanBase Vector 和千帆 BCE Reranker;ARCH-006
进一步确认 canonical 关系数据与向量索引同处 OceanBase MySQL 数据库。关系表保存 Document、
Chunk、ChunkIndexState 和 Index Profile;向量表只保存可以重建的检索行。旧 Mochow 运行配置、依赖和代码已移除,但本流程不会自动删除
旧外部索引,物理退役必须由受控运维流程另行确认。
## Profile 与迁移前提
默认新 Profile 为:
```text
demo-bge-large-zh-structure-v1-oceanbase-v1
```
它与旧 Demo Profile 使用不同 ID,避免已有 Mochow `ChunkIndexState` 被误认为 OceanBase
写入证据。自定义 ID 仍由 `INDEX_PROFILE_ID` 注入,并且必须是全新的、稳定的版本标识。
执行前必须:
1. 运行 `uv run alembic upgrade head`,创建 canonical `index_profiles` 表;
2. 使用 `deploy/oceanbase/001_knowledge_chunks_v1.sql` 预创建目标表和 HNSW 索引;
3. 仅在运行时注入已授权的 OceanBase 与千帆凭据,不写入仓库或日志;
4. 确认目标 OceanBase 为 4.3.5.4+,且应用账号只有目标数据库/表所需权限;
5. 在重建期间避免修改 Chunk 策略;激活事务会阻止 canonical Chunk 并发写入穿过最终核对。
## 顺序执行
每一步只输出 Profile ID 和计数,不输出 Endpoint、SQL、Chunk 内容或凭据。
```bash
uv run ekp-index-rebuild build
uv run ekp-index-rebuild verify
uv run ekp-index-rebuild activate
```
- `build` 注册 Shadow Profile,从 OceanBase canonical Chunk 分文档生成 Embedding 并写入
OceanBase。命令中断后可以原样重试;已完整文档会跳过,部分目标 Profile 数据会先定向
清理再重建,不触碰 Active Profile;
- `verify` 要求目标策略下至少存在一个 canonical Chunk,并核对全部 Chunk 与匹配模型、
维度的 ChunkIndexState。输出中的 `complete` 必须为 `true`;
- `activate` 在 OceanBase 事务内再次验证,锁定 Chunk 写入,将原 Active Profile 标为
`retired`,再将目标 Shadow Profile 标为 `active`。重复执行已成功的激活是幂等的。
激活后 API 从 OceanBase canonical 关系表解析 Active Profile,不再仅依赖进程环境中的 ID。新上传与查询
统一使用 OceanBase Active Profile。部署仍应保留前一版本制品和旧索引回滚窗口;本 Task
不自动销毁任何旧外部数据。
## 验证与已知边界
离线测试覆盖多文档构建、重试跳过、完整性失败关闭、单 Active 约束、原子激活、迁移和
CLI 输出脱敏。真实 BOS/OceanBase/Qianfan 连通性、DDL 权限、30~50 条脱敏 Golden Query、
召回质量和延迟属于 ARCH-006,且只能使用运行时授权凭据与受控测试资源。

View File

@ -0,0 +1,143 @@
# Kubernetes / Helm 部署说明
## 1. 交付范围
DEPLOY-002 提供单一 Helm Chart,将以下运行角色部署到已有 Kubernetes 集群:
- API/Admin:默认两个副本,提供 `/healthz`、`/admin/*` 和 `/api/v1/*`;
- MCP Gateway:默认两个副本,提供经 Bearer Token 保护的 `/mcp`;
- Async Worker:默认两个副本,只消费 `ingestion` 队列;
- Recovery Scheduler:固定单副本,以 `Recreate` 策略运行 Celery Beat;
- Migration Job:每次安装/升级前串行执行 `alembic upgrade head`;
- 可选 RabbitMQ:只供开发/演示集群显式启用,生产默认使用外部 Broker。
OceanBase MySQL、BOS、Embedding/Reranker 服务继续使用集群外已有资源。Chart 不创建数据库、
BOS Bucket、域名、证书或云账号,也没有扩大公共知识库阶段的业务范围。
## 2. 前置条件
操作者需要提前准备:
1. 可用的 Kubernetes Namespace 和对应 `kubectl` 上下文;
2. Helm 3/4 与 Kubernetes 客户端;
3. 已推送到目标 Registry 的 API、MCP、Worker 不可变 tag 或 digest;
4. Namespace 内的运行时 Secret、TLS Secret,以及私有 Registry 所需的 ImagePullSecret;
5. 集群到 OceanBase、BOS、RabbitMQ 和模型服务的网络连通性。
Chart 默认引用 `knowledge-platform-runtime`,但不会创建它。该 Secret 至少包含以下键:
```text
DATABASE_URL
RABBITMQ_URL
BOS_ACCESS_KEY_ID
BOS_SECRET_ACCESS_KEY
BAIDU_QIANFAN_API_KEY
OCEANBASE_VECTOR_PASSWORD
MCP_AUTH_TOKEN
```
`RERANKER_API_KEY` 可单独提供;未提供时应用按既有契约复用
`BAIDU_QIANFAN_API_KEY`。`MCP_AUTH_TOKEN` 至少 32 个字符,只具备
`knowledge:search` 服务范围。不要使用 `kubectl --from-literal`、Shell 参数或 Helm
`--set` 传递以上值;应由受控 Secret 管理系统或运维流程预先写入 Namespace。
## 3. 环境 Values
复制 [`values.example.yaml`](../../deploy/helm/environments/values.example.yaml) 到仓库外的受控
路径,填写以下非敏感信息:
- 三个镜像的 Registry 路径与不可变 tag/digest;
- `existingSecret` 和 `imagePullSecrets` 的资源名称;
- OceanBase/BOS 的非敏感 Endpoint、数据库标识和 Bucket;
- Admin/API 与 MCP 的 Ingress Host、TLS Secret 名称;
- 与外部地址一致的 MCP Public URL、Issuer、Allowed Host/Origin;
- 副本、资源、超时和批量上传限制。
`values.schema.json` 拒绝未知顶层字段、`latest` tag、无效 digest、单副本 API/MCP、错误类型和
空的关键引用;模板还会校验 Chunk、检索和入库限制之间的大小关系。Values 只允许非敏感配置,
模板逐项生成 ConfigMap,不提供任意 `extraEnv` 入口,避免把凭据误写进渲染产物。
## 4. 安装与幂等升级
先做离线校验和清单检查:
```bash
make deploy-lint VALUES=/secure/path/environment-values.yaml
make deploy-render VALUES=/secure/path/environment-values.yaml
```
安装或升级:
```bash
NAMESPACE=knowledge-platform \
RELEASE=knowledge-platform \
make deploy VALUES=/secure/path/environment-values.yaml
```
`make deploy` 只接收 Values 文件路径。脚本会先验证 Chart、Namespace、运行时 Secret 及必需
键名、可选 RabbitMQ Secret 键名,以及启用 Ingress 时的两个 TLS Secret 引用,然后执行带失败
回滚和等待语义的 `helm upgrade --install`;脚本对 Helm 3 使用 `--atomic`,对 Helm 4 使用等价
的 `--rollback-on-failure`。它只验证 Secret 资源和键是否存在,不读取或输出 Secret 值。
Migration 是 `pre-install,pre-upgrade` Hook,固定名称配合
`before-hook-creation` 保证同一 Release 内不会并发执行。Helm 等待 Migration 成功后才更新业务
工作负载;失败时 Helm 保留上一版本。完成的 Job 最长保留一天供日志追查,并在下次升级前删除。
安装命令可以重复执行;数据库迁移必须保持 forward-only 与幂等。
## 5. Ingress、探针与 Pod 安全
生产环境启用 Ingress 后,Admin/API 与 MCP 使用独立稳定 HTTPS Host,也可由同一 Ingress
Controller 托管。证书只通过预建 TLS Secret 引用,不进入 Values。
- API 的 startup/readiness/liveness 调用无凭据 `/healthz`;
- MCP 的三类探针使用 TCP 端口,不发送或记录 Token;
- Worker/Scheduler 使用 PID 1 存活探针,业务恢复正确性由持久化 Job/Attempt 保证;
- 全部应用容器固定 UID/GID `10001`、禁止提权、丢弃 Linux capabilities、使用
`RuntimeDefault` seccomp 和只读根文件系统;
- `/tmp` 使用有大小上限的 `emptyDir`,ServiceAccount 不挂载 API Token;
- API/MCP 滚动更新使用 `maxUnavailable: 0`,未 Ready Pod 不进入 Service Endpoint。
`/healthz` 当前只表示 API 进程存活,不探测所有外部 Provider;真实依赖故障、滚动重启和
Worker 恢复行为由 DEPLOY-003 在目标集群验收。
## 6. 可选 Demo RabbitMQ
生产默认 `rabbitmq.enabled=false`,`RABBITMQ_URL` 指向外部受管 Broker。仅开发/演示时可开启
本地子 Chart;它使用单节点 StatefulSet 和 PVC,并要求预建 Secret 包含:
```text
username
password
erlang-cookie
```
同时必须让运行时 Secret 中的 `RABBITMQ_URL` 指向该 Service。子 Chart 使用固定
`rabbitmq:4.1.4-alpine` tag 与 multi-architecture manifest digest;它不具备生产 RabbitMQ 的
高可用、备份或升级保证,禁止作为生产默认值。卸载 Release 时 StatefulSet PVC 默认保留,
删除前必须由数据责任人确认。
## 7. 回滚、卸载与 Secret 轮换
查看历史并回滚时不要输出 Pod 环境变量:
```bash
helm -n knowledge-platform history knowledge-platform
helm -n knowledge-platform rollback knowledge-platform <revision> --wait
```
更新外部 Secret 后,Secret 名称不变;用户和 WorkBuddy 配置无需重新注入。当前应通过受控
滚动重启让 Pod 读取新值,目标集群的无中断验证属于 DEPLOY-003。
卸载只删除本 Release 管理的 Kubernetes 对象:
```bash
helm -n knowledge-platform uninstall knowledge-platform
```
外部 OceanBase、BOS、RabbitMQ、TLS/运行时 Secret 和可选 RabbitMQ PVC 不由卸载命令删除。
## 8. DEPLOY-002 验收边界
本任务完成 Chart lint、双路径渲染、严格 Kubernetes Schema、Values 失败关闭、Secret 泄漏扫描
以及本地集群两次安装/升级验证。目标企业集群的稳定 HTTPS、滚动重启、Worker 故障恢复、真实
Provider 和远程 WorkBuddy 验收统一属于后续 `DEPLOY-003`,不得提前宣称完成。

View File

@ -0,0 +1,67 @@
# ARCH-006 受控真实链路验收
本验收覆盖 OceanBase canonical relational metadata → BOS immutable source → 千帆
`bge-large-zh` → OceanBase Vector → 千帆 `bce-reranker-base` → canonical Source Result。数据集包含
32 条完全合成、脱敏 Query,不含真实客户、产品、政策或内部知识。
## 强制安全门禁
只有同时满足以下条件才允许执行:
1. 数据库、BOS 和千帆凭据已经所有者明确授权;配置清单中的现有值只允许本次 test
验收临时注入,禁止保存为 `.env`、写入日志或用于 fat/prod,完成后仍必须轮换;
2. OceanBase Database/Table 和 BOS Bucket/Prefix 已明确授权用于测试;
3. `APP_ENV=test`;
4. `BOS_KEY_PREFIX` 以 `live-acceptance/` 开头,避免写入共享业务前缀;
5. 显式设置 `EKP_RUN_LIVE_TESTS=1`、`EKP_CREDENTIALS_AUTHORIZED=1` 和
`EKP_LIVE_RESOURCE_ACK=1`。
任一条件不满足时,命令在创建 Provider 或外部连接前失败关闭,并且不输出 Endpoint、用户
名、SQL、响应正文、Chunk 内容或 Secret。仓库 `.env.example` 中三个开关固定为 `0`。
## 执行顺序
先完成 ARCH-005 切换:
```bash
uv run alembic upgrade head
uv run ekp-index-rebuild build
uv run ekp-index-rebuild verify
uv run ekp-index-rebuild activate
```
确认 `verify` 和 `activate` 输出的 `complete` 均为 `true` 后,再运行:
```bash
make live-acceptance
```
命令会使用随机运行前缀上传两份合成 Markdown,执行全部 32 条 Query,并在 `finally` 清理
本次运行的 OceanBase Vector 行、canonical Document/Chunk/State 以及 BOS 对象。清理失败会使验收
失败,且不会尝试删除不带本次随机文件名前缀的数据。
## 验收指标
默认门槛:
```text
Recall@1 >= 0.90
Recall@5 = 1.00
MRR >= 0.90
NDCG@5 >= 0.90
Search P95 <= 10000 ms
```
输出只包含 Query 数、Recall@1/3/5、MRR、NDCG@5、P50/P95 和 `passed`。如受控环境需要调整
门槛,只能通过 `EKP_LIVE_MIN_*` / `EKP_LIVE_MAX_P95_MS` 显式注入,并把实际门槛与结果记录
到受保护的验收日志;不要把 Endpoint、账号、凭据或原始响应提交到仓库。
## 验收结果(2026-09-02)
目标 test OceanBase 已完成 migration、专用向量表/HNSW、PRE-FILTER、BOS、Embedding、
Reranker 和 canonical Source Result 验收。32 条查询结果:Recall@1/3/5、MRR、NDCG@5 均为
`1.0`,P50 `3787.61 ms`,P95 `4759.21 ms`,`passed=true`。正式命令退出码为 `0`,数据库
复核 `live-` Document/Chunk/Vector 残留为 `0`。
首次正式运行出现过一次无细节瞬时失败;分阶段 Provider/单文档/32 查询诊断以及随后正式
运行均通过。后续稳定性任务应补充有界重试、限流指标与告警;这不阻塞本次受控验收。

View File

@ -0,0 +1,102 @@
# DEMO-012 MCP Gateway 使用说明
## 范围
DEMO-012 提供独立的 Enterprise Knowledge MCP Gateway,通过标准 Streamable HTTP 暴露一个
只读工具:`search_knowledge`。Gateway 直接复用 API 进程使用的 `SearchService` 组合,不复制
Dense Retrieval、Reranker 或 Source Enrichment 逻辑。
当前 Demo 只建立认证和身份传递边界:一个运行时注入的专用 Bearer Token 映射到一个
user/service identity,并且只有 `knowledge:search` scope。它不是共享超级管理员 Token,也不
代表 V1 OAuth、Document ACL 或完整 MCP P0 工具已经完成。
## 配置
必须通过运行环境提供至少 32 字符的随机 Token:
```bash
export MCP_AUTH_TOKEN="<runtime-secret-with-at-least-32-characters>"
make run-mcp
```
默认地址为 `http://127.0.0.1:8766/mcp`。除现有数据库、BOS、Embedding、Vector 和 Reranker
配置外,Gateway 支持:
| 变量 | 默认值 | 说明 |
|---|---|---|
| `MCP_HOST` | `127.0.0.1` | 监听地址;本地默认不暴露到其他网卡 |
| `MCP_PORT` | `8766` | Gateway 端口 |
| `MCP_PATH` | `/mcp` | Streamable HTTP 单一端点 |
| `MCP_PUBLIC_URL` | `http://127.0.0.1:8766/mcp` | MCP Resource Server URL |
| `MCP_ISSUER_URL` | `http://127.0.0.1:8766` | Demo Token 的身份颁发方标识 |
| `MCP_CLIENT_ID` | `workbuddy-enterprise` | MCP Client 类型标识 |
| `MCP_SERVICE_IDENTITY` | `workbuddy-demo` | 传入业务边界的服务身份 |
| `MCP_REQUIRED_SCOPE` | `knowledge:search` | Tool 所需最小 scope |
| `MCP_ALLOWED_HOSTS` | 仅本机/test | DNS rebinding Host allowlist,JSON 数组 |
| `MCP_ALLOWED_ORIGINS` | 仅本机 | Origin allowlist,JSON 数组 |
远程部署必须使用 TLS 反向代理,并把公开域名加入 `MCP_ALLOWED_HOSTS`;有 `Origin` 的客户端还
必须显式配置 `MCP_ALLOWED_ORIGINS`。Token、真实域名和内部 Endpoint 不得提交 Git。
## WorkBuddy / CodeBuddy 配置
WorkBuddy Enterprise 官方客户端支持 `type: "http"`、远程 URL、自定义 Header 和环境变量
展开。可复制 `deploy/workbuddy-mcp.example.json` 作为项目配置,内容如下:
```jsonc
{
"mcpServers": {
"enterprise-knowledge": {
"type": "http",
"url": "${EKP_MCP_URL:-http://127.0.0.1:8766/mcp}",
"headers": {
"Authorization": "Bearer ${EKP_MCP_TOKEN}"
},
"description": "企业知识检索"
}
}
}
```
只提交包含环境变量占位符的配置;`EKP_MCP_TOKEN` 必须由每个运行环境安全注入。连接后应只
发现 `mcp__enterprise_knowledge__search_knowledge` 一个工具。
## 安全和错误语义
- 缺失或错误 Token 返回 `401`,缺少 scope 返回 `403`;
- 未批准 Host 返回 `421`,未批准 Origin 返回 `403`;
- Query 长度和 `top_k` 在 MCP schema 层校验;
- 预期业务错误返回安全 Tool Error;未知 Provider/数据库错误收敛为
`knowledge search failed`;
- MCP 输出和 Gateway 日志不得包含 Token、Provider Endpoint、数据库表名、模型名或
Index Profile。
## 验证
```bash
uv run pytest tests/unit/test_mcp_auth.py tests/unit/test_mcp_search.py
uv run pytest tests/integration/test_mcp_server.py
```
集成测试使用官方 MCP Python Client 通过 Streamable HTTP 完成 initialize、tools/list 和
tools/call,并使用与 WorkBuddy 相同的 Bearer Header 配置验证身份传递、无结果、参数错误、
认证失败、Origin 防护和错误脱敏。
### WorkBuddy 现场验收记录
2026-09-02 使用 WorkBuddy Desktop `5.3.14` 自带的 CodeBuddy CLI `2.115.0` 完成兼容性
验收。客户端直接加载 `deploy/workbuddy-mcp.example.json`,健康检查为 `Connected`;工具
发现结果只有 `mcp__enterprise-knowledge__search_knowledge`,随后以字符串 `query` 和
`top_k=1` 完成 Tool 调用并读取结构化 `filename`、`section`、`content` 等来源字段。
现场验收使用 DEMO-012 的真实 Gateway、Bearer 验证、service identity 传递、输入 Schema 和
Tool 注册代码。为避免外部数据库与模型波动掩盖客户端兼容性,检索应用使用确定性验收返回;
真实 Provider 检索链路由 `ARCH-006` 的受控验收负责。非交互 CLI 需要批准 MCP Tool 调用,
本次只在单次会话中允许 `DeferExecuteTool`,没有永久启用跳过权限。测试结束后已删除临时 MCP
配置并停止本地 Gateway,未写入真实凭据。
参考:
- [MCP Python SDK](https://github.com/modelcontextprotocol/python-sdk)
- [MCP Streamable HTTP 规范](https://modelcontextprotocol.io/specification/2025-11-25/basic/transports)
- [WorkBuddy Enterprise MCP 使用文档](https://cloud.tencent.cn/document/product/1831/137039)

View File

@ -1,90 +0,0 @@
# DEMO-006/007 千帆与 Mochow 配置
本文记录 Knowledge Platform Demo 的安全连接方式和 Mochow 表契约。配置参考现有
`Embedding与数据库配置说明.md`,但不复制其中的环境地址或任何凭证。
## Canonical 数据库
本项目继续使用 `DATABASE_URL` 连接 PostgreSQL。参考项目中的 MySQL 配置不适用于
本仓库,也不会替换 Accepted ADR 中的 PostgreSQL canonical storage。
## 千帆 Embedding
```dotenv
EMBEDDING_PROVIDER=qianfan
EMBEDDING_MODEL=bge-large-zh
EMBEDDING_ENDPOINT=https://qianfan.baidubce.com/v2/embeddings
BAIDU_QIANFAN_API_KEY=<运行环境注入>
EMBEDDING_TIMEOUT_SECONDS=30
```
`bge-large-zh` 固定输出 1024 维向量。Adapter 将每次请求限制为最多 16 条,并校验
输入字符数、响应数量、顺序、维度和有限数值。API Key 使用 `SecretStr` 保存,异常中
不记录请求头、请求正文或响应正文。
## Mochow 连接
```dotenv
VECTOR_STORE_PROVIDER=mochow
BAIDU_MOCHOW_ACCOUNT=root
BAIDU_MOCHOW_API_KEY=<运行环境注入>
BAIDU_MOCHOW_ENDPOINT=<已授权实例地址>
KNOWLEDGE_VDB_DB_NAME=<专用数据库名>
MOCHOW_TABLE=knowledge_chunks
MOCHOW_BATCH_SIZE=100
MOCHOW_SEARCH_EF=200
MOCHOW_TIMEOUT_SECONDS=30
INDEX_PROFILE_ID=demo-bge-large-zh-structure-v1
DENSE_TOP_N=20
```
配置说明中的 endpoint 可作为运行环境的值,但不得提交到 `.env.example` 或代码。
建议为本 Demo 使用独立表,禁止直接写入字段契约未知的既有业务表。
## Mochow 表契约
`MOCHOW_TABLE` 必须由受控的部署/运维步骤预先创建。应用不会在启动或入库时隐式创建、
修改或删除数据库结构。
| 字段 | Mochow 类型 | 要求 |
| --- | --- | --- |
| `index_record_id` | STRING | 主键、分区键、非空;由 Chunk/profile 确定性生成 |
| `chunk_id` | STRING | 非空 |
| `document_id` | STRING | 非空,建议建立过滤索引 |
| `filename` | STRING | 非空 |
| `page` | UINT32 | 可空,一基页码 |
| `page_end` | UINT32 | 可空,一基页码 |
| `section` | TEXT | 可空 |
| `chunk_index` | UINT32 | 非空 |
| `content` | TEXT | 非空 |
| `embedding` | FLOAT_VECTOR | 非空,dimension=1024 |
| `embedding_model` | STRING | 非空 |
| `index_profile_id` | STRING | 非空,建议建立过滤索引 |
向量字段建议建立 HNSW/COSINE 索引。`document_id` 与 `index_profile_id` 必须支持标量
过滤,以便失败补偿和后续按文档删除。切换模型或 Chunk 策略时应使用新的
`INDEX_PROFILE_ID` 和 shadow index,不得原地覆盖已记录的 profile。
## Dense Retrieval 契约
DEMO-007 使用同一 `EmbeddingProvider` 将 Query 转换为 1024 维向量,然后调用 Mochow
`VectorTopkSearchRequest`:
- 向量字段固定为 `embedding`;
- 默认 `DENSE_TOP_N=20`,调用方可在 1~1000 范围内覆盖;
- HNSW 搜索候选参数默认 `MOCHOW_SEARCH_EF=200`,运行时至少提升到本次 Top-N;
- `index_profile_id` 过滤随 Dense Search 请求下推,禁止混入其他模型/Chunk 版本;
- 返回值是尚未 Rerank 的候选集,`score` 原样保留 Mochow 返回值,不在此阶段设置阈值。
Demo 明确不实现 RBAC/Document ACL;正式 V1 引入权限后,允许范围仍必须在召回阶段下推,
不得先做全局 Top-K 再过滤。Reranker 与最终 Source Result 的配置参见
`docs/development/RERANKER_SETUP.md`;REST API 参见
`docs/development/REST_API_SETUP.md`。
## 验证边界
默认测试使用 `FakeEmbeddingProvider` 和 `FakeVectorStore`,不访问互联网或生产凭证。
真实联调前需确认:专用表已按上述 Schema 创建、1024 维向量索引正常、Endpoint 网络
可达、运行身份拥有 upsert/delete/vector_search 权限。默认回归已离线验证 Query
Embedding → profile 过滤 → Dense Top20 及来源元数据保持;真实 Mochow 的网络联调需要由
受控运行环境注入凭证后单独执行。

View File

@ -0,0 +1,72 @@
# ARCH-003 OceanBase Vector 配置与索引契约
ARCH-003 在既有 `VectorStore` 契约后新增 OceanBase Vector Adapter。ARCH-006 确认 OceanBase
MySQL 关系表保存 canonical Document、Chunk、Index Profile 和 Chunk Index State;同库向量表
只保存可由这些数据重建的检索行。ARCH-005 已完成运行时默认切换,并移除 Mochow 运行配置与依赖。
## 数据库前提
- OceanBase MySQL 租户版本至少为 4.3.5.4:ARCH-006 已在目标 test 集群用只读
`EXPLAIN` 验证 `VECTOR_INDEX(... pre_filter)` 产生 PRE-FILTER 执行计划;
- 目标数据库和最小权限账号由受控部署流程预创建;
- Vector 内存、资源规格和 HNSW 参数由 DBA 在受控环境确认;
- 应用启动和请求路径不会创建、修改或删除表结构。
使用 [001_knowledge_chunks_v1.sql](../../deploy/oceanbase/001_knowledge_chunks_v1.sql)
在专用数据库中创建索引表。脚本固定:
```text
VECTOR(1024)
DISTANCE = COSINE
TYPE = HNSW
EF_CONSTRUCTION = 256
EF_SEARCH = 200
```
表的 `index_record_id` 是确定性的 Chunk/Profile 主键;`document_id + index_profile_id` 和
`index_profile_id` 是清理与 Profile 过滤的标量索引。表名或向量维度变化必须建立新表或
Shadow Profile,验证后再激活,禁止在生产索引上原地修改。
## 运行配置
配置只能由部署环境或 Secret Manager 注入,禁止把真实地址、用户名和密码写入 `.env`、
镜像、日志、测试快照或提交记录。
```text
VECTOR_STORE_PROVIDER=oceanbase
OCEANBASE_VECTOR_URI=<host:port>
OCEANBASE_VECTOR_USER=<运行时注入>
OCEANBASE_VECTOR_PASSWORD=<运行时注入>
OCEANBASE_VECTOR_DATABASE=<预创建数据库>
OCEANBASE_VECTOR_TABLE=knowledge_chunks_v1
OCEANBASE_VECTOR_INDEX=vidx_knowledge_chunks_embedding
OCEANBASE_VECTOR_BATCH_SIZE=100
OCEANBASE_VECTOR_TIMEOUT_SECONDS=30
```
- URI 只接受 `host:port`,不得包含 Scheme、用户信息、Path、Query 或 Fragment;
- Database/Table/Index 只接受安全标识符;
- Password 使用 `SecretStr`,Provider 异常只返回收敛后的错误,不回显 SQL、连接信息、
Chunk 内容或 SDK Response;
- `pyobvector` 只存在于 Adapter/Factory 内,业务服务不依赖其 Row、Engine 或距离函数类型。
## 写入、删除和检索语义
- Upsert 使用 `index_record_id` 主键,按可配置批次写入;
- 删除使用参数化的 `document_id + index_profile_id` 条件,不拼接用户输入;
- Dense Search 在同一 ANN 查询中使用 `VECTOR_INDEX(... pre_filter)`,保证
`index_profile_id` 在候选近邻搜索前过滤,并返回 Dense Top20;禁止依赖普通 `WHERE`
的自适应过滤语义,也禁止全局 TopK 后再过滤;
- OceanBase `cosine_distance` 是低值优先距离,Adapter 统一转换为
`score = 1 - cosine_distance`,保持 `VectorStore` 的高分优先契约;
- 返回 UUID、Profile、字段数、距离范围和排序不满足契约时失败关闭,不生成 Source Result。
默认测试使用 Fake Client,不连接 OceanBase。真实版本、DDL、权限、Cosine HNSW 召回、过滤
行为与 30~50 条脱敏 Golden Query 统一在 ARCH-006 的受控资源中验证。凭据只能在运行时
临时注入,禁止进入仓库、日志或测试快照。
官方参考:
- [pyobvector 官方仓库](https://github.com/oceanbase/pyobvector)
- [OceanBase 创建向量索引](https://en.oceanbase.com/docs/common-oceanbase-database-10000000003075357)
- [OceanBase 向量索引查询 Hint](https://en.oceanbase.com/docs/common-oceanbase-database-10000000004311163)

View File

@ -1,4 +1,4 @@
# DEMO-008 千帆 Reranker 与 Source Result 配置 # ARCH-004 千帆 BCE Reranker 与 Source Result 配置
本文记录 Knowledge Platform Demo 的重排序连接方式、请求边界和 canonical Source Result 本文记录 Knowledge Platform Demo 的重排序连接方式、请求边界和 canonical Source Result
契约。真实 API Key 只允许由运行环境注入,不得写入代码、示例配置、日志或提交记录。 契约。真实 API Key 只允许由运行环境注入,不得写入代码、示例配置、日志或提交记录。
@ -7,7 +7,7 @@
```dotenv ```dotenv
RERANKER_PROVIDER=qianfan RERANKER_PROVIDER=qianfan
RERANKER_MODEL=qwen3-reranker-0.6b RERANKER_MODEL=bce-reranker-base
RERANKER_ENDPOINT=https://qianfan.baidubce.com/v2/rerank RERANKER_ENDPOINT=https://qianfan.baidubce.com/v2/rerank
RERANKER_API_KEY=<运行环境注入,可空> RERANKER_API_KEY=<运行环境注入,可空>
RERANKER_TIMEOUT_SECONDS=30 RERANKER_TIMEOUT_SECONDS=30
@ -21,8 +21,8 @@ Provider 创建失败。凭证使用 `SecretStr`,HTTP/JSON 异常不会包含
## Reranker 请求契约 ## Reranker 请求契约
DEMO-008 固定使用千帆 `POST /v2/rerank`,模型基线为 ARCH-004 固定使用千帆 `POST /v2/rerank`,模型基线为
`qwen3-reranker-0.6b`: `bce-reranker-base`:
- Query 去除首尾空白后不能为空,最多 1600 字符; - Query 去除首尾空白后不能为空,最多 1600 字符;
- documents 不能为空,最多 64 条,每条最多 4096 字符; - documents 不能为空,最多 64 条,每条最多 4096 字符;
@ -31,13 +31,18 @@ DEMO-008 固定使用千帆 `POST /v2/rerank`,模型基线为
- `relevance_score` 必须是有限数值,并保持降序; - `relevance_score` 必须是有限数值,并保持降序;
- 响应若回显 document,必须与对应输入文本完全一致。 - 响应若回显 document,必须与对应输入文本完全一致。
上述边界与千帆重排序接口当前约束一致。BCE 和原 Demo Qwen3 模型共享请求/响应结构,
因此本次变更不修改 `RerankerProvider`、Search REST API 或最终 Top5 行为。Qwen3-Reranker
保留为 PRD 中的后续 Benchmark 候选,不作为当前运行默认值。
Adapter 只向业务层返回原候选索引和 Reranker 分数,不暴露千帆响应结构。默认测试使用 Adapter 只向业务层返回原候选索引和 Reranker 分数,不暴露千帆响应结构。默认测试使用
`FakeRerankerProvider` 的确定性字符覆盖率排序,不发起网络请求。 `FakeRerankerProvider` 的确定性字符覆盖率排序,不发起网络请求。
## Source Result 契约 ## Source Result 契约
Reranker 对 Mochow Dense Candidate 的 `content` 评分;选出 Top5 后,`SearchService` 使用 Reranker 对 provider-neutral Dense Candidate 的 `content` 评分;选出 Top5 后,
`chunk_id` 一次性回查 PostgreSQL canonical `chunks` 表,并由 canonical row 生成: `SearchService` 使用
`chunk_id` 一次性回查 OceanBase canonical `chunks` 关系表,并由 canonical row 生成:
```text ```text
document_id document_id
@ -51,8 +56,8 @@ rerank_score
``` ```
其中 `score` 保留 Dense 分数,`rerank_score` 保存精排分数。若 Chunk 缺失、document_id 其中 `score` 保留 Dense 分数,`rerank_score` 保存精排分数。若 Chunk 缺失、document_id
不一致或 Mochow content 已与 canonical content 不一致,搜索会失败关闭,避免返回来源与 不一致或向量索引 content 已与 canonical content 不一致,搜索会失败关闭,避免返回来源
评分文本不一致的结果。 与评分文本不一致的结果。
DEMO-009 已通过 `POST /api/v1/search` 提供序列化和错误映射,参见 DEMO-009 已通过 `POST /api/v1/search` 提供序列化和错误映射,参见
`docs/development/REST_API_SETUP.md`。Demo 仍不实现 RBAC/Document ACL,正式 V1 权限 `docs/development/REST_API_SETUP.md`。Demo 仍不实现 RBAC/Document ACL,正式 V1 权限
@ -67,3 +72,5 @@ Golden Dataset 验证预期文档和章节升至首位。真实千帆联调前
- Endpoint 网络可达; - Endpoint 网络可达;
- 模型 API 名称与控制台服务一致; - 模型 API 名称与控制台服务一致;
- 真实中文 Query 的延迟、配额和 Top5 质量满足 Demo 演示需求。 - 真实中文 Query 的延迟、配额和 Top5 质量满足 Demo 演示需求。
官方接口参考:[千帆重排序 API](https://cloud.baidu.com/doc/qianfan-api/s/2m7u4zt74)。

View File

@ -2,17 +2,21 @@
## 范围 ## 范围
DEMO-009 将已经完成的同步入库和检索服务公开为版本化 REST API。API 层只负责输入校验、 DEMO-009 将入库和检索服务公开为版本化 REST API,DEMO-015 将单文件入库切换为持久化异步
错误映射和响应序列化,不重复实现 Parser、Embedding、Mochow 或 Reranker 业务逻辑。 Job,DEMO-016 增加批量接收与聚合查询。API 层只负责输入校验、
错误映射和响应序列化,不重复实现 Parser、Embedding、VectorStore 或 Reranker 业务逻辑。
本任务不包含 Admin Console、ACL/RBAC、异步任务、软删除或 Hybrid Search。 当前不包含 ACL/RBAC、软删除、目录/压缩包上传、断点续传或 Hybrid Search。
## 启动前提 ## 启动前提
1. 按 `docs/development/MOCHOW_SETUP.md` 配置 PostgreSQL、Mochow 和千帆 Embedding; 1. 按 `docs/development/BOS_SETUP.md` 配置 BOS;
2. 按 `docs/development/RERANKER_SETUP.md` 配置 Reranker; 2. 按 `docs/development/OCEANBASE_VECTOR_SETUP.md` 配置 OceanBase Vector 和千帆 Embedding;
3. 执行 `uv run alembic upgrade head`; 3. 按 `docs/development/RERANKER_SETUP.md` 配置 Reranker;
4. 执行 `make run`。 4. 按 `docs/development/ASYNC_INGESTION_SETUP.md` 配置 RabbitMQ 并启动 Worker;
5. 执行 `uv run alembic upgrade head`;
6. 新旧索引迁移按 `docs/development/INDEX_REBUILD.md` 构建、验证并激活 Profile;
7. 执行 `make run`。
`/healthz` 和 OpenAPI 可以在没有云端凭证时启动和访问。业务依赖在首次访问 P0 业务接口时 `/healthz` 和 OpenAPI 可以在没有云端凭证时启动和访问。业务依赖在首次访问 P0 业务接口时
延迟创建;配置不完整时返回 `503`,不会将凭证名以外的配置值暴露给客户端。 延迟创建;配置不完整时返回 `503`,不会将凭证名以外的配置值暴露给客户端。
@ -23,13 +27,29 @@ Swagger UI:`/api/v1/docs`
## 示例 ## 示例
上传并同步入库: 上传并异步排队:
```bash ```bash
curl -F 'file=@./policy.md;type=text/markdown' \ curl -F 'file=@./policy.md;type=text/markdown' \
http://127.0.0.1:8000/api/v1/documents http://127.0.0.1:8000/api/v1/documents
``` ```
使用响应中的 `job_id` 查询进度:
```bash
curl http://127.0.0.1:8000/api/v1/jobs/<job-id>
```
批量上传并查询聚合进度:
```bash
curl -F 'files=@./policy.md;type=text/markdown' \
-F 'files=@./manual.pdf;type=application/pdf' \
http://127.0.0.1:8000/api/v1/ingestion-batches
curl http://127.0.0.1:8000/api/v1/ingestion-batches/<batch-id>
```
检索: 检索:
```bash ```bash
@ -38,17 +58,19 @@ curl -H 'Content-Type: application/json' \
http://127.0.0.1:8000/api/v1/search http://127.0.0.1:8000/api/v1/search
``` ```
Demo 上传请求会一直等待 Parse、Clean、Chunk、Embedding 和 Mochow Upsert 完成。大文件或 上传只等待不可变源文件、Document 与 Job 持久化,成功返回 `202`。Worker 后台执行 Parse、
云端调用较慢时,请求耗时会相应增加;异步任务属于 Demo 后置能力。 Clean、Chunk、Embedding 和 OceanBase Upsert;客户端通过 Job 状态判断何时可检索。
## 删除一致性 ## 删除一致性
删除按以下顺序执行: 删除按以下顺序执行:
1. 删除所有已记录 Index Profile 以及当前 Profile 的 Mochow 行; 1. 删除所有已记录 Index Profile 以及当前 Profile 的 OceanBase 行;
2. 删除 PostgreSQL 中的 ChunkIndexState、Chunk 和 Document; 2. 删除 OceanBase canonical 关系表中的 ChunkIndexState、Chunk 和 Document;
3. 删除解析 JSON 与不可变原始文件。 3. 删除解析 JSON 与不可变原始文件。
`queued`、`running` 或 `retrying` Job 对应的 Document 返回 `409`,避免删除与 Worker 并发。
向量清理失败时保留 canonical Metadata 和原始文件。Metadata 提交后若文件清理失败,接口 向量清理失败时保留 canonical Metadata 和原始文件。Metadata 提交后若文件清理失败,接口
返回明确的 `500`,可能留下不可访问的孤立文件;这是 Demo 直接删除的已知边界,正式 V1 返回明确的 `500`,可能留下不可访问的孤立文件;这是 Demo 直接删除的已知边界,正式 V1
应通过可重试清理任务和审计记录强化。 应通过可重试清理任务和审计记录强化。

View File

@ -1,6 +1,6 @@
# Test Strategy # Test Strategy
Unit tests must run without Mochow, Qianfan, internet, GPU or production credentials. Unit tests must run without OceanBase, BOS, Qianfan, internet, GPU or production credentials.
Provide deterministic fakes: Provide deterministic fakes:
- FakeEmbeddingProvider - FakeEmbeddingProvider
@ -11,6 +11,11 @@ Provide deterministic fakes:
External integrations use separate integration suites/markers. External integrations use separate integration suites/markers.
Async ingestion tests run with an in-process recording dispatcher and deterministic providers. They
must cover persisted Job/Attempt history, safe error summaries, bounded retry, cleanup-before-retry,
active-lease duplicate delivery, expired-lease recovery, immutable-source preservation and terminal
cleanup failure without requiring a live RabbitMQ broker.
E2E covers upload -> parse -> chunk -> embed -> index -> retrieve -> source trace, and source -> compilation draft -> review -> publish -> retrieval. E2E covers upload -> parse -> chunk -> embed -> index -> retrieve -> source trace, and source -> compilation draft -> review -> publish -> retrieval.
Any parsing/chunking/embedding/fusion/retrieval/reranking/query-routing change must run the Golden Dataset evaluation. Any parsing/chunking/embedding/fusion/retrieval/reranking/query-routing change must run the Golden Dataset evaluation.

View File

@ -6,4 +6,20 @@ Benchmark the full pipeline: Embedding × Retrieval Strategy × Fusion × Rerank
Initial embedding candidates: bge-large-zh, Qwen3-Embedding-0.6B, Qwen3-Embedding-4B, optional 8B. Initial embedding candidates: bge-large-zh, Qwen3-Embedding-0.6B, Qwen3-Embedding-4B, optional 8B.
Use synthetic/public/anonymized fixtures only. Use synthetic/public/anonymized fixtures only. ARCH-006 expands the checked-in dataset to 32
fully synthetic queries across refund, technical support, lodging, and reimbursement-material
sources. Offline Fake Provider regression must remain deterministic; target-provider acceptance
uses the same records through `make live-acceptance` and reports aggregate metrics only.
Current live gates:
```text
Recall@1 >= 0.90
Recall@5 = 1.00
MRR >= 0.90
NDCG@5 >= 0.90
Latency P95 <= 10000 ms
```
Threshold overrides are allowed only as explicit test-run environment inputs and must be recorded
with the run result. See `docs/development/LIVE_ACCEPTANCE.md` for safety and cleanup requirements.

View File

@ -2,13 +2,13 @@
## 目标基线 ## 目标基线
本计划依据 2026-09-01 核对的内部平台配置现状调整 EKP Provider 选型,同时保留本仓库 本计划依据 2026-09-02 对最新内部配置与 test 环境的只读核验调整 EKP Provider 和数据库
已经 Accepted 的 canonical-data 和 API-first 原则。 选型,同时保留 canonical-data 和 API-first 原则。
```text ```text
PostgreSQL canonical Metadata / Chunk / Index State OceanBase MySQL canonical Metadata / Chunk / Index State
Baidu BOS immutable source and derived artifacts Baidu BOS immutable source and derived artifacts
OceanBase Vector rebuildable Dense retrieval index OceanBase Vector co-located rebuildable Dense retrieval index
Qianfan bge-large-zh 1024-dimensional Embedding Qianfan bge-large-zh 1024-dimensional Embedding
Qianfan BCE Reranker bce-reranker-base Qianfan BCE Reranker bce-reranker-base
``` ```
@ -19,7 +19,7 @@ FakeRerankerProvider。
## 明确保留的行为 ## 明确保留的行为
- PostgreSQL 保持 canonical database,OceanBase 只保存可重建索引; - OceanBase 关系表保持 canonical database,专用向量表只保存可重建索引;
- 原始文件不可变; - 原始文件不可变;
- Dense Top20 → Reranker Top5; - Dense Top20 → Reranker Top5;
- 当前 PDF/DOCX/TXT/Markdown Parser 和 `structure_v1` Chunk; - 当前 PDF/DOCX/TXT/Markdown Parser 和 `structure_v1` Chunk;
@ -29,7 +29,7 @@ FakeRerankerProvider。
## 明确不随迁移引入 ## 明确不随迁移引入
- OceanBase 取代 PostgreSQL canonical database; - 将 OceanBase 向量表当作 canonical source;
- Top50、距离阈值、标准答案捷径、双查询和 RRF; - Top50、距离阈值、标准答案捷径、双查询和 RRF;
- Marker/Pandoc、LLM 摘要/问题/关键词索引; - Marker/Pandoc、LLM 摘要/问题/关键词索引;
- RBAC/ACL、异步队列、MCP、Hybrid/BM25 或 Admin Console 扩展; - RBAC/ACL、异步队列、MCP、Hybrid/BM25 或 Admin Console 扩展;
@ -64,19 +64,22 @@ Evaluation;保持 Top5 API 行为。
### ARCH-005 Runtime Cutover ### ARCH-005 Runtime Cutover
将 API 运行时切换到 BOS、OceanBase Vector 和 BCE Reranker;提供从 PostgreSQL canonical 将 API 运行时切换到 BOS、OceanBase Vector 和 BCE Reranker;提供从 canonical
Chunk 构建新 shadow index 的可重试命令,验证后激活新 Profile,再清理 Mochow 运行配置。 Chunk 构建新 shadow index 的可重试命令,验证后激活新 Profile,再清理 Mochow 运行配置。
依赖:ARCH-004。 依赖:ARCH-004。
### ARCH-006 真实验收 ### ARCH-006 真实验收
在受控测试资源中执行 PostgreSQL → BOS → Qianfan Embedding → OceanBase Vector → BCE 修正先前错误的 PostgreSQL 前提:将 canonical SQLAlchemy/Alembic 迁移到 OceanBase MySQL,
Reranker → canonical Source Result 全链路,并完成 30~50 条脱敏 Golden Query 验收。 同库保留逻辑隔离的可重建向量表;在受控 test 资源中执行 OceanBase canonical → BOS →
Qianfan Embedding → OceanBase Vector → BCE Reranker → canonical Source Result 全链路,并完成
30~50 条脱敏 Golden Query 验收。
依赖:ARCH-005、轮换后的运行凭据、已授权的测试数据库/表和 BOS 前缀。 依赖:ARCH-005、已授权的 test 运行凭据、数据库/表和 BOS 专用前缀。
## 安全边界 ## 安全边界
内部配置清单中的现有凭据按已暴露处理,只能使用轮换后的运行时 Secret。禁止读取凭据后 内部配置清单中的现有凭据按已暴露处理。经所有者明确要求,本次只允许把 test 凭据临时
生成 `.env`、测试快照或调试日志;真实集成测试必须使用专用测试资源并清理临时对象。 注入 ARCH-006 进程;不得生成 `.env`、测试快照或调试日志,不得连接 fat/prod,验收后仍应
轮换。真实集成测试必须使用专用表/BOS 前缀并清理本次临时对象。

View File

@ -0,0 +1,123 @@
# MCP 与异步批量入库优先级方案
**Task:** PLAN-001
**Status:** Accepted planning baseline
**Date:** 2026-09-02
## 1. 现状判断
| 能力 | 当前实现 | 结论 |
|---|---|---|
| 单文件上传 | `POST /api/v1/documents` 持久化 Source/Document/Job 后返回 `202`,Worker 后台处理 | DEMO-015 已实现 |
| 批量上传 | 批次 API、逐文件 Job 状态和 Admin Console 自动轮询 | DEMO-016 已实现 |
| 失败补偿 | 阶段内补偿 + 重试前清理全部派生数据,保留不可变源和执行历史 | DEMO-015 已实现 |
| 自动重试 | OceanBase Job/Attempt + Celery/RabbitMQ + 有界退避 + Worker 租约恢复 | DEMO-015 已实现 |
| 重试前清理 | 重试前删除向量、索引状态、Chunk、解析产物,保留源文件与历史 | DEMO-015 已实现 |
| MCP | 独立 Gateway、认证边界、`search_knowledge` 与 WorkBuddy 协议级验收 | DEMO-012 已实现 |
PLAN-001 的异步执行缺口已由 DEMO-015 补齐,批量提交与状态 UI 已由 DEMO-016 补齐。
## 2. Task 归并结论
### DEMO-012:MCP `search_knowledge`(提升为最高优先级)
保留既有 Task ID,不新建重复任务。范围限定为:
- 提供可运行的标准 MCP Gateway;
- 只暴露 `search_knowledge`;
- 复用现有 Search Service 或正式 `/api/v1/search`,不复制检索逻辑;
- 建立 WorkBuddy 所需的 user/service identity 边界,不使用共享超级管理员凭据;
- 完成协议契约、错误收敛、凭据脱敏和 WorkBuddy 兼容性验证。
该 Task 是 `EKP-070`、`EKP-071`、`EKP-072` 的最小垂直切片,不能据此把三个完整 V1 Task 标记为完成。
### DEMO-015:异步入库 Job 与自动重试清理(新增)
从 `EKP-017` 和 `EKP-023` 抽取 Demo 所需的最小基础能力,并复用 DEMO-003~006 的处理服务:
- 新增持久化 Ingestion Job/Attempt,记录阶段、状态、尝试次数、安全错误和时间戳;
- 源文件成功写入 BOS 且 Document 已持久化后立即排队,API 不等待解析和索引;
- 同一 Document 同时最多一个有效执行,任务投递和消费具备幂等保护;
- 对可重试的瞬时错误自动执行有界重试和退避;Demo 默认最多 3 次总尝试,允许通过非敏感配置调整;
- 格式不支持、文件为空、无可提取内容等确定性错误直接失败,不进行无意义重试;
- 每次重新执行前,按顺序清理该 Document 的向量行、`ChunkIndexState`、Chunk 和解析产物,再从不可变原始文件重新处理;
- 原始文件、Document 标识和审计所需的 Job/Attempt 历史不得被重试清理删除;
- 任一清理步骤失败时禁止启动下一次尝试,并记录为终止失败,避免在未知脏状态上重复执行。
- 通过执行器抽象接入 Celery/RabbitMQ,使 API 和 Worker 可独立部署;业务应用服务不得直接
依赖 Celery API,确保执行器仍可替换;
- Worker 重启、消息重复投递和确认超时必须保持持久化 Job 状态一致,不能产生双重有效
Attempt 或重复派生数据。
该 Task 只完成公共知识库入库所需的 Job/Retry 与队列基线,不扩展到完整审计系统、
Compilation、OCR、跨区域高可用或通用企业级 Job 平台。
### DEMO-016:批量异步上传与状态展示(已完成)
建立在 DEMO-015 之上,不与 Job 基础合并:
- 新增批量上传 API,一次接收多个 Demo 支持类型文件;
- 每个文件独立生成 Document 和 Ingestion Job,单个文件失败不回滚已成功接收的其他文件;
- 返回批次 ID、逐文件 Document ID、Job ID 和接收状态,不等待后台处理完成;
- 提供批次/Job 查询能力,并在 Admin Console 展示逐文件阶段、重试次数和最终结果;
- 文件数量、单文件大小和请求总大小均受可配置限制;
- 不在本 Task 引入目录上传、压缩包解包、断点续传或跨批次去重。
该 Task 对应 `EKP-021` 的批量异步扩展及 `EKP-080`/`EKP-081` 的最小状态 UI 切片,不能据此把完整 V1 Task 标记为完成。
## 3. 实施顺序与依赖
```text
DEMO-009 + ARCH-006 → DEMO-012(已完成)
PLAN-002(已完成)
└──→ DEMO-003 → DEMO-004 → DEMO-005 → DEMO-006
└──→ DEMO-015 Async Job + Celery/RabbitMQ + Retry Cleanup
└──→ DEMO-016 Batch Async Upload + UI
└──→ DEPLOY-001 → DEPLOY-002 → DEPLOY-003
```
推荐严格按以下顺序执行并逐项闭环:
1. `PLAN-002`(已完成)
2. `DEMO-015`(已完成)
3. `DEMO-016`(已完成)
4. `DEPLOY-001`
5. `DEPLOY-002`
6. `DEPLOY-003`
MCP 与异步入库没有代码级前置依赖,因此提高 MCP 优先级不会阻塞后续批量上传。`DEMO-016` 必须等待 `DEMO-015`,否则会把同步流水线简单并发放大,缺少可靠的状态、隔离和恢复边界。
## 4. 验收边界
### DEMO-012
- MCP Inspector/标准客户端能发现并调用 `search_knowledge`;
- WorkBuddy 使用已映射的 user/service identity 完成一次端到端检索;
- REST 与 MCP 对同一查询复用同一业务逻辑并返回一致的来源字段;
- 响应与日志不泄露 OceanBase、BOS、模型名、Endpoint 或凭据。
### DEMO-015
- 上传接口在源文件和 Job 持久化后返回 `202`,后台完成现有全流水线;
- 注入一次可重试故障后可观察到自动重试,并最终成功;
- 第二次尝试开始前,旧解析产物、Chunk、索引状态和向量均已清理,原始源文件仍存在;
- 永久错误不重试,达到次数上限后稳定进入 `failed`;
- 并发或重复投递不会产生双重有效执行和重复派生数据。
- RabbitMQ 消息确认超时或 Worker 重启后,Job 可恢复到终态且不会重复生成派生数据。
### DEMO-016
- 一次提交多个文件后立即得到逐文件 Document/Job 标识;
- 批次内成功、处理中、重试中和失败状态互不影响且可查询;
- Admin Console 无需刷新即可轮询展示逐文件进度和最终状态;
- 批次边界、大小限制、部分失败和重试清理均有集成测试。
## 5. 非目标与 ADR 影响
- PLAN-002 已把 Celery/RabbitMQ 纳入 `DEMO-015`,并以独立 `DEPLOY-*` Task 纳入最小
Kubernetes、多副本和滚动升级能力;跨区域容灾、自动云资源编排和完整 Observability
仍为非目标。
- 不改变 OceanBase/BOS canonical、向量索引可重建、原始文档不可变、Provider 抽象和 MCP 隐藏基础设施等 Accepted ADR。
- 公共知识库只是阶段性单一可见域;ADR-007 保持 Accepted,用户/ACL 实现后置且不得用全局
Top-K 后过滤替代。
- 本次任务拆分和交付边界不需要新增或重开 Accepted ADR。

View File

@ -1,15 +1,15 @@
# 项目进度 # 项目进度
当前阶段:Provider 架构迁移 当前阶段:生产容器镜像闭环完成,准备实施 Kubernetes/Helm 一键部署
已完成:DEMO-001~DEMO-009、ARCH-001、ARCH-002 已完成:DEMO-001~DEMO-010、DEMO-012、DEMO-014~DEMO-016、ARCH-001~ARCH-006、PLAN-001~PLAN-002、DEPLOY-001
进行中:无 进行中:无
阻塞:无 阻塞:无
下一任务:ARCH-003 下一任务:`DEPLOY-002`,完成并合并后执行 `DEPLOY-003`。
Provider 架构迁移按 `ARCH-001 → ARCH-002 → ARCH-003 → ARCH-004 → ARCH-005 → ARCH-006` Provider 架构迁移按 `ARCH-001 → ARCH-002 → ARCH-003 → ARCH-004 → ARCH-005 → ARCH-006`
执行;每个 Task 验证后独立推送并合并 `main`。全部迁移完成后再继续 DEMO-010。 执行并已完成。DEMO-010 在迁移闭环验收后实施,未扩大为完整 V1 Admin Console。
## 任务记录 ## 任务记录
@ -106,7 +106,8 @@ Provider 架构迁移按 `ARCH-001 → ARCH-002 → ARCH-003 → ARCH-004 → AR
- 新增与厂商无关的 `RerankerProvider` / `RerankResult` 契约及确定性 Fake,使用原始 - 新增与厂商无关的 `RerankerProvider` / `RerankResult` 契约及确定性 Fake,使用原始
candidate index 映射精排结果,业务服务不依赖千帆响应结构。 candidate index 映射精排结果,业务服务不依赖千帆响应结构。
- 新增千帆 v2 `qwen3-reranker-0.6b` Adapter,调用 `/v2/rerank`,严格限制 Query、 - 新增千帆 v2 Reranker Adapter(初始为 `qwen3-reranker-0.6b`,ARCH-004 已迁移至
`bce-reranker-base`),调用 `/v2/rerank`,严格限制 Query、
documents 和 Top-N,校验响应数量、唯一索引、回显文本、有限分数与降序关系;错误不会 documents 和 Top-N,校验响应数量、唯一索引、回显文本、有限分数与降序关系;错误不会
回显请求内容或凭证。 回显请求内容或凭证。
- 新增 `SearchService`,完成 Dense Top20 → Reranker → Top5。Reranker 选中候选后按 - 新增 `SearchService`,完成 Dense Top20 → Reranker → Top5。Reranker 选中候选后按
@ -154,3 +155,272 @@ Provider 架构迁移按 `ARCH-001 → ARCH-002 → ARCH-003 → ARCH-004 → AR
Prefix 和超时均由环境注入。Local 仍是测试/开发及 ARCH-005 前的默认实现。 Prefix 和超时均由环境注入。Local 仍是测试/开发及 ARCH-005 前的默认实现。
- 新增完全离线的 BOS Adapter、配置、Provider wiring 测试和中文配置说明;未使用内部配置 - 新增完全离线的 BOS Adapter、配置、Provider wiring 测试和中文配置说明;未使用内部配置
清单中的真实地址或凭据,真实 BOS 联调留在 ARCH-006。 清单中的真实地址或凭据,真实 BOS 联调留在 ARCH-006。
### ARCH-003
- 新增官方 `pyobvector` 驱动的 `OceanBaseVectorStore`,覆盖批量 Upsert、按 Document/Profile
参数化删除和 Profile 条件下推的 Cosine ANN TopN;SDK 类型不进入业务服务。检索固定使用
OceanBase 4.6 的 `VECTOR_INDEX(... pre_filter)`,不依赖普通 `WHERE` 的自适应过滤,也不做
全局 TopK 后过滤。
- 将低值优先的 `cosine_distance` 转为 `score = 1 - distance`,并对向量维度、有限值、
非零范数、返回字段、UUID、Profile、距离范围和排序执行失败关闭校验。
- 新增 `VECTOR(1024)`、Cosine HNSW、Profile/Document 标量索引的预置 DDL;应用不自动执行
外部索引 DDL,PostgreSQL 继续保存 canonical 数据,OceanBase 表可重建。
- 新增 `mochow|oceanbase` 类型化配置、SecretStr 密码和运行时工厂;Mochow 保持 ARCH-005
前的默认实现。全部 Adapter、配置、Schema 和 wiring 测试均离线运行,真实数据库验收留在
ARCH-006。
### ARCH-004
- 将千帆 Reranker 运行基线从 `qwen3-reranker-0.6b` 切换为已接受的
`bce-reranker-base`;配置使用受限 Literal,业务 API、Provider 抽象和 Dense Top20 →
Reranker Top5 行为保持不变。
- 按千帆 `POST /v2/rerank` 官方约束复核并保留 Query 1600 字符、64 个候选、单文档
4096 字符、Top-N 及响应排序/索引/分数失败关闭校验;真实 API Key 不进入代码和测试。
- 更新 Adapter、环境示例、Provider wiring、配置与请求契约测试;Golden Dataset 继续使用
确定性 Fake 离线评估,但显式绑定 `bce-reranker-base` 当前模型标识。真实 BCE 质量与延迟
统一留在 ARCH-006 使用轮换凭据验收。
### ARCH-005
- 将正式运行默认值切换为 BOS、OceanBase Vector 和 `bce-reranker-base`,新 Profile ID 为
`demo-bge-large-zh-structure-v1-oceanbase-v1`;Local/Fake 仍用于显式本地和离线测试。
- 新增 PostgreSQL canonical `index_profiles` 迁移与 `shadow / active / retired` 生命周期,
通过局部唯一索引保证至多一个 Active Profile;API 启动时优先解析数据库 Active Profile,
无记录的新部署才使用配置基线。
- 新增 `ekp-index-rebuild build|verify|activate`:从 canonical Chunk 分文档重建,确定性 Upsert,
重试时跳过完整文档并清理目标 Profile 的部分写入;激活前核对全部 ChunkIndexState,并在
PostgreSQL 事务内锁定 Chunk 写入、退役旧 Profile、激活新 Profile。
- 移除 Mochow 运行配置、公开 Adapter、测试、配置文档及 `pymochow` 依赖;不自动删除旧
外部索引,保留受控回滚和运维退役边界。
- 新增重建/重试/完整性失败/单 Active/激活/迁移/CLI 脱敏测试及中文操作文档。真实资源
构建、BOS/OceanBase/Qianfan 连通性和 30~50 条脱敏查询验收仍属于 ARCH-006。
### ARCH-006(已完成)
- Golden Dataset 已扩展为 32 条完全合成 Query,覆盖退款、技术支持、住宿标准和报销材料;
Dense、Reranker、canonical Source Result 的离线回归全部通过。
- 新增 `ekp-live-acceptance` / `make live-acceptance`,实际覆盖上传、解析、Chunk、Embedding、
OceanBase Dense Top20、BCE Reranker Top5、canonical 来源核对及本次运行数据清理。
- live 命令强制要求 `APP_ENV=test`、专用 `live-acceptance/` BOS Prefix、凭据授权确认、资源
授权确认和显式运行开关;当前环境下已验证会在创建 Provider 前失败关闭且不泄露细节。
- 根据最新配置清单和 test 环境只读探测,确认没有独立 PostgreSQL;目标为 OceanBase
4.3.5.4 MySQL 模式,同一数据库承载 canonical 关系数据和逻辑隔离的向量表。ADR-012 已
取代先前 PostgreSQL 部署假设,业务事实来源与可重建索引的逻辑边界保持不变。
- 已在目标 test 集群只读验证数据库连通、版本与兼容模式,并通过 `EXPLAIN` 确认
`VECTOR_INDEX(... pre_filter)` 生成 `VECTOR INDEX ADAPTIVE SCAN (PRE-FILTER)`;未读取
业务正文或输出任何连接信息/凭据。
- canonical SQLAlchemy 默认已切换为 `mysql+pymysql`,数据库 URL 使用 `SecretStr`,新增
有界连接池/超时配置;新增 forward-only migration,以生成列唯一索引保证 OceanBase 与
SQLite 下至多一个 Active Profile;PostgreSQL 驱动已移除。
- 配置清单中的 test 凭据仅按用户明确授权临时注入本次验收,不保存为 `.env`、不提交、
不连接 fat/prod;完成后仍记录为待轮换风险。
- 已在 test OceanBase 执行全部 forward-only migration,并对本项目表完成列、类型、默认值、
主键的 scoped schema drift 核对;OceanBase 扩展语法导致标准 MySQL 反射器不能可靠比较的
二级索引/唯一约束/外键,由 SQLite 全量 drift、真实唯一约束事务和 live 级联删除补充验证。创建
`knowledge_chunks_v1` 12 列专用向量表和 Cosine HNSW,确认 PRE-FILTER 执行计划。迁移前
已确认所有目标表均不存在,没有覆盖原有 31 张业务表;事务内双 Active 写入实测被唯一
索引拒绝并完整回滚。
- 已完成 OceanBase canonical → BOS → `bge-large-zh` → OceanBase Vector →
`bce-reranker-base` → canonical Source Result 正式验收:32 条查询的 Recall@1/3/5、MRR、
NDCG@5 均为 `1.0`,P50 `3787.61 ms`、P95 `4759.21 ms`,通过 P95 `10000 ms` 门槛。
- 正式命令退出码为 `0`;两份合成文档、canonical Chunk/State、向量行与 BOS 对象均走随机
前缀清理,数据库复核 `live-` Document/Chunk/Vector 残留均为 `0`。首次 live 运行曾出现
一次瞬时失败,分阶段诊断及随后完整 32 查询均成功,仍需在后续稳定性任务关注 Provider
重试、限流和告警。
### DEMO-010(已完成)
- 新增同源轻量 Admin Console,提供 `/admin/upload`、`/admin/documents` 和
`/admin/search` 三个直接可访问页面;FastAPI 只负责静态文件路由,页面业务操作全部复用
`/api/v1`,没有复制上传、文档管理或检索逻辑。
- 上传页支持 PDF、DOCX、TXT、Markdown 文件选择、处理状态和结果摘要;文档页展示文件名、
类型、上传时间、状态,支持详情与确认删除;Search Lab 显式展示 Rank、Filename、Page、
Section、Chunk Content、Dense Score 和 Rerank Score。
- 前端使用安全 DOM 文本节点渲染 API 数据,不注入 HTML,不展示 storage key、Index Profile、
Provider Endpoint 或凭据;只实现 Demo 规定的三个页面,没有引入 ACL/RBAC、任务、审计、
Compiled Knowledge、MCP、Hybrid/BM25 或额外前端构建工具链。
- 新增静态路由、资产、范围和 API First 契约测试;浏览器使用本地确定性 Provider 完成
`README.md` 上传 → Ready → 列表/详情 → Top5 检索,并验证桌面和 390px 移动布局无横向
溢出、无前端警告或错误。删除的 UI 确认和服务端清理语义由现有 REST 全链路测试覆盖。
- 完整验证结果:Ruff Format/Lint、Mypy、JavaScript 语法、Git diff、依赖锁、Alembic Head/
离线升级均通过;Unit + Integration `150 passed`,RAG Evaluation `5 passed`。本任务没有
修改检索算法或 Provider,不重复调用真实资源验收;ARCH-006 的 32 条真实查询结果继续
作为底层链路基线。
- 已知非阻塞风险保持不变:配置清单中的历史 test 凭据仍应轮换;真实 Provider 瞬时失败的
重试、限流与告警留待后续明确 Task。Demo Console 本身没有新增凭据或生产 Endpoint。
### DEMO-014(已完成)
- 将 DEMO-010 通用绿色界面升级为地瓜机器人品牌化 Admin Console,使用唯一品牌强调色
`#FF3C00`、黑白高对比、无衬线中文排版、低圆角工业组件和与三步操作对应的计算信号轨;
没有增加页面、接口或业务能力。
- 品牌标记由内联 SVG 与 CSS 原生绘制,不下载、复制或热链官网资产。页面没有新增外部
Runtime、字体请求或第三方脚本,继续保持原生 HTML/CSS/JS 和同源 `/api/v1` 调用。
- GitHub 调研确认官方 `openai/skills` 当前没有通用前端设计 Skill;选择并安装了 Apache-2.0
的 `anthropics/skills/skills/frontend-design`。该 Skill 只影响视觉策划和自检方法,不进入
项目运行依赖或仓库产物。
- 浏览器使用本地确定性 Provider 再次完成 README 上传 → Ready → 文档列表 → Top5 检索;
1440px 桌面端和移动端均无根页面横向溢出、无前端警告或错误。复验中发现隐藏表头将移动
页面撑宽至 503px,已改为 `aria-label`,最终根宽度与 463px 视口一致,表格只在容器内滚动。
- 完整验证结果:Ruff Format/Lint、Mypy、JavaScript 语法与 Git diff 通过;Unit、Integration
和 RAG Evaluation 合计 `155 passed`;Alembic Head 与离线升级通过。本任务不修改 API、
检索算法、Provider 或数据库,不重复调用真实资源验收。
### PLAN-001(已完成)
- 审计确认当前 `/api/v1/documents` 仅接受一个文件,并在请求线程同步执行解析、Chunk、
Embedding 和索引;API 与 Admin Console 均没有批量提交、Job 查询或后台状态轮询。
- 当前上传、解析和向量索引具备阶段内失败补偿,影子索引重建具备目标 Profile 的重试安全
清理,但正常上传流水线没有持久化 Job/Attempt、自动调度退避或统一的重试前派生数据清理,
因而不能视为已实现自动重试。
- 保留既有 `DEMO-012` 并提升为最高优先级;新增 `DEMO-015` 承担 Async Job、自动重试和
重试前清理,新增 `DEMO-016` 承担批量异步上传和逐文件状态 UI。严格执行顺序为
`DEMO-012 → DEMO-015 → DEMO-016`。
- 已更新 PRD、Architecture、API/MCP Contract、Execution Plan、Task List、Demo Scope 与 DoD;
本任务只形成规划和待实现契约,没有修改业务代码、数据库 Migration 或 Accepted ADR。
- 验证结果:Ruff Format/Lint、Mypy、Git diff 与 Alembic Head/离线升级通过;Unit + Integration
`150 passed`,RAG Evaluation `5 passed`。首次受限网络运行因无法解析 PyPI 失败,获准联网
获取已锁定的构建依赖后全套复验通过。
- 下一任务:`DEMO-012`。详细依赖、范围和验收标准见
`docs/plans/ASYNC_INGESTION_MCP_PLAN.md`。
### DEMO-012(已完成)
- 新增独立 `ekp-mcp` Streamable HTTP Gateway 和 `make run-mcp` 入口,使用官方 MCP Python
SDK v2;默认只监听 `127.0.0.1:8766/mcp`,且只注册 `search_knowledge` 一个业务 Tool。
- Gateway 直接复用现有 `SearchService`,没有复制 Dense Retrieval、Reranker 或 Source
Enrichment;结构化输出与 REST 保持相同来源字段,不返回 Index Profile 或 Provider 信息。
- 新增运行时专用 Bearer Token 验证,将凭据映射为显式 client/service identity 和最小
`knowledge:search` scope;Token 至少 32 字符并通过 `SecretStr` 管理,不允许写入仓库。
当前身份层是 Demo 边界,不宣称完成 V1 OAuth、Document ACL 或用户委托授权。
- 启用 Host/Origin allowlist 和请求体限制;缺失/错误认证、scope、参数和检索异常均按安全
边界收敛,响应和日志不回显 Token、内部 Endpoint、数据库、模型或索引实现。
- 新增官方 MCP Client 的协议级测试,覆盖 initialize、tools/list、tools/call、唯一 Tool、
identity 传递、REST 字段一致性、无结果、参数拒绝、认证失败、Origin 防护和错误脱敏;
WorkBuddy 的 HTTP + Bearer + 环境变量配置记录于 `docs/development/MCP_SETUP.md`。
- 验证结果:Ruff Format/Lint、Mypy、Git diff、Alembic Head/离线升级全部通过;Unit +
Integration `165 passed`,RAG Evaluation `5 passed`。本任务未改变检索算法或数据库结构,
不重复消耗真实 Embedding/OceanBase/Reranker 资源。
- 2026-09-02 补充完成 WorkBuddy 现场验收:本机 WorkBuddy Desktop `5.3.14` 自带的
CodeBuddy CLI `2.115.0` 通过仓库交付的 `deploy/workbuddy-mcp.example.json` 建立
Streamable HTTP + Bearer 连接,健康检查为 `Connected`,仅发现
`mcp__enterprise-knowledge__search_knowledge`,并以 `query` + `top_k=1` 完成真实 Tool
调用、读取结构化来源字段。验收使用真实 Gateway、认证、身份传递、Schema 和 Tool
注册代码,检索应用返回确定性数据,以隔离 WorkBuddy 兼容性与外部 Provider 波动。
- 非交互客户端调用使用仅对当前会话放行 `DeferExecuteTool` 的权限设置,没有持久化放宽权限;
临时客户端配置已删除、Gateway 已停止,仓库和用户配置中均未留下测试 Token。至此
`DEMO-012` 约定的 WorkBuddy 客户端验收项闭环;公网 HTTPS 部署验证属于部署环境验收,
不阻塞本地 Demo Task 完成。
- 下一任务:`DEMO-015`。
### PLAN-002(已完成)
- 将当前交付目标明确为企业受控网络内的单一公共知识库:本阶段所有知识共享一个可见域,
暂缓用户、角色、知识库空间和 Document ACL,但不废弃 ADR-007,也不把未实现项标记完成。
- 保持 Accepted ADR 的模块化单体边界,确定 API/Admin、MCP Gateway 和 Async Worker 独立
部署;OceanBase/BOS/AI Provider 使用外部服务,当前云端模型链路不启用 GPU Worker。
- 将 Celery/RabbitMQ 正式并入 `DEMO-015`,并新增 `DEPLOY-001~003`,严格顺序为
`DEMO-015 → DEMO-016 → DEPLOY-001 → DEPLOY-002 → DEPLOY-003`。
- 定义 `make deploy VALUES=...` 的一键部署边界:操作者预先提供集群访问与 Secret,仓库负责
生产镜像、串行 Migration、Helm 安装/升级、Ingress/TLS、探针、资源与 Pod 安全配置;不得
将凭据写入 Git、镜像、命令行或日志。
- 明确现有 `docker-compose.dev.yml` 的 PostgreSQL/MinIO 假设已被 ADR-012 取代,不作为集群
交付模板;新增 `docs/plans/PUBLIC_KNOWLEDGE_CLUSTER_PLAN.md` 作为任务范围和验收基线。
- 本任务仅修改规划、产品、架构、API/MCP 契约、Demo Scope 和 DoD 文档,没有修改业务代码、
数据库 Migration 或 Accepted ADR 状态。
- 验证结果:Markdown 相对链接、敏感值模式和 Git diff 检查通过;Ruff Format/Lint、Mypy、
Alembic Head/离线升级通过;Unit + Integration `165 passed`,RAG Evaluation `5 passed`。
- 下一任务:`DEMO-015`。
### DEMO-015(已完成)
- 新增 OceanBase canonical `ingestion_jobs` / `ingestion_attempts` 与 forward-only migration;
Document 唯一 Job、Attempt 编号上限和单一运行中 Attempt 均由数据库约束保护。
- `POST /api/v1/documents` 改为持久化不可变 Source、Document、Job 后返回 `202`,新增
`GET /api/v1/jobs/{job_id}`;活动 Job 的 Document 删除返回 `409`,避免与 Worker 竞态。
- 新增队列无关的 Dispatcher 与 Job Runner,首个适配器使用 Celery/RabbitMQ;消息只包含 Job
ID,启用 late ack、worker-lost reject、单任务 prefetch 与周期性持久化 Job 恢复扫描。
- 瞬时错误按配置执行指数退避和最多三次总 Attempt;Worker 租约有效时重复投递不创建新
Attempt,租约过期后关闭旧 Attempt 并恢复。错误记录为固定安全摘要,不保存 Provider 正文。
- 第二次及后续 Attempt 前依次清理向量、ChunkIndexState、Chunk 与 parsed JSON,再从不可变
源文件重建;清理失败直接终止且不创建新 Attempt。Job 创建失败会补偿 Document 与源对象。
- Admin Console 仅将上传提示调整为“已接收并进入队列”;批量选择、Job 轮询和逐文件状态 UI
没有提前实现,继续属于 `DEMO-016`。
- 验证结果:Ruff Format/Lint、Mypy、JavaScript 语法、依赖锁、Alembic Head/离线 MySQL SQL
与 SQLite schema drift 均通过;Unit + Integration `179 passed`,RAG Evaluation `5 passed`;
Worker 入口确认注册 `ekp.ingestion.run` 与 `ekp.ingestion.recover`。
- 环境限制:当前工作站没有 Docker/RabbitMQ,也没有持久化注入目标 OceanBase URL,因此未做
真实 Broker 断连/确认超时和目标库 migration;这些真实环境项必须在 DEPLOY-003 集群验收
重跑。本任务以 Celery 配置测试、SQLite 持久化/唯一约束和故障注入集成测试闭环代码语义。
- 下一任务:`DEMO-016`。
### DEMO-016(已完成)
- 新增 canonical `ingestion_batches` / `ingestion_batch_items` 及 forward-only migration,保存
输入顺序、逐文件接收结果与 Document/Job 引用;异步阶段和最终结果仍以 Job 为事实源,
批次不是跨文件事务。
- 新增 `POST /api/v1/ingestion-batches` 与 `GET /api/v1/ingestion-batches/{batch_id}`;批量
请求字段为 `files`,成功返回 `202`、聚合状态和逐文件接收/Job 状态。类型、大小或内容错误
只拒绝当前文件,错误映射为固定安全摘要,其他文件继续接收。
- 新增 `BATCH_MAX_FILES` 与 `BATCH_MAX_TOTAL_BYTES`,复用 `MAX_UPLOAD_BYTES` 约束单文件;
上传流共享总字节预算,不把整批文件读入内存。数量超限在创建批次前返回 `413`。
- Admin Console 支持多选提交,显示当前批次的接收数量、完成数量、逐文件阶段、Attempt 与安全
错误,并每 1.5 秒自动轮询;进入 `completed`、`partial_failure` 或 `failed` 后停止轮询。
DOM 数据继续只通过文本节点渲染。
- 集成测试覆盖顺序稳定、部分拒绝不回滚、数量/单文件/批次总大小、全成功/部分失败聚合、
未知批次、OpenAPI 和前端契约;迁移 drift 增加两张新表。Ruff Format/Lint、Mypy、
JavaScript 语法、依赖锁、Alembic Head/离线升级均通过;Unit + Integration `183 passed`,
RAG Evaluation `5 passed`。
- 浏览器验收确认桌面布局和 390px 窄屏布局正常、无根页面横向溢出、无控制台警告或错误;
本地页面不注入云端凭据,批次结果行为由确定性 API 集成测试验证。
- 未扩展目录上传、压缩包解包、断点续传、手动重试、去重、用户或 ACL;自动重试和清理继续
复用 DEMO-015,Accepted ADR 未变更。
- 下一任务:`DEPLOY-001`。
### DEPLOY-001(已完成)
- 新增共享生产 Dockerfile 的 `api`、`mcp`、`worker` 三个 target,使用精确 Python/uv 版本、
frozen production dependencies、固定非 root UID/GID、exec-form 启动命令和逐 target 健康检查。
- 新增 amd64/arm64 Buildx Bake 计划、镜像构建/渲染/推送 Make 入口和 `.dockerignore`;构建上下文
排除凭据、本地状态、缓存、测试与开发产物,镜像不包含环境 Endpoint 或 Secret。
- Admin 静态目录新增类型化运行时覆盖,使非 editable 安装后的 API 镜像仍能提供同源页面;
源码运行继续使用既有默认目录。
- 新增不依赖 Docker daemon 的镜像契约测试与中文运行文档;基础 Python/uv 镜像同时固定精确
tag 与 manifest digest,Registry 地址和本项目发布 digest 继续由部署环境提供。
- 使用 Docker Buildx `0.36.1` + Colima 分别实际构建并加载全部三个 `linux/arm64` 和三个
`linux/amd64` target;最终 digest 固定版本再次构建命中相同业务 layer。没有执行 Registry
push,也没有在构建参数、history 或镜像文件系统中发现凭据。
- API 镜像以 `10001:10001` 启动并进入 healthy,`/healthz`、`/admin/upload`、OpenAPI 及
DEMO-016 批量路径 smoke test 通过;MCP/Worker 镜像以同一非 root 用户完成安装入口导入,
默认 CMD 与 HEALTHCHECK 元数据核对通过。真实 Provider/Broker 启动仍属于 DEPLOY-003。
- Ruff Format/Lint、Mypy、Docker Bake 渲染、JavaScript/依赖锁/Alembic 与完整测试门禁通过;
Unit + Integration `187 passed`,RAG Evaluation `5 passed`。下一任务:`DEPLOY-002`。
### DEPLOY-002(已完成)
- 新增 `deploy/helm/knowledge-platform` Helm Chart,部署 API/Admin、MCP Gateway、Async Worker、
单实例 Recovery Scheduler、串行 Migration Job、Service、Ingress/TLS 和可选 Demo RabbitMQ;
API/MCP 默认各两个副本,Worker 可独立扩缩,Beat 使用 `Recreate` 避免重复调度。
- 新增严格 `values.schema.json` 与仓库外环境 Values 示例,拒绝未知顶层字段、`latest` tag、
非法 digest、API/MCP 单副本和错误类型;跨字段模板校验继续约束批量大小、重试、Chunk 和
Top-K 关系。应用和可选 RabbitMQ 镜像均支持 digest,Demo RabbitMQ 已固定 manifest digest。
- 敏感配置只通过预建 `existingSecret` 注入;ConfigMap 仅渲染显式白名单内的非敏感字段,
没有任意 `extraEnv` 或 Secret 对象。`scripts/deploy.sh` 在部署前校验 Values、Namespace、运行时
Secret/必需键、可选 RabbitMQ Secret/键和启用 Ingress 时的 TLS Secret 引用,不读取 Secret
值;Helm 3/4 分别使用等价的原子失败回滚参数。
- Migration 使用 `pre-install,pre-upgrade` Hook、固定名称和 `before-hook-creation`,安装/升级在
Migration 成功前不更新业务 Pod。应用容器固定非 root UID/GID、只读根文件系统、丢弃全部
capabilities、禁用 ServiceAccount Token,并为临时目录、RabbitMQ 日志和 Beat 状态提供
`emptyDir`;API/MCP/Worker/Scheduler 均配置启动和存活/就绪探针及资源 requests/limits。
- 本机使用 K3s `v1.35.0+k3s1`、Helm `4.2.4` 和 DEPLOY-001 ARM64 镜像完成真实安装;Migration
成功,API/MCP 为 `2/2`,Worker、Scheduler 与可选 RabbitMQ 全部 Ready。首次安装暴露的
RabbitMQ 只读日志目录和 Beat 调度文件权限问题已修复,随后相同 Values 连续升级至 Revision
3~5 均成功;API `/healthz` 和 Admin 上传页 smoke test 通过。
- 默认、环境示例和 RabbitMQ 三种 Helm lint 均通过;含 Ingress/TLS/RabbitMQ 的 14 个最终
资源通过 kubeconform strict 校验。仓库、Helm manifest、Pod 日志三路 Secret 值扫描均为 `0`;
Ruff Format/Lint、Mypy、Alembic、Bash 语法与完整门禁通过,Unit + Integration `195 passed`,
RAG Evaluation `5 passed`。
- 目标企业集群的稳定 HTTPS、滚动重启、Worker 故障恢复、真实外部依赖监测与远程 WorkBuddy
仍属于 `DEPLOY-003`;本任务没有把这些项提前标记完成,也没有改变 Accepted ADR 或公共知识库
范围。下一任务:`DEPLOY-003`。

View File

@ -0,0 +1,123 @@
# 公共知识库集群交付计划
**Task:** PLAN-002
**Status:** Accepted planning baseline
**Date:** 2026-09-02
## 1. 阶段目标与边界
当前阶段只交付一个面向企业内部受控网络的公共知识库,先保证服务持续可用、可重复部署,
并让使用者打开 Admin Console 或连接 WorkBuddy 后即可使用,无需感知后端进程启停。
“公共”表示本阶段所有已接收知识处于同一个可见域,不代表面向公网匿名开放,也不代表
ADR-007 被废弃。用户、用户组、角色、知识库多空间、Document ACL、SSO、OAuth 用户委托等
完整 V1 能力继续后置;一旦需要接收非公共知识,必须先实现 Permission Resolver 和检索前/
检索中 ACL 下推,禁止先做全局 Top-K 再过滤。
Admin 写操作只部署在企业受控网络或统一接入网关后。MCP 继续使用运行时注入、只具备
`knowledge:search` scope 的服务凭据;静态 Demo credential 不得演变为共享超级管理员 Token。
## 2. 目标部署拓扑
```text
Ingress / TLS
│
┌──────────────┴──────────────┐
│ │
Admin Console + Knowledge API MCP Gateway
replicas replicas
│ │
└──────── Application Services ────────┐
│
┌─────────────────────────────┼─────────────┐
│ │ │
OceanBase MySQL BOS RabbitMQ
canonical + vector │
▼
Async Worker(s)
```
沿用 ADR-002:业务核心仍为模块化单体,不拆分额外微服务;API、MCP Gateway 和异步 Worker
分别作为可独立扩缩容的进程/容器交付。当前 Parser 与云端 Embedding/Reranker 不需要本地 GPU,
因此 GPU Worker 不进入本轮最小部署,未来引入本地 OCR/模型时再按既有架构角色启用。
OceanBase、BOS 和模型服务使用集群外已有服务。RabbitMQ 默认使用外部地址;Helm Chart 可以
提供显式开关安装开发/演示依赖,但生产值不得把队列数据生命周期绑定到应用 Pod。
## 3. “一键部署”的定义
在操作者已具备 Kubernetes 访问权限、目标 Namespace 和 Secret 的前提下,仓库提供单一入口:
```text
make deploy VALUES=<environment-values.yaml>
```
该入口负责:
- 校验非敏感配置和 `existingSecret` 引用,不把凭据写入 Git、镜像、命令行或日志;
- 运行一次性、可追踪且串行的数据库 Migration Job;
- 安装或升级 API/Admin、MCP Gateway、Async Worker、Service 和可选 RabbitMQ 依赖;
- 配置 Ingress/TLS、资源 requests/limits、非 root、只读根文件系统和最小 ServiceAccount;
- 配置 readiness/liveness probe、副本数和滚动升级策略;
- 支持重复执行和幂等升级,并提供对应卸载/回滚说明。
本定义不承诺自动创建 OceanBase、BOS、域名证书或云厂商账号,也不允许在仓库内保存这些
资源的敏感凭据。
## 4. Task 顺序与依赖
```text
PLAN-002
└──→ DEMO-015 Celery/RabbitMQ Async Job + Retry Cleanup
└──→ DEMO-016 Batch Async Upload + Status UI
└──→ DEPLOY-001 Production Images
└──→ DEPLOY-002 Kubernetes/Helm One-click Deploy
└──→ DEPLOY-003 Availability + Remote Acceptance
```
### DEMO-015
实现持久化 Job/Attempt、重试前清理、幂等消费和有界自动重试,并把执行器正式接入
Celery/RabbitMQ。队列调用保持在抽象边界后,不允许业务服务直接依赖 Celery API。
### DEMO-016
**已完成。** 基于 DEMO-015 提供逐文件隔离的批量异步提交、批次/Job 查询和 Admin Console
状态展示;下一交付门槛为 DEPLOY-001 的生产镜像。
### DEPLOY-001
**已完成。** 交付 API/Admin、MCP Gateway、Async Worker 的生产容器镜像与启动约定;镜像使用固定依赖、
非 root 用户、健康检查和多架构可构建基线,不内置凭据。
### DEPLOY-002
**已完成。** 交付 Kubernetes/Helm Chart、严格环境 Values Schema、`existingSecret`、串行
Migration Hook、Ingress/TLS、Pod 安全、资源限制、探针和 `make deploy` 入口;Worker 与单实例
恢复调度器已分离。现有 `docker-compose.dev.yml` 仍仅作为旧的本地开发文件,因其
PostgreSQL/MinIO 假设与 ADR-012 不一致,不得作为集群交付模板。下一交付门槛为 DEPLOY-003
的目标集群可用性与远程验收。
### DEPLOY-003
在目标集群完成安装、幂等升级、滚动重启、Worker 故障恢复和远程 WorkBuddy 验收;形成运行
手册、备份/恢复责任边界和最小日志/指标检查。灾备演练与完整 Observability 平台仍属后置项。
## 5. 最小验收标准
- 新 Namespace 可由单一命令完成安装,重复执行不会破坏已有数据;
- Migration 在应用升级前串行成功,失败时新版本不继续接管流量;
- Admin/API 与 MCP 使用稳定 HTTPS 地址,重启或滚动升级不要求最终用户重新注入配置;
- API/MCP 支持至少两个副本,readiness 未通过的 Pod 不接收流量;
- Worker 重启后持久化 Job 可继续恢复,重复消息不产生双重有效执行或重复派生数据;
- 远程 WorkBuddy 仅发现批准的 `search_knowledge` Tool,并完成一次真实检索;
- 清单、镜像、日志和验收产物不包含凭据,敏感值均来自 Secret;
- 关键请求、Job 和外部依赖失败具备可定位的结构化日志和最小指标。
## 6. 明确后置能力
- 用户、用户组、角色、多租户、知识库空间隔离和 Document ACL;
- LDAP/SSO、OAuth 用户委托、公共互联网匿名访问;
- GPU Worker、本地模型、OCR/MinerU;
- 跨区域容灾、自动云资源编排和完整 Observability 平台;
- 未被既有 Demo/V1 明确批准的业务能力。

View File

@ -4,10 +4,10 @@
- [x] ARCH-001 Architecture decision, PRD, Scope and execution baseline - [x] ARCH-001 Architecture decision, PRD, Scope and execution baseline
- [x] ARCH-002 BOS ObjectStorage adapter and runtime configuration - [x] ARCH-002 BOS ObjectStorage adapter and runtime configuration
- [ ] ARCH-003 OceanBase VectorStore adapter and index schema - [x] ARCH-003 OceanBase VectorStore adapter and index schema
- [ ] ARCH-004 Qianfan `bce-reranker-base` alignment - [x] ARCH-004 Qianfan `bce-reranker-base` alignment
- [ ] ARCH-005 Runtime cutover, shadow rebuild workflow and legacy configuration cleanup - [x] ARCH-005 Runtime cutover, shadow rebuild workflow and legacy configuration cleanup
- [ ] ARCH-006 Controlled live integration and RAG acceptance - [x] ARCH-006 OceanBase canonical cutover, controlled live integration and RAG acceptance
Dependencies: Dependencies:
@ -23,6 +23,30 @@ ARCH-001
Each architecture task uses an independent branch and is merged into `main` only after its own Each architecture task uses an independent branch and is merged into `main` only after its own
validation. Provider credentials and internal endpoints are runtime inputs and never task artifacts. validation. Provider credentials and internal endpoints are runtime inputs and never task artifacts.
## Post-Demo priority plan
- [x] PLAN-001 MCP 与异步批量入库现状审计、任务拆分和契约基线
- [x] DEMO-012 MCP `search_knowledge` Gateway + identity + WorkBuddy verification
- [x] PLAN-002 公共知识库集群交付范围、任务拆分和一键部署基线
- [x] DEMO-015 Async ingestion Job + Celery/RabbitMQ + bounded retry + pre-retry cleanup
- [x] DEMO-016 Batch async upload API + per-file Job status + Admin Console
- [x] DEPLOY-001 Production images for API/Admin, MCP Gateway and Async Worker
- [x] DEPLOY-002 Kubernetes/Helm one-click deployment + Secret/Migration/Ingress baseline
- [ ] DEPLOY-003 Availability, rolling upgrade and remote WorkBuddy acceptance
Dependencies and acceptance boundaries are defined in
[`ASYNC_INGESTION_MCP_PLAN.md`](ASYNC_INGESTION_MCP_PLAN.md) and
[`PUBLIC_KNOWLEDGE_CLUSTER_PLAN.md`](PUBLIC_KNOWLEDGE_CLUSTER_PLAN.md). These Demo/Deploy tasks are
vertical slices of the V1 backlog below; completing a slice does not automatically complete its
corresponding `EKP-*` task. The current public-knowledge phase defers, but does not complete or
supersede, the unchecked user/ACL tasks.
```text
DEMO-009 + ARCH-006 → DEMO-012
PLAN-002
└→ DEMO-003~006 → DEMO-015 → DEMO-016 → DEPLOY-001 → DEPLOY-002 → DEPLOY-003
```
## Phase 0 ## Phase 0
- [ ] EKP-001 Repository bootstrap - [ ] EKP-001 Repository bootstrap
- [ ] EKP-002 Backend Python toolchain - [ ] EKP-002 Backend Python toolchain
@ -34,7 +58,7 @@ validation. Provider credentials and internal endpoints are runtime inputs and n
## Canonical domain / ingestion ## Canonical domain / ingestion
- [ ] EKP-010 Core domain package - [ ] EKP-010 Core domain package
- [ ] EKP-011 PostgreSQL schema and migrations - [ ] EKP-011 OceanBase MySQL schema and migrations
- [ ] EKP-012 User/group/role model - [ ] EKP-012 User/group/role model
- [ ] EKP-013 Knowledge Base model - [ ] EKP-013 Knowledge Base model
- [ ] EKP-014 Folder model - [ ] EKP-014 Folder model

View File

@ -9,3 +9,26 @@
7. Admin Console 7. Admin Console
8. Knowledge Compilation 8. Knowledge Compilation
9. Evaluation, security and release hardening 9. Evaluation, security and release hardening
## 当前增量实施顺序(2026-09-02)
`DEMO-012` 已完成 MCP `search_knowledge` 与本地 WorkBuddy 验收。根据 PLAN-002,当前阶段按
“单一公共知识可见域、企业受控网络、持续可用和一键集群部署”推进,不提前实现用户/权限隔离:
1. `PLAN-002`:确定公共知识库交付边界、目标拓扑、Task 拆分与验收门槛;
2. `DEMO-015`(已完成):持久化异步 Job、Celery/RabbitMQ 执行器、自动重试和重试前清理;
3. `DEMO-016`(已完成):批量异步上传、批次/Job 查询和 Admin Console 状态展示;
4. `DEPLOY-001`(已完成):API/Admin、MCP Gateway、Async Worker 生产镜像;
5. `DEPLOY-002`(已完成):Kubernetes/Helm、Secret、Migration、Ingress/TLS 和一键部署入口;
6. `DEPLOY-003`:多副本可用性、滚动升级、Worker 恢复和远程 WorkBuddy 验收。
严格依赖为:
```text
PLAN-002 → DEMO-015 → DEMO-016 → DEPLOY-001 → DEPLOY-002 → DEPLOY-003
```
`DEMO-016` 不得在缺少持久化状态、幂等消费和恢复边界时直接并发调用同步流水线;集群交付也
不得继续使用 `docker-compose.dev.yml` 中已被 ADR-012 取代的 PostgreSQL/MinIO 假设。详细
范围和验收标准见 [`ASYNC_INGESTION_MCP_PLAN.md`](ASYNC_INGESTION_MCP_PLAN.md) 与
[`PUBLIC_KNOWLEDGE_CLUSTER_PLAN.md`](PUBLIC_KNOWLEDGE_CLUSTER_PLAN.md)。

View File

@ -10,6 +10,21 @@
--- ---
## 当前阶段交付覆盖(PLAN-002)
为优先达成服务可用和一键集群部署,当前阶段只交付企业受控网络内的单一公共知识库:所有
已接收知识处于同一个可见域,用户打开 Admin Console 或连接 WorkBuddy 即可使用,无需感知
后端进程启停。用户、用户组、角色、知识库多空间、Document ACL、LDAP/SSO 和 OAuth 用户
委托仍是本 PRD 的后续 V1 要求,本阶段不实现,也不得标记为完成。
该阶段覆盖不会取代安全原则:公共知识库不等于匿名公网开放;Admin 写操作必须位于受控网络
或统一接入网关后,MCP 使用运行时注入的最小 `knowledge:search` 服务凭据。一旦平台需要接收
非公共知识,必须先完成 Permission Resolver 与检索前/检索中 ACL 下推,禁止全局 Top-K 后置
过滤。实施顺序和集群验收边界见
[`PUBLIC_KNOWLEDGE_CLUSTER_PLAN.md`](../plans/PUBLIC_KNOWLEDGE_CLUSTER_PLAN.md)。
---
# 1. 项目背景 # 1. 项目背景
企业内部存在大量分散于 PDF、Word、PPT、Excel、Markdown、图片、扫描件及其他文件中的知识资产。 企业内部存在大量分散于 PDF、Word、PPT、Excel、Markdown、图片、扫描件及其他文件中的知识资产。
@ -447,6 +462,19 @@ V1 必须支持:
Document processing 必须异步执行。 Document processing 必须异步执行。
源文件与 Document/Job 成功持久化后返回 `202 Accepted`。批量上传中的每个文件拥有独立的
Document ID 和 Ingestion Job;单个文件处理失败不得阻塞或回滚批次内其他已接收文件。
当前 Demo 已实现多文件 `202` 接收、批次聚合查询和逐文件 Job 状态;目录上传、压缩包解包、
断点续传以及完整 V1 的 Hash/Version/ACL 能力仍按各自 Task 后置。
公共知识库集群交付使用分别可扩缩的 API/Admin、MCP Gateway 和 Async Worker 生产镜像;
镜像只包含锁定代码与非敏感默认值,环境凭据必须在部署时注入。
集群编排由 Helm Chart 统一管理:API/MCP 默认多副本,Worker 与单实例恢复调度器分离;敏感值
只引用预建 Kubernetes Secret,数据库 Migration 在安装/升级前串行完成。Ingress/TLS、探针、
资源限制和 Pod 安全默认值属于部署契约,不改变 REST/MCP 业务接口。
**REQ-INGEST-004** **REQ-INGEST-004**
展示处理阶段: 展示处理阶段:
@ -468,10 +496,18 @@ Failed
失败必须展示: 失败必须展示:
- Failed Stage; - Failed Stage;
- Attempt Count;
- 是否仍会自动重试;
- Error Code; - Error Code;
- Error Message; - Error Message;
- Retry。 - Retry。
**REQ-INGEST-006**
可重试的瞬时执行失败必须自动进行有界重试。每次重新执行前必须清理该 Document 的旧解析
产物、Chunk、Chunk Index State 和向量/稀疏索引数据,再从不可变原始文件开始处理。清理不得
删除原始文件、Document 标识或 Job/Attempt 历史;清理失败时不得继续重试。
--- ---
# 11. 文件版本和去重 # 11. 文件版本和去重
@ -1814,6 +1850,9 @@ Cancelled
Retry Retry
``` ```
Retry 必须记录 Attempt 历史并区分可重试的瞬时错误与确定性永久错误;重试执行前遵循
`REQ-INGEST-006` 的派生数据清理规则。
--- ---
# 57. 删除语义 # 57. 删除语义
@ -2374,6 +2413,10 @@ Backend:
API First API First
``` ```
运行时切换约束:索引 Provider 或 Profile 变更必须以新 Profile 构建 Shadow Index;Profile
生命周期和激活状态保存在 OceanBase canonical 关系表。只有 canonical Chunk 与 ChunkIndexState 完整核对
通过后才能原子激活,禁止复用旧 Profile ID 或原地覆盖当前 Active Index。
--- ---
# 74. 已确认产品决策 # 74. 已确认产品决策
@ -2393,7 +2436,7 @@ API First + Admin Console
**DEC-002** **DEC-002**
OceanBase Vector 作为当前生产主检索索引,同时必须保留 Vector Store 替换能力。 OceanBase Vector 作为当前生产主检索索引,同时必须保留 Vector Store 替换能力。
PostgreSQL 继续保存 canonical 数据,Vector Store 只保存可重建索引。 OceanBase 关系表保存 canonical 数据,同库 Vector Store 表只保存可重建索引。
--- ---
@ -2449,7 +2492,7 @@ PRD 评审通过后进入《总体技术架构设计》。
1. Backend 技术栈; 1. Backend 技术栈;
2. Admin Console 技术栈; 2. Admin Console 技术栈;
3. PostgreSQL 数据模型; 3. OceanBase MySQL canonical 数据模型;
4. Object Storage; 4. Object Storage;
5. Task Queue; 5. Task Queue;
6. MinerU vs Docling; 6. MinerU vs Docling;
@ -2535,8 +2578,8 @@ Cloud LLM
# 77. 调研依据备注 # 77. 调研依据备注
内部最新配置基线确认现有知识平台主链使用千帆 `bge-large-zh`、`bce-reranker-base`、 内部最新配置基线确认现有知识平台主链使用千帆 `bge-large-zh`、`bce-reranker-base`、
BOS 与 OceanBase Vector。该信息只用于确定 Provider 选型,不把连接地址、账号、密钥或 BOS 与 OceanBase MySQL 模式;OceanBase 同时承载 canonical 关系表和逻辑隔离的可重建
历史环境加载方式写入产品文档和代码。 向量表。连接地址、账号和密钥仍只能作为运行时 Secret,不进入产品文档或提交记录。
OceanBase Vector 的生产接入必须验证向量检索、标量 Filter、索引参数和版本兼容性; OceanBase Vector 的生产接入必须验证向量检索、标量 Filter、索引参数和版本兼容性;
BM25/Hybrid 在相应任务验收前仍按产品能力要求保留,不在本轮 Provider 迁移中提前扩展。 BM25/Hybrid 在相应任务验收前仍按产品能力要求保留,不在本轮 Provider 迁移中提前扩展。

View File

@ -1,2 +1,32 @@
{"id":"sample-001","query":"员工出差住宿标准是多少?","expected_documents":["sample-travel-policy.md"],"expected_sections":["住宿标准"],"tags":["制度","自然语言"]} {"id":"refund-001","query":"产品A退款周期","expected_documents":["sample-product-a-policy.md"],"expected_sections":["退款"],"tags":["产品","短查询"]}
{"id":"sample-002","query":"产品A退款周期","expected_documents":["sample-product-a-policy.md"],"expected_sections":["退款"],"tags":["产品","短查询"]} {"id":"refund-002","query":"示例产品 A 的退款需要几个自然日?","expected_documents":["sample-product-a-policy.md"],"expected_sections":["退款"],"tags":["产品","自然语言"]}
{"id":"refund-003","query":"申请产品A退款后多久完成?","expected_documents":["sample-product-a-policy.md"],"expected_sections":["退款"],"tags":["产品","同义表达"]}
{"id":"refund-004","query":"退款规则里写的处理周期是什么?","expected_documents":["sample-product-a-policy.md"],"expected_sections":["退款"],"tags":["产品","规则"]}
{"id":"refund-005","query":"产品 A 提交退款申请后的等待时间","expected_documents":["sample-product-a-policy.md"],"expected_sections":["退款"],"tags":["产品","长查询"]}
{"id":"refund-006","query":"退款是否按自然日计算?","expected_documents":["sample-product-a-policy.md"],"expected_sections":["退款"],"tags":["产品","精确术语"]}
{"id":"refund-007","query":"请查找产品A有关退款时限的说明","expected_documents":["sample-product-a-policy.md"],"expected_sections":["退款"],"tags":["产品","指令式"]}
{"id":"refund-008","query":"示例服务政策中的退款周期是多少天?","expected_documents":["sample-product-a-policy.md"],"expected_sections":["退款"],"tags":["产品","文档定位"]}
{"id":"support-001","query":"产品A遇到问题如何联系技术支持?","expected_documents":["sample-product-a-policy.md"],"expected_sections":["技术支持"],"tags":["支持","自然语言"]}
{"id":"support-002","query":"测试用户在哪里提交支持问题?","expected_documents":["sample-product-a-policy.md"],"expected_sections":["技术支持"],"tags":["支持","入口"]}
{"id":"support-003","query":"技术支持入口是否提供真实联系方式?","expected_documents":["sample-product-a-policy.md"],"expected_sections":["技术支持"],"tags":["支持","边界"]}
{"id":"support-004","query":"我想提交产品问题,支持说明是什么?","expected_documents":["sample-product-a-policy.md"],"expected_sections":["技术支持"],"tags":["支持","同义表达"]}
{"id":"support-005","query":"测试用户的虚构支持入口","expected_documents":["sample-product-a-policy.md"],"expected_sections":["技术支持"],"tags":["支持","短查询"]}
{"id":"support-006","query":"支持入口能否用于测试用户反馈问题?","expected_documents":["sample-product-a-policy.md"],"expected_sections":["技术支持"],"tags":["支持","多条件"]}
{"id":"support-007","query":"请定位服务政策里关于提交问题的段落","expected_documents":["sample-product-a-policy.md"],"expected_sections":["技术支持"],"tags":["支持","文档定位"]}
{"id":"support-008","query":"产品技术支持部分有没有联系电话?","expected_documents":["sample-product-a-policy.md"],"expected_sections":["技术支持"],"tags":["支持","否定事实"]}
{"id":"lodging-001","query":"员工出差住宿标准是多少?","expected_documents":["sample-travel-policy.md"],"expected_sections":["住宿标准"],"tags":["制度","自然语言"]}
{"id":"lodging-002","query":"出差时应如何选择住宿?","expected_documents":["sample-travel-policy.md"],"expected_sections":["住宿标准"],"tags":["制度","行为"]}
{"id":"lodging-003","query":"住宿地点需要满足哪些安全和交通条件?","expected_documents":["sample-travel-policy.md"],"expected_sections":["住宿标准"],"tags":["制度","多条件"]}
{"id":"lodging-004","query":"出差住宿是否必须在批准预算内?","expected_documents":["sample-travel-policy.md"],"expected_sections":["住宿标准"],"tags":["制度","预算"]}
{"id":"lodging-005","query":"批准预算范围内的安全住宿","expected_documents":["sample-travel-policy.md"],"expected_sections":["住宿标准"],"tags":["制度","短查询"]}
{"id":"lodging-006","query":"员工出差选酒店时要考虑什么?","expected_documents":["sample-travel-policy.md"],"expected_sections":["住宿标准"],"tags":["制度","同义表达"]}
{"id":"lodging-007","query":"请找出关于安全且交通便利住宿的说明","expected_documents":["sample-travel-policy.md"],"expected_sections":["住宿标准"],"tags":["制度","文档定位"]}
{"id":"lodging-008","query":"出差住宿示例标准能否作为实际报销依据?","expected_documents":["sample-travel-policy.md"],"expected_sections":["住宿标准"],"tags":["制度","边界"]}
{"id":"expense-001","query":"出差报销需要提交哪些材料?","expected_documents":["sample-travel-policy.md"],"expected_sections":["报销材料"],"tags":["报销","自然语言"]}
{"id":"expense-002","query":"报销时是否需要行程单?","expected_documents":["sample-travel-policy.md"],"expected_sections":["报销材料"],"tags":["报销","材料"]}
{"id":"expense-003","query":"合规票据是不是报销必需材料?","expected_documents":["sample-travel-policy.md"],"expected_sections":["报销材料"],"tags":["报销","票据"]}
{"id":"expense-004","query":"提交报销要附上什么凭证?","expected_documents":["sample-travel-policy.md"],"expected_sections":["报销材料"],"tags":["报销","同义表达"]}
{"id":"expense-005","query":"提交报销的示例行程单和合规票据","expected_documents":["sample-travel-policy.md"],"expected_sections":["报销材料"],"tags":["报销","短查询"]}
{"id":"expense-006","query":"行程单和票据应在什么时候提交?","expected_documents":["sample-travel-policy.md"],"expected_sections":["报销材料"],"tags":["报销","多条件"]}
{"id":"expense-007","query":"请定位管理制度中有关合规票据的段落","expected_documents":["sample-travel-policy.md"],"expected_sections":["报销材料"],"tags":["报销","文档定位"]}
{"id":"expense-008","query":"报销材料段落是否只用于自动化测试?","expected_documents":["sample-travel-policy.md"],"expected_sections":["报销材料"],"tags":["报销","边界"]}

View File

@ -1,6 +1,27 @@
# Database migrations # Database migrations
Migrations are forward-only and operate on PostgreSQL canonical metadata. Run Migrations are forward-only and operate on OceanBase MySQL canonical metadata. Run
`make migration-check` to render the full upgrade path without changing a `make migration-check` to render the full upgrade path without changing a
database. Integration tests also apply the migration to a temporary database database. Integration tests also apply the migration to a temporary database
and compare it with SQLAlchemy metadata. and compare it with SQLAlchemy metadata.
The target OceanBase database is shared. Alembic autogenerate and drift checks
are intentionally scoped to tables present in EKP `Base.metadata`; unrelated
application tables and the separately provisioned vector-index table are never
reflected or treated as migration targets.
OceanBase adds `BLOCK_SIZE ... LOCAL` syntax that SQLAlchemy's stock MySQL
reflector does not recognize for secondary indexes, unique constraints, and
foreign keys. Real OceanBase drift checks therefore compare columns, types,
defaults, and primary keys while excluding those three falsely reported object
types. Full constraint comparison still runs against SQLite in integration
tests; target-cluster checks separately verify HNSW/PRE-FILTER and the
single-active generated unique index.
DEMO-015 adds `ingestion_jobs` and `ingestion_attempts`. The former is the canonical execution state;
the latter preserves numbered history and uses a generated nullable active marker plus a unique index
to reject two running Attempts for the same Job on both OceanBase MySQL and SQLite.
DEMO-016 adds `ingestion_batches` and `ingestion_batch_items`. They retain input order and per-file
acceptance references while Job rows remain the processing state source of truth; a batch is an aggregate,
not a cross-file transaction.

View File

@ -1,6 +1,7 @@
"""Alembic environment for canonical document metadata migrations.""" """Alembic environment for canonical document metadata migrations."""
from logging.config import fileConfig from logging.config import fileConfig
from typing import Any
from alembic import context from alembic import context
from sqlalchemy import engine_from_config, pool from sqlalchemy import engine_from_config, pool
@ -13,12 +14,47 @@ if config.config_file_name is not None:
fileConfig(config.config_file_name) fileConfig(config.config_file_name)
target_metadata = Base.metadata target_metadata = Base.metadata
TARGET_TABLE_NAMES = frozenset(target_metadata.tables)
def _include_name(
name: str | None,
type_: str,
parent_names: dict[str, str | None],
) -> bool:
"""Keep Alembic reflection inside EKP-owned canonical tables.
The target OceanBase database is shared with another application and
contains provider-specific VECTOR columns. Excluding unrelated tables
before reflection prevents schema checks from inspecting or proposing
changes to objects outside this repository's ownership boundary.
"""
del parent_names
return type_ != "table" or name in TARGET_TABLE_NAMES
def _include_oceanbase_object(
object_: Any,
name: str | None,
type_: str,
reflected: bool,
compare_to: Any,
) -> bool:
"""Skip constraints the stock MySQL reflector cannot parse from OceanBase.
OceanBase appends `BLOCK_SIZE ... LOCAL` to secondary constraints. SQLAlchemy
consequently reports model constraints as missing even when they exist.
Columns, types, defaults and primary keys remain in the real drift check;
constraints are covered by migration tests and targeted live invariants.
"""
del object_, name, reflected, compare_to
return type_ not in {"index", "unique_constraint", "foreign_key_constraint"}
def _database_url() -> str: def _database_url() -> str:
"""Allow tests to override the URL without changing runtime settings.""" """Allow tests to override the URL without changing runtime settings."""
override = config.attributes.get("database_url") override = config.attributes.get("database_url")
return override if isinstance(override, str) else get_settings().database_url return override if isinstance(override, str) else get_settings().database_url.get_secret_value()
def run_migrations_offline() -> None: def run_migrations_offline() -> None:
@ -29,6 +65,7 @@ def run_migrations_offline() -> None:
literal_binds=True, literal_binds=True,
dialect_opts={"paramstyle": "named"}, dialect_opts={"paramstyle": "named"},
compare_type=True, compare_type=True,
include_name=_include_name,
) )
with context.begin_transaction(): with context.begin_transaction():
@ -48,6 +85,10 @@ def run_migrations_online() -> None:
connection=connection, connection=connection,
target_metadata=target_metadata, target_metadata=target_metadata,
compare_type=True, compare_type=True,
include_name=_include_name,
include_object=(
_include_oceanbase_object if connection.dialect.name == "mysql" else None
),
) )
with context.begin_transaction(): with context.begin_transaction():

View File

@ -0,0 +1,77 @@
"""Create canonical index-profile lifecycle records.
Revision ID: 20260902_0005
Revises: 20260901_0004
Create Date: 2026-09-02
"""
from collections.abc import Sequence
import sqlalchemy as sa
from alembic import op
revision: str = "20260902_0005"
down_revision: str | None = "20260901_0004"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def upgrade() -> None:
"""Persist shadow/active/retired profiles without storing vector data."""
op.create_table(
"index_profiles",
sa.Column("profile_id", sa.String(length=128), nullable=False),
sa.Column("chunk_strategy_version", sa.String(length=64), nullable=False),
sa.Column("embedding_model", sa.String(length=128), nullable=False),
sa.Column("vector_dimension", sa.Integer(), nullable=False),
sa.Column(
"status",
sa.Enum(
"shadow",
"active",
"retired",
name="index_profile_status",
native_enum=False,
create_constraint=False,
),
nullable=False,
),
sa.Column(
"created_at",
sa.DateTime(timezone=True),
server_default=sa.text("CURRENT_TIMESTAMP"),
nullable=False,
),
sa.Column("activated_at", sa.DateTime(timezone=True), nullable=True),
sa.CheckConstraint("length(profile_id) > 0", name="profile_id_not_empty"),
sa.CheckConstraint(
"length(chunk_strategy_version) > 0",
name="profile_chunk_strategy_not_empty",
),
sa.CheckConstraint(
"length(embedding_model) > 0",
name="profile_embedding_model_not_empty",
),
sa.CheckConstraint(
"vector_dimension > 0",
name="profile_dimension_positive",
),
sa.CheckConstraint(
"status IN ('shadow', 'active', 'retired')",
name="profile_status_valid",
),
sa.PrimaryKeyConstraint("profile_id", name="pk_index_profiles"),
)
op.create_index(
"uq_index_profiles_single_active",
"index_profiles",
["status"],
unique=True,
postgresql_where=sa.text("status = 'active'"),
sqlite_where=sa.text("status = 'active'"),
)
def downgrade() -> None:
"""Reject downgrade because project migrations are forward-only."""
raise NotImplementedError("Database migrations are forward-only")

View File

@ -0,0 +1,41 @@
"""Make single-active profile enforcement portable to OceanBase MySQL.
Revision ID: 20260902_0006
Revises: 20260902_0005
Create Date: 2026-09-02
"""
from collections.abc import Sequence
import sqlalchemy as sa
from alembic import op
revision: str = "20260902_0006"
down_revision: str | None = "20260902_0005"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def upgrade() -> None:
"""Replace dialect-specific partial uniqueness with a generated marker."""
op.drop_index("uq_index_profiles_single_active", table_name="index_profiles")
op.add_column(
"index_profiles",
sa.Column(
"active_marker",
sa.String(length=6),
sa.Computed("CASE WHEN status = 'active' THEN 'active' ELSE NULL END"),
nullable=True,
),
)
op.create_index(
"uq_index_profiles_single_active",
"index_profiles",
["active_marker"],
unique=True,
)
def downgrade() -> None:
"""Reject downgrade because project migrations are forward-only."""
raise NotImplementedError("Database migrations are forward-only")

View File

@ -0,0 +1,174 @@
"""Create durable ingestion jobs and attempt history.
Revision ID: 20260902_0007
Revises: 20260902_0006
Create Date: 2026-09-02
"""
from collections.abc import Sequence
import sqlalchemy as sa
from alembic import op
revision: str = "20260902_0007"
down_revision: str | None = "20260902_0006"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def upgrade() -> None:
"""Persist queue-independent Job state and bounded execution attempts."""
op.create_table(
"ingestion_jobs",
sa.Column("id", sa.Uuid(), nullable=False),
sa.Column("document_id", sa.Uuid(), nullable=False),
sa.Column(
"status",
sa.Enum(
"queued",
"running",
"retrying",
"succeeded",
"failed",
"cancelled",
name="ingestion_job_status",
native_enum=False,
create_constraint=False,
),
nullable=False,
),
sa.Column(
"stage",
sa.Enum(
"parsing",
"cleaning",
"chunking",
"embedding",
"indexing",
name="ingestion_stage",
native_enum=False,
create_constraint=False,
),
nullable=True,
),
sa.Column("attempt_count", sa.Integer(), nullable=False),
sa.Column("max_attempts", sa.Integer(), nullable=False),
sa.Column("error_message", sa.Text(), nullable=True),
sa.Column("next_run_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("lease_expires_at", sa.DateTime(timezone=True), nullable=True),
sa.Column(
"created_at",
sa.DateTime(timezone=True),
server_default=sa.text("CURRENT_TIMESTAMP"),
nullable=False,
),
sa.Column(
"updated_at",
sa.DateTime(timezone=True),
server_default=sa.text("CURRENT_TIMESTAMP"),
nullable=False,
),
sa.Column("completed_at", sa.DateTime(timezone=True), nullable=True),
sa.CheckConstraint("attempt_count >= 0", name="attempt_count_non_negative"),
sa.CheckConstraint("max_attempts > 0", name="max_attempts_positive"),
sa.CheckConstraint("attempt_count <= max_attempts", name="attempt_count_bounded"),
sa.CheckConstraint(
"status IN ('queued', 'running', 'retrying', 'succeeded', 'failed', 'cancelled')",
name="status_valid",
),
sa.CheckConstraint(
"stage IS NULL OR stage IN "
"('parsing', 'cleaning', 'chunking', 'embedding', 'indexing')",
name="stage_valid",
),
sa.ForeignKeyConstraint(
["document_id"],
["documents.id"],
name="fk_ingestion_jobs_document_id_documents",
ondelete="CASCADE",
),
sa.PrimaryKeyConstraint("id", name="pk_ingestion_jobs"),
sa.UniqueConstraint("document_id", name="uq_ingestion_jobs_document_id"),
)
op.create_index(
"ix_ingestion_jobs_status_next_run",
"ingestion_jobs",
["status", "next_run_at"],
unique=False,
)
op.create_table(
"ingestion_attempts",
sa.Column("id", sa.Uuid(), nullable=False),
sa.Column("job_id", sa.Uuid(), nullable=False),
sa.Column("attempt_number", sa.Integer(), nullable=False),
sa.Column(
"status",
sa.Enum(
"running",
"succeeded",
"failed",
name="ingestion_attempt_status",
native_enum=False,
create_constraint=False,
),
nullable=False,
),
sa.Column(
"stage",
sa.Enum(
"parsing",
"cleaning",
"chunking",
"embedding",
"indexing",
name="ingestion_attempt_stage",
native_enum=False,
create_constraint=False,
),
nullable=True,
),
sa.Column("retryable", sa.Boolean(), nullable=True),
sa.Column("error_message", sa.Text(), nullable=True),
sa.Column(
"active_marker",
sa.Integer(),
sa.Computed("CASE WHEN status = 'running' THEN 1 ELSE NULL END"),
nullable=True,
),
sa.Column(
"started_at",
sa.DateTime(timezone=True),
server_default=sa.text("CURRENT_TIMESTAMP"),
nullable=False,
),
sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True),
sa.CheckConstraint("attempt_number > 0", name="attempt_number_positive"),
sa.CheckConstraint(
"status IN ('running', 'succeeded', 'failed')",
name="status_valid",
),
sa.CheckConstraint(
"stage IS NULL OR stage IN "
"('parsing', 'cleaning', 'chunking', 'embedding', 'indexing')",
name="stage_valid",
),
sa.ForeignKeyConstraint(
["job_id"],
["ingestion_jobs.id"],
name="fk_ingestion_attempts_job_id_ingestion_jobs",
ondelete="CASCADE",
),
sa.PrimaryKeyConstraint("id", name="pk_ingestion_attempts"),
sa.UniqueConstraint("job_id", "attempt_number", name="uq_ingestion_attempts_job_number"),
)
op.create_index(
"uq_ingestion_attempts_active",
"ingestion_attempts",
["job_id", "active_marker"],
unique=True,
)
def downgrade() -> None:
"""Reject downgrade because project migrations are forward-only."""
raise NotImplementedError("Database migrations are forward-only")

View File

@ -0,0 +1,109 @@
"""Create ingestion batch aggregation metadata.
Revision ID: 20260902_0008
Revises: 20260902_0007
Create Date: 2026-09-02
"""
from collections.abc import Sequence
import sqlalchemy as sa
from alembic import op
revision: str = "20260902_0008"
down_revision: str | None = "20260902_0007"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def upgrade() -> None:
"""Persist ordered per-file acceptance without making a batch transactional."""
op.create_table(
"ingestion_batches",
sa.Column("id", sa.Uuid(), nullable=False),
sa.Column("total_files", sa.Integer(), nullable=False),
sa.Column(
"created_at",
sa.DateTime(timezone=True),
server_default=sa.text("CURRENT_TIMESTAMP"),
nullable=False,
),
sa.CheckConstraint("total_files > 0", name="total_files_positive"),
sa.PrimaryKeyConstraint("id", name="pk_ingestion_batches"),
)
op.create_table(
"ingestion_batch_items",
sa.Column("id", sa.Uuid(), nullable=False),
sa.Column("batch_id", sa.Uuid(), nullable=False),
sa.Column("item_index", sa.Integer(), nullable=False),
sa.Column("filename", sa.String(length=255), nullable=False),
sa.Column(
"acceptance_status",
sa.Enum(
"pending",
"accepted",
"rejected",
name="batch_item_acceptance_status",
native_enum=False,
create_constraint=False,
),
nullable=False,
),
sa.Column("document_id", sa.Uuid(), nullable=True),
sa.Column("job_id", sa.Uuid(), nullable=True),
sa.Column("error_message", sa.Text(), nullable=True),
sa.Column(
"created_at",
sa.DateTime(timezone=True),
server_default=sa.text("CURRENT_TIMESTAMP"),
nullable=False,
),
sa.Column(
"updated_at",
sa.DateTime(timezone=True),
server_default=sa.text("CURRENT_TIMESTAMP"),
nullable=False,
),
sa.CheckConstraint("item_index >= 0", name="item_index_non_negative"),
sa.CheckConstraint("length(filename) > 0", name="filename_not_empty"),
sa.CheckConstraint(
"acceptance_status IN ('pending', 'accepted', 'rejected')",
name="acceptance_status_valid",
),
sa.ForeignKeyConstraint(
["batch_id"],
["ingestion_batches.id"],
name="fk_ingestion_batch_items_batch_id_ingestion_batches",
ondelete="CASCADE",
),
sa.ForeignKeyConstraint(
["document_id"],
["documents.id"],
name="fk_ingestion_batch_items_document_id_documents",
ondelete="SET NULL",
),
sa.ForeignKeyConstraint(
["job_id"],
["ingestion_jobs.id"],
name="fk_ingestion_batch_items_job_id_ingestion_jobs",
ondelete="SET NULL",
),
sa.PrimaryKeyConstraint("id", name="pk_ingestion_batch_items"),
)
op.create_index(
"uq_ingestion_batch_items_order",
"ingestion_batch_items",
["batch_id", "item_index"],
unique=True,
)
op.create_index(
"uq_ingestion_batch_items_job_id",
"ingestion_batch_items",
["job_id"],
unique=True,
)
def downgrade() -> None:
"""Reject downgrade because project migrations are forward-only."""
raise NotImplementedError("Database migrations are forward-only")

View File

@ -4,8 +4,9 @@ from functools import lru_cache
from pathlib import Path from pathlib import Path
from typing import Literal from typing import Literal
from pydantic import Field, SecretStr, model_validator from pydantic import AnyHttpUrl, Field, SecretStr, model_validator
from pydantic_settings import BaseSettings, SettingsConfigDict from pydantic_settings import BaseSettings, SettingsConfigDict
from sqlalchemy.engine import make_url
class Settings(BaseSettings): class Settings(BaseSettings):
@ -24,8 +25,24 @@ class Settings(BaseSettings):
api_host: str = "0.0.0.0" api_host: str = "0.0.0.0"
api_port: int = Field(default=8000, ge=1, le=65535) api_port: int = Field(default=8000, ge=1, le=65535)
api_prefix: str = "/api/v1" api_prefix: str = "/api/v1"
database_url: str = "postgresql+psycopg://localhost:5432/ekp" admin_web_root: Path | None = None
object_storage_provider: Literal["local", "bos"] = "local" # SQLAlchemy URLs commonly embed a password, so configuration rendering
# must mask this value and unwrap it only at the database driver boundary.
database_url: SecretStr = SecretStr("mysql+pymysql://localhost:2881/ekp")
database_pool_size: int = Field(default=10, ge=1, le=100)
database_max_overflow: int = Field(default=10, ge=0, le=100)
database_pool_recycle_seconds: int = Field(default=600, ge=1)
database_pool_timeout_seconds: float = Field(default=30.0, gt=0)
database_connect_timeout_seconds: int = Field(default=30, ge=1)
rabbitmq_url: SecretStr | None = None
ingestion_max_attempts: int = Field(default=3, ge=1, le=10)
ingestion_retry_backoff_seconds: int = Field(default=2, ge=1, le=3600)
ingestion_retry_backoff_max_seconds: int = Field(default=60, ge=1, le=86400)
ingestion_worker_lease_seconds: int = Field(default=300, ge=30, le=86400)
ingestion_recovery_interval_seconds: int = Field(default=30, ge=5, le=3600)
batch_max_files: int = Field(default=20, ge=1, le=100)
batch_max_total_bytes: int = Field(default=200 * 1024 * 1024, gt=0)
object_storage_provider: Literal["local", "bos"] = "bos"
object_storage_path: Path = Path("storage") object_storage_path: Path = Path("storage")
bos_endpoint: str | None = None bos_endpoint: str | None = None
bos_access_key_id: SecretStr | None = None bos_access_key_id: SecretStr | None = None
@ -42,23 +59,39 @@ class Settings(BaseSettings):
embedding_endpoint: str = "https://qianfan.baidubce.com/v2/embeddings" embedding_endpoint: str = "https://qianfan.baidubce.com/v2/embeddings"
baidu_qianfan_api_key: SecretStr | None = None baidu_qianfan_api_key: SecretStr | None = None
embedding_timeout_seconds: float = Field(default=30.0, gt=0) embedding_timeout_seconds: float = Field(default=30.0, gt=0)
vector_store_provider: Literal["mochow"] = "mochow" vector_store_provider: Literal["oceanbase"] = "oceanbase"
baidu_mochow_account: str = "root" oceanbase_vector_uri: str | None = None
baidu_mochow_api_key: SecretStr | None = None oceanbase_vector_user: str | None = None
baidu_mochow_endpoint: str | None = None oceanbase_vector_password: SecretStr | None = None
knowledge_vdb_db_name: str | None = None oceanbase_vector_database: str | None = None
mochow_table: str = "knowledge_chunks" oceanbase_vector_table: str = "knowledge_chunks_v1"
mochow_batch_size: int = Field(default=100, ge=1, le=1000) oceanbase_vector_index: str = "vidx_knowledge_chunks_embedding"
mochow_search_ef: int = Field(default=200, ge=1, le=10000) oceanbase_vector_batch_size: int = Field(default=100, ge=1, le=1000)
mochow_timeout_seconds: float = Field(default=30.0, gt=0) oceanbase_vector_timeout_seconds: float = Field(default=30.0, gt=0)
index_profile_id: str = "demo-bge-large-zh-structure-v1" index_profile_id: str = "demo-bge-large-zh-structure-v1-oceanbase-v1"
dense_top_n: int = Field(default=20, ge=1, le=64) dense_top_n: int = Field(default=20, ge=1, le=64)
reranker_provider: Literal["qianfan"] = "qianfan" reranker_provider: Literal["qianfan"] = "qianfan"
reranker_model: Literal["qwen3-reranker-0.6b"] = "qwen3-reranker-0.6b" reranker_model: Literal["bce-reranker-base"] = "bce-reranker-base"
reranker_endpoint: str = "https://qianfan.baidubce.com/v2/rerank" reranker_endpoint: str = "https://qianfan.baidubce.com/v2/rerank"
reranker_api_key: SecretStr | None = None reranker_api_key: SecretStr | None = None
reranker_timeout_seconds: float = Field(default=30.0, gt=0) reranker_timeout_seconds: float = Field(default=30.0, gt=0)
search_top_k: int = Field(default=5, ge=1, le=64) search_top_k: int = Field(default=5, ge=1, le=64)
mcp_host: str = "127.0.0.1"
mcp_port: int = Field(default=8766, ge=1, le=65535)
mcp_path: str = "/mcp"
mcp_public_url: AnyHttpUrl = AnyHttpUrl("http://127.0.0.1:8766/mcp")
mcp_issuer_url: AnyHttpUrl = AnyHttpUrl("http://127.0.0.1:8766")
mcp_auth_token: SecretStr | None = None
mcp_client_id: str = Field(default="workbuddy-enterprise", min_length=1, max_length=200)
mcp_service_identity: str = Field(default="workbuddy-demo", min_length=1, max_length=200)
mcp_required_scope: str = Field(default="knowledge:search", min_length=1, max_length=200)
mcp_allowed_hosts: list[str] = ["127.0.0.1:*", "localhost:*", "[::1]:*", "testserver"]
mcp_allowed_origins: list[str] = [
"http://127.0.0.1:*",
"http://localhost:*",
"http://[::1]:*",
]
mcp_max_request_body_bytes: int = Field(default=1024 * 1024, ge=1024, le=4 * 1024 * 1024)
@model_validator(mode="after") @model_validator(mode="after")
def validate_chunk_window(self) -> "Settings": def validate_chunk_window(self) -> "Settings":
@ -67,6 +100,29 @@ class Settings(BaseSettings):
raise ValueError("chunk_overlap_tokens must be less than chunk_max_tokens") raise ValueError("chunk_overlap_tokens must be less than chunk_max_tokens")
if self.search_top_k > self.dense_top_n: if self.search_top_k > self.dense_top_n:
raise ValueError("search_top_k must not exceed dense_top_n") raise ValueError("search_top_k must not exceed dense_top_n")
if self.ingestion_retry_backoff_max_seconds < self.ingestion_retry_backoff_seconds:
raise ValueError(
"ingestion_retry_backoff_max_seconds must be at least the initial backoff"
)
longest_provider_timeout = max(
self.bos_timeout_seconds,
self.embedding_timeout_seconds,
self.oceanbase_vector_timeout_seconds,
)
if self.ingestion_worker_lease_seconds <= longest_provider_timeout:
raise ValueError("ingestion_worker_lease_seconds must exceed provider timeouts")
if self.batch_max_total_bytes < self.max_upload_bytes:
raise ValueError("batch_max_total_bytes must be at least max_upload_bytes")
if not self.mcp_path.startswith("/") or self.mcp_path.endswith("/"):
raise ValueError("mcp_path must start with '/' and must not end with '/'")
if not self.mcp_allowed_hosts:
raise ValueError("mcp_allowed_hosts must not be empty")
database_url = make_url(self.database_url.get_secret_value())
if database_url.get_backend_name() == "sqlite":
if self.app_env != "test":
raise ValueError("SQLite database URLs are restricted to APP_ENV=test")
elif database_url.drivername != "mysql+pymysql":
raise ValueError("canonical database must use OceanBase through mysql+pymysql")
return self return self

View File

@ -1,14 +1,35 @@
"""SQLAlchemy engine and session construction for canonical metadata.""" """SQLAlchemy engine and session construction for canonical metadata."""
from sqlalchemy import Engine, create_engine from sqlalchemy import Engine, create_engine
from sqlalchemy.engine import make_url
from sqlalchemy.orm import Session, sessionmaker from sqlalchemy.orm import Session, sessionmaker
from ekp_core.config import Settings from ekp_core.config import Settings
def create_database_engine(settings: Settings) -> Engine: def create_database_engine(settings: Settings) -> Engine:
"""Create the metadata engine without opening a connection eagerly.""" """Create the canonical engine without opening a connection eagerly.
return create_engine(settings.database_url, pool_pre_ping=True)
OceanBase uses the synchronous PyMySQL driver in this Demo. Pool and
socket timeouts are explicit so an outage cannot hold API workers
indefinitely. SQLite remains available only for isolated tests.
"""
database_url = settings.database_url.get_secret_value()
url = make_url(database_url)
options: dict[str, object] = {"pool_pre_ping": True}
if url.get_backend_name() == "mysql":
options.update(
pool_size=settings.database_pool_size,
max_overflow=settings.database_max_overflow,
pool_recycle=settings.database_pool_recycle_seconds,
pool_timeout=settings.database_pool_timeout_seconds,
connect_args={
"connect_timeout": settings.database_connect_timeout_seconds,
"read_timeout": settings.database_connect_timeout_seconds,
"write_timeout": settings.database_connect_timeout_seconds,
},
)
return create_engine(database_url, **options)
def create_session_factory(engine: Engine) -> sessionmaker[Session]: def create_session_factory(engine: Engine) -> sessionmaker[Session]:

View File

@ -1,5 +1,6 @@
"""Application services for building and persisting document chunks.""" """Application services for building and persisting document chunks."""
from collections.abc import Callable
from dataclasses import dataclass from dataclasses import dataclass
from hashlib import sha256 from hashlib import sha256
from uuid import UUID, uuid5 from uuid import UUID, uuid5
@ -11,7 +12,7 @@ from ekp_core.chunking import ChunkDraft, ChunkingStrategy
from ekp_core.document_parsing import DocumentIngestionService, DocumentParsingService from ekp_core.document_parsing import DocumentIngestionService, DocumentParsingService
from ekp_core.document_upload import DocumentUploadService from ekp_core.document_upload import DocumentUploadService
from ekp_core.parsed_document_artifact import deserialize_parsed_document from ekp_core.parsed_document_artifact import deserialize_parsed_document
from ekp_domain import Chunk, Document, DocumentStatus, ParsedDocument from ekp_domain import Chunk, Document, DocumentStatus, IngestionStage, ParsedDocument
from ekp_object_storage import BinarySource, ObjectStorage from ekp_object_storage import BinarySource, ObjectStorage
MAX_CHUNKING_ERROR_LENGTH = 2000 MAX_CHUNKING_ERROR_LENGTH = 2000
@ -54,10 +55,16 @@ class DocumentChunkingService:
self._storage = storage self._storage = storage
self._strategy = strategy self._strategy = strategy
def process(self, document_id: UUID) -> tuple[Chunk, ...]: def process(
"""Run DEMO-005 while reserving READY for the later index stage.""" self,
document_id: UUID,
stage_callback: Callable[[IngestionStage], None] | None = None,
) -> tuple[Chunk, ...]:
"""Build chunks while optionally reporting the durable Job stage."""
filename, parsed_key = self._start_chunking(document_id) filename, parsed_key = self._start_chunking(document_id)
try: try:
if stage_callback is not None:
stage_callback(IngestionStage.CHUNKING)
with self._storage.open(parsed_key) as source: with self._storage.open(parsed_key) as source:
parsed = deserialize_parsed_document(source.read()) parsed = deserialize_parsed_document(source.read())
if parsed.document_id != document_id or parsed.filename != filename: if parsed.document_id != document_id or parsed.filename != filename:

View File

@ -1,5 +1,6 @@
"""Embedding and vector-index orchestration for canonical document chunks.""" """Embedding and vector-index orchestration for canonical document chunks."""
from collections.abc import Callable
from dataclasses import dataclass from dataclasses import dataclass
from uuid import UUID, uuid5 from uuid import UUID, uuid5
@ -7,7 +8,14 @@ from sqlalchemy import select
from sqlalchemy.orm import Session, sessionmaker from sqlalchemy.orm import Session, sessionmaker
from ekp_core.document_chunking import DocumentPipelineResult, DocumentPipelineService from ekp_core.document_chunking import DocumentPipelineResult, DocumentPipelineService
from ekp_domain import Chunk, ChunkIndexState, Document, DocumentStatus, IndexProfile from ekp_domain import (
Chunk,
ChunkIndexState,
Document,
DocumentStatus,
IndexProfile,
IngestionStage,
)
from ekp_embedding import EmbeddingProvider from ekp_embedding import EmbeddingProvider
from ekp_object_storage import BinarySource from ekp_object_storage import BinarySource
from ekp_vector_store import VectorRecord, VectorStore from ekp_vector_store import VectorRecord, VectorStore
@ -67,12 +75,16 @@ class DocumentIndexingService:
self._vector_store = vector_store self._vector_store = vector_store
self._index_profile = index_profile self._index_profile = index_profile
def process(self, document_id: UUID) -> DocumentIndexingResult: def process(
"""Index all chunks for one profile, compensating any partial Mochow write.""" self,
document_id: UUID,
stage_callback: Callable[[IngestionStage], None] | None = None,
) -> DocumentIndexingResult:
"""Embed/index chunks while optionally reporting durable Job stages."""
chunks = self._start_indexing(document_id) chunks = self._start_indexing(document_id)
upsert_attempted = False upsert_attempted = False
try: try:
vectors = self._embed_chunks(chunks) vectors = self._embed_chunks(chunks, stage_callback)
records = tuple( records = tuple(
VectorRecord( VectorRecord(
index_record_id=uuid5( index_record_id=uuid5(
@ -93,6 +105,8 @@ class DocumentIndexingService:
) )
for chunk, vector in zip(chunks, vectors, strict=True) for chunk, vector in zip(chunks, vectors, strict=True)
) )
if stage_callback is not None:
stage_callback(IngestionStage.INDEXING)
upsert_attempted = True upsert_attempted = True
self._vector_store.upsert(records) self._vector_store.upsert(records)
self._record_success(document_id, chunks) self._record_success(document_id, chunks)
@ -152,13 +166,19 @@ class DocumentIndexingService:
session.commit() session.commit()
return chunks return chunks
def _embed_chunks(self, chunks: tuple[Chunk, ...]) -> tuple[tuple[float, ...], ...]: def _embed_chunks(
"""Respect provider batch limits and validate every returned vector.""" self,
chunks: tuple[Chunk, ...],
stage_callback: Callable[[IngestionStage], None] | None,
) -> tuple[tuple[float, ...], ...]:
"""Embed bounded batches and renew an optional Job lease before each call."""
vectors: list[tuple[float, ...]] = [] vectors: list[tuple[float, ...]] = []
batch_size = self._embedding_provider.max_batch_size batch_size = self._embedding_provider.max_batch_size
if batch_size <= 0: if batch_size <= 0:
raise EmbeddingResultError("embedding provider has an invalid batch size") raise EmbeddingResultError("embedding provider has an invalid batch size")
for start in range(0, len(chunks), batch_size): for start in range(0, len(chunks), batch_size):
if stage_callback is not None:
stage_callback(IngestionStage.EMBEDDING)
batch = chunks[start : start + batch_size] batch = chunks[start : start + batch_size]
embedded = self._embedding_provider.embed([chunk.content for chunk in batch]) embedded = self._embedding_provider.embed([chunk.content for chunk in batch])
if len(embedded) != len(batch): if len(embedded) != len(batch):

View File

@ -5,7 +5,7 @@ from uuid import UUID
from sqlalchemy import delete, select from sqlalchemy import delete, select
from sqlalchemy.orm import Session, sessionmaker from sqlalchemy.orm import Session, sessionmaker
from ekp_domain import Chunk, ChunkIndexState, Document from ekp_domain import Chunk, ChunkIndexState, Document, IngestionJob, IngestionJobStatus
from ekp_object_storage import ObjectStorage from ekp_object_storage import ObjectStorage
from ekp_vector_store import VectorStore from ekp_vector_store import VectorStore
@ -18,6 +18,10 @@ class DocumentArtifactCleanupError(RuntimeError):
"""Raised when deleted metadata leaves an external storage artifact behind.""" """Raised when deleted metadata leaves an external storage artifact behind."""
class DocumentIngestionActiveError(RuntimeError):
"""Raised when direct deletion would race an active asynchronous Job."""
class DocumentManagementService: class DocumentManagementService:
"""Read canonical documents and coordinate the Demo's direct deletion. """Read canonical documents and coordinate the Demo's direct deletion.
@ -63,6 +67,22 @@ class DocumentManagementService:
document = session.get(Document, document_id) document = session.get(Document, document_id)
if document is None: if document is None:
raise DocumentNotFoundError(str(document_id)) raise DocumentNotFoundError(str(document_id))
active_job_id = session.scalar(
select(IngestionJob.id)
.where(
IngestionJob.document_id == document_id,
IngestionJob.status.in_(
(
IngestionJobStatus.QUEUED,
IngestionJobStatus.RUNNING,
IngestionJobStatus.RETRYING,
)
),
)
.limit(1)
)
if active_job_id is not None:
raise DocumentIngestionActiveError(str(document_id))
source_keys = tuple( source_keys = tuple(
key key

View File

@ -1,5 +1,6 @@
"""Application services for parsing and persisting derived document content.""" """Application services for parsing and persisting derived document content."""
from collections.abc import Callable
from io import BytesIO from io import BytesIO
from uuid import UUID from uuid import UUID
@ -8,7 +9,7 @@ from sqlalchemy.orm import Session, sessionmaker
from ekp_core.document_cleaner import clean_parsed_document from ekp_core.document_cleaner import clean_parsed_document
from ekp_core.document_upload import DocumentUploadService from ekp_core.document_upload import DocumentUploadService
from ekp_core.parsed_document_artifact import serialize_parsed_document from ekp_core.parsed_document_artifact import serialize_parsed_document
from ekp_domain import Document, DocumentStatus, ParsedDocument from ekp_domain import Document, DocumentStatus, IngestionStage, ParsedDocument
from ekp_object_storage import BinarySource, ObjectStorage from ekp_object_storage import BinarySource, ObjectStorage
from ekp_parser import DocumentParseError, Parser from ekp_parser import DocumentParseError, Parser
@ -48,15 +49,23 @@ class DocumentParsingService:
self._storage = storage self._storage = storage
self._parser = parser self._parser = parser
def process(self, document_id: UUID) -> ParsedDocument: def process(
"""Run DEMO-004 synchronously and persist success or failure state.""" self,
document_id: UUID,
stage_callback: Callable[[IngestionStage], None] | None = None,
) -> ParsedDocument:
"""Parse and clean while optionally reporting durable Job stage boundaries."""
filename, source_key = self._start_processing(document_id) filename, source_key = self._start_processing(document_id)
parsed_key = f"parsed/{document_id}/document.json" parsed_key = f"parsed/{document_id}/document.json"
parsed_object_stored = False parsed_object_stored = False
try: try:
if stage_callback is not None:
stage_callback(IngestionStage.PARSING)
with self._storage.open(source_key) as source: with self._storage.open(source_key) as source:
parsed = self._parser.parse(document_id, filename, source) parsed = self._parser.parse(document_id, filename, source)
if stage_callback is not None:
stage_callback(IngestionStage.CLEANING)
cleaned = clean_parsed_document(parsed) cleaned = clean_parsed_document(parsed)
if not cleaned.blocks: if not cleaned.blocks:
raise EmptyParsedDocumentError( raise EmptyParsedDocumentError(

View File

@ -1,8 +1,9 @@
"""Validated, transactional source-document upload application service.""" """Validated, transactional source-document upload application service."""
from pathlib import Path from pathlib import Path
from uuid import uuid4 from uuid import UUID, uuid4
from sqlalchemy import delete
from sqlalchemy.orm import Session, sessionmaker from sqlalchemy.orm import Session, sessionmaker
from ekp_domain import Document from ekp_domain import Document
@ -132,6 +133,26 @@ class DocumentUploadService:
return document return document
def rollback_acceptance(self, document_id: UUID) -> None:
"""Compensate a just-uploaded source when durable Job creation fails.
This narrow method is only valid before parsing starts; normal document
deletion remains coordinated by ``DocumentManagementService``.
"""
with self._session_factory() as session:
document = session.get(Document, document_id)
if document is None:
return
storage_key = document.storage_key
session.execute(delete(Document).where(Document.id == document.id))
session.commit()
try:
self._storage.delete(storage_key)
except Exception as cleanup_error:
raise UploadRollbackError(
f"job creation failed and source cleanup failed for {storage_key}"
) from cleanup_error
def _validate_filename(filename: str) -> str: def _validate_filename(filename: str) -> str:
"""Accept a plain basename that can be traced without path ambiguity.""" """Accept a plain basename that can be traced without path ambiguity."""

View File

@ -0,0 +1,279 @@
"""Retry-safe shadow index rebuild, verification, and activation services."""
from dataclasses import dataclass
from datetime import UTC, datetime
from uuid import UUID
from sqlalchemy import delete, select
from sqlalchemy.orm import Session, sessionmaker
from ekp_core.document_indexing import DocumentIndexingService
from ekp_domain import (
Chunk,
ChunkIndexState,
IndexProfile,
IndexProfileStatus,
StoredIndexProfile,
)
from ekp_embedding import EmbeddingProvider
from ekp_vector_store import VectorStore
class IndexProfileLifecycleError(RuntimeError):
"""Raised when a shadow profile cannot safely move through its lifecycle."""
class IndexProfileMetadataConflictError(IndexProfileLifecycleError):
"""Raised when a profile ID is reused with different immutable metadata."""
class IndexProfileNotFoundError(IndexProfileLifecycleError):
"""Raised when verification or activation targets an unknown profile."""
class IndexProfileIncompleteError(IndexProfileLifecycleError):
"""Raised when activation is attempted before every canonical chunk is indexed."""
@dataclass(frozen=True, slots=True)
class IndexBuildSummary:
"""Safe progress counters for rebuild commands and deployment checks."""
profile_id: str
expected_documents: int
built_documents: int
skipped_documents: int
expected_chunks: int
indexed_chunks: int
complete: bool
class IndexRebuildService:
"""Build a new profile from canonical chunks without mutating an active index."""
def __init__(
self,
session_factory: sessionmaker[Session],
embedding_provider: EmbeddingProvider,
vector_store: VectorStore,
target_profile: IndexProfile,
) -> None:
self._session_factory = session_factory
self._vector_store = vector_store
self._target_profile = target_profile
self._indexing = DocumentIndexingService(
session_factory,
embedding_provider,
vector_store,
target_profile,
)
def rebuild(self) -> IndexBuildSummary:
"""Register and populate a shadow profile, skipping verified documents on retry."""
self._register_shadow()
document_ids = self._target_document_ids()
built_documents = 0
skipped_documents = 0
for document_id in document_ids:
if self._document_is_complete(document_id):
skipped_documents += 1
continue
self._reset_partial_document(document_id)
self._indexing.process(document_id)
built_documents += 1
verified = self.verify()
return IndexBuildSummary(
profile_id=verified.profile_id,
expected_documents=verified.expected_documents,
built_documents=built_documents,
skipped_documents=skipped_documents,
expected_chunks=verified.expected_chunks,
indexed_chunks=verified.indexed_chunks,
complete=verified.complete,
)
def verify(self) -> IndexBuildSummary:
"""Compare canonical chunks and successful index states for the target profile."""
with self._session_factory() as session:
self._require_registered_profile(session)
return self._verification_summary(session)
def activate(self) -> IndexBuildSummary:
"""Atomically retire the previous profile after target completeness is proven."""
with self._session_factory() as session:
target = self._require_registered_profile(session, for_update=True)
self._lock_canonical_chunks(session)
summary = self._verification_summary(session)
if not summary.complete:
raise IndexProfileIncompleteError(
"shadow index is incomplete and cannot be activated"
)
if target.status is IndexProfileStatus.ACTIVE:
return summary
if target.status is not IndexProfileStatus.SHADOW:
raise IndexProfileLifecycleError("only a shadow profile can be activated")
active_profiles = session.scalars(
select(StoredIndexProfile)
.where(StoredIndexProfile.status == IndexProfileStatus.ACTIVE)
.with_for_update()
).all()
for active in active_profiles:
active.status = IndexProfileStatus.RETIRED
target.status = IndexProfileStatus.ACTIVE
target.activated_at = datetime.now(UTC)
session.commit()
return summary
def _register_shadow(self) -> None:
"""Create an idempotent shadow registration without reusing active IDs."""
with self._session_factory() as session:
stored = session.get(StoredIndexProfile, self._target_profile.profile_id)
if stored is None:
session.add(
StoredIndexProfile(
profile_id=self._target_profile.profile_id,
chunk_strategy_version=self._target_profile.chunk_strategy_version,
embedding_model=self._target_profile.embedding_model,
vector_dimension=self._target_profile.vector_dimension,
status=IndexProfileStatus.SHADOW,
)
)
session.commit()
return
self._validate_stored_metadata(stored)
if stored.status is not IndexProfileStatus.SHADOW:
raise IndexProfileLifecycleError(
"rebuild target must be a shadow profile with a new profile ID"
)
def _target_document_ids(self) -> tuple[UUID, ...]:
"""List documents that have canonical chunks for the target strategy."""
with self._session_factory() as session:
return tuple(
session.scalars(
select(Chunk.document_id)
.where(Chunk.strategy_version == self._target_profile.chunk_strategy_version)
.distinct()
.order_by(Chunk.document_id)
).all()
)
def _document_is_complete(self, document_id: UUID) -> bool:
"""Return true only when every selected chunk has matching profile evidence."""
with self._session_factory() as session:
chunks = session.scalars(
select(Chunk.chunk_id).where(
Chunk.document_id == document_id,
Chunk.strategy_version == self._target_profile.chunk_strategy_version,
)
).all()
states = session.scalars(
select(ChunkIndexState).where(
ChunkIndexState.chunk_id.in_(chunks),
ChunkIndexState.index_profile_id == self._target_profile.profile_id,
)
).all()
return len(states) == len(chunks) and all(
self._state_matches_profile(state) for state in states
)
def _reset_partial_document(self, document_id: UUID) -> None:
"""Remove only target-profile remnants before retrying one document."""
self._vector_store.delete_document(document_id, self._target_profile.profile_id)
with self._session_factory() as session:
chunk_ids = select(Chunk.chunk_id).where(Chunk.document_id == document_id)
session.execute(
delete(ChunkIndexState).where(
ChunkIndexState.chunk_id.in_(chunk_ids),
ChunkIndexState.index_profile_id == self._target_profile.profile_id,
)
)
session.commit()
def _require_registered_profile(
self,
session: Session,
*,
for_update: bool = False,
) -> StoredIndexProfile:
statement = select(StoredIndexProfile).where(
StoredIndexProfile.profile_id == self._target_profile.profile_id
)
if for_update:
statement = statement.with_for_update()
stored = session.scalar(statement)
if stored is None:
raise IndexProfileNotFoundError(self._target_profile.profile_id)
self._validate_stored_metadata(stored)
return stored
def _validate_stored_metadata(self, stored: StoredIndexProfile) -> None:
expected = self._target_profile
if (
stored.chunk_strategy_version != expected.chunk_strategy_version
or stored.embedding_model != expected.embedding_model
or stored.vector_dimension != expected.vector_dimension
):
raise IndexProfileMetadataConflictError(
"index profile ID is already registered with different metadata"
)
def _verification_summary(self, session: Session) -> IndexBuildSummary:
expected_rows = session.execute(
select(Chunk.chunk_id, Chunk.document_id).where(
Chunk.strategy_version == self._target_profile.chunk_strategy_version
)
).all()
expected_chunk_ids = {row.chunk_id for row in expected_rows}
expected_document_ids = {row.document_id for row in expected_rows}
states = session.scalars(
select(ChunkIndexState).where(
ChunkIndexState.index_profile_id == self._target_profile.profile_id
)
).all()
indexed_chunk_ids = {
state.chunk_id
for state in states
if state.chunk_id in expected_chunk_ids and self._state_matches_profile(state)
}
complete = (
bool(expected_chunk_ids)
and len(states) == len(expected_chunk_ids)
and indexed_chunk_ids == expected_chunk_ids
)
return IndexBuildSummary(
profile_id=self._target_profile.profile_id,
expected_documents=len(expected_document_ids),
built_documents=0,
skipped_documents=0,
expected_chunks=len(expected_chunk_ids),
indexed_chunks=len(indexed_chunk_ids),
complete=complete,
)
@staticmethod
def _lock_canonical_chunks(session: Session) -> None:
"""Prevent canonical chunk writes between verification and activation."""
bind = session.get_bind()
if bind.dialect.name == "mysql":
session.execute(select(Chunk.chunk_id).with_for_update()).all()
def _state_matches_profile(self, state: ChunkIndexState) -> bool:
return (
state.embedding_model == self._target_profile.embedding_model
and state.vector_dimension == self._target_profile.vector_dimension
)
def resolve_active_index_profile(
session_factory: sessionmaker[Session],
fallback: IndexProfile,
) -> IndexProfile:
"""Resolve the canonical active profile, with a fresh-deployment fallback."""
with session_factory() as session:
stored = session.scalar(
select(StoredIndexProfile).where(StoredIndexProfile.status == IndexProfileStatus.ACTIVE)
)
return fallback if stored is None else stored.to_profile()

View File

@ -0,0 +1,270 @@
"""Batch submission and aggregate status over independent ingestion Jobs."""
from dataclasses import dataclass
from enum import StrEnum
from typing import Protocol
from uuid import UUID
from sqlalchemy import select
from sqlalchemy.orm import Session, sessionmaker
from ekp_core.document_upload import (
DocumentTooLargeError,
DocumentUploadError,
EmptyDocumentError,
MediaTypeMismatchError,
UnsupportedDocumentTypeError,
)
from ekp_core.ingestion_jobs import AsyncIngestionSubmissionService
from ekp_domain import (
BatchItemAcceptanceStatus,
IngestionBatch,
IngestionBatchItem,
IngestionJob,
IngestionJobStatus,
)
from ekp_object_storage import BinarySource
SAFE_UNSUPPORTED_ERROR = "unsupported document type or media type"
SAFE_FILE_SIZE_ERROR = "document exceeds the configured upload limit"
SAFE_BATCH_SIZE_ERROR = "batch exceeds the configured total upload limit"
SAFE_EMPTY_ERROR = "document must contain content"
SAFE_ACCEPTANCE_ERROR = "document could not be accepted"
class BatchNotFoundError(LookupError):
"""Raised when a requested ingestion batch does not exist."""
class BatchFileCountLimitError(ValueError):
"""Raised before persistence when the multipart file count is invalid."""
class BatchAggregateStatus(StrEnum):
"""Computed batch state derived from item acceptance and durable Jobs."""
PROCESSING = "processing"
COMPLETED = "completed"
PARTIAL_FAILURE = "partial_failure"
FAILED = "failed"
class BatchTotalSizeLimitError(ValueError):
"""Raised while streaming the file that crosses the shared batch budget."""
class _ReadableSource(Protocol):
def read(self, size: int = -1, /) -> bytes:
"""Read bytes from an uploaded file."""
@dataclass(frozen=True, slots=True)
class BatchSource:
"""Provider-neutral input for one ordered multipart file."""
filename: str
media_type: str | None
source: _ReadableSource
@dataclass(frozen=True, slots=True)
class BatchItemView:
"""Detached batch item plus its current optional Job state."""
item: IngestionBatchItem
job: IngestionJob | None
@dataclass(frozen=True, slots=True)
class BatchView:
"""Detached aggregate used by REST without exposing persistence sessions."""
batch: IngestionBatch
items: tuple[BatchItemView, ...]
@property
def status(self) -> BatchAggregateStatus:
"""Aggregate terminal results without copying Job state into batch rows."""
if any(
view.item.acceptance_status == BatchItemAcceptanceStatus.PENDING
or (
view.item.acceptance_status == BatchItemAcceptanceStatus.ACCEPTED
and (
view.job is None
or view.job.status
in {
IngestionJobStatus.QUEUED,
IngestionJobStatus.RUNNING,
IngestionJobStatus.RETRYING,
}
)
)
for view in self.items
):
return BatchAggregateStatus.PROCESSING
succeeded = sum(
view.job is not None and view.job.status == IngestionJobStatus.SUCCEEDED
for view in self.items
)
failed = len(self.items) - succeeded
if failed == 0:
return BatchAggregateStatus.COMPLETED
if succeeded == 0:
return BatchAggregateStatus.FAILED
return BatchAggregateStatus.PARTIAL_FAILURE
class _BatchByteBudget:
"""Count bytes across independently streamed files without buffering them."""
def __init__(self, limit_bytes: int) -> None:
self._limit_bytes = limit_bytes
self._consumed_bytes = 0
def wrap(self, source: _ReadableSource) -> BinarySource:
return _BatchLimitedSource(source, self)
def consume(self, size_bytes: int) -> None:
self._consumed_bytes += size_bytes
if self._consumed_bytes > self._limit_bytes:
raise BatchTotalSizeLimitError(SAFE_BATCH_SIZE_ERROR)
class _BatchLimitedSource:
"""Apply one shared request budget through the ObjectStorage stream contract."""
def __init__(self, source: _ReadableSource, budget: _BatchByteBudget) -> None:
self._source = source
self._budget = budget
def read(self, size: int = -1, /) -> bytes:
chunk = self._source.read(size)
self._budget.consume(len(chunk))
return chunk
class IngestionBatchService:
"""Accept files independently and persist ordered aggregate status."""
def __init__(
self,
session_factory: sessionmaker[Session],
submissions: AsyncIngestionSubmissionService,
*,
max_files: int,
max_total_bytes: int,
) -> None:
if max_files <= 0 or max_total_bytes <= 0:
raise ValueError("batch limits must be positive")
self._session_factory = session_factory
self._submissions = submissions
self._max_files = max_files
self._max_total_bytes = max_total_bytes
def submit(self, sources: tuple[BatchSource, ...]) -> BatchView:
"""Create a batch and independently accept or reject every ordered file."""
if not sources or len(sources) > self._max_files:
raise BatchFileCountLimitError(
f"batch must contain between 1 and {self._max_files} files"
)
batch_id, item_ids = self._create_pending_batch(sources)
budget = _BatchByteBudget(self._max_total_bytes)
for item_id, source in zip(item_ids, sources, strict=True):
try:
submission = self._submissions.submit(
source.filename,
source.media_type,
budget.wrap(source.source),
)
except Exception as error:
self._record_rejected(item_id, _safe_acceptance_error(error))
continue
self._record_accepted(
item_id,
document_id=submission.document.id,
job_id=submission.job.id,
)
return self.get(batch_id)
def get(self, batch_id: UUID) -> BatchView:
"""Return ordered items and fresh Job state for polling clients."""
with self._session_factory() as session:
batch = session.get(IngestionBatch, batch_id)
if batch is None:
raise BatchNotFoundError(str(batch_id))
items = tuple(
session.scalars(
select(IngestionBatchItem)
.where(IngestionBatchItem.batch_id == batch_id)
.order_by(IngestionBatchItem.item_index)
).all()
)
views = tuple(
BatchItemView(
item=item,
job=session.get(IngestionJob, item.job_id) if item.job_id is not None else None,
)
for item in items
)
return BatchView(batch=batch, items=views)
def _create_pending_batch(
self, sources: tuple[BatchSource, ...]
) -> tuple[UUID, tuple[UUID, ...]]:
"""Persist every input position before starting external storage writes."""
with self._session_factory() as session:
batch = IngestionBatch(total_files=len(sources))
session.add(batch)
session.flush()
items = tuple(
IngestionBatchItem(
batch_id=batch.id,
item_index=index,
filename=_display_filename(source.filename),
)
for index, source in enumerate(sources)
)
session.add_all(items)
session.commit()
return batch.id, tuple(item.id for item in items)
def _record_accepted(self, item_id: UUID, *, document_id: UUID, job_id: UUID) -> None:
with self._session_factory() as session:
item = session.get(IngestionBatchItem, item_id)
if item is None:
raise BatchNotFoundError(str(item_id))
item.acceptance_status = BatchItemAcceptanceStatus.ACCEPTED
item.document_id = document_id
item.job_id = job_id
item.error_message = None
session.commit()
def _record_rejected(self, item_id: UUID, error_message: str) -> None:
with self._session_factory() as session:
item = session.get(IngestionBatchItem, item_id)
if item is None:
raise BatchNotFoundError(str(item_id))
item.acceptance_status = BatchItemAcceptanceStatus.REJECTED
item.error_message = error_message
session.commit()
def _display_filename(filename: str) -> str:
"""Store a bounded safe label even when the upload filename is invalid."""
display = filename.replace("\x00", "<EFBFBD>").strip()
return display[:255] or "<unnamed>"
def _safe_acceptance_error(error: Exception) -> str:
"""Map per-file failures without persisting exception text or credentials."""
if isinstance(error, (UnsupportedDocumentTypeError, MediaTypeMismatchError)):
return SAFE_UNSUPPORTED_ERROR
if isinstance(error, DocumentTooLargeError):
return SAFE_FILE_SIZE_ERROR
if isinstance(error, BatchTotalSizeLimitError):
return SAFE_BATCH_SIZE_ERROR
if isinstance(error, EmptyDocumentError):
return SAFE_EMPTY_ERROR
if isinstance(error, DocumentUploadError):
return SAFE_ACCEPTANCE_ERROR
return SAFE_ACCEPTANCE_ERROR

View File

@ -0,0 +1,588 @@
"""Durable asynchronous ingestion orchestration and retry cleanup."""
from dataclasses import dataclass
from datetime import UTC, datetime, timedelta
from enum import StrEnum
from math import ceil
from typing import Protocol
from uuid import UUID
from sqlalchemy import delete, select
from sqlalchemy.orm import Session, sessionmaker
from ekp_core.document_chunking import (
DocumentAlreadyChunkedError,
DocumentChunkingService,
DocumentNotParsedError,
EmptyChunkSetError,
ParsedDocumentMismatchError,
)
from ekp_core.document_chunking import (
DocumentNotFoundError as ChunkingDocumentNotFoundError,
)
from ekp_core.document_indexing import (
DocumentAlreadyIndexedError,
DocumentHasNoChunksError,
DocumentIndexingService,
)
from ekp_core.document_indexing import (
DocumentNotFoundError as IndexingDocumentNotFoundError,
)
from ekp_core.document_parsing import (
DocumentAlreadyParsedError,
DocumentParsingService,
EmptyParsedDocumentError,
)
from ekp_core.document_parsing import (
DocumentNotFoundError as ParsingDocumentNotFoundError,
)
from ekp_core.document_upload import DocumentUploadService
from ekp_core.parsed_document_artifact import ParsedDocumentArtifactError
from ekp_domain import (
Chunk,
ChunkIndexState,
Document,
DocumentStatus,
IngestionAttempt,
IngestionAttemptStatus,
IngestionJob,
IngestionJobStatus,
IngestionStage,
)
from ekp_object_storage import BinarySource, ObjectStorage
from ekp_parser import DocumentParseError
from ekp_vector_store import VectorStore
SAFE_TRANSIENT_ERROR = "temporary ingestion failure"
SAFE_PERMANENT_ERROR = "document content could not be processed"
SAFE_CLEANUP_ERROR = "retry cleanup failed"
SAFE_LEASE_ERROR = "worker execution lease expired"
class IngestionJobNotFoundError(LookupError):
"""Raised when a requested ingestion Job does not exist."""
class IngestionCleanupError(RuntimeError):
"""Raised when derived data cannot be restored to a known clean state."""
class JobDispatcher(Protocol):
"""Replaceable message-delivery boundary used by submission services."""
def enqueue(self, job_id: UUID) -> None:
"""Request asynchronous execution of one already-persisted Job."""
@dataclass(frozen=True, slots=True)
class IngestionSubmission:
"""Stable identifiers returned after source and Job persistence."""
document: Document
job: IngestionJob
class JobRunAction(StrEnum):
"""Instruction returned to a queue adapter after one delivery."""
COMPLETE = "complete"
RETRY = "retry"
@dataclass(frozen=True, slots=True)
class JobRunResult:
"""Queue-neutral result for acknowledging or rescheduling a delivery."""
action: JobRunAction
status: IngestionJobStatus
retry_delay_seconds: int | None = None
@dataclass(frozen=True, slots=True)
class _Claim:
"""Internal durable claim outcome used before external provider work."""
run: bool
cleanup_required: bool = False
result: JobRunResult | None = None
class IngestionJobService:
"""Create and query Job state independently from the broker implementation."""
def __init__(self, session_factory: sessionmaker[Session], max_attempts: int) -> None:
if max_attempts <= 0:
raise ValueError("max_attempts must be positive")
self._session_factory = session_factory
self._max_attempts = max_attempts
def create(self, document_id: UUID) -> IngestionJob:
"""Create exactly one queued ingestion Job for a canonical Document."""
with self._session_factory() as session:
job = IngestionJob(document_id=document_id, max_attempts=self._max_attempts)
session.add(job)
session.commit()
return job
def get(self, job_id: UUID) -> IngestionJob:
"""Return one detached Job or a stable not-found error."""
with self._session_factory() as session:
job = session.get(IngestionJob, job_id)
if job is None:
raise IngestionJobNotFoundError(str(job_id))
return job
def list_dispatchable(self, limit: int = 100) -> tuple[UUID, ...]:
"""Find due queued/retrying Jobs and expired running leases for recovery."""
if not 1 <= limit <= 1000:
raise ValueError("dispatch limit must be between 1 and 1000")
now = datetime.now(UTC)
with self._session_factory() as session:
jobs = session.scalars(
select(IngestionJob)
.where(
(IngestionJob.status == IngestionJobStatus.QUEUED)
| (
(IngestionJob.status == IngestionJobStatus.RETRYING)
& ((IngestionJob.next_run_at.is_(None)) | (IngestionJob.next_run_at <= now))
)
| (
(IngestionJob.status == IngestionJobStatus.RUNNING)
& (
(IngestionJob.lease_expires_at.is_(None))
| (IngestionJob.lease_expires_at <= now)
)
)
)
.order_by(IngestionJob.created_at, IngestionJob.id)
.limit(limit)
).all()
return tuple(job.id for job in jobs)
class AsyncIngestionSubmissionService:
"""Persist immutable source and Job before best-effort broker delivery."""
def __init__(
self,
upload_service: DocumentUploadService,
jobs: IngestionJobService,
dispatcher: JobDispatcher,
) -> None:
self._upload_service = upload_service
self._jobs = jobs
self._dispatcher = dispatcher
def submit(
self,
filename: str,
media_type: str | None,
source: BinarySource,
) -> IngestionSubmission:
"""Accept one source and return without executing expensive stages inline.
Broker delivery is deliberately best effort after the canonical commit.
A periodic dispatcher recovers queued Jobs when the broker was temporarily
unavailable, so the request never treats a message as the source of truth.
"""
document = self._upload_service.upload(filename, media_type, source)
try:
job = self._jobs.create(document.id)
except Exception:
self._upload_service.rollback_acceptance(document.id)
raise
try:
self._dispatcher.enqueue(job.id)
except Exception:
# Keeping the Job queued is intentional. The recovery dispatcher can
# safely redeliver it without asking the client to upload again.
pass
return IngestionSubmission(document=document, job=job)
class DerivedDocumentCleanupService:
"""Remove only rebuildable derivatives before a retry begins."""
def __init__(
self,
session_factory: sessionmaker[Session],
storage: ObjectStorage,
vector_store: VectorStore,
active_index_profile_id: str,
) -> None:
self._session_factory = session_factory
self._storage = storage
self._vector_store = vector_store
self._active_index_profile_id = active_index_profile_id
def cleanup(self, document_id: UUID) -> None:
"""Clean vector, relational and parsed artifacts while preserving source/Job."""
try:
with self._session_factory() as session:
document = session.get(Document, document_id)
if document is None:
raise IngestionCleanupError("document metadata is missing")
parsed_key = document.parsed_storage_key
profile_ids = set(
session.scalars(
select(ChunkIndexState.index_profile_id)
.join(Chunk, Chunk.chunk_id == ChunkIndexState.chunk_id)
.where(Chunk.document_id == document_id)
.distinct()
).all()
)
profile_ids.add(self._active_index_profile_id)
# External vector rows are deleted before their canonical evidence.
for profile_id in sorted(profile_ids):
self._vector_store.delete_document(document_id, profile_id)
with self._session_factory() as session:
chunk_ids = select(Chunk.chunk_id).where(Chunk.document_id == document_id)
session.execute(
delete(ChunkIndexState).where(ChunkIndexState.chunk_id.in_(chunk_ids))
)
session.execute(delete(Chunk).where(Chunk.document_id == document_id))
session.commit()
if parsed_key is not None and self._storage.exists(parsed_key):
self._storage.delete(parsed_key)
with self._session_factory() as session:
document = session.get(Document, document_id)
if document is None:
raise IngestionCleanupError("document metadata is missing")
document.parsed_storage_key = None
document.status = DocumentStatus.UPLOADED
document.error_message = None
session.commit()
except IngestionCleanupError:
raise
except Exception as error:
raise IngestionCleanupError(SAFE_CLEANUP_ERROR) from error
class IngestionJobRunner:
"""Execute one durable attempt and return queue-neutral retry instructions."""
def __init__(
self,
session_factory: sessionmaker[Session],
parsing: DocumentParsingService,
chunking: DocumentChunkingService,
indexing: DocumentIndexingService,
cleanup: DerivedDocumentCleanupService,
*,
retry_backoff_seconds: int,
retry_backoff_max_seconds: int,
lease_seconds: int,
) -> None:
if retry_backoff_seconds <= 0 or retry_backoff_max_seconds <= 0 or lease_seconds <= 0:
raise ValueError("retry and lease durations must be positive")
self._session_factory = session_factory
self._parsing = parsing
self._chunking = chunking
self._indexing = indexing
self._cleanup = cleanup
self._retry_backoff_seconds = retry_backoff_seconds
self._retry_backoff_max_seconds = retry_backoff_max_seconds
self._lease_seconds = lease_seconds
def run(self, job_id: UUID) -> JobRunResult:
"""Claim, optionally clean, execute and durably finalize one delivery."""
claim = self._claim(job_id)
if not claim.run:
assert claim.result is not None
return claim.result
if claim.cleanup_required:
try:
document_id = self._job_document_id(job_id)
self._cleanup.cleanup(document_id)
except Exception:
self._finish_cleanup_failure(job_id)
return JobRunResult(JobRunAction.COMPLETE, IngestionJobStatus.FAILED)
attempt_number, document_id = self._start_attempt(job_id)
def stage_callback(stage: IngestionStage) -> None:
"""Bind this attempt while pipeline stages remain Job-agnostic."""
self._record_stage(job_id, attempt_number, stage)
try:
self._parsing.process(document_id, stage_callback=stage_callback)
self._chunking.process(document_id, stage_callback=stage_callback)
self._indexing.process(document_id, stage_callback=stage_callback)
except Exception as error:
return self._finish_failure(job_id, attempt_number, document_id, error)
self._finish_success(job_id, attempt_number, document_id)
return JobRunResult(JobRunAction.COMPLETE, IngestionJobStatus.SUCCEEDED)
def _claim(self, job_id: UUID) -> _Claim:
"""Serialize deliveries and recover only after an execution lease expires."""
now = datetime.now(UTC)
with self._session_factory() as session:
job = session.scalar(
select(IngestionJob).where(IngestionJob.id == job_id).with_for_update()
)
if job is None:
raise IngestionJobNotFoundError(str(job_id))
if job.status in {
IngestionJobStatus.SUCCEEDED,
IngestionJobStatus.FAILED,
IngestionJobStatus.CANCELLED,
}:
return _Claim(
run=False,
result=JobRunResult(JobRunAction.COMPLETE, job.status),
)
if job.status == IngestionJobStatus.RUNNING:
lease_expiry = _as_utc(job.lease_expires_at)
if lease_expiry is not None and lease_expiry > now:
delay = max(1, ceil((lease_expiry - now).total_seconds()))
return _Claim(
run=False,
result=JobRunResult(
JobRunAction.RETRY,
IngestionJobStatus.RUNNING,
delay,
),
)
self._expire_active_attempt(session, job, now)
if job.attempt_count >= job.max_attempts:
job.status = IngestionJobStatus.FAILED
job.lease_expires_at = None
job.completed_at = now
document = session.get(Document, job.document_id)
if document is not None:
document.status = DocumentStatus.FAILED
document.error_message = SAFE_LEASE_ERROR
session.commit()
return _Claim(
run=False,
result=JobRunResult(JobRunAction.COMPLETE, job.status),
)
job.status = IngestionJobStatus.RETRYING
next_run = _as_utc(job.next_run_at)
if (
job.status == IngestionJobStatus.RETRYING
and next_run is not None
and next_run > now
):
delay = max(1, ceil((next_run - now).total_seconds()))
return _Claim(
run=False,
result=JobRunResult(
JobRunAction.RETRY,
IngestionJobStatus.RETRYING,
delay,
),
)
cleanup_required = job.attempt_count > 0
job.status = IngestionJobStatus.RUNNING
job.lease_expires_at = now + timedelta(seconds=self._lease_seconds)
job.next_run_at = None
session.commit()
return _Claim(run=True, cleanup_required=cleanup_required)
def _expire_active_attempt(self, session: Session, job: IngestionJob, now: datetime) -> None:
"""Close an abandoned Attempt before a replacement can be created."""
attempt = session.scalar(
select(IngestionAttempt)
.where(
IngestionAttempt.job_id == job.id,
IngestionAttempt.status == IngestionAttemptStatus.RUNNING,
)
.with_for_update()
)
if attempt is not None:
attempt.status = IngestionAttemptStatus.FAILED
attempt.retryable = True
attempt.error_message = SAFE_LEASE_ERROR
attempt.finished_at = now
job.error_message = SAFE_LEASE_ERROR
def _job_document_id(self, job_id: UUID) -> UUID:
with self._session_factory() as session:
job = session.get(IngestionJob, job_id)
if job is None:
raise IngestionJobNotFoundError(str(job_id))
return job.document_id
def _start_attempt(self, job_id: UUID) -> tuple[int, UUID]:
"""Create the next numbered Attempt only after retry cleanup succeeds."""
now = datetime.now(UTC)
with self._session_factory() as session:
job = session.scalar(
select(IngestionJob).where(IngestionJob.id == job_id).with_for_update()
)
if job is None:
raise IngestionJobNotFoundError(str(job_id))
attempt_number = job.attempt_count + 1
job.attempt_count = attempt_number
job.stage = IngestionStage.PARSING
job.error_message = None
job.lease_expires_at = now + timedelta(seconds=self._lease_seconds)
session.add(
IngestionAttempt(
job_id=job.id,
attempt_number=attempt_number,
stage=IngestionStage.PARSING,
)
)
session.commit()
return attempt_number, job.document_id
def _record_stage(self, job_id: UUID, attempt_number: int, stage: IngestionStage) -> None:
"""Persist progress and renew the worker lease at every stage boundary."""
with self._session_factory() as session:
job = session.get(IngestionJob, job_id)
attempt = session.scalar(
select(IngestionAttempt).where(
IngestionAttempt.job_id == job_id,
IngestionAttempt.attempt_number == attempt_number,
)
)
if job is None or attempt is None:
raise IngestionJobNotFoundError(str(job_id))
job.stage = stage
job.lease_expires_at = datetime.now(UTC) + timedelta(seconds=self._lease_seconds)
attempt.stage = stage
session.commit()
def _finish_success(self, job_id: UUID, attempt_number: int, document_id: UUID) -> None:
"""Atomically close Job/Attempt after the document becomes ready."""
now = datetime.now(UTC)
with self._session_factory() as session:
job, attempt = self._load_job_attempt(session, job_id, attempt_number)
document = session.get(Document, document_id)
if document is None:
raise IngestionJobNotFoundError(str(job_id))
attempt.status = IngestionAttemptStatus.SUCCEEDED
attempt.retryable = False
attempt.finished_at = now
job.status = IngestionJobStatus.SUCCEEDED
job.error_message = None
job.lease_expires_at = None
job.completed_at = now
document.status = DocumentStatus.READY
document.error_message = None
session.commit()
def _finish_failure(
self,
job_id: UUID,
attempt_number: int,
document_id: UUID,
error: Exception,
) -> JobRunResult:
"""Classify safely, bound retry count, and schedule exponential backoff."""
retryable = _is_retryable(error)
safe_error = SAFE_TRANSIENT_ERROR if retryable else SAFE_PERMANENT_ERROR
now = datetime.now(UTC)
with self._session_factory() as session:
job, attempt = self._load_job_attempt(session, job_id, attempt_number)
document = session.get(Document, document_id)
attempt.status = IngestionAttemptStatus.FAILED
attempt.retryable = retryable
attempt.error_message = safe_error
attempt.finished_at = now
job.error_message = safe_error
job.lease_expires_at = None
if retryable and job.attempt_count < job.max_attempts:
delay = min(
self._retry_backoff_seconds * (2 ** (job.attempt_count - 1)),
self._retry_backoff_max_seconds,
)
job.status = IngestionJobStatus.RETRYING
job.next_run_at = now + timedelta(seconds=delay)
if document is not None:
document.status = DocumentStatus.PROCESSING
document.error_message = safe_error
session.commit()
return JobRunResult(JobRunAction.RETRY, job.status, delay)
job.status = IngestionJobStatus.FAILED
job.next_run_at = None
job.completed_at = now
if document is not None:
document.status = DocumentStatus.FAILED
document.error_message = safe_error
session.commit()
return JobRunResult(JobRunAction.COMPLETE, job.status)
def _finish_cleanup_failure(self, job_id: UUID) -> None:
"""Fail closed without creating a new Attempt on unknown dirty state."""
now = datetime.now(UTC)
with self._session_factory() as session:
job = session.get(IngestionJob, job_id)
if job is None:
raise IngestionJobNotFoundError(str(job_id))
document = session.get(Document, job.document_id)
job.status = IngestionJobStatus.FAILED
job.error_message = SAFE_CLEANUP_ERROR
job.next_run_at = None
job.lease_expires_at = None
job.completed_at = now
if document is not None:
document.status = DocumentStatus.FAILED
document.error_message = SAFE_CLEANUP_ERROR
session.commit()
@staticmethod
def _load_job_attempt(
session: Session, job_id: UUID, attempt_number: int
) -> tuple[IngestionJob, IngestionAttempt]:
job = session.get(IngestionJob, job_id)
attempt = session.scalar(
select(IngestionAttempt).where(
IngestionAttempt.job_id == job_id,
IngestionAttempt.attempt_number == attempt_number,
)
)
if job is None or attempt is None:
raise IngestionJobNotFoundError(str(job_id))
return job, attempt
def dispatch_pending_jobs(jobs: IngestionJobService, dispatcher: JobDispatcher) -> int:
"""Best-effort redelivery loop used by Celery beat after broker/API failures."""
dispatched = 0
for job_id in jobs.list_dispatchable():
try:
dispatcher.enqueue(job_id)
except Exception:
continue
dispatched += 1
return dispatched
def _is_retryable(error: Exception) -> bool:
"""Treat known content/state violations as permanent and providers as transient."""
permanent_errors = (
DocumentParseError,
EmptyParsedDocumentError,
EmptyChunkSetError,
ParsedDocumentMismatchError,
ParsedDocumentArtifactError,
DocumentAlreadyParsedError,
DocumentAlreadyChunkedError,
DocumentAlreadyIndexedError,
DocumentNotParsedError,
DocumentHasNoChunksError,
ParsingDocumentNotFoundError,
ChunkingDocumentNotFoundError,
IndexingDocumentNotFoundError,
)
return not isinstance(error, permanent_errors)
def _as_utc(value: datetime | None) -> datetime | None:
"""Normalize SQLite's naive DateTime values to the UTC runtime contract."""
if value is None:
return None
if value.tzinfo is None:
return value.replace(tzinfo=UTC)
return value.astimezone(UTC)

View File

@ -59,7 +59,7 @@ class SearchResult:
class SearchService: class SearchService:
"""Compose Dense Top-N, reranking, and PostgreSQL source enrichment.""" """Compose Dense Top-N, reranking, and canonical source enrichment."""
def __init__( def __init__(
self, self,
@ -158,7 +158,7 @@ class SearchService:
self, self,
selected: tuple[tuple[DenseSearchCandidate, float], ...], selected: tuple[tuple[DenseSearchCandidate, float], ...],
) -> tuple[SourceResult, ...]: ) -> tuple[SourceResult, ...]:
"""Rehydrate final content and provenance from canonical PostgreSQL rows.""" """Rehydrate final content and provenance from canonical relational rows."""
chunk_ids = [candidate.chunk_id for candidate, _ in selected] chunk_ids = [candidate.chunk_id for candidate, _ in selected]
with self._session_factory() as session: with self._session_factory() as session:
chunks = { chunks = {

View File

@ -2,17 +2,44 @@
from ekp_domain.chunk import Chunk from ekp_domain.chunk import Chunk
from ekp_domain.document import Base, Document, DocumentStatus from ekp_domain.document import Base, Document, DocumentStatus
from ekp_domain.index_profile import ChunkIndexState, IndexProfile from ekp_domain.index_profile import (
ChunkIndexState,
IndexProfile,
IndexProfileStatus,
StoredIndexProfile,
)
from ekp_domain.ingestion_batch import (
BatchItemAcceptanceStatus,
IngestionBatch,
IngestionBatchItem,
)
from ekp_domain.ingestion_job import (
IngestionAttempt,
IngestionAttemptStatus,
IngestionJob,
IngestionJobStatus,
IngestionStage,
)
from ekp_domain.parsed_document import ParsedBlock, ParsedBlockType, ParsedDocument from ekp_domain.parsed_document import ParsedBlock, ParsedBlockType, ParsedDocument
__all__ = [ __all__ = [
"Base", "Base",
"BatchItemAcceptanceStatus",
"Chunk", "Chunk",
"ChunkIndexState", "ChunkIndexState",
"Document", "Document",
"DocumentStatus", "DocumentStatus",
"IndexProfile", "IndexProfile",
"IndexProfileStatus",
"IngestionAttempt",
"IngestionAttemptStatus",
"IngestionJob",
"IngestionJobStatus",
"IngestionStage",
"IngestionBatch",
"IngestionBatchItem",
"ParsedBlock", "ParsedBlock",
"ParsedBlockType", "ParsedBlockType",
"ParsedDocument", "ParsedDocument",
"StoredIndexProfile",
] ]

View File

@ -3,9 +3,11 @@
import re import re
from dataclasses import dataclass from dataclasses import dataclass
from datetime import datetime from datetime import datetime
from enum import StrEnum
from uuid import UUID from uuid import UUID
from sqlalchemy import CheckConstraint, DateTime, ForeignKey, Integer, String, func from sqlalchemy import CheckConstraint, Computed, DateTime, ForeignKey, Index, Integer, String, func
from sqlalchemy import Enum as SqlEnum
from sqlalchemy.orm import Mapped, mapped_column from sqlalchemy.orm import Mapped, mapped_column
from ekp_domain.document import Base from ekp_domain.document import Base
@ -13,6 +15,14 @@ from ekp_domain.document import Base
PROFILE_ID_PATTERN = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$") PROFILE_ID_PATTERN = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$")
class IndexProfileStatus(StrEnum):
"""Lifecycle state for a canonical index-profile registration."""
SHADOW = "shadow"
ACTIVE = "active"
RETIRED = "retired"
@dataclass(frozen=True, slots=True) @dataclass(frozen=True, slots=True)
class IndexProfile: class IndexProfile:
"""Small Demo profile binding chunk and embedding versions together.""" """Small Demo profile binding chunk and embedding versions together."""
@ -31,6 +41,72 @@ class IndexProfile:
raise ValueError("vector_dimension must be positive") raise ValueError("vector_dimension must be positive")
class StoredIndexProfile(Base):
"""Canonical registration used to build, verify, and activate shadow indexes."""
__tablename__ = "index_profiles"
__table_args__ = (
CheckConstraint("length(profile_id) > 0", name="profile_id_not_empty"),
CheckConstraint(
"length(chunk_strategy_version) > 0",
name="profile_chunk_strategy_not_empty",
),
CheckConstraint(
"length(embedding_model) > 0",
name="profile_embedding_model_not_empty",
),
CheckConstraint("vector_dimension > 0", name="profile_dimension_positive"),
CheckConstraint(
"status IN ('shadow', 'active', 'retired')",
name="profile_status_valid",
),
# A generated nullable marker provides portable partial-uniqueness:
# every active row maps to the same value, while NULL shadow/retired
# markers may repeat under OceanBase MySQL and SQLite.
Index("uq_index_profiles_single_active", "active_marker", unique=True),
)
profile_id: Mapped[str] = mapped_column(String(128), primary_key=True)
chunk_strategy_version: Mapped[str] = mapped_column(String(64), nullable=False)
embedding_model: Mapped[str] = mapped_column(String(128), nullable=False)
vector_dimension: Mapped[int] = mapped_column(Integer, nullable=False)
status: Mapped[IndexProfileStatus] = mapped_column(
SqlEnum(
IndexProfileStatus,
name="index_profile_status",
native_enum=False,
create_constraint=False,
validate_strings=True,
values_callable=lambda enum_type: [item.value for item in enum_type],
),
nullable=False,
default=IndexProfileStatus.SHADOW,
)
active_marker: Mapped[str | None] = mapped_column(
String(6),
Computed("CASE WHEN status = 'active' THEN 'active' ELSE NULL END"),
nullable=True,
)
created_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True),
nullable=False,
server_default=func.now(),
)
activated_at: Mapped[datetime | None] = mapped_column(
DateTime(timezone=True),
nullable=True,
)
def to_profile(self) -> IndexProfile:
"""Return the immutable business value used by indexing and retrieval."""
return IndexProfile(
profile_id=self.profile_id,
chunk_strategy_version=self.chunk_strategy_version,
embedding_model=self.embedding_model,
vector_dimension=self.vector_dimension,
)
class ChunkIndexState(Base): class ChunkIndexState(Base):
"""Canonical evidence that one chunk was written to a rebuildable index.""" """Canonical evidence that one chunk was written to a rebuildable index."""

View File

@ -0,0 +1,96 @@
"""Canonical batch submission metadata for per-file ingestion tracking."""
from datetime import datetime
from enum import StrEnum
from uuid import UUID, uuid4
from sqlalchemy import (
CheckConstraint,
DateTime,
ForeignKey,
Index,
Integer,
String,
Text,
Uuid,
func,
)
from sqlalchemy import Enum as SqlEnum
from sqlalchemy.orm import Mapped, mapped_column
from ekp_domain.document import Base
class BatchItemAcceptanceStatus(StrEnum):
"""Source acceptance state, independent from its asynchronous Job state."""
PENDING = "pending"
ACCEPTED = "accepted"
REJECTED = "rejected"
class IngestionBatch(Base):
"""Stable aggregation identifier for one bounded multipart submission."""
__tablename__ = "ingestion_batches"
__table_args__ = (CheckConstraint("total_files > 0", name="total_files_positive"),)
id: Mapped[UUID] = mapped_column(Uuid(as_uuid=True), primary_key=True, default=uuid4)
total_files: Mapped[int] = mapped_column(Integer, nullable=False)
created_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), nullable=False, server_default=func.now()
)
class IngestionBatchItem(Base):
"""One ordered input and its optional accepted Document/Job references."""
__tablename__ = "ingestion_batch_items"
__table_args__ = (
CheckConstraint("item_index >= 0", name="item_index_non_negative"),
CheckConstraint("length(filename) > 0", name="filename_not_empty"),
CheckConstraint(
"acceptance_status IN ('pending', 'accepted', 'rejected')",
name="acceptance_status_valid",
),
Index("uq_ingestion_batch_items_order", "batch_id", "item_index", unique=True),
Index("uq_ingestion_batch_items_job_id", "job_id", unique=True),
)
id: Mapped[UUID] = mapped_column(Uuid(as_uuid=True), primary_key=True, default=uuid4)
batch_id: Mapped[UUID] = mapped_column(
Uuid(as_uuid=True),
ForeignKey("ingestion_batches.id", ondelete="CASCADE"),
nullable=False,
)
item_index: Mapped[int] = mapped_column(Integer, nullable=False)
filename: Mapped[str] = mapped_column(String(255), nullable=False)
acceptance_status: Mapped[BatchItemAcceptanceStatus] = mapped_column(
SqlEnum(
BatchItemAcceptanceStatus,
name="batch_item_acceptance_status",
native_enum=False,
create_constraint=False,
validate_strings=True,
values_callable=lambda values: [item.value for item in values],
),
nullable=False,
default=BatchItemAcceptanceStatus.PENDING,
)
document_id: Mapped[UUID | None] = mapped_column(
Uuid(as_uuid=True),
ForeignKey("documents.id", ondelete="SET NULL"),
nullable=True,
)
job_id: Mapped[UUID | None] = mapped_column(
Uuid(as_uuid=True),
ForeignKey("ingestion_jobs.id", ondelete="SET NULL"),
nullable=True,
)
error_message: Mapped[str | None] = mapped_column(Text, nullable=True)
created_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), nullable=False, server_default=func.now()
)
updated_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), nullable=False, server_default=func.now(), onupdate=func.now()
)

View File

@ -0,0 +1,166 @@
"""Durable ingestion Job and Attempt state for asynchronous processing."""
from datetime import datetime
from enum import StrEnum
from uuid import UUID, uuid4
from sqlalchemy import (
Boolean,
CheckConstraint,
Computed,
DateTime,
ForeignKey,
Index,
Integer,
Text,
UniqueConstraint,
Uuid,
func,
)
from sqlalchemy import Enum as SqlEnum
from sqlalchemy.orm import Mapped, mapped_column
from ekp_domain.document import Base
class IngestionJobStatus(StrEnum):
"""Public lifecycle states for one document ingestion Job."""
QUEUED = "queued"
RUNNING = "running"
RETRYING = "retrying"
SUCCEEDED = "succeeded"
FAILED = "failed"
CANCELLED = "cancelled"
class IngestionStage(StrEnum):
"""Observable stages within one ingestion attempt."""
PARSING = "parsing"
CLEANING = "cleaning"
CHUNKING = "chunking"
EMBEDDING = "embedding"
INDEXING = "indexing"
class IngestionAttemptStatus(StrEnum):
"""Terminal or active state of one bounded execution attempt."""
RUNNING = "running"
SUCCEEDED = "succeeded"
FAILED = "failed"
def _enum_column(enum_type: type[StrEnum], name: str) -> SqlEnum:
"""Build consistent portable string enums for OceanBase and SQLite tests."""
return SqlEnum(
enum_type,
name=name,
native_enum=False,
create_constraint=False,
validate_strings=True,
values_callable=lambda values: [item.value for item in values],
)
class IngestionJob(Base):
"""Canonical orchestration state; broker messages are only delivery hints."""
__tablename__ = "ingestion_jobs"
__table_args__ = (
CheckConstraint("attempt_count >= 0", name="attempt_count_non_negative"),
CheckConstraint("max_attempts > 0", name="max_attempts_positive"),
CheckConstraint("attempt_count <= max_attempts", name="attempt_count_bounded"),
CheckConstraint(
"status IN ('queued', 'running', 'retrying', 'succeeded', 'failed', 'cancelled')",
name="status_valid",
),
CheckConstraint(
"stage IS NULL OR stage IN "
"('parsing', 'cleaning', 'chunking', 'embedding', 'indexing')",
name="stage_valid",
),
UniqueConstraint("document_id", name="uq_ingestion_jobs_document_id"),
Index("ix_ingestion_jobs_status_next_run", "status", "next_run_at"),
)
id: Mapped[UUID] = mapped_column(Uuid(as_uuid=True), primary_key=True, default=uuid4)
document_id: Mapped[UUID] = mapped_column(
Uuid(as_uuid=True),
ForeignKey("documents.id", ondelete="CASCADE"),
nullable=False,
)
status: Mapped[IngestionJobStatus] = mapped_column(
_enum_column(IngestionJobStatus, "ingestion_job_status"),
nullable=False,
default=IngestionJobStatus.QUEUED,
)
stage: Mapped[IngestionStage | None] = mapped_column(
_enum_column(IngestionStage, "ingestion_stage"),
nullable=True,
)
attempt_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
max_attempts: Mapped[int] = mapped_column(Integer, nullable=False)
error_message: Mapped[str | None] = mapped_column(Text, nullable=True)
next_run_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
lease_expires_at: Mapped[datetime | None] = mapped_column(
DateTime(timezone=True), nullable=True
)
created_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), nullable=False, server_default=func.now()
)
updated_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), nullable=False, server_default=func.now(), onupdate=func.now()
)
completed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
class IngestionAttempt(Base):
"""Immutable-numbered execution history with at most one active row per Job."""
__tablename__ = "ingestion_attempts"
__table_args__ = (
CheckConstraint("attempt_number > 0", name="attempt_number_positive"),
CheckConstraint(
"status IN ('running', 'succeeded', 'failed')",
name="status_valid",
),
CheckConstraint(
"stage IS NULL OR stage IN "
"('parsing', 'cleaning', 'chunking', 'embedding', 'indexing')",
name="stage_valid",
),
UniqueConstraint("job_id", "attempt_number", name="uq_ingestion_attempts_job_number"),
# NULL values may repeat, while every running attempt maps to 1. This
# provides portable partial uniqueness on OceanBase MySQL and SQLite.
Index("uq_ingestion_attempts_active", "job_id", "active_marker", unique=True),
)
id: Mapped[UUID] = mapped_column(Uuid(as_uuid=True), primary_key=True, default=uuid4)
job_id: Mapped[UUID] = mapped_column(
Uuid(as_uuid=True),
ForeignKey("ingestion_jobs.id", ondelete="CASCADE"),
nullable=False,
)
attempt_number: Mapped[int] = mapped_column(Integer, nullable=False)
status: Mapped[IngestionAttemptStatus] = mapped_column(
_enum_column(IngestionAttemptStatus, "ingestion_attempt_status"),
nullable=False,
default=IngestionAttemptStatus.RUNNING,
)
stage: Mapped[IngestionStage | None] = mapped_column(
_enum_column(IngestionStage, "ingestion_attempt_stage"),
nullable=True,
)
retryable: Mapped[bool | None] = mapped_column(Boolean, nullable=True)
error_message: Mapped[str | None] = mapped_column(Text, nullable=True)
active_marker: Mapped[int | None] = mapped_column(
Integer,
Computed("CASE WHEN status = 'running' THEN 1 ELSE NULL END"),
nullable=True,
)
started_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), nullable=False, server_default=func.now()
)
finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)

View File

@ -147,7 +147,7 @@ def create_bos_object_storage(
"""Build the official BOS client from runtime-injected, validated settings.""" """Build the official BOS client from runtime-injected, validated settings."""
if not access_key_id or not secret_access_key: if not access_key_id or not secret_access_key:
raise BosConfigurationError("BOS access key ID and secret access key are required") raise BosConfigurationError("BOS access key ID and secret access key are required")
_validate_endpoint(endpoint) normalized_endpoint = _normalize_endpoint(endpoint)
if not BOS_BUCKET_NAME.fullmatch(bucket_name): if not BOS_BUCKET_NAME.fullmatch(bucket_name):
raise BosConfigurationError("invalid BOS bucket name") raise BosConfigurationError("invalid BOS bucket name")
if timeout_seconds <= 0: if timeout_seconds <= 0:
@ -164,7 +164,7 @@ def create_bos_object_storage(
config = BceClientConfiguration( config = BceClientConfiguration(
credentials=BceCredentials(access_key_id, secret_access_key), credentials=BceCredentials(access_key_id, secret_access_key),
endpoint=endpoint, endpoint=normalized_endpoint,
connection_timeout_in_mills=int(timeout_seconds * 1000), connection_timeout_in_mills=int(timeout_seconds * 1000),
) )
client = BosClient(config) client = BosClient(config)
@ -192,9 +192,10 @@ def _status_code(error: Exception) -> int | None:
return status_code if isinstance(status_code, int) else None return status_code if isinstance(status_code, int) else None
def _validate_endpoint(endpoint: str) -> None: def _normalize_endpoint(endpoint: str) -> str:
"""Require an explicit HTTP(S) origin without embedded credentials or paths.""" """Validate a BOS origin and force host-only legacy settings onto HTTPS."""
parsed = urlsplit(endpoint) candidate = endpoint if "://" in endpoint else f"https://{endpoint}"
parsed = urlsplit(candidate)
if ( if (
parsed.scheme not in {"http", "https"} parsed.scheme not in {"http", "https"}
or not parsed.hostname or not parsed.hostname
@ -205,3 +206,4 @@ def _validate_endpoint(endpoint: str) -> None:
or parsed.fragment or parsed.fragment
): ):
raise BosConfigurationError("invalid BOS endpoint") raise BosConfigurationError("invalid BOS endpoint")
return candidate.rstrip("/")

View File

@ -1,4 +1,4 @@
"""Baidu Qianfan v2 adapter for the Demo Qwen3 reranker baseline.""" """Baidu Qianfan v2 adapter for the Demo BCE reranker baseline."""
import json import json
from collections.abc import Mapping, Sequence from collections.abc import Mapping, Sequence
@ -10,7 +10,7 @@ from urllib.request import Request, urlopen
from ekp_reranker.base import RerankerProviderError, RerankResult from ekp_reranker.base import RerankerProviderError, RerankResult
QWEN3_RERANKER_MODEL = "qwen3-reranker-0.6b" BCE_RERANKER_MODEL = "bce-reranker-base"
QIANFAN_RERANK_ENDPOINT = "https://qianfan.baidubce.com/v2/rerank" QIANFAN_RERANK_ENDPOINT = "https://qianfan.baidubce.com/v2/rerank"
QIANFAN_MAX_DOCUMENTS = 64 QIANFAN_MAX_DOCUMENTS = 64
QIANFAN_MAX_QUERY_CHARACTERS = 1600 QIANFAN_MAX_QUERY_CHARACTERS = 1600
@ -65,7 +65,7 @@ class QianfanRerankerProvider:
self, self,
api_key: str, api_key: str,
endpoint: str = QIANFAN_RERANK_ENDPOINT, endpoint: str = QIANFAN_RERANK_ENDPOINT,
model_name: str = QWEN3_RERANKER_MODEL, model_name: str = BCE_RERANKER_MODEL,
timeout_seconds: float = 30.0, timeout_seconds: float = 30.0,
http_client: JsonHttpClient | None = None, http_client: JsonHttpClient | None = None,
) -> None: ) -> None:

Some files were not shown because too many files have changed in this diff Show More