Compare commits
42 Commits
codex/docs
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
03dd5ffbfd | ||
|
|
5100d98e3c | ||
|
|
d727d0a118 | ||
|
|
72dc248bbc | ||
|
|
a1eae16fb1 | ||
|
|
0fad825eb0 | ||
|
|
a8dfbe0cfa | ||
|
|
7938c0b9cf | ||
|
|
4d73465888 | ||
|
|
f47634e104 | ||
|
|
6f16c971e4 | ||
|
|
123864b80d | ||
|
|
9d53bdbe4b | ||
|
|
77119f4a80 | ||
|
|
6af4ab465d | ||
|
|
e3b5b4d50f | ||
|
|
fe9d85b6da | ||
|
|
f16c738035 | ||
|
|
8fba051da4 | ||
|
|
c13ae39543 | ||
|
|
9931224b46 | ||
|
|
2558264d21 | ||
|
|
c6421cc9d4 | ||
|
|
df4b0ae91e | ||
|
|
4ce74604ba | ||
|
|
0d21fae173 | ||
|
|
92d098cc25 | ||
|
|
6efc33e868 | ||
|
|
b98b04af17 | ||
|
|
c95bd8a118 | ||
|
|
d5b1d4fe4e | ||
|
|
48e6732d4f | ||
|
|
03f0f2b0e3 | ||
|
|
cb1cdfd07d | ||
|
|
ac054eaf94 | ||
|
|
f696aeec2b | ||
|
|
a0aa30c41c | ||
|
|
43b649c46a | ||
|
|
be69b16d18 | ||
|
|
3349c97f59 | ||
|
|
934c755901 | ||
|
|
3c0440abc8 |
21
.dockerignore
Normal file
21
.dockerignore
Normal file
@ -0,0 +1,21 @@
|
|||||||
|
# Keep credentials, local state, caches, and developer artifacts out of every image context.
|
||||||
|
.env
|
||||||
|
.env.*
|
||||||
|
!.env.example
|
||||||
|
.git
|
||||||
|
.github
|
||||||
|
.idea
|
||||||
|
.vscode
|
||||||
|
.DS_Store
|
||||||
|
.mypy_cache
|
||||||
|
.pytest_cache
|
||||||
|
.ruff_cache
|
||||||
|
.venv
|
||||||
|
__pycache__
|
||||||
|
*.py[cod]
|
||||||
|
*.sqlite*
|
||||||
|
dist
|
||||||
|
storage
|
||||||
|
tests
|
||||||
|
docs
|
||||||
|
docker-compose.dev.yml
|
||||||
89
.env.example
89
.env.example
@ -4,35 +4,86 @@ LOG_LEVEL=INFO
|
|||||||
API_HOST=0.0.0.0
|
API_HOST=0.0.0.0
|
||||||
API_PORT=8000
|
API_PORT=8000
|
||||||
API_PREFIX=/api/v1
|
API_PREFIX=/api/v1
|
||||||
|
# Optional outside source checkouts; production image fixes this to /app/apps/admin-web.
|
||||||
|
# ADMIN_WEB_ROOT=/absolute/path/to/apps/admin-web
|
||||||
|
|
||||||
DATABASE_URL=postgresql+psycopg://ekp:ekp@localhost:5432/ekp
|
DATABASE_URL=mysql+pymysql://ekp:change-me@localhost:2881/ekp
|
||||||
REDIS_URL=redis://localhost:6379/0
|
DATABASE_POOL_SIZE=10
|
||||||
RABBITMQ_URL=amqp://guest:guest@localhost:5672//
|
DATABASE_MAX_OVERFLOW=10
|
||||||
|
DATABASE_POOL_RECYCLE_SECONDS=600
|
||||||
|
DATABASE_POOL_TIMEOUT_SECONDS=30
|
||||||
|
DATABASE_CONNECT_TIMEOUT_SECONDS=30
|
||||||
|
# DEMO-015 queue. Inject the real credential at runtime; never commit its value.
|
||||||
|
RABBITMQ_URL=
|
||||||
|
INGESTION_MAX_ATTEMPTS=3
|
||||||
|
INGESTION_RETRY_BACKOFF_SECONDS=2
|
||||||
|
INGESTION_RETRY_BACKOFF_MAX_SECONDS=60
|
||||||
|
INGESTION_WORKER_LEASE_SECONDS=300
|
||||||
|
INGESTION_RECOVERY_INTERVAL_SECONDS=30
|
||||||
|
BATCH_MAX_FILES=20
|
||||||
|
BATCH_MAX_TOTAL_BYTES=209715200
|
||||||
|
|
||||||
OBJECT_STORAGE_PROVIDER=local
|
OBJECT_STORAGE_PROVIDER=bos
|
||||||
OBJECT_STORAGE_PATH=storage
|
OBJECT_STORAGE_PATH=storage
|
||||||
|
BOS_ENDPOINT=
|
||||||
|
BOS_ACCESS_KEY_ID=
|
||||||
|
BOS_SECRET_ACCESS_KEY=
|
||||||
|
BOS_BUCKET=
|
||||||
|
BOS_KEY_PREFIX=knowledge-platform
|
||||||
|
BOS_TIMEOUT_SECONDS=30
|
||||||
|
MAX_UPLOAD_BYTES=52428800
|
||||||
|
CHUNK_MAX_TOKENS=450
|
||||||
|
CHUNK_OVERLAP_TOKENS=75
|
||||||
|
CHUNK_MAX_CHARACTERS=2000
|
||||||
|
|
||||||
# Reserved for a future S3/MinIO adapter; DEMO-002 uses local storage.
|
VECTOR_STORE_PROVIDER=oceanbase
|
||||||
OBJECT_STORAGE_ENDPOINT=http://localhost:9000
|
OCEANBASE_VECTOR_URI=
|
||||||
OBJECT_STORAGE_ACCESS_KEY=
|
OCEANBASE_VECTOR_USER=
|
||||||
OBJECT_STORAGE_SECRET_KEY=
|
OCEANBASE_VECTOR_PASSWORD=
|
||||||
OBJECT_STORAGE_BUCKET=ekp
|
OCEANBASE_VECTOR_DATABASE=
|
||||||
|
OCEANBASE_VECTOR_TABLE=knowledge_chunks_v1
|
||||||
VECTOR_STORE_PROVIDER=mochow
|
OCEANBASE_VECTOR_INDEX=vidx_knowledge_chunks_embedding
|
||||||
MOCHOW_ENDPOINT=
|
OCEANBASE_VECTOR_BATCH_SIZE=100
|
||||||
MOCHOW_API_KEY=
|
OCEANBASE_VECTOR_TIMEOUT_SECONDS=30
|
||||||
MOCHOW_DATABASE=
|
|
||||||
MOCHOW_TABLE=
|
|
||||||
|
|
||||||
EMBEDDING_PROVIDER=qianfan
|
EMBEDDING_PROVIDER=qianfan
|
||||||
EMBEDDING_MODEL=bge-large-zh
|
EMBEDDING_MODEL=bge-large-zh
|
||||||
EMBEDDING_ENDPOINT=
|
EMBEDDING_ENDPOINT=https://qianfan.baidubce.com/v2/embeddings
|
||||||
EMBEDDING_API_KEY=
|
BAIDU_QIANFAN_API_KEY=
|
||||||
|
EMBEDDING_TIMEOUT_SECONDS=30
|
||||||
|
INDEX_PROFILE_ID=demo-bge-large-zh-structure-v1-oceanbase-v1
|
||||||
|
DENSE_TOP_N=20
|
||||||
|
|
||||||
RERANKER_PROVIDER=qianfan
|
RERANKER_PROVIDER=qianfan
|
||||||
RERANKER_MODEL=
|
RERANKER_MODEL=bce-reranker-base
|
||||||
RERANKER_ENDPOINT=
|
RERANKER_ENDPOINT=https://qianfan.baidubce.com/v2/rerank
|
||||||
RERANKER_API_KEY=
|
RERANKER_API_KEY=
|
||||||
|
RERANKER_TIMEOUT_SECONDS=30
|
||||||
|
SEARCH_TOP_K=5
|
||||||
|
|
||||||
|
# DEMO-012 MCP gateway. Inject MCP_AUTH_TOKEN at runtime; never commit its value.
|
||||||
|
MCP_HOST=127.0.0.1
|
||||||
|
MCP_PORT=8766
|
||||||
|
MCP_PATH=/mcp
|
||||||
|
MCP_PUBLIC_URL=http://127.0.0.1:8766/mcp
|
||||||
|
MCP_ISSUER_URL=http://127.0.0.1:8766
|
||||||
|
MCP_AUTH_TOKEN=
|
||||||
|
MCP_CLIENT_ID=workbuddy-enterprise
|
||||||
|
MCP_SERVICE_IDENTITY=workbuddy-demo
|
||||||
|
MCP_REQUIRED_SCOPE=knowledge:search
|
||||||
|
MCP_ALLOWED_HOSTS='["127.0.0.1:*","localhost:*","[::1]:*","testserver"]'
|
||||||
|
MCP_ALLOWED_ORIGINS='["http://127.0.0.1:*","http://localhost:*","http://[::1]:*"]'
|
||||||
|
MCP_MAX_REQUEST_BODY_BYTES=1048576
|
||||||
|
|
||||||
|
# Controlled ARCH-006 acceptance; keep disabled outside an authorized test run.
|
||||||
|
EKP_RUN_LIVE_TESTS=0
|
||||||
|
EKP_CREDENTIALS_AUTHORIZED=0
|
||||||
|
EKP_LIVE_RESOURCE_ACK=0
|
||||||
|
EKP_LIVE_MIN_RECALL_AT_1=0.90
|
||||||
|
EKP_LIVE_MIN_RECALL_AT_5=1.0
|
||||||
|
EKP_LIVE_MIN_MRR=0.90
|
||||||
|
EKP_LIVE_MIN_NDCG_AT_5=0.90
|
||||||
|
EKP_LIVE_MAX_P95_MS=10000
|
||||||
|
|
||||||
LLM_PROVIDER=qianfan
|
LLM_PROVIDER=qianfan
|
||||||
LLM_MODEL=
|
LLM_MODEL=
|
||||||
|
|||||||
@ -19,8 +19,8 @@ Do not redesign accepted architecture unless the task explicitly asks for an arc
|
|||||||
|
|
||||||
1. API First.
|
1. API First.
|
||||||
2. V1 uses a modular monolith plus independent async/GPU workers and an MCP gateway.
|
2. V1 uses a modular monolith plus independent async/GPU workers and an MCP gateway.
|
||||||
3. PostgreSQL + object storage hold canonical data.
|
3. OceanBase relational tables + object storage hold canonical data.
|
||||||
4. Vector databases are rebuildable indexes, not sources of truth.
|
4. OceanBase vector tables are rebuildable indexes, not sources of truth, even when co-located with canonical relational tables.
|
||||||
5. Business services depend on `VectorStore` abstractions, never directly on Mochow.
|
5. Business services depend on `VectorStore` abstractions, never directly on Mochow.
|
||||||
6. Embedding, reranking, parsing, OCR, object storage, and LLMs use provider interfaces.
|
6. Embedding, reranking, parsing, OCR, object storage, and LLMs use provider interfaces.
|
||||||
7. Original source documents are immutable.
|
7. Original source documents are immutable.
|
||||||
|
|||||||
33
Makefile
33
Makefile
@ -1,4 +1,4 @@
|
|||||||
.PHONY: help install run format format-check lint typecheck test test-unit test-integration migration-check eval up down
|
.PHONY: help install run run-mcp run-worker images images-print images-push deploy deploy-lint deploy-render format format-check lint typecheck test test-unit test-integration migration-check eval live-acceptance up down
|
||||||
|
|
||||||
UV_CACHE_DIR ?= /tmp/ekp-uv-cache
|
UV_CACHE_DIR ?= /tmp/ekp-uv-cache
|
||||||
export UV_CACHE_DIR
|
export UV_CACHE_DIR
|
||||||
@ -7,6 +7,14 @@ help:
|
|||||||
@echo "Enterprise Knowledge Platform"
|
@echo "Enterprise Knowledge Platform"
|
||||||
@echo " install Install development dependencies"
|
@echo " install Install development dependencies"
|
||||||
@echo " run Start the development API"
|
@echo " run Start the development API"
|
||||||
|
@echo " run-mcp Start the authenticated MCP gateway"
|
||||||
|
@echo " run-worker Start the ingestion worker and recovery scheduler"
|
||||||
|
@echo " images Build production images for the current builder"
|
||||||
|
@echo " images-print Render the multi-architecture image build plan"
|
||||||
|
@echo " images-push Build and push amd64/arm64 production images"
|
||||||
|
@echo " deploy Install or upgrade Kubernetes with VALUES=<file>"
|
||||||
|
@echo " deploy-lint Validate Helm values and templates"
|
||||||
|
@echo " deploy-render Render non-secret Kubernetes manifests"
|
||||||
@echo " format Format code"
|
@echo " format Format code"
|
||||||
@echo " format-check Check formatting without modifying files"
|
@echo " format-check Check formatting without modifying files"
|
||||||
@echo " lint Run linters"
|
@echo " lint Run linters"
|
||||||
@ -16,6 +24,7 @@ help:
|
|||||||
@echo " test-integration Run integration tests"
|
@echo " test-integration Run integration tests"
|
||||||
@echo " migration-check Validate the forward migration path"
|
@echo " migration-check Validate the forward migration path"
|
||||||
@echo " eval Run RAG evaluation"
|
@echo " eval Run RAG evaluation"
|
||||||
|
@echo " live-acceptance Run controlled target-provider acceptance"
|
||||||
@echo " up Start local dependencies"
|
@echo " up Start local dependencies"
|
||||||
@echo " down Stop local dependencies"
|
@echo " down Stop local dependencies"
|
||||||
|
|
||||||
@ -23,6 +32,24 @@ install:
|
|||||||
uv sync --all-groups
|
uv sync --all-groups
|
||||||
run:
|
run:
|
||||||
uv run uvicorn ekp_api.main:app --host "$${API_HOST:-0.0.0.0}" --port "$${API_PORT:-8000}"
|
uv run uvicorn ekp_api.main:app --host "$${API_HOST:-0.0.0.0}" --port "$${API_PORT:-8000}"
|
||||||
|
run-mcp:
|
||||||
|
uv run ekp-mcp
|
||||||
|
run-worker:
|
||||||
|
uv run ekp-worker
|
||||||
|
images:
|
||||||
|
docker buildx bake -f deploy/docker/docker-bake.hcl
|
||||||
|
images-print:
|
||||||
|
docker buildx bake -f deploy/docker/docker-bake.hcl --print
|
||||||
|
images-push:
|
||||||
|
docker buildx bake -f deploy/docker/docker-bake.hcl --push
|
||||||
|
deploy:
|
||||||
|
./scripts/deploy.sh "$(VALUES)"
|
||||||
|
deploy-lint:
|
||||||
|
@test -n "$(VALUES)" || (echo "VALUES must point to an environment values file" >&2; exit 2)
|
||||||
|
helm lint deploy/helm/knowledge-platform --values "$(VALUES)"
|
||||||
|
deploy-render:
|
||||||
|
@test -n "$(VALUES)" || (echo "VALUES must point to an environment values file" >&2; exit 2)
|
||||||
|
helm template "$${RELEASE:-knowledge-platform}" deploy/helm/knowledge-platform --namespace "$${NAMESPACE:-knowledge-platform}" --values "$(VALUES)"
|
||||||
format:
|
format:
|
||||||
uv run ruff format .
|
uv run ruff format .
|
||||||
format-check:
|
format-check:
|
||||||
@ -30,7 +57,7 @@ format-check:
|
|||||||
lint:
|
lint:
|
||||||
uv run ruff check .
|
uv run ruff check .
|
||||||
typecheck:
|
typecheck:
|
||||||
uv run mypy apps/api/src packages tests
|
uv run mypy apps/api/src apps/mcp-server/src packages tests
|
||||||
test:
|
test:
|
||||||
uv run pytest tests/unit tests/integration
|
uv run pytest tests/unit tests/integration
|
||||||
test-unit:
|
test-unit:
|
||||||
@ -42,6 +69,8 @@ migration-check:
|
|||||||
uv run alembic upgrade head --sql >/dev/null
|
uv run alembic upgrade head --sql >/dev/null
|
||||||
eval:
|
eval:
|
||||||
uv run pytest tests/evaluation
|
uv run pytest tests/evaluation
|
||||||
|
live-acceptance:
|
||||||
|
uv run ekp-live-acceptance
|
||||||
up:
|
up:
|
||||||
docker compose -f docker-compose.dev.yml up -d
|
docker compose -f docker-compose.dev.yml up -d
|
||||||
down:
|
down:
|
||||||
|
|||||||
91
README.md
91
README.md
@ -19,8 +19,9 @@ Enterprise internal knowledge infrastructure.
|
|||||||
|
|
||||||
- API First + Admin Console
|
- API First + Admin Console
|
||||||
- Modular Monolith + async workers + MCP gateway
|
- Modular Monolith + async workers + MCP gateway
|
||||||
- PostgreSQL + object storage as canonical source of truth
|
- OceanBase MySQL relational tables + object storage as canonical source of truth
|
||||||
- Mochow as the initial retrieval index, behind a replaceable adapter
|
- Co-located OceanBase Vector tables as rebuildable retrieval indexes behind an adapter
|
||||||
|
- Baidu BOS as target object storage; local storage remains for tests/development
|
||||||
- Embedding / reranker / parser / LLM providers are replaceable
|
- Embedding / reranker / parser / LLM providers are replaceable
|
||||||
- Raw source is immutable
|
- Raw source is immutable
|
||||||
- Compiled Knowledge is derived and traceable
|
- Compiled Knowledge is derived and traceable
|
||||||
@ -45,19 +46,40 @@ The current implementation target is the retrieval demo defined in
|
|||||||
Prerequisites:
|
Prerequisites:
|
||||||
|
|
||||||
- `uv`
|
- `uv`
|
||||||
- Docker (for PostgreSQL and later integration dependencies)
|
- Docker (for optional local integration dependencies)
|
||||||
|
|
||||||
Install dependencies and run the API:
|
Install dependencies and run the API:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
cp .env.example .env
|
cp .env.example .env
|
||||||
make install
|
make install
|
||||||
|
make run-worker
|
||||||
make run
|
make run
|
||||||
```
|
```
|
||||||
|
|
||||||
|
Run the independent authenticated MCP Gateway after injecting a dedicated
|
||||||
|
runtime token of at least 32 characters:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
export MCP_AUTH_TOKEN="<runtime-secret-with-at-least-32-characters>"
|
||||||
|
make run-mcp
|
||||||
|
```
|
||||||
|
|
||||||
|
The Streamable HTTP endpoint defaults to `http://127.0.0.1:8766/mcp` and exposes
|
||||||
|
only `search_knowledge`. See `docs/development/MCP_SETUP.md` for WorkBuddy configuration,
|
||||||
|
identity boundaries, remote allowlists, and protocol verification.
|
||||||
|
|
||||||
The health endpoint is available at `http://localhost:8000/healthz`, and the
|
The health endpoint is available at `http://localhost:8000/healthz`, and the
|
||||||
versioned OpenAPI document is available at
|
versioned OpenAPI document is available at
|
||||||
`http://localhost:8000/api/v1/openapi.json`.
|
`http://localhost:8000/api/v1/openapi.json`.
|
||||||
|
The Demo Admin Console is available at `http://localhost:8000/admin` and contains
|
||||||
|
only document upload, document management, and Search Lab pages. See
|
||||||
|
`docs/development/ADMIN_CONSOLE.md` for the operation and scope boundaries.
|
||||||
|
|
||||||
|
DEPLOY-001 provides locked, non-root `api`, `mcp`, and `worker` production image targets with an
|
||||||
|
amd64/arm64 Buildx Bake plan. DEPLOY-002 adds the Kubernetes/Helm Chart, runtime Secret references,
|
||||||
|
serialized Migration, Ingress/TLS and the `make deploy VALUES=<file>` entrypoint. See
|
||||||
|
`docs/development/CONTAINER_IMAGES.md` and `docs/development/KUBERNETES_DEPLOYMENT.md`.
|
||||||
|
|
||||||
Run the development checks with:
|
Run the development checks with:
|
||||||
|
|
||||||
@ -65,7 +87,64 @@ Run the development checks with:
|
|||||||
make format-check lint typecheck test migration-check eval
|
make format-check lint typecheck test migration-check eval
|
||||||
```
|
```
|
||||||
|
|
||||||
Document metadata is canonical in PostgreSQL. Demo source files use the local
|
Controlled target-provider acceptance uses 32 synthetic queries and is disabled by
|
||||||
|
default. It requires rotated secrets, dedicated test resources, and all safety
|
||||||
|
acknowledgements described in `docs/development/LIVE_ACCEPTANCE.md`; only then run
|
||||||
|
`make live-acceptance`.
|
||||||
|
|
||||||
|
Document metadata is canonical in OceanBase MySQL relational tables. Demo source files use the local
|
||||||
`storage/` directory through the provider-neutral `ObjectStorage` interface;
|
`storage/` directory through the provider-neutral `ObjectStorage` interface;
|
||||||
existing objects cannot be overwritten. Upload endpoints are introduced by
|
existing objects cannot be overwritten. The upload service accepts PDF, DOCX,
|
||||||
DEMO-003, not by the current storage foundation.
|
TXT, and MD sources, streams at most `MAX_UPLOAD_BYTES`, and compensates stored
|
||||||
|
bytes if metadata persistence fails. The ingestion service then routes PDF to
|
||||||
|
PyMuPDF, DOCX to python-docx, and UTF-8 TXT/Markdown to lightweight parsers. It
|
||||||
|
normalizes whitespace, retains available page/section context, and stores a
|
||||||
|
versioned parsed JSON artifact under `parsed/{document_id}/document.json`.
|
||||||
|
The `structure_v1` chunk strategy keeps heading/section boundaries, combines
|
||||||
|
paragraphs, and creates configurable 450-token windows with a 75-token overlap.
|
||||||
|
Chunk metadata, including deterministic IDs, filename, page range, section,
|
||||||
|
index, and strategy version, is stored atomically in canonical relational tables. The lightweight
|
||||||
|
token counter is replaceable so DEMO-006 can inject model-specific behavior.
|
||||||
|
Documents remain `processing` until embedding/indexing completes; pipeline
|
||||||
|
failures become `failed` without deleting the immutable original. DEMO-015 now
|
||||||
|
accepts a source plus durable Job with HTTP `202`, while a Celery/RabbitMQ worker
|
||||||
|
runs the pipeline. Bounded transient retries clean every derived artifact before
|
||||||
|
starting the next Attempt; broker redelivery and expired worker leases converge on
|
||||||
|
OceanBase Job state. See `docs/development/ASYNC_INGESTION_SETUP.md`.
|
||||||
|
|
||||||
|
DEMO-006 added a provider-neutral `EmbeddingProvider` and `VectorStore`, plus a
|
||||||
|
validated Qianfan `bge-large-zh` adapter. ARCH-005 now wires that boundary to OceanBase.
|
||||||
|
Chunk vectors are upserted with source metadata under a versioned index profile;
|
||||||
|
OceanBase relational tables record canonical index state while vector rows remain rebuildable. A
|
||||||
|
document becomes `ready` only after every chunk is indexed. See
|
||||||
|
`docs/development/OCEANBASE_VECTOR_SETUP.md` for the required environment variables
|
||||||
|
and pre-provisioned table schema.
|
||||||
|
|
||||||
|
The accepted provider migration is tracked by `ARCH-001` through `ARCH-006`:
|
||||||
|
Baidu BOS, OceanBase Vector, and Qianfan `bce-reranker-base` replace the initial
|
||||||
|
demo providers, and ARCH-006 aligns canonical ownership with the verified OceanBase target. BOS and OceanBase
|
||||||
|
are now the runtime defaults; LocalObjectStorage and FakeVectorStore remain available
|
||||||
|
for explicit local/offline tests. ARCH-005 also adds the controlled shadow rebuild
|
||||||
|
workflow documented in `docs/development/INDEX_REBUILD.md`.
|
||||||
|
|
||||||
|
DEMO-007 adds query embedding and profile-scoped Dense Top-N retrieval. The
|
||||||
|
default is Top 20; the in-memory fake uses
|
||||||
|
exact cosine search for deterministic offline regression. Returned objects are
|
||||||
|
raw dense candidates only.
|
||||||
|
|
||||||
|
DEMO-008 adds a provider-neutral reranker, and ARCH-004 aligns its validated Qianfan
|
||||||
|
adapter with the `bce-reranker-base` baseline. The final Top20-to-Top5 pipeline and
|
||||||
|
source results are rehydrated from canonical relational chunks and retain both
|
||||||
|
Dense and reranker scores; stale vector content fails closed. See
|
||||||
|
`docs/development/RERANKER_SETUP.md` for configuration and validation limits.
|
||||||
|
DEMO-009 exposes document and search REST endpoints under `/api/v1`; DEMO-015
|
||||||
|
changes upload to asynchronous acceptance and adds Job status. DEMO-016 adds bounded multi-file
|
||||||
|
submission, batch/per-file status, and automatic Admin Console polling. Upload, list, detail, direct delete,
|
||||||
|
and reranked source search are documented in
|
||||||
|
`docs/development/REST_API_SETUP.md`; provider and storage internals remain
|
||||||
|
outside the public response contract.
|
||||||
|
|
||||||
|
DEMO-012 adds the independent authenticated MCP Gateway. It uses the official
|
||||||
|
MCP Python SDK, maps a runtime-only Bearer credential to a least-privilege service
|
||||||
|
identity, and delegates `search_knowledge` to the same `SearchService` used by REST.
|
||||||
|
No provider, storage, database, model, or index-profile details are exposed.
|
||||||
|
|||||||
411
apps/admin-web/assets/app.js
Normal file
411
apps/admin-web/assets/app.js
Normal file
@ -0,0 +1,411 @@
|
|||||||
|
// The Admin Console owns presentation state only. Batch acceptance, Job state,
|
||||||
|
// and retrieval remain behind the versioned public REST contract.
|
||||||
|
const API_BASE = "/api/v1";
|
||||||
|
const BATCH_POLL_INTERVAL_MS = 1500;
|
||||||
|
const terminalBatchStatuses = new Set(["completed", "partial_failure", "failed"]);
|
||||||
|
let batchPollTimer;
|
||||||
|
let activeBatchId;
|
||||||
|
|
||||||
|
const statusLabels = {
|
||||||
|
uploaded: "Uploaded",
|
||||||
|
processing: "Processing",
|
||||||
|
ready: "Ready",
|
||||||
|
failed: "Failed",
|
||||||
|
pending: "Pending",
|
||||||
|
accepted: "Accepted",
|
||||||
|
rejected: "Rejected",
|
||||||
|
queued: "Queued",
|
||||||
|
running: "Running",
|
||||||
|
retrying: "Retrying",
|
||||||
|
succeeded: "Succeeded",
|
||||||
|
completed: "Completed",
|
||||||
|
partial_failure: "Partial failure",
|
||||||
|
};
|
||||||
|
|
||||||
|
const stageLabels = {
|
||||||
|
parsing: "解析",
|
||||||
|
cleaning: "清理",
|
||||||
|
chunking: "切分",
|
||||||
|
embedding: "向量化",
|
||||||
|
indexing: "索引",
|
||||||
|
};
|
||||||
|
|
||||||
|
const pageTitles = {
|
||||||
|
upload: "上传文档",
|
||||||
|
documents: "文档列表",
|
||||||
|
search: "Search Lab",
|
||||||
|
};
|
||||||
|
|
||||||
|
function currentPage() {
|
||||||
|
const route = window.location.pathname.split("/").filter(Boolean).at(-1);
|
||||||
|
return Object.hasOwn(pageTitles, route) ? route : "upload";
|
||||||
|
}
|
||||||
|
|
||||||
|
function showPage(page) {
|
||||||
|
document.querySelectorAll("[data-page]").forEach((section) => {
|
||||||
|
section.hidden = section.dataset.page !== page;
|
||||||
|
});
|
||||||
|
document.querySelectorAll("[data-nav]").forEach((link) => {
|
||||||
|
const active = link.dataset.nav === page;
|
||||||
|
link.classList.toggle("active", active);
|
||||||
|
if (active) link.setAttribute("aria-current", "page");
|
||||||
|
else link.removeAttribute("aria-current");
|
||||||
|
});
|
||||||
|
document.querySelector("#page-title").textContent = pageTitles[page];
|
||||||
|
}
|
||||||
|
|
||||||
|
function showNotice(message, kind = "success") {
|
||||||
|
const notice = document.querySelector("#notice");
|
||||||
|
notice.textContent = message;
|
||||||
|
notice.className = `notice ${kind}`;
|
||||||
|
notice.hidden = false;
|
||||||
|
window.clearTimeout(showNotice.timeout);
|
||||||
|
showNotice.timeout = window.setTimeout(() => {
|
||||||
|
notice.hidden = true;
|
||||||
|
}, 6000);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function requestJson(path, options = {}) {
|
||||||
|
const response = await fetch(`${API_BASE}${path}`, options);
|
||||||
|
if (!response.ok) {
|
||||||
|
let message = `请求失败(HTTP ${response.status})`;
|
||||||
|
try {
|
||||||
|
const body = await response.json();
|
||||||
|
if (body.detail) message = body.detail;
|
||||||
|
} catch {
|
||||||
|
// A non-JSON upstream error is represented by the stable HTTP message above.
|
||||||
|
}
|
||||||
|
throw new Error(message);
|
||||||
|
}
|
||||||
|
return response.status === 204 ? null : response.json();
|
||||||
|
}
|
||||||
|
|
||||||
|
function formatDate(value) {
|
||||||
|
if (!value) return "—";
|
||||||
|
return new Intl.DateTimeFormat("zh-CN", {
|
||||||
|
dateStyle: "medium",
|
||||||
|
timeStyle: "short",
|
||||||
|
}).format(new Date(value));
|
||||||
|
}
|
||||||
|
|
||||||
|
function formatSize(bytes) {
|
||||||
|
if (bytes < 1024) return `${bytes} B`;
|
||||||
|
if (bytes < 1024 ** 2) return `${(bytes / 1024).toFixed(1)} KB`;
|
||||||
|
return `${(bytes / 1024 ** 2).toFixed(1)} MB`;
|
||||||
|
}
|
||||||
|
|
||||||
|
function statusPill(status) {
|
||||||
|
const pill = document.createElement("span");
|
||||||
|
pill.className = `status-pill status-${status}`;
|
||||||
|
pill.textContent = statusLabels[status] ?? status;
|
||||||
|
return pill;
|
||||||
|
}
|
||||||
|
|
||||||
|
function setButtonBusy(button, busy, busyText) {
|
||||||
|
if (!button.dataset.label) button.dataset.label = button.textContent;
|
||||||
|
button.disabled = busy;
|
||||||
|
button.textContent = busy ? busyText : button.dataset.label;
|
||||||
|
}
|
||||||
|
|
||||||
|
function initializeUpload() {
|
||||||
|
const form = document.querySelector("#upload-form");
|
||||||
|
const input = document.querySelector("#file-input");
|
||||||
|
const selected = document.querySelector("#selected-file");
|
||||||
|
const button = document.querySelector("#upload-button");
|
||||||
|
|
||||||
|
const updateFilename = () => {
|
||||||
|
const files = Array.from(input.files);
|
||||||
|
if (!files.length) {
|
||||||
|
selected.textContent = "尚未选择文件";
|
||||||
|
} else if (files.length === 1) {
|
||||||
|
selected.textContent = files[0].name;
|
||||||
|
} else {
|
||||||
|
selected.textContent = `已选择 ${files.length} 个文件`;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
input.addEventListener("change", updateFilename);
|
||||||
|
|
||||||
|
form.addEventListener("submit", async (event) => {
|
||||||
|
event.preventDefault();
|
||||||
|
if (!input.files.length) return;
|
||||||
|
window.clearTimeout(batchPollTimer);
|
||||||
|
activeBatchId = undefined;
|
||||||
|
setButtonBusy(button, true, "提交中…");
|
||||||
|
try {
|
||||||
|
const formData = new FormData();
|
||||||
|
Array.from(input.files).forEach((file) => formData.append("files", file));
|
||||||
|
const batch = await requestJson("/ingestion-batches", {
|
||||||
|
method: "POST",
|
||||||
|
body: formData,
|
||||||
|
});
|
||||||
|
activeBatchId = batch.batch_id;
|
||||||
|
renderBatchResult(batch);
|
||||||
|
scheduleBatchPoll(batch);
|
||||||
|
showNotice(
|
||||||
|
`批次已接收:${batch.accepted_files} 个进入后台队列,${batch.rejected_files} 个被拒绝。`,
|
||||||
|
batch.rejected_files ? "error" : "success",
|
||||||
|
);
|
||||||
|
form.reset();
|
||||||
|
updateFilename();
|
||||||
|
} catch (error) {
|
||||||
|
showNotice(`上传失败:${error.message}`, "error");
|
||||||
|
} finally {
|
||||||
|
setButtonBusy(button, false, "");
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function scheduleBatchPoll(batch) {
|
||||||
|
window.clearTimeout(batchPollTimer);
|
||||||
|
if (batch.batch_id !== activeBatchId) return;
|
||||||
|
if (terminalBatchStatuses.has(batch.status)) return;
|
||||||
|
batchPollTimer = window.setTimeout(async () => {
|
||||||
|
try {
|
||||||
|
const latest = await requestJson(`/ingestion-batches/${batch.batch_id}`);
|
||||||
|
renderBatchResult(latest);
|
||||||
|
scheduleBatchPoll(latest);
|
||||||
|
} catch (error) {
|
||||||
|
showNotice(`批次状态刷新失败:${error.message}`, "error");
|
||||||
|
scheduleBatchPoll(batch);
|
||||||
|
}
|
||||||
|
}, BATCH_POLL_INTERVAL_MS);
|
||||||
|
}
|
||||||
|
|
||||||
|
function renderBatchResult(batch) {
|
||||||
|
const card = document.querySelector("#upload-result");
|
||||||
|
document.querySelector("#upload-batch-title").textContent =
|
||||||
|
`批次 ${batch.batch_id.slice(0, 8)} · ${formatDate(batch.created_at)}`;
|
||||||
|
document.querySelector("#upload-total").textContent = `${batch.total_files} 个`;
|
||||||
|
document.querySelector("#upload-completed").textContent =
|
||||||
|
`${batch.completed_files} / ${batch.total_files}`;
|
||||||
|
document.querySelector("#upload-accepted").textContent =
|
||||||
|
`${batch.accepted_files} / ${batch.rejected_files}`;
|
||||||
|
document.querySelector("#upload-status").replaceWith(statusPill(batch.status));
|
||||||
|
const status = card.querySelector(".status-pill");
|
||||||
|
status.id = "upload-status";
|
||||||
|
const items = document.querySelector("#batch-items");
|
||||||
|
items.replaceChildren(...batch.items.map(batchItem));
|
||||||
|
card.hidden = false;
|
||||||
|
}
|
||||||
|
|
||||||
|
function batchItem(item) {
|
||||||
|
const row = document.createElement("article");
|
||||||
|
row.className = "batch-item";
|
||||||
|
|
||||||
|
const order = document.createElement("span");
|
||||||
|
order.className = "batch-item-order";
|
||||||
|
order.textContent = String(item.item_index + 1).padStart(2, "0");
|
||||||
|
|
||||||
|
const body = document.createElement("div");
|
||||||
|
body.className = "batch-item-body";
|
||||||
|
const name = document.createElement("strong");
|
||||||
|
name.textContent = item.filename;
|
||||||
|
const detail = document.createElement("span");
|
||||||
|
const stage = item.stage ? ` · ${stageLabels[item.stage] ?? item.stage}` : "";
|
||||||
|
const attempts = item.max_attempts == null ? "" : ` · 尝试 ${item.attempt}/${item.max_attempts}`;
|
||||||
|
detail.textContent = item.error_message ?? `后台任务${stage}${attempts}`;
|
||||||
|
body.append(name, detail);
|
||||||
|
|
||||||
|
const state = item.job_status ?? item.acceptance_status;
|
||||||
|
row.append(order, body, statusPill(state));
|
||||||
|
return row;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function loadDocuments() {
|
||||||
|
const refresh = document.querySelector("#refresh-documents");
|
||||||
|
setButtonBusy(refresh, true, "加载中…");
|
||||||
|
try {
|
||||||
|
const documents = await requestJson("/documents");
|
||||||
|
renderDocuments(documents);
|
||||||
|
} catch (error) {
|
||||||
|
showNotice(`文档列表加载失败:${error.message}`, "error");
|
||||||
|
} finally {
|
||||||
|
setButtonBusy(refresh, false, "");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function renderDocuments(documents) {
|
||||||
|
const rows = document.querySelector("#document-rows");
|
||||||
|
const empty = document.querySelector("#document-empty");
|
||||||
|
rows.replaceChildren();
|
||||||
|
empty.hidden = documents.length !== 0;
|
||||||
|
document.querySelector("#document-count").textContent = `${documents.length} 个文档`;
|
||||||
|
document.querySelector("#document-refresh-time").textContent =
|
||||||
|
`更新于 ${new Intl.DateTimeFormat("zh-CN", { timeStyle: "short" }).format(new Date())}`;
|
||||||
|
|
||||||
|
documents.forEach((documentData) => {
|
||||||
|
const row = document.createElement("tr");
|
||||||
|
row.append(
|
||||||
|
tableCell(documentData.filename, "filename-cell"),
|
||||||
|
tableCell(documentData.media_type),
|
||||||
|
tableCell(formatDate(documentData.created_at)),
|
||||||
|
);
|
||||||
|
const statusCell = document.createElement("td");
|
||||||
|
statusCell.append(statusPill(documentData.status));
|
||||||
|
row.append(statusCell);
|
||||||
|
|
||||||
|
const actions = document.createElement("td");
|
||||||
|
actions.className = "row-actions";
|
||||||
|
const view = actionButton("查看", () => showDocumentDetail(documentData));
|
||||||
|
const remove = actionButton("删除", () => deleteDocument(documentData), "danger-link");
|
||||||
|
actions.append(view, remove);
|
||||||
|
row.append(actions);
|
||||||
|
rows.append(row);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function tableCell(value, className = "") {
|
||||||
|
const cell = document.createElement("td");
|
||||||
|
cell.textContent = value;
|
||||||
|
cell.className = className;
|
||||||
|
return cell;
|
||||||
|
}
|
||||||
|
|
||||||
|
function actionButton(label, handler, className = "") {
|
||||||
|
const button = document.createElement("button");
|
||||||
|
button.type = "button";
|
||||||
|
button.className = `table-action ${className}`;
|
||||||
|
button.textContent = label;
|
||||||
|
button.addEventListener("click", handler);
|
||||||
|
return button;
|
||||||
|
}
|
||||||
|
|
||||||
|
function showDocumentDetail(documentData) {
|
||||||
|
const dialog = document.querySelector("#document-dialog");
|
||||||
|
document.querySelector("#dialog-filename").textContent = documentData.filename;
|
||||||
|
const metadata = document.querySelector("#dialog-metadata");
|
||||||
|
metadata.replaceChildren();
|
||||||
|
[
|
||||||
|
["文档 ID", documentData.id],
|
||||||
|
["文件类型", documentData.media_type],
|
||||||
|
["文件大小", formatSize(documentData.size_bytes)],
|
||||||
|
["当前状态", statusLabels[documentData.status] ?? documentData.status],
|
||||||
|
["上传时间", formatDate(documentData.created_at)],
|
||||||
|
["更新时间", formatDate(documentData.updated_at)],
|
||||||
|
["错误信息", documentData.error_message ?? "无"],
|
||||||
|
].forEach(([term, value]) => {
|
||||||
|
const wrapper = document.createElement("div");
|
||||||
|
const dt = document.createElement("dt");
|
||||||
|
const dd = document.createElement("dd");
|
||||||
|
dt.textContent = term;
|
||||||
|
dd.textContent = value;
|
||||||
|
wrapper.append(dt, dd);
|
||||||
|
metadata.append(wrapper);
|
||||||
|
});
|
||||||
|
dialog.showModal();
|
||||||
|
}
|
||||||
|
|
||||||
|
async function deleteDocument(documentData) {
|
||||||
|
if (!window.confirm(`确认删除“${documentData.filename}”及其索引数据?`)) return;
|
||||||
|
try {
|
||||||
|
await requestJson(`/documents/${documentData.id}`, { method: "DELETE" });
|
||||||
|
showNotice(`已删除 ${documentData.filename}`, "success");
|
||||||
|
await loadDocuments();
|
||||||
|
} catch (error) {
|
||||||
|
showNotice(`删除失败:${error.message}`, "error");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function initializeSearch() {
|
||||||
|
const form = document.querySelector("#search-form");
|
||||||
|
const button = document.querySelector("#search-button");
|
||||||
|
form.addEventListener("submit", async (event) => {
|
||||||
|
event.preventDefault();
|
||||||
|
const query = document.querySelector("#search-query").value.trim();
|
||||||
|
const topK = Number(document.querySelector("#search-top-k").value);
|
||||||
|
if (!query) return;
|
||||||
|
setButtonBusy(button, true, "检索中…");
|
||||||
|
try {
|
||||||
|
const startedAt = performance.now();
|
||||||
|
const response = await requestJson("/search", {
|
||||||
|
method: "POST",
|
||||||
|
headers: { "Content-Type": "application/json" },
|
||||||
|
body: JSON.stringify({ query, top_k: topK }),
|
||||||
|
});
|
||||||
|
renderSearchResults(response.results, performance.now() - startedAt);
|
||||||
|
} catch (error) {
|
||||||
|
showNotice(`检索失败:${error.message}`, "error");
|
||||||
|
} finally {
|
||||||
|
setButtonBusy(button, false, "");
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function renderSearchResults(results, elapsedMs) {
|
||||||
|
const container = document.querySelector("#search-results");
|
||||||
|
container.replaceChildren();
|
||||||
|
document.querySelector("#search-summary").textContent =
|
||||||
|
`${results.length} 条结果 · ${(elapsedMs / 1000).toFixed(2)} 秒`;
|
||||||
|
if (!results.length) {
|
||||||
|
const empty = document.createElement("div");
|
||||||
|
empty.className = "empty-state search-empty";
|
||||||
|
const strong = document.createElement("strong");
|
||||||
|
strong.textContent = "没有找到匹配内容";
|
||||||
|
const detail = document.createElement("span");
|
||||||
|
detail.textContent = "可以换一种问法,或先确认文档状态为 Ready。";
|
||||||
|
empty.append(strong, detail);
|
||||||
|
container.append(empty);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
results.forEach((result, index) => {
|
||||||
|
const card = document.createElement("article");
|
||||||
|
card.className = "search-result-card";
|
||||||
|
const rank = document.createElement("span");
|
||||||
|
rank.className = "rank";
|
||||||
|
rank.textContent = String(index + 1).padStart(2, "0");
|
||||||
|
const body = document.createElement("div");
|
||||||
|
body.className = "result-body";
|
||||||
|
const heading = document.createElement("div");
|
||||||
|
heading.className = "result-title-row";
|
||||||
|
const title = document.createElement("h4");
|
||||||
|
title.textContent = result.filename;
|
||||||
|
const sourceMeta = document.createElement("div");
|
||||||
|
sourceMeta.className = "source-meta";
|
||||||
|
sourceMeta.append(
|
||||||
|
metadataItem("Page", result.page == null ? "—" : String(result.page)),
|
||||||
|
metadataItem("Section", result.section ?? "—"),
|
||||||
|
);
|
||||||
|
heading.append(title, sourceMeta);
|
||||||
|
const content = document.createElement("p");
|
||||||
|
content.className = "chunk-content";
|
||||||
|
content.textContent = result.content;
|
||||||
|
const scores = document.createElement("div");
|
||||||
|
scores.className = "score-row";
|
||||||
|
scores.append(
|
||||||
|
scoreItem("Dense Score", result.score),
|
||||||
|
scoreItem("Rerank Score", result.rerank_score),
|
||||||
|
);
|
||||||
|
body.append(heading, content, scores);
|
||||||
|
card.append(rank, body);
|
||||||
|
container.append(card);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function scoreItem(label, value) {
|
||||||
|
const item = document.createElement("span");
|
||||||
|
const labelNode = document.createElement("small");
|
||||||
|
const valueNode = document.createElement("strong");
|
||||||
|
labelNode.textContent = label;
|
||||||
|
valueNode.textContent = Number(value).toFixed(4);
|
||||||
|
item.append(labelNode, valueNode);
|
||||||
|
return item;
|
||||||
|
}
|
||||||
|
|
||||||
|
function metadataItem(label, value) {
|
||||||
|
const item = document.createElement("span");
|
||||||
|
const labelNode = document.createElement("small");
|
||||||
|
const valueNode = document.createElement("strong");
|
||||||
|
labelNode.textContent = label;
|
||||||
|
valueNode.textContent = value;
|
||||||
|
item.append(labelNode, valueNode);
|
||||||
|
return item;
|
||||||
|
}
|
||||||
|
|
||||||
|
const page = currentPage();
|
||||||
|
showPage(page);
|
||||||
|
initializeUpload();
|
||||||
|
initializeSearch();
|
||||||
|
document.querySelector("#refresh-documents").addEventListener("click", loadDocuments);
|
||||||
|
if (page === "documents") loadDocuments();
|
||||||
|
window.addEventListener("pagehide", () => window.clearTimeout(batchPollTimer));
|
||||||
1039
apps/admin-web/assets/styles.css
Normal file
1039
apps/admin-web/assets/styles.css
Normal file
File diff suppressed because it is too large
Load Diff
201
apps/admin-web/index.html
Normal file
201
apps/admin-web/index.html
Normal file
@ -0,0 +1,201 @@
|
|||||||
|
<!doctype html>
|
||||||
|
<html lang="zh-CN">
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||||
|
<meta
|
||||||
|
name="description"
|
||||||
|
content="Enterprise Knowledge Platform Demo 管理控制台"
|
||||||
|
/>
|
||||||
|
<title>Enterprise Knowledge Platform</title>
|
||||||
|
<link rel="stylesheet" href="/admin/assets/styles.css" />
|
||||||
|
<script type="module" src="/admin/assets/app.js"></script>
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
<div class="app-shell">
|
||||||
|
<aside class="sidebar" aria-label="管理控制台导航">
|
||||||
|
<a class="brand" href="/admin/upload" aria-label="Enterprise Knowledge Platform 首页">
|
||||||
|
<svg class="brand-mark" viewBox="0 0 48 36" aria-hidden="true">
|
||||||
|
<path d="M15 3h17a15 15 0 0 1 0 30H15v-9h17a6 6 0 0 0 0-12H15z" />
|
||||||
|
<circle cx="11" cy="18" r="8" />
|
||||||
|
</svg>
|
||||||
|
<span>
|
||||||
|
<strong>地瓜机器人</strong>
|
||||||
|
<small>Knowledge Platform</small>
|
||||||
|
</span>
|
||||||
|
</a>
|
||||||
|
<div class="workspace-label">
|
||||||
|
<span>CONTROL DESK</span>
|
||||||
|
<strong>企业知识运营台</strong>
|
||||||
|
</div>
|
||||||
|
<nav class="primary-nav">
|
||||||
|
<a data-nav="upload" href="/admin/upload">
|
||||||
|
<span aria-hidden="true">01</span>上传文档
|
||||||
|
</a>
|
||||||
|
<a data-nav="documents" href="/admin/documents">
|
||||||
|
<span aria-hidden="true">02</span>文档列表
|
||||||
|
</a>
|
||||||
|
<a data-nav="search" href="/admin/search">
|
||||||
|
<span aria-hidden="true">03</span>Search Lab
|
||||||
|
</a>
|
||||||
|
</nav>
|
||||||
|
<div class="sidebar-foot">
|
||||||
|
<span class="environment-dot" aria-hidden="true"></span>
|
||||||
|
DEMO ENVIRONMENT
|
||||||
|
</div>
|
||||||
|
</aside>
|
||||||
|
|
||||||
|
<main class="workspace">
|
||||||
|
<header class="topbar">
|
||||||
|
<div>
|
||||||
|
<p class="eyebrow">KNOWLEDGE OPERATIONS</p>
|
||||||
|
<h1 id="page-title">上传文档</h1>
|
||||||
|
</div>
|
||||||
|
<div class="topbar-actions">
|
||||||
|
<span class="system-state"><i aria-hidden="true"></i>API FIRST · V1</span>
|
||||||
|
<a class="api-link" href="/api/v1/docs" target="_blank" rel="noreferrer">
|
||||||
|
API 文档 <span aria-hidden="true">↗</span>
|
||||||
|
</a>
|
||||||
|
</div>
|
||||||
|
</header>
|
||||||
|
<div class="signal-line" aria-hidden="true"><span></span><span></span><span></span></div>
|
||||||
|
|
||||||
|
<div id="notice" class="notice" role="status" aria-live="polite" hidden></div>
|
||||||
|
|
||||||
|
<section id="upload-page" class="page" data-page="upload" aria-labelledby="page-title">
|
||||||
|
<div class="page-heading">
|
||||||
|
<div>
|
||||||
|
<p class="section-label">01 / INGESTION</p>
|
||||||
|
<h2>让知识进入检索链路</h2>
|
||||||
|
<p>批量提交 PDF、DOCX、TXT 或 Markdown 文件,系统将逐个接收并在后台独立完成解析、切分与索引。</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<form id="upload-form" class="upload-panel">
|
||||||
|
<label id="drop-zone" class="drop-zone" for="file-input">
|
||||||
|
<input
|
||||||
|
id="file-input"
|
||||||
|
name="files"
|
||||||
|
type="file"
|
||||||
|
multiple
|
||||||
|
accept=".pdf,.docx,.txt,.md,application/pdf,application/vnd.openxmlformats-officedocument.wordprocessingml.document,text/plain,text/markdown"
|
||||||
|
required
|
||||||
|
/>
|
||||||
|
<span class="upload-icon" aria-hidden="true">↑</span>
|
||||||
|
<strong>选择一批知识文件</strong>
|
||||||
|
<span>支持 PDF、DOCX、TXT、Markdown · 每个文件独立处理</span>
|
||||||
|
<span id="selected-file" class="selected-file">尚未选择文件</span>
|
||||||
|
</label>
|
||||||
|
<button id="upload-button" class="primary-button" type="submit">上传并处理</button>
|
||||||
|
</form>
|
||||||
|
|
||||||
|
<article id="upload-result" class="result-card" hidden>
|
||||||
|
<div>
|
||||||
|
<p class="section-label">LATEST BATCH</p>
|
||||||
|
<h3 id="upload-batch-title"></h3>
|
||||||
|
</div>
|
||||||
|
<span id="upload-status" class="status-pill"></span>
|
||||||
|
<dl class="metadata-grid">
|
||||||
|
<div><dt>总文件</dt><dd id="upload-total"></dd></div>
|
||||||
|
<div><dt>已完成</dt><dd id="upload-completed"></dd></div>
|
||||||
|
<div><dt>接收 / 拒绝</dt><dd id="upload-accepted"></dd></div>
|
||||||
|
</dl>
|
||||||
|
<div id="batch-items" class="batch-items" aria-live="polite"></div>
|
||||||
|
<a class="text-link" href="/admin/search">前往 Search Lab 检索 <span>→</span></a>
|
||||||
|
</article>
|
||||||
|
</section>
|
||||||
|
|
||||||
|
<section id="documents-page" class="page" data-page="documents" aria-labelledby="page-title" hidden>
|
||||||
|
<div class="page-heading heading-row">
|
||||||
|
<div>
|
||||||
|
<p class="section-label">02 / DOCUMENTS</p>
|
||||||
|
<h2>文档列表</h2>
|
||||||
|
<p>查看知识文档的状态、元数据并管理 Demo 数据。</p>
|
||||||
|
</div>
|
||||||
|
<button id="refresh-documents" class="secondary-button" type="button">刷新列表</button>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="table-card">
|
||||||
|
<div class="table-meta">
|
||||||
|
<span id="document-count">0 个文档</span>
|
||||||
|
<span id="document-refresh-time">尚未加载</span>
|
||||||
|
</div>
|
||||||
|
<div class="table-scroll">
|
||||||
|
<table>
|
||||||
|
<thead>
|
||||||
|
<tr>
|
||||||
|
<th>文件名</th>
|
||||||
|
<th>类型</th>
|
||||||
|
<th>上传时间</th>
|
||||||
|
<th>状态</th>
|
||||||
|
<th aria-label="操作"></th>
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
|
<tbody id="document-rows"></tbody>
|
||||||
|
</table>
|
||||||
|
</div>
|
||||||
|
<div id="document-empty" class="empty-state" hidden>
|
||||||
|
<strong>还没有文档</strong>
|
||||||
|
<span>上传第一个文件后,它会出现在这里。</span>
|
||||||
|
<a class="primary-button compact" href="/admin/upload">上传文档</a>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</section>
|
||||||
|
|
||||||
|
<section id="search-page" class="page" data-page="search" aria-labelledby="page-title" hidden>
|
||||||
|
<div class="page-heading">
|
||||||
|
<div>
|
||||||
|
<p class="section-label">03 / RETRIEVAL</p>
|
||||||
|
<h2>Search Lab</h2>
|
||||||
|
<p>输入问题,检查 Dense 召回与 Reranker 精排后的 Top-K 来源。</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<form id="search-form" class="search-panel">
|
||||||
|
<label for="search-query">检索问题</label>
|
||||||
|
<div class="search-controls">
|
||||||
|
<textarea
|
||||||
|
id="search-query"
|
||||||
|
name="query"
|
||||||
|
rows="3"
|
||||||
|
maxlength="1600"
|
||||||
|
placeholder="例如:产品 A 的退款周期是多少?"
|
||||||
|
required
|
||||||
|
></textarea>
|
||||||
|
<label class="top-k-control" for="search-top-k">
|
||||||
|
Top-K
|
||||||
|
<select id="search-top-k" name="top_k">
|
||||||
|
<option value="3">3</option>
|
||||||
|
<option value="5" selected>5</option>
|
||||||
|
<option value="10">10</option>
|
||||||
|
</select>
|
||||||
|
</label>
|
||||||
|
<button id="search-button" class="primary-button" type="submit">开始检索</button>
|
||||||
|
</div>
|
||||||
|
</form>
|
||||||
|
|
||||||
|
<div class="results-heading">
|
||||||
|
<div>
|
||||||
|
<p class="section-label">RESULTS</p>
|
||||||
|
<h3>来源结果</h3>
|
||||||
|
</div>
|
||||||
|
<span id="search-summary">等待检索</span>
|
||||||
|
</div>
|
||||||
|
<div id="search-results" class="search-results" aria-live="polite">
|
||||||
|
<div class="empty-state search-empty">
|
||||||
|
<strong>检索结果会显示在这里</strong>
|
||||||
|
<span>结果包含排名、文件、页码、章节、Chunk 内容和两阶段分数。</span>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</section>
|
||||||
|
</main>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<dialog id="document-dialog" class="document-dialog">
|
||||||
|
<form method="dialog"><button class="icon-button" aria-label="关闭详情">×</button></form>
|
||||||
|
<p class="section-label">DOCUMENT DETAIL</p>
|
||||||
|
<h2 id="dialog-filename"></h2>
|
||||||
|
<dl id="dialog-metadata" class="dialog-metadata"></dl>
|
||||||
|
</dialog>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
55
apps/api/src/ekp_api/celery_queue.py
Normal file
55
apps/api/src/ekp_api/celery_queue.py
Normal file
@ -0,0 +1,55 @@
|
|||||||
|
"""Celery/RabbitMQ adapter behind the queue-neutral ingestion dispatcher."""
|
||||||
|
|
||||||
|
from uuid import UUID
|
||||||
|
|
||||||
|
from celery import Celery # type: ignore[import-untyped]
|
||||||
|
|
||||||
|
from ekp_core.config import Settings
|
||||||
|
|
||||||
|
INGESTION_TASK_NAME = "ekp.ingestion.run"
|
||||||
|
RECOVERY_TASK_NAME = "ekp.ingestion.recover"
|
||||||
|
|
||||||
|
|
||||||
|
def create_celery_app(settings: Settings) -> Celery:
|
||||||
|
"""Create a secret-safe broker client without opening a connection eagerly."""
|
||||||
|
if settings.rabbitmq_url is None or not settings.rabbitmq_url.get_secret_value().strip():
|
||||||
|
raise ValueError("RABBITMQ_URL is required for asynchronous ingestion")
|
||||||
|
app = Celery(
|
||||||
|
"enterprise-knowledge-platform",
|
||||||
|
broker=settings.rabbitmq_url.get_secret_value(),
|
||||||
|
)
|
||||||
|
app.conf.update(
|
||||||
|
task_ignore_result=True,
|
||||||
|
task_acks_late=True,
|
||||||
|
task_reject_on_worker_lost=True,
|
||||||
|
worker_prefetch_multiplier=1,
|
||||||
|
broker_connection_retry_on_startup=True,
|
||||||
|
timezone="UTC",
|
||||||
|
enable_utc=True,
|
||||||
|
task_routes={
|
||||||
|
INGESTION_TASK_NAME: {"queue": "ingestion"},
|
||||||
|
RECOVERY_TASK_NAME: {"queue": "ingestion"},
|
||||||
|
},
|
||||||
|
beat_schedule={
|
||||||
|
"recover-durable-ingestion-jobs": {
|
||||||
|
"task": RECOVERY_TASK_NAME,
|
||||||
|
"schedule": float(settings.ingestion_recovery_interval_seconds),
|
||||||
|
}
|
||||||
|
},
|
||||||
|
)
|
||||||
|
return app
|
||||||
|
|
||||||
|
|
||||||
|
class CeleryIngestionDispatcher:
|
||||||
|
"""Publish only a durable Job id; source bytes and state stay canonical."""
|
||||||
|
|
||||||
|
def __init__(self, app: Celery) -> None:
|
||||||
|
self._app = app
|
||||||
|
|
||||||
|
def enqueue(self, job_id: UUID) -> None:
|
||||||
|
"""Deliver one idempotent execution hint to the ingestion queue."""
|
||||||
|
self._app.send_task(
|
||||||
|
INGESTION_TASK_NAME,
|
||||||
|
args=[str(job_id)],
|
||||||
|
queue="ingestion",
|
||||||
|
)
|
||||||
67
apps/api/src/ekp_api/index_rebuild.py
Normal file
67
apps/api/src/ekp_api/index_rebuild.py
Normal file
@ -0,0 +1,67 @@
|
|||||||
|
"""Controlled CLI for the ARCH-005 shadow index cutover workflow."""
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import json
|
||||||
|
import sys
|
||||||
|
from collections.abc import Sequence
|
||||||
|
from dataclasses import asdict
|
||||||
|
|
||||||
|
from ekp_api.providers import (
|
||||||
|
create_embedding_provider,
|
||||||
|
create_index_profile,
|
||||||
|
create_vector_store,
|
||||||
|
)
|
||||||
|
from ekp_core.config import Settings
|
||||||
|
from ekp_core.database import create_database_engine, create_session_factory
|
||||||
|
from ekp_core.index_rebuild import IndexBuildSummary, IndexRebuildService
|
||||||
|
|
||||||
|
|
||||||
|
def _parser() -> argparse.ArgumentParser:
|
||||||
|
parser = argparse.ArgumentParser(
|
||||||
|
description="Build, verify, and activate a versioned shadow vector index."
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"command",
|
||||||
|
choices=("build", "verify", "activate"),
|
||||||
|
help="Lifecycle step to execute for INDEX_PROFILE_ID.",
|
||||||
|
)
|
||||||
|
return parser
|
||||||
|
|
||||||
|
|
||||||
|
def _create_service(settings: Settings) -> IndexRebuildService:
|
||||||
|
"""Compose the command from the same provider settings as the API runtime."""
|
||||||
|
engine = create_database_engine(settings)
|
||||||
|
session_factory = create_session_factory(engine)
|
||||||
|
return IndexRebuildService(
|
||||||
|
session_factory=session_factory,
|
||||||
|
embedding_provider=create_embedding_provider(settings),
|
||||||
|
vector_store=create_vector_store(settings),
|
||||||
|
target_profile=create_index_profile(settings),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _execute(service: IndexRebuildService, command: str) -> IndexBuildSummary:
|
||||||
|
"""Run exactly one explicit lifecycle step."""
|
||||||
|
if command == "build":
|
||||||
|
return service.rebuild()
|
||||||
|
if command == "verify":
|
||||||
|
return service.verify()
|
||||||
|
if command == "activate":
|
||||||
|
return service.activate()
|
||||||
|
raise ValueError("unsupported index rebuild command")
|
||||||
|
|
||||||
|
|
||||||
|
def main(argv: Sequence[str] | None = None) -> int:
|
||||||
|
"""Return a shell-safe status without printing endpoints, SQL, or credentials."""
|
||||||
|
args = _parser().parse_args(argv)
|
||||||
|
try:
|
||||||
|
summary = _execute(_create_service(Settings()), args.command)
|
||||||
|
except Exception:
|
||||||
|
print("index profile command failed; inspect sanitized service logs", file=sys.stderr)
|
||||||
|
return 1
|
||||||
|
print(json.dumps(asdict(summary), sort_keys=True, separators=(",", ":")))
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
279
apps/api/src/ekp_api/live_acceptance.py
Normal file
279
apps/api/src/ekp_api/live_acceptance.py
Normal file
@ -0,0 +1,279 @@
|
|||||||
|
"""Controlled end-to-end acceptance for the target provider chain."""
|
||||||
|
|
||||||
|
import json
|
||||||
|
import math
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
from collections.abc import Sequence
|
||||||
|
from dataclasses import asdict, dataclass
|
||||||
|
from io import BytesIO
|
||||||
|
from pathlib import Path
|
||||||
|
from time import perf_counter
|
||||||
|
from typing import cast
|
||||||
|
from uuid import uuid4
|
||||||
|
|
||||||
|
from ekp_api.providers import ApiServices, create_api_services
|
||||||
|
from ekp_core.config import Settings
|
||||||
|
from ekp_core.search import SourceResult
|
||||||
|
|
||||||
|
MIN_GOLDEN_QUERIES = 30
|
||||||
|
MAX_GOLDEN_QUERIES = 50
|
||||||
|
DEFAULT_MIN_RECALL_AT_1 = 0.90
|
||||||
|
DEFAULT_MIN_RECALL_AT_5 = 1.0
|
||||||
|
DEFAULT_MIN_MRR = 0.90
|
||||||
|
DEFAULT_MIN_NDCG_AT_5 = 0.90
|
||||||
|
DEFAULT_MAX_P95_MS = 10_000.0
|
||||||
|
|
||||||
|
|
||||||
|
class LiveAcceptanceError(RuntimeError):
|
||||||
|
"""Raised for a safe precondition or acceptance failure."""
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class GoldenQuery:
|
||||||
|
"""One anonymized expected source and section."""
|
||||||
|
|
||||||
|
query_id: str
|
||||||
|
query: str
|
||||||
|
expected_document: str
|
||||||
|
expected_sections: tuple[str, ...]
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class AcceptanceReport:
|
||||||
|
"""Provider-neutral quality and latency metrics without retrieved content."""
|
||||||
|
|
||||||
|
query_count: int
|
||||||
|
recall_at_1: float
|
||||||
|
recall_at_3: float
|
||||||
|
recall_at_5: float
|
||||||
|
mrr: float
|
||||||
|
ndcg_at_5: float
|
||||||
|
latency_p50_ms: float
|
||||||
|
latency_p95_ms: float
|
||||||
|
passed: bool
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class AcceptanceThresholds:
|
||||||
|
"""Explicit gates for controlled live acceptance."""
|
||||||
|
|
||||||
|
min_recall_at_1: float = DEFAULT_MIN_RECALL_AT_1
|
||||||
|
min_recall_at_5: float = DEFAULT_MIN_RECALL_AT_5
|
||||||
|
min_mrr: float = DEFAULT_MIN_MRR
|
||||||
|
min_ndcg_at_5: float = DEFAULT_MIN_NDCG_AT_5
|
||||||
|
max_p95_ms: float = DEFAULT_MAX_P95_MS
|
||||||
|
|
||||||
|
def __post_init__(self) -> None:
|
||||||
|
quality_values = (
|
||||||
|
self.min_recall_at_1,
|
||||||
|
self.min_recall_at_5,
|
||||||
|
self.min_mrr,
|
||||||
|
self.min_ndcg_at_5,
|
||||||
|
)
|
||||||
|
if any(not 0 <= value <= 1 for value in quality_values):
|
||||||
|
raise ValueError("live quality thresholds must be between zero and one")
|
||||||
|
if self.max_p95_ms <= 0:
|
||||||
|
raise ValueError("live latency threshold must be positive")
|
||||||
|
|
||||||
|
|
||||||
|
def _project_root() -> Path:
|
||||||
|
return Path(__file__).resolve().parents[4]
|
||||||
|
|
||||||
|
|
||||||
|
def _load_golden_queries(path: Path) -> tuple[GoldenQuery, ...]:
|
||||||
|
"""Load only the minimal fields needed by the live evaluator."""
|
||||||
|
records: list[GoldenQuery] = []
|
||||||
|
for line in path.read_text(encoding="utf-8").splitlines():
|
||||||
|
if not line:
|
||||||
|
continue
|
||||||
|
raw: object = json.loads(line)
|
||||||
|
if not isinstance(raw, dict):
|
||||||
|
raise LiveAcceptanceError("Golden Query record must be an object")
|
||||||
|
documents = raw.get("expected_documents")
|
||||||
|
sections = raw.get("expected_sections")
|
||||||
|
if (
|
||||||
|
not isinstance(raw.get("id"), str)
|
||||||
|
or not isinstance(raw.get("query"), str)
|
||||||
|
or not isinstance(documents, list)
|
||||||
|
or len(documents) != 1
|
||||||
|
or not isinstance(documents[0], str)
|
||||||
|
or not isinstance(sections, list)
|
||||||
|
or not sections
|
||||||
|
or any(not isinstance(section, str) for section in sections)
|
||||||
|
):
|
||||||
|
raise LiveAcceptanceError("Golden Query record violates the live contract")
|
||||||
|
records.append(
|
||||||
|
GoldenQuery(
|
||||||
|
query_id=cast(str, raw["id"]),
|
||||||
|
query=cast(str, raw["query"]),
|
||||||
|
expected_document=documents[0],
|
||||||
|
expected_sections=tuple(cast(list[str], sections)),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
if not MIN_GOLDEN_QUERIES <= len(records) <= MAX_GOLDEN_QUERIES:
|
||||||
|
raise LiveAcceptanceError("live Golden Dataset must contain 30 to 50 queries")
|
||||||
|
if len({record.query_id for record in records}) != len(records):
|
||||||
|
raise LiveAcceptanceError("live Golden Query IDs must be unique")
|
||||||
|
return tuple(records)
|
||||||
|
|
||||||
|
|
||||||
|
def _matching_rank(
|
||||||
|
results: Sequence[SourceResult],
|
||||||
|
*,
|
||||||
|
expected_filename: str,
|
||||||
|
expected_sections: tuple[str, ...],
|
||||||
|
) -> int | None:
|
||||||
|
"""Return the one-based rank of the first matching canonical source."""
|
||||||
|
for rank, result in enumerate(results, start=1):
|
||||||
|
if result.filename != expected_filename or result.section is None:
|
||||||
|
continue
|
||||||
|
if any(section in result.section for section in expected_sections):
|
||||||
|
return rank
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _percentile(values: Sequence[float], percentile: float) -> float:
|
||||||
|
"""Use nearest-rank percentiles for a small, deterministic acceptance set."""
|
||||||
|
if not values:
|
||||||
|
raise ValueError("latency sample must not be empty")
|
||||||
|
ordered = sorted(values)
|
||||||
|
index = max(0, math.ceil(percentile * len(ordered)) - 1)
|
||||||
|
return ordered[index]
|
||||||
|
|
||||||
|
|
||||||
|
def _build_report(
|
||||||
|
ranks: Sequence[int | None],
|
||||||
|
latencies_ms: Sequence[float],
|
||||||
|
thresholds: AcceptanceThresholds,
|
||||||
|
) -> AcceptanceReport:
|
||||||
|
"""Calculate binary-relevance metrics and apply explicit acceptance gates."""
|
||||||
|
if not ranks or len(ranks) != len(latencies_ms):
|
||||||
|
raise ValueError("live ranks and latencies must have equal non-zero length")
|
||||||
|
query_count = len(ranks)
|
||||||
|
|
||||||
|
def recall_at(limit: int) -> float:
|
||||||
|
return sum(rank is not None and rank <= limit for rank in ranks) / query_count
|
||||||
|
|
||||||
|
reciprocal_ranks = [0.0 if rank is None else 1.0 / rank for rank in ranks]
|
||||||
|
discounted_gains = [
|
||||||
|
0.0 if rank is None or rank > 5 else 1.0 / math.log2(rank + 1) for rank in ranks
|
||||||
|
]
|
||||||
|
recall_at_1 = recall_at(1)
|
||||||
|
recall_at_3 = recall_at(3)
|
||||||
|
recall_at_5 = recall_at(5)
|
||||||
|
mrr = sum(reciprocal_ranks) / query_count
|
||||||
|
ndcg_at_5 = sum(discounted_gains) / query_count
|
||||||
|
p50 = _percentile(latencies_ms, 0.50)
|
||||||
|
p95 = _percentile(latencies_ms, 0.95)
|
||||||
|
passed = (
|
||||||
|
recall_at_1 >= thresholds.min_recall_at_1
|
||||||
|
and recall_at_5 >= thresholds.min_recall_at_5
|
||||||
|
and mrr >= thresholds.min_mrr
|
||||||
|
and ndcg_at_5 >= thresholds.min_ndcg_at_5
|
||||||
|
and p95 <= thresholds.max_p95_ms
|
||||||
|
)
|
||||||
|
return AcceptanceReport(
|
||||||
|
query_count=query_count,
|
||||||
|
recall_at_1=recall_at_1,
|
||||||
|
recall_at_3=recall_at_3,
|
||||||
|
recall_at_5=recall_at_5,
|
||||||
|
mrr=mrr,
|
||||||
|
ndcg_at_5=ndcg_at_5,
|
||||||
|
latency_p50_ms=p50,
|
||||||
|
latency_p95_ms=p95,
|
||||||
|
passed=passed,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _thresholds_from_environment() -> AcceptanceThresholds:
|
||||||
|
return AcceptanceThresholds(
|
||||||
|
min_recall_at_1=float(os.getenv("EKP_LIVE_MIN_RECALL_AT_1", str(DEFAULT_MIN_RECALL_AT_1))),
|
||||||
|
min_recall_at_5=float(os.getenv("EKP_LIVE_MIN_RECALL_AT_5", str(DEFAULT_MIN_RECALL_AT_5))),
|
||||||
|
min_mrr=float(os.getenv("EKP_LIVE_MIN_MRR", str(DEFAULT_MIN_MRR))),
|
||||||
|
min_ndcg_at_5=float(os.getenv("EKP_LIVE_MIN_NDCG_AT_5", str(DEFAULT_MIN_NDCG_AT_5))),
|
||||||
|
max_p95_ms=float(os.getenv("EKP_LIVE_MAX_P95_MS", str(DEFAULT_MAX_P95_MS))),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _require_safe_live_settings(settings: Settings) -> None:
|
||||||
|
"""Refuse external writes unless rotation and resource authorization are explicit."""
|
||||||
|
required_flags = (
|
||||||
|
"EKP_RUN_LIVE_TESTS",
|
||||||
|
"EKP_CREDENTIALS_AUTHORIZED",
|
||||||
|
"EKP_LIVE_RESOURCE_ACK",
|
||||||
|
)
|
||||||
|
if any(os.getenv(name) != "1" for name in required_flags):
|
||||||
|
raise LiveAcceptanceError("live acceptance safety flags are not enabled")
|
||||||
|
if settings.app_env != "test":
|
||||||
|
raise LiveAcceptanceError("live acceptance requires APP_ENV=test")
|
||||||
|
if settings.object_storage_provider != "bos" or settings.vector_store_provider != "oceanbase":
|
||||||
|
raise LiveAcceptanceError("live acceptance requires the target provider baseline")
|
||||||
|
if not settings.bos_key_prefix.startswith("live-acceptance/"):
|
||||||
|
raise LiveAcceptanceError("BOS_KEY_PREFIX must use a dedicated live-acceptance prefix")
|
||||||
|
|
||||||
|
|
||||||
|
def _cleanup_run_documents(services: ApiServices, filename_prefix: str) -> None:
|
||||||
|
"""Delete only documents created under this random acceptance-run prefix."""
|
||||||
|
documents = tuple(
|
||||||
|
document
|
||||||
|
for document in services.documents.list_documents()
|
||||||
|
if document.filename.startswith(filename_prefix)
|
||||||
|
)
|
||||||
|
for document in documents:
|
||||||
|
services.documents.delete_document(document.id)
|
||||||
|
|
||||||
|
|
||||||
|
def _run_live(settings: Settings) -> AcceptanceReport:
|
||||||
|
"""Upload synthetic sources, execute every query, and clean run-owned data."""
|
||||||
|
_require_safe_live_settings(settings)
|
||||||
|
project_root = _project_root()
|
||||||
|
golden = _load_golden_queries(project_root / "fixtures/evaluation/golden.jsonl")
|
||||||
|
services = create_api_services(settings)
|
||||||
|
filename_prefix = f"live-{uuid4().hex[:12]}-"
|
||||||
|
uploaded_names: dict[str, str] = {}
|
||||||
|
cleanup_error: Exception | None = None
|
||||||
|
try:
|
||||||
|
source_names = sorted({record.expected_document for record in golden})
|
||||||
|
for source_name in source_names:
|
||||||
|
live_name = f"{filename_prefix}{source_name}"
|
||||||
|
payload = (project_root / "fixtures/documents" / source_name).read_bytes()
|
||||||
|
services.ingestion.ingest(live_name, "text/markdown", BytesIO(payload))
|
||||||
|
uploaded_names[source_name] = live_name
|
||||||
|
|
||||||
|
ranks: list[int | None] = []
|
||||||
|
latencies_ms: list[float] = []
|
||||||
|
for record in golden:
|
||||||
|
started = perf_counter()
|
||||||
|
result = services.search.search(record.query, top_k=5)
|
||||||
|
latencies_ms.append((perf_counter() - started) * 1000)
|
||||||
|
ranks.append(
|
||||||
|
_matching_rank(
|
||||||
|
result.results,
|
||||||
|
expected_filename=uploaded_names[record.expected_document],
|
||||||
|
expected_sections=record.expected_sections,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return _build_report(ranks, latencies_ms, _thresholds_from_environment())
|
||||||
|
finally:
|
||||||
|
try:
|
||||||
|
_cleanup_run_documents(services, filename_prefix)
|
||||||
|
except Exception as error:
|
||||||
|
cleanup_error = error
|
||||||
|
if cleanup_error is not None:
|
||||||
|
raise LiveAcceptanceError("live acceptance cleanup failed") from cleanup_error
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
"""Print only aggregate metrics and return non-zero for unsafe or failed runs."""
|
||||||
|
try:
|
||||||
|
report = _run_live(Settings())
|
||||||
|
except Exception:
|
||||||
|
print("live acceptance failed; no provider details were emitted", file=sys.stderr)
|
||||||
|
return 1
|
||||||
|
print(json.dumps(asdict(report), sort_keys=True, separators=(",", ":")))
|
||||||
|
return 0 if report.passed else 2
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
@ -1,10 +1,18 @@
|
|||||||
"""FastAPI application entry point."""
|
"""FastAPI application entry point."""
|
||||||
|
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
from fastapi import FastAPI
|
from fastapi import FastAPI
|
||||||
|
from fastapi.responses import FileResponse
|
||||||
|
from fastapi.staticfiles import StaticFiles
|
||||||
from pydantic import BaseModel
|
from pydantic import BaseModel
|
||||||
|
|
||||||
|
from ekp_api.providers import ApiServices
|
||||||
|
from ekp_api.routes import router
|
||||||
from ekp_core.config import Settings, get_settings
|
from ekp_core.config import Settings, get_settings
|
||||||
|
|
||||||
|
ADMIN_WEB_ROOT = Path(__file__).resolve().parents[3] / "admin-web"
|
||||||
|
|
||||||
|
|
||||||
class HealthResponse(BaseModel):
|
class HealthResponse(BaseModel):
|
||||||
"""Health endpoint response."""
|
"""Health endpoint response."""
|
||||||
@ -14,7 +22,10 @@ class HealthResponse(BaseModel):
|
|||||||
environment: str
|
environment: str
|
||||||
|
|
||||||
|
|
||||||
def create_app(settings: Settings | None = None) -> FastAPI:
|
def create_app(
|
||||||
|
settings: Settings | None = None,
|
||||||
|
api_services: ApiServices | None = None,
|
||||||
|
) -> FastAPI:
|
||||||
"""Create the API application with explicit, testable settings."""
|
"""Create the API application with explicit, testable settings."""
|
||||||
resolved_settings = settings or get_settings()
|
resolved_settings = settings or get_settings()
|
||||||
application = FastAPI(
|
application = FastAPI(
|
||||||
@ -23,6 +34,26 @@ def create_app(settings: Settings | None = None) -> FastAPI:
|
|||||||
docs_url=f"{resolved_settings.api_prefix}/docs",
|
docs_url=f"{resolved_settings.api_prefix}/docs",
|
||||||
openapi_url=f"{resolved_settings.api_prefix}/openapi.json",
|
openapi_url=f"{resolved_settings.api_prefix}/openapi.json",
|
||||||
)
|
)
|
||||||
|
application.state.settings = resolved_settings
|
||||||
|
application.state.api_services = api_services
|
||||||
|
application.include_router(router, prefix=resolved_settings.api_prefix)
|
||||||
|
admin_web_root = (resolved_settings.admin_web_root or ADMIN_WEB_ROOT).resolve()
|
||||||
|
|
||||||
|
# The Demo console is a deliberately thin, same-origin client. It contains
|
||||||
|
# no business logic and calls only the versioned public REST API.
|
||||||
|
application.mount(
|
||||||
|
"/admin/assets",
|
||||||
|
StaticFiles(directory=admin_web_root / "assets"),
|
||||||
|
name="admin-assets",
|
||||||
|
)
|
||||||
|
|
||||||
|
@application.get("/admin", include_in_schema=False)
|
||||||
|
@application.get("/admin/upload", include_in_schema=False)
|
||||||
|
@application.get("/admin/documents", include_in_schema=False)
|
||||||
|
@application.get("/admin/search", include_in_schema=False)
|
||||||
|
async def admin_console() -> FileResponse:
|
||||||
|
"""Serve the three-page Demo Admin Console shell."""
|
||||||
|
return FileResponse(admin_web_root / "index.html")
|
||||||
|
|
||||||
@application.get("/healthz", response_model=HealthResponse, tags=["system"])
|
@application.get("/healthz", response_model=HealthResponse, tags=["system"])
|
||||||
async def health() -> HealthResponse:
|
async def health() -> HealthResponse:
|
||||||
|
|||||||
347
apps/api/src/ekp_api/providers.py
Normal file
347
apps/api/src/ekp_api/providers.py
Normal file
@ -0,0 +1,347 @@
|
|||||||
|
"""Runtime provider and application-service composition from safe settings."""
|
||||||
|
|
||||||
|
from dataclasses import dataclass
|
||||||
|
|
||||||
|
from sqlalchemy.orm import Session, sessionmaker
|
||||||
|
|
||||||
|
from ekp_api.celery_queue import CeleryIngestionDispatcher, create_celery_app
|
||||||
|
from ekp_core.chunking import StructureAwareChunker
|
||||||
|
from ekp_core.config import Settings
|
||||||
|
from ekp_core.database import create_database_engine, create_session_factory
|
||||||
|
from ekp_core.dense_retrieval import DenseRetrievalService
|
||||||
|
from ekp_core.document_chunking import DocumentChunkingService, DocumentPipelineService
|
||||||
|
from ekp_core.document_indexing import (
|
||||||
|
DocumentIndexingPipelineService,
|
||||||
|
DocumentIndexingService,
|
||||||
|
)
|
||||||
|
from ekp_core.document_management import DocumentManagementService
|
||||||
|
from ekp_core.document_parsing import DocumentParsingService
|
||||||
|
from ekp_core.document_upload import DocumentUploadService
|
||||||
|
from ekp_core.index_rebuild import resolve_active_index_profile
|
||||||
|
from ekp_core.ingestion_batches import IngestionBatchService
|
||||||
|
from ekp_core.ingestion_jobs import (
|
||||||
|
AsyncIngestionSubmissionService,
|
||||||
|
DerivedDocumentCleanupService,
|
||||||
|
IngestionJobRunner,
|
||||||
|
IngestionJobService,
|
||||||
|
)
|
||||||
|
from ekp_core.search import SearchService
|
||||||
|
from ekp_domain import IndexProfile
|
||||||
|
from ekp_embedding import EmbeddingProvider, QianfanEmbeddingProvider
|
||||||
|
from ekp_object_storage import ObjectStorage, create_bos_object_storage
|
||||||
|
from ekp_object_storage.local import LocalObjectStorage
|
||||||
|
from ekp_parser import ParserRouter
|
||||||
|
from ekp_reranker import QianfanRerankerProvider, RerankerProvider
|
||||||
|
from ekp_vector_store import (
|
||||||
|
VectorStore,
|
||||||
|
create_oceanbase_vector_store,
|
||||||
|
)
|
||||||
|
|
||||||
|
BGE_LARGE_ZH_DIMENSIONS = 1024
|
||||||
|
DEMO_CHUNK_STRATEGY_VERSION = "structure_v1"
|
||||||
|
|
||||||
|
|
||||||
|
class ProviderConfigurationError(ValueError):
|
||||||
|
"""Raised when a runtime provider lacks required injected configuration."""
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class ApiServices:
|
||||||
|
"""Business services injected into REST routes without provider leakage."""
|
||||||
|
|
||||||
|
ingestion: DocumentIndexingPipelineService
|
||||||
|
submissions: AsyncIngestionSubmissionService
|
||||||
|
batches: IngestionBatchService
|
||||||
|
jobs: IngestionJobService
|
||||||
|
documents: DocumentManagementService
|
||||||
|
search: SearchService
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class WorkerServices:
|
||||||
|
"""Provider graph required by Celery without REST search-only services."""
|
||||||
|
|
||||||
|
runner: IngestionJobRunner
|
||||||
|
jobs: IngestionJobService
|
||||||
|
dispatcher: CeleryIngestionDispatcher
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class _IngestionComponents:
|
||||||
|
"""Shared composition result used by API and independent worker processes."""
|
||||||
|
|
||||||
|
session_factory: sessionmaker[Session]
|
||||||
|
storage: ObjectStorage
|
||||||
|
vector_store: VectorStore
|
||||||
|
embedding_provider: EmbeddingProvider
|
||||||
|
index_profile: IndexProfile
|
||||||
|
upload: DocumentUploadService
|
||||||
|
parsing: DocumentParsingService
|
||||||
|
chunking: DocumentChunkingService
|
||||||
|
indexing: DocumentIndexingService
|
||||||
|
jobs: IngestionJobService
|
||||||
|
dispatcher: CeleryIngestionDispatcher
|
||||||
|
|
||||||
|
|
||||||
|
def create_index_profile(
|
||||||
|
settings: Settings,
|
||||||
|
session_factory: sessionmaker[Session] | None = None,
|
||||||
|
) -> IndexProfile:
|
||||||
|
"""Resolve the canonical active profile or the fresh-deployment baseline."""
|
||||||
|
configured = IndexProfile(
|
||||||
|
profile_id=settings.index_profile_id,
|
||||||
|
chunk_strategy_version=DEMO_CHUNK_STRATEGY_VERSION,
|
||||||
|
embedding_model=settings.embedding_model,
|
||||||
|
vector_dimension=BGE_LARGE_ZH_DIMENSIONS,
|
||||||
|
)
|
||||||
|
if session_factory is None:
|
||||||
|
return configured
|
||||||
|
return resolve_active_index_profile(session_factory, configured)
|
||||||
|
|
||||||
|
|
||||||
|
def create_embedding_provider(settings: Settings) -> QianfanEmbeddingProvider:
|
||||||
|
"""Create Qianfan only when its API key is supplied by the runtime."""
|
||||||
|
if settings.baidu_qianfan_api_key is None:
|
||||||
|
raise ProviderConfigurationError("BAIDU_QIANFAN_API_KEY is required")
|
||||||
|
return QianfanEmbeddingProvider(
|
||||||
|
api_key=settings.baidu_qianfan_api_key.get_secret_value(),
|
||||||
|
endpoint=settings.embedding_endpoint,
|
||||||
|
timeout_seconds=settings.embedding_timeout_seconds,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def create_object_storage(settings: Settings) -> ObjectStorage:
|
||||||
|
"""Create local or BOS storage while keeping credentials out of core services."""
|
||||||
|
if settings.object_storage_provider == "local":
|
||||||
|
return LocalObjectStorage(settings.object_storage_path)
|
||||||
|
if settings.bos_endpoint is None:
|
||||||
|
raise ProviderConfigurationError("BOS_ENDPOINT is required")
|
||||||
|
if settings.bos_access_key_id is None:
|
||||||
|
raise ProviderConfigurationError("BOS_ACCESS_KEY_ID is required")
|
||||||
|
if settings.bos_secret_access_key is None:
|
||||||
|
raise ProviderConfigurationError("BOS_SECRET_ACCESS_KEY is required")
|
||||||
|
if settings.bos_bucket is None:
|
||||||
|
raise ProviderConfigurationError("BOS_BUCKET is required")
|
||||||
|
return create_bos_object_storage(
|
||||||
|
endpoint=settings.bos_endpoint,
|
||||||
|
access_key_id=settings.bos_access_key_id.get_secret_value(),
|
||||||
|
secret_access_key=settings.bos_secret_access_key.get_secret_value(),
|
||||||
|
bucket_name=settings.bos_bucket,
|
||||||
|
key_prefix=settings.bos_key_prefix,
|
||||||
|
timeout_seconds=settings.bos_timeout_seconds,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def create_vector_store(settings: Settings) -> VectorStore:
|
||||||
|
"""Create the OceanBase adapter from explicit, secret-safe settings."""
|
||||||
|
if settings.oceanbase_vector_uri is None:
|
||||||
|
raise ProviderConfigurationError("OCEANBASE_VECTOR_URI is required")
|
||||||
|
if settings.oceanbase_vector_user is None:
|
||||||
|
raise ProviderConfigurationError("OCEANBASE_VECTOR_USER is required")
|
||||||
|
if settings.oceanbase_vector_password is None:
|
||||||
|
raise ProviderConfigurationError("OCEANBASE_VECTOR_PASSWORD is required")
|
||||||
|
if settings.oceanbase_vector_database is None:
|
||||||
|
raise ProviderConfigurationError("OCEANBASE_VECTOR_DATABASE is required")
|
||||||
|
return create_oceanbase_vector_store(
|
||||||
|
uri=settings.oceanbase_vector_uri,
|
||||||
|
user=settings.oceanbase_vector_user,
|
||||||
|
password=settings.oceanbase_vector_password.get_secret_value(),
|
||||||
|
database_name=settings.oceanbase_vector_database,
|
||||||
|
table_name=settings.oceanbase_vector_table,
|
||||||
|
index_name=settings.oceanbase_vector_index,
|
||||||
|
vector_dimension=BGE_LARGE_ZH_DIMENSIONS,
|
||||||
|
batch_size=settings.oceanbase_vector_batch_size,
|
||||||
|
timeout_seconds=settings.oceanbase_vector_timeout_seconds,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def create_dense_retrieval_service(
|
||||||
|
settings: Settings,
|
||||||
|
embedding_provider: EmbeddingProvider,
|
||||||
|
vector_store: VectorStore,
|
||||||
|
index_profile: IndexProfile,
|
||||||
|
) -> DenseRetrievalService:
|
||||||
|
"""Compose the profile-scoped dense stage reused by ingestion search APIs."""
|
||||||
|
return DenseRetrievalService(
|
||||||
|
embedding_provider=embedding_provider,
|
||||||
|
vector_store=vector_store,
|
||||||
|
index_profile=index_profile,
|
||||||
|
default_top_n=settings.dense_top_n,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def create_reranker_provider(settings: Settings) -> QianfanRerankerProvider:
|
||||||
|
"""Create the Demo BCE reranker using an explicit or shared Qianfan key."""
|
||||||
|
secret = settings.reranker_api_key or settings.baidu_qianfan_api_key
|
||||||
|
if secret is None:
|
||||||
|
raise ProviderConfigurationError("RERANKER_API_KEY or BAIDU_QIANFAN_API_KEY is required")
|
||||||
|
return QianfanRerankerProvider(
|
||||||
|
api_key=secret.get_secret_value(),
|
||||||
|
endpoint=settings.reranker_endpoint,
|
||||||
|
model_name=settings.reranker_model,
|
||||||
|
timeout_seconds=settings.reranker_timeout_seconds,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def create_search_service(
|
||||||
|
settings: Settings,
|
||||||
|
session_factory: sessionmaker[Session],
|
||||||
|
dense_retriever: DenseRetrievalService,
|
||||||
|
reranker_provider: RerankerProvider,
|
||||||
|
) -> SearchService:
|
||||||
|
"""Compose reranked search independently from REST response schemas."""
|
||||||
|
return SearchService(
|
||||||
|
session_factory=session_factory,
|
||||||
|
dense_retriever=dense_retriever,
|
||||||
|
reranker_provider=reranker_provider,
|
||||||
|
default_top_k=settings.search_top_k,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def create_runtime_search_service(settings: Settings) -> SearchService:
|
||||||
|
"""Compose only read-path providers for the independent MCP Gateway."""
|
||||||
|
engine = create_database_engine(settings)
|
||||||
|
session_factory = create_session_factory(engine)
|
||||||
|
index_profile = create_index_profile(settings, session_factory)
|
||||||
|
embedding_provider = create_embedding_provider(settings)
|
||||||
|
vector_store = create_vector_store(settings)
|
||||||
|
dense_retrieval = create_dense_retrieval_service(
|
||||||
|
settings,
|
||||||
|
embedding_provider,
|
||||||
|
vector_store,
|
||||||
|
index_profile,
|
||||||
|
)
|
||||||
|
return create_search_service(
|
||||||
|
settings,
|
||||||
|
session_factory,
|
||||||
|
dense_retrieval,
|
||||||
|
create_reranker_provider(settings),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _create_ingestion_components(settings: Settings) -> _IngestionComponents:
|
||||||
|
"""Compose provider-neutral ingestion stages and the Celery delivery adapter."""
|
||||||
|
if settings.rabbitmq_url is None or not settings.rabbitmq_url.get_secret_value().strip():
|
||||||
|
raise ProviderConfigurationError("RABBITMQ_URL is required")
|
||||||
|
engine = create_database_engine(settings)
|
||||||
|
session_factory = create_session_factory(engine)
|
||||||
|
storage = create_object_storage(settings)
|
||||||
|
index_profile = create_index_profile(settings, session_factory)
|
||||||
|
embedding_provider = create_embedding_provider(settings)
|
||||||
|
vector_store = create_vector_store(settings)
|
||||||
|
|
||||||
|
upload_service = DocumentUploadService(
|
||||||
|
session_factory,
|
||||||
|
storage,
|
||||||
|
max_upload_bytes=settings.max_upload_bytes,
|
||||||
|
)
|
||||||
|
parsing_service = DocumentParsingService(session_factory, storage, ParserRouter())
|
||||||
|
chunking_service = DocumentChunkingService(
|
||||||
|
session_factory,
|
||||||
|
storage,
|
||||||
|
StructureAwareChunker(
|
||||||
|
max_tokens=settings.chunk_max_tokens,
|
||||||
|
overlap_tokens=settings.chunk_overlap_tokens,
|
||||||
|
max_characters=settings.chunk_max_characters,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
indexing_service = DocumentIndexingService(
|
||||||
|
session_factory,
|
||||||
|
embedding_provider,
|
||||||
|
vector_store,
|
||||||
|
index_profile,
|
||||||
|
)
|
||||||
|
jobs = IngestionJobService(session_factory, settings.ingestion_max_attempts)
|
||||||
|
dispatcher = CeleryIngestionDispatcher(create_celery_app(settings))
|
||||||
|
return _IngestionComponents(
|
||||||
|
session_factory=session_factory,
|
||||||
|
storage=storage,
|
||||||
|
vector_store=vector_store,
|
||||||
|
embedding_provider=embedding_provider,
|
||||||
|
index_profile=index_profile,
|
||||||
|
upload=upload_service,
|
||||||
|
parsing=parsing_service,
|
||||||
|
chunking=chunking_service,
|
||||||
|
indexing=indexing_service,
|
||||||
|
jobs=jobs,
|
||||||
|
dispatcher=dispatcher,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _create_runner(settings: Settings, components: _IngestionComponents) -> IngestionJobRunner:
|
||||||
|
"""Compose retry cleanup and execution policy from non-sensitive settings."""
|
||||||
|
cleanup = DerivedDocumentCleanupService(
|
||||||
|
components.session_factory,
|
||||||
|
components.storage,
|
||||||
|
components.vector_store,
|
||||||
|
components.index_profile.profile_id,
|
||||||
|
)
|
||||||
|
return IngestionJobRunner(
|
||||||
|
components.session_factory,
|
||||||
|
components.parsing,
|
||||||
|
components.chunking,
|
||||||
|
components.indexing,
|
||||||
|
cleanup,
|
||||||
|
retry_backoff_seconds=settings.ingestion_retry_backoff_seconds,
|
||||||
|
retry_backoff_max_seconds=settings.ingestion_retry_backoff_max_seconds,
|
||||||
|
lease_seconds=settings.ingestion_worker_lease_seconds,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def create_worker_services(settings: Settings) -> WorkerServices:
|
||||||
|
"""Build the independent worker runtime from the same application boundaries."""
|
||||||
|
components = _create_ingestion_components(settings)
|
||||||
|
return WorkerServices(
|
||||||
|
runner=_create_runner(settings, components),
|
||||||
|
jobs=components.jobs,
|
||||||
|
dispatcher=components.dispatcher,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def create_api_services(settings: Settings) -> ApiServices:
|
||||||
|
"""Compose REST services while ingestion execution remains in workers."""
|
||||||
|
components = _create_ingestion_components(settings)
|
||||||
|
document_pipeline = DocumentPipelineService(
|
||||||
|
components.upload,
|
||||||
|
components.parsing,
|
||||||
|
components.chunking,
|
||||||
|
)
|
||||||
|
ingestion = DocumentIndexingPipelineService(document_pipeline, components.indexing)
|
||||||
|
submissions = AsyncIngestionSubmissionService(
|
||||||
|
components.upload,
|
||||||
|
components.jobs,
|
||||||
|
components.dispatcher,
|
||||||
|
)
|
||||||
|
batches = IngestionBatchService(
|
||||||
|
components.session_factory,
|
||||||
|
submissions,
|
||||||
|
max_files=settings.batch_max_files,
|
||||||
|
max_total_bytes=settings.batch_max_total_bytes,
|
||||||
|
)
|
||||||
|
dense_retrieval = create_dense_retrieval_service(
|
||||||
|
settings,
|
||||||
|
components.embedding_provider,
|
||||||
|
components.vector_store,
|
||||||
|
components.index_profile,
|
||||||
|
)
|
||||||
|
reranker_provider = create_reranker_provider(settings)
|
||||||
|
search = create_search_service(
|
||||||
|
settings,
|
||||||
|
components.session_factory,
|
||||||
|
dense_retrieval,
|
||||||
|
reranker_provider,
|
||||||
|
)
|
||||||
|
documents = DocumentManagementService(
|
||||||
|
components.session_factory,
|
||||||
|
components.storage,
|
||||||
|
components.vector_store,
|
||||||
|
active_index_profile_id=components.index_profile.profile_id,
|
||||||
|
)
|
||||||
|
return ApiServices(
|
||||||
|
ingestion=ingestion,
|
||||||
|
submissions=submissions,
|
||||||
|
batches=batches,
|
||||||
|
jobs=components.jobs,
|
||||||
|
documents=documents,
|
||||||
|
search=search,
|
||||||
|
)
|
||||||
224
apps/api/src/ekp_api/routes.py
Normal file
224
apps/api/src/ekp_api/routes.py
Normal file
@ -0,0 +1,224 @@
|
|||||||
|
"""Versioned document and search routes for the retrieval Demo."""
|
||||||
|
|
||||||
|
from typing import Annotated
|
||||||
|
from uuid import UUID
|
||||||
|
|
||||||
|
from fastapi import APIRouter, Depends, File, HTTPException, Request, Response, UploadFile, status
|
||||||
|
|
||||||
|
from ekp_api.providers import ApiServices, ProviderConfigurationError, create_api_services
|
||||||
|
from ekp_api.schemas import (
|
||||||
|
DocumentResponse,
|
||||||
|
IngestionAcceptedResponse,
|
||||||
|
IngestionBatchResponse,
|
||||||
|
IngestionJobResponse,
|
||||||
|
SearchRequest,
|
||||||
|
SearchResponse,
|
||||||
|
)
|
||||||
|
from ekp_core.dense_retrieval import InvalidRetrievalQueryError
|
||||||
|
from ekp_core.document_management import (
|
||||||
|
DocumentArtifactCleanupError,
|
||||||
|
DocumentIngestionActiveError,
|
||||||
|
DocumentNotFoundError,
|
||||||
|
)
|
||||||
|
from ekp_core.document_upload import (
|
||||||
|
DocumentTooLargeError,
|
||||||
|
DocumentUploadError,
|
||||||
|
MediaTypeMismatchError,
|
||||||
|
UnsupportedDocumentTypeError,
|
||||||
|
)
|
||||||
|
from ekp_core.ingestion_batches import BatchFileCountLimitError, BatchNotFoundError, BatchSource
|
||||||
|
from ekp_core.ingestion_jobs import IngestionJobNotFoundError
|
||||||
|
|
||||||
|
router = APIRouter()
|
||||||
|
|
||||||
|
|
||||||
|
def get_api_services(request: Request) -> ApiServices:
|
||||||
|
"""Resolve provider composition lazily so health checks need no cloud credentials."""
|
||||||
|
configured = getattr(request.app.state, "api_services", None)
|
||||||
|
if isinstance(configured, ApiServices):
|
||||||
|
return configured
|
||||||
|
try:
|
||||||
|
services = create_api_services(request.app.state.settings)
|
||||||
|
except ProviderConfigurationError as error:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
|
||||||
|
detail="API providers are not fully configured",
|
||||||
|
) from error
|
||||||
|
request.app.state.api_services = services
|
||||||
|
return services
|
||||||
|
|
||||||
|
|
||||||
|
Services = Annotated[ApiServices, Depends(get_api_services)]
|
||||||
|
|
||||||
|
|
||||||
|
@router.post(
|
||||||
|
"/documents",
|
||||||
|
response_model=IngestionAcceptedResponse,
|
||||||
|
status_code=status.HTTP_202_ACCEPTED,
|
||||||
|
tags=["documents"],
|
||||||
|
)
|
||||||
|
def upload_document(
|
||||||
|
services: Services,
|
||||||
|
file: Annotated[UploadFile, File(description="One PDF, DOCX, TXT, or Markdown file")],
|
||||||
|
) -> IngestionAcceptedResponse:
|
||||||
|
"""Persist one immutable source and durable Job, then return immediately."""
|
||||||
|
try:
|
||||||
|
submission = services.submissions.submit(file.filename or "", file.content_type, file.file)
|
||||||
|
except DocumentTooLargeError as error:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=status.HTTP_413_CONTENT_TOO_LARGE,
|
||||||
|
detail="document exceeds the configured upload limit",
|
||||||
|
) from error
|
||||||
|
except (UnsupportedDocumentTypeError, MediaTypeMismatchError) as error:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=status.HTTP_415_UNSUPPORTED_MEDIA_TYPE,
|
||||||
|
detail="unsupported document type or media type",
|
||||||
|
) from error
|
||||||
|
except DocumentUploadError as error:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=status.HTTP_422_UNPROCESSABLE_CONTENT,
|
||||||
|
detail="document could not be accepted",
|
||||||
|
) from error
|
||||||
|
except Exception as error:
|
||||||
|
# Provider and persistence exceptions are intentionally not reflected to clients.
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=status.HTTP_502_BAD_GATEWAY,
|
||||||
|
detail="document acceptance failed",
|
||||||
|
) from error
|
||||||
|
return IngestionAcceptedResponse.from_submission(submission)
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/jobs/{job_id}", response_model=IngestionJobResponse, tags=["jobs"])
|
||||||
|
def get_ingestion_job(job_id: UUID, services: Services) -> IngestionJobResponse:
|
||||||
|
"""Return durable stage, attempt and safe failure state for one Job."""
|
||||||
|
try:
|
||||||
|
job = services.jobs.get(job_id)
|
||||||
|
except IngestionJobNotFoundError as error:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=status.HTTP_404_NOT_FOUND, detail="job not found"
|
||||||
|
) from error
|
||||||
|
return IngestionJobResponse.from_job(job)
|
||||||
|
|
||||||
|
|
||||||
|
@router.post(
|
||||||
|
"/ingestion-batches",
|
||||||
|
response_model=IngestionBatchResponse,
|
||||||
|
status_code=status.HTTP_202_ACCEPTED,
|
||||||
|
tags=["jobs"],
|
||||||
|
)
|
||||||
|
def upload_ingestion_batch(
|
||||||
|
services: Services,
|
||||||
|
files: Annotated[
|
||||||
|
list[UploadFile],
|
||||||
|
File(description="One or more PDF, DOCX, TXT, or Markdown files"),
|
||||||
|
],
|
||||||
|
) -> IngestionBatchResponse:
|
||||||
|
"""Accept a bounded batch while isolating every file's result."""
|
||||||
|
try:
|
||||||
|
view = services.batches.submit(
|
||||||
|
tuple(
|
||||||
|
BatchSource(
|
||||||
|
filename=file.filename or "",
|
||||||
|
media_type=file.content_type,
|
||||||
|
source=file.file,
|
||||||
|
)
|
||||||
|
for file in files
|
||||||
|
)
|
||||||
|
)
|
||||||
|
except BatchFileCountLimitError as error:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=status.HTTP_413_CONTENT_TOO_LARGE,
|
||||||
|
detail="batch file count exceeds the configured limit",
|
||||||
|
) from error
|
||||||
|
except Exception as error:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=status.HTTP_502_BAD_GATEWAY,
|
||||||
|
detail="batch could not be accepted",
|
||||||
|
) from error
|
||||||
|
return IngestionBatchResponse.from_view(view)
|
||||||
|
|
||||||
|
|
||||||
|
@router.get(
|
||||||
|
"/ingestion-batches/{batch_id}",
|
||||||
|
response_model=IngestionBatchResponse,
|
||||||
|
tags=["jobs"],
|
||||||
|
)
|
||||||
|
def get_ingestion_batch(batch_id: UUID, services: Services) -> IngestionBatchResponse:
|
||||||
|
"""Return aggregate and per-file status for Admin Console polling."""
|
||||||
|
try:
|
||||||
|
view = services.batches.get(batch_id)
|
||||||
|
except BatchNotFoundError as error:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=status.HTTP_404_NOT_FOUND,
|
||||||
|
detail="ingestion batch not found",
|
||||||
|
) from error
|
||||||
|
return IngestionBatchResponse.from_view(view)
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/documents", response_model=list[DocumentResponse], tags=["documents"])
|
||||||
|
def list_documents(services: Services) -> list[DocumentResponse]:
|
||||||
|
"""List canonical document metadata for the Demo admin page."""
|
||||||
|
return [
|
||||||
|
DocumentResponse.from_document(document) for document in services.documents.list_documents()
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/documents/{document_id}", response_model=DocumentResponse, tags=["documents"])
|
||||||
|
def get_document(document_id: UUID, services: Services) -> DocumentResponse:
|
||||||
|
"""Return one document's status and safe processing diagnostic."""
|
||||||
|
try:
|
||||||
|
document = services.documents.get_document(document_id)
|
||||||
|
except DocumentNotFoundError as error:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=status.HTTP_404_NOT_FOUND, detail="document not found"
|
||||||
|
) from error
|
||||||
|
return DocumentResponse.from_document(document)
|
||||||
|
|
||||||
|
|
||||||
|
@router.delete(
|
||||||
|
"/documents/{document_id}",
|
||||||
|
status_code=status.HTTP_204_NO_CONTENT,
|
||||||
|
tags=["documents"],
|
||||||
|
)
|
||||||
|
def delete_document(document_id: UUID, services: Services) -> Response:
|
||||||
|
"""Directly remove Demo metadata, source objects, and vector rows."""
|
||||||
|
try:
|
||||||
|
services.documents.delete_document(document_id)
|
||||||
|
except DocumentNotFoundError as error:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=status.HTTP_404_NOT_FOUND, detail="document not found"
|
||||||
|
) from error
|
||||||
|
except DocumentIngestionActiveError as error:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=status.HTTP_409_CONFLICT,
|
||||||
|
detail="document ingestion is still active",
|
||||||
|
) from error
|
||||||
|
except DocumentArtifactCleanupError as error:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||||
|
detail="document metadata was deleted but artifact cleanup failed",
|
||||||
|
) from error
|
||||||
|
except Exception as error:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=status.HTTP_502_BAD_GATEWAY,
|
||||||
|
detail="document deletion failed",
|
||||||
|
) from error
|
||||||
|
return Response(status_code=status.HTTP_204_NO_CONTENT)
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/search", response_model=SearchResponse, tags=["search"])
|
||||||
|
def search_knowledge(payload: SearchRequest, services: Services) -> SearchResponse:
|
||||||
|
"""Return reranked Top-K sources without leaking provider or index details."""
|
||||||
|
try:
|
||||||
|
result = services.search.search(payload.query, payload.top_k)
|
||||||
|
except InvalidRetrievalQueryError as error:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=status.HTTP_422_UNPROCESSABLE_CONTENT,
|
||||||
|
detail="search query must contain non-whitespace text",
|
||||||
|
) from error
|
||||||
|
except Exception as error:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=status.HTTP_502_BAD_GATEWAY,
|
||||||
|
detail="search providers failed",
|
||||||
|
) from error
|
||||||
|
return SearchResponse.from_result(result)
|
||||||
215
apps/api/src/ekp_api/schemas.py
Normal file
215
apps/api/src/ekp_api/schemas.py
Normal file
@ -0,0 +1,215 @@
|
|||||||
|
"""Provider-neutral REST request and response schemas for the Demo API."""
|
||||||
|
|
||||||
|
from datetime import datetime
|
||||||
|
from uuid import UUID
|
||||||
|
|
||||||
|
from pydantic import BaseModel, ConfigDict, Field
|
||||||
|
|
||||||
|
from ekp_core.ingestion_batches import BatchAggregateStatus, BatchItemView, BatchView
|
||||||
|
from ekp_core.ingestion_jobs import IngestionSubmission
|
||||||
|
from ekp_core.search import SearchResult
|
||||||
|
from ekp_domain import (
|
||||||
|
BatchItemAcceptanceStatus,
|
||||||
|
Document,
|
||||||
|
DocumentStatus,
|
||||||
|
IngestionJob,
|
||||||
|
IngestionJobStatus,
|
||||||
|
IngestionStage,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class DocumentResponse(BaseModel):
|
||||||
|
"""Public document metadata without internal storage or provider fields."""
|
||||||
|
|
||||||
|
model_config = ConfigDict(from_attributes=True)
|
||||||
|
|
||||||
|
id: UUID
|
||||||
|
filename: str
|
||||||
|
media_type: str
|
||||||
|
size_bytes: int
|
||||||
|
status: DocumentStatus
|
||||||
|
error_message: str | None
|
||||||
|
created_at: datetime
|
||||||
|
updated_at: datetime
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def from_document(cls, document: Document) -> "DocumentResponse":
|
||||||
|
"""Serialize a detached canonical model through the explicit API shape."""
|
||||||
|
return cls.model_validate(document)
|
||||||
|
|
||||||
|
|
||||||
|
class IngestionAcceptedResponse(DocumentResponse):
|
||||||
|
"""Single-file acceptance with the durable Job needed for status polling."""
|
||||||
|
|
||||||
|
job_id: UUID
|
||||||
|
job_status: IngestionJobStatus
|
||||||
|
max_attempts: int
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def from_submission(cls, submission: IngestionSubmission) -> "IngestionAcceptedResponse":
|
||||||
|
"""Flatten identifiers while retaining the existing document fields."""
|
||||||
|
document = submission.document
|
||||||
|
job = submission.job
|
||||||
|
return cls(
|
||||||
|
id=document.id,
|
||||||
|
filename=document.filename,
|
||||||
|
media_type=document.media_type,
|
||||||
|
size_bytes=document.size_bytes,
|
||||||
|
status=document.status,
|
||||||
|
error_message=document.error_message,
|
||||||
|
created_at=document.created_at,
|
||||||
|
updated_at=document.updated_at,
|
||||||
|
job_id=job.id,
|
||||||
|
job_status=job.status,
|
||||||
|
max_attempts=job.max_attempts,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class IngestionJobResponse(BaseModel):
|
||||||
|
"""Public queue-independent Job status without provider or broker details."""
|
||||||
|
|
||||||
|
id: UUID
|
||||||
|
document_id: UUID
|
||||||
|
status: IngestionJobStatus
|
||||||
|
stage: IngestionStage | None
|
||||||
|
attempt: int
|
||||||
|
max_attempts: int
|
||||||
|
error_message: str | None
|
||||||
|
created_at: datetime
|
||||||
|
updated_at: datetime
|
||||||
|
completed_at: datetime | None
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def from_job(cls, job: IngestionJob) -> "IngestionJobResponse":
|
||||||
|
"""Serialize only client-actionable state and safe diagnostics."""
|
||||||
|
return cls(
|
||||||
|
id=job.id,
|
||||||
|
document_id=job.document_id,
|
||||||
|
status=job.status,
|
||||||
|
stage=job.stage,
|
||||||
|
attempt=job.attempt_count,
|
||||||
|
max_attempts=job.max_attempts,
|
||||||
|
error_message=job.error_message,
|
||||||
|
created_at=job.created_at,
|
||||||
|
updated_at=job.updated_at,
|
||||||
|
completed_at=job.completed_at,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class IngestionBatchItemResponse(BaseModel):
|
||||||
|
"""Per-file acceptance and latest asynchronous processing state."""
|
||||||
|
|
||||||
|
item_index: int
|
||||||
|
filename: str
|
||||||
|
acceptance_status: BatchItemAcceptanceStatus
|
||||||
|
document_id: UUID | None
|
||||||
|
job_id: UUID | None
|
||||||
|
job_status: IngestionJobStatus | None
|
||||||
|
stage: IngestionStage | None
|
||||||
|
attempt: int
|
||||||
|
max_attempts: int | None
|
||||||
|
error_message: str | None
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def from_view(cls, view: BatchItemView) -> "IngestionBatchItemResponse":
|
||||||
|
job = view.job
|
||||||
|
return cls(
|
||||||
|
item_index=view.item.item_index,
|
||||||
|
filename=view.item.filename,
|
||||||
|
acceptance_status=view.item.acceptance_status,
|
||||||
|
document_id=view.item.document_id,
|
||||||
|
job_id=view.item.job_id,
|
||||||
|
job_status=job.status if job is not None else None,
|
||||||
|
stage=job.stage if job is not None else None,
|
||||||
|
attempt=job.attempt_count if job is not None else 0,
|
||||||
|
max_attempts=job.max_attempts if job is not None else None,
|
||||||
|
error_message=(job.error_message if job is not None else view.item.error_message),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class IngestionBatchResponse(BaseModel):
|
||||||
|
"""Batch aggregate with ordered independently processed files."""
|
||||||
|
|
||||||
|
batch_id: UUID
|
||||||
|
status: BatchAggregateStatus
|
||||||
|
total_files: int
|
||||||
|
accepted_files: int
|
||||||
|
rejected_files: int
|
||||||
|
completed_files: int
|
||||||
|
created_at: datetime
|
||||||
|
items: list[IngestionBatchItemResponse]
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def from_view(cls, view: BatchView) -> "IngestionBatchResponse":
|
||||||
|
items = [IngestionBatchItemResponse.from_view(item) for item in view.items]
|
||||||
|
accepted = sum(
|
||||||
|
item.acceptance_status == BatchItemAcceptanceStatus.ACCEPTED for item in items
|
||||||
|
)
|
||||||
|
rejected = sum(
|
||||||
|
item.acceptance_status == BatchItemAcceptanceStatus.REJECTED for item in items
|
||||||
|
)
|
||||||
|
completed = sum(
|
||||||
|
item.acceptance_status == BatchItemAcceptanceStatus.REJECTED
|
||||||
|
or item.job_status
|
||||||
|
in {
|
||||||
|
IngestionJobStatus.SUCCEEDED,
|
||||||
|
IngestionJobStatus.FAILED,
|
||||||
|
IngestionJobStatus.CANCELLED,
|
||||||
|
}
|
||||||
|
for item in items
|
||||||
|
)
|
||||||
|
return cls(
|
||||||
|
batch_id=view.batch.id,
|
||||||
|
status=view.status,
|
||||||
|
total_files=view.batch.total_files,
|
||||||
|
accepted_files=accepted,
|
||||||
|
rejected_files=rejected,
|
||||||
|
completed_files=completed,
|
||||||
|
created_at=view.batch.created_at,
|
||||||
|
items=items,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class SearchRequest(BaseModel):
|
||||||
|
"""Validated public search input; normalization remains in the core service."""
|
||||||
|
|
||||||
|
query: str = Field(min_length=1, max_length=1600)
|
||||||
|
top_k: int = Field(default=5, ge=1, le=64, strict=True)
|
||||||
|
|
||||||
|
|
||||||
|
class SourceResponse(BaseModel):
|
||||||
|
"""Traceable source returned after dense retrieval and reranking."""
|
||||||
|
|
||||||
|
document_id: UUID
|
||||||
|
filename: str
|
||||||
|
page: int | None
|
||||||
|
section: str | None
|
||||||
|
chunk_id: UUID
|
||||||
|
content: str
|
||||||
|
score: float
|
||||||
|
rerank_score: float
|
||||||
|
|
||||||
|
|
||||||
|
class SearchResponse(BaseModel):
|
||||||
|
"""Minimal response contract consumed by the DEMO-010 Search Lab."""
|
||||||
|
|
||||||
|
results: list[SourceResponse]
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def from_result(cls, result: SearchResult) -> "SearchResponse":
|
||||||
|
"""Hide index-profile internals while preserving every required source field."""
|
||||||
|
return cls(
|
||||||
|
results=[
|
||||||
|
SourceResponse(
|
||||||
|
document_id=source.document_id,
|
||||||
|
filename=source.filename,
|
||||||
|
page=source.page,
|
||||||
|
section=source.section,
|
||||||
|
chunk_id=source.chunk_id,
|
||||||
|
content=source.content,
|
||||||
|
score=source.score,
|
||||||
|
rerank_score=source.rerank_score,
|
||||||
|
)
|
||||||
|
for source in result.results
|
||||||
|
]
|
||||||
|
)
|
||||||
51
apps/api/src/ekp_api/worker.py
Normal file
51
apps/api/src/ekp_api/worker.py
Normal file
@ -0,0 +1,51 @@
|
|||||||
|
"""Celery worker entry point for durable ingestion Jobs."""
|
||||||
|
|
||||||
|
from functools import lru_cache
|
||||||
|
from typing import Any
|
||||||
|
from uuid import UUID
|
||||||
|
|
||||||
|
from ekp_api.celery_queue import (
|
||||||
|
INGESTION_TASK_NAME,
|
||||||
|
RECOVERY_TASK_NAME,
|
||||||
|
create_celery_app,
|
||||||
|
)
|
||||||
|
from ekp_api.providers import WorkerServices, create_worker_services
|
||||||
|
from ekp_core.config import get_settings
|
||||||
|
from ekp_core.ingestion_jobs import JobRunAction, dispatch_pending_jobs
|
||||||
|
|
||||||
|
_settings = get_settings()
|
||||||
|
celery_app = create_celery_app(_settings)
|
||||||
|
|
||||||
|
|
||||||
|
@lru_cache
|
||||||
|
def _worker_services() -> WorkerServices:
|
||||||
|
"""Build one provider/service graph per worker process."""
|
||||||
|
return create_worker_services(_settings)
|
||||||
|
|
||||||
|
|
||||||
|
@celery_app.task( # type: ignore[untyped-decorator]
|
||||||
|
bind=True,
|
||||||
|
name=INGESTION_TASK_NAME,
|
||||||
|
max_retries=None,
|
||||||
|
)
|
||||||
|
def run_ingestion(self: Any, job_id: str) -> None:
|
||||||
|
"""Execute one delivery and ask Celery to redeliver only when required.
|
||||||
|
|
||||||
|
Celery delivery retries are unbounded because valid-lease deferrals are not
|
||||||
|
business Attempts; the durable runner independently enforces max_attempts.
|
||||||
|
"""
|
||||||
|
result = _worker_services().runner.run(UUID(job_id))
|
||||||
|
if result.action == JobRunAction.RETRY:
|
||||||
|
raise self.retry(countdown=result.retry_delay_seconds or 1, max_retries=None)
|
||||||
|
|
||||||
|
|
||||||
|
@celery_app.task(name=RECOVERY_TASK_NAME) # type: ignore[untyped-decorator]
|
||||||
|
def recover_ingestion_jobs() -> int:
|
||||||
|
"""Redeliver due durable Jobs after API/broker or worker interruptions."""
|
||||||
|
services = _worker_services()
|
||||||
|
return dispatch_pending_jobs(services.jobs, services.dispatcher)
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> None:
|
||||||
|
"""Run one worker with the embedded Demo recovery scheduler."""
|
||||||
|
celery_app.worker_main(["worker", "--loglevel=INFO", "--queues=ingestion", "--beat"])
|
||||||
5
apps/mcp-server/src/ekp_mcp/__init__.py
Normal file
5
apps/mcp-server/src/ekp_mcp/__init__.py
Normal file
@ -0,0 +1,5 @@
|
|||||||
|
"""Enterprise Knowledge MCP gateway."""
|
||||||
|
|
||||||
|
from ekp_mcp.server import create_mcp_http_app, create_mcp_server
|
||||||
|
|
||||||
|
__all__ = ["create_mcp_http_app", "create_mcp_server"]
|
||||||
76
apps/mcp-server/src/ekp_mcp/auth.py
Normal file
76
apps/mcp-server/src/ekp_mcp/auth.py
Normal file
@ -0,0 +1,76 @@
|
|||||||
|
"""Request-scoped MCP service identity and bearer-token verification."""
|
||||||
|
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from secrets import compare_digest
|
||||||
|
|
||||||
|
from mcp.server.auth.middleware.auth_context import get_access_token
|
||||||
|
from mcp.server.auth.provider import AccessToken
|
||||||
|
from mcp.server.mcpserver.exceptions import ToolError
|
||||||
|
from pydantic import SecretStr
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class McpIdentity:
|
||||||
|
"""Authenticated client/service identity passed across the application boundary."""
|
||||||
|
|
||||||
|
client_id: str
|
||||||
|
subject: str
|
||||||
|
scopes: frozenset[str]
|
||||||
|
|
||||||
|
|
||||||
|
class StaticServiceTokenVerifier:
|
||||||
|
"""Validate one runtime-injected, least-privilege Demo service credential.
|
||||||
|
|
||||||
|
The verifier is intentionally small for DEMO-012. It establishes the MCP
|
||||||
|
identity boundary without pretending to be the V1 OAuth or ACL solution.
|
||||||
|
"""
|
||||||
|
|
||||||
|
__slots__ = (
|
||||||
|
"_client_id",
|
||||||
|
"_expected_token",
|
||||||
|
"_issuer",
|
||||||
|
"_scope",
|
||||||
|
"_subject",
|
||||||
|
)
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
expected_token: SecretStr,
|
||||||
|
*,
|
||||||
|
client_id: str,
|
||||||
|
subject: str,
|
||||||
|
scope: str,
|
||||||
|
issuer: str,
|
||||||
|
) -> None:
|
||||||
|
self._expected_token = expected_token
|
||||||
|
self._client_id = client_id
|
||||||
|
self._subject = subject
|
||||||
|
self._scope = scope
|
||||||
|
self._issuer = issuer
|
||||||
|
|
||||||
|
async def verify_token(self, token: str) -> AccessToken | None:
|
||||||
|
"""Return scoped identity metadata only for a constant-time token match."""
|
||||||
|
if not compare_digest(token, self._expected_token.get_secret_value()):
|
||||||
|
return None
|
||||||
|
return AccessToken(
|
||||||
|
token=token,
|
||||||
|
client_id=self._client_id,
|
||||||
|
subject=self._subject,
|
||||||
|
scopes=[self._scope],
|
||||||
|
claims={"iss": self._issuer},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def current_mcp_identity() -> McpIdentity:
|
||||||
|
"""Resolve the SDK-authenticated principal for the active tool request."""
|
||||||
|
access_token = get_access_token()
|
||||||
|
if access_token is None:
|
||||||
|
# HTTP middleware normally rejects this earlier; retaining the guard
|
||||||
|
# prevents accidental unauthenticated in-process tool invocation.
|
||||||
|
raise ToolError("authentication required")
|
||||||
|
subject = access_token.subject or access_token.client_id
|
||||||
|
return McpIdentity(
|
||||||
|
client_id=access_token.client_id,
|
||||||
|
subject=subject,
|
||||||
|
scopes=frozenset(access_token.scopes),
|
||||||
|
)
|
||||||
30
apps/mcp-server/src/ekp_mcp/main.py
Normal file
30
apps/mcp-server/src/ekp_mcp/main.py
Normal file
@ -0,0 +1,30 @@
|
|||||||
|
"""Command-line entry point for the independent MCP gateway process."""
|
||||||
|
|
||||||
|
from mcp.server.transport_security import TransportSecuritySettings
|
||||||
|
|
||||||
|
from ekp_core.config import get_settings
|
||||||
|
from ekp_mcp.server import create_mcp_server
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> None:
|
||||||
|
"""Run the authenticated Streamable HTTP server from environment settings."""
|
||||||
|
settings = get_settings()
|
||||||
|
server = create_mcp_server(settings)
|
||||||
|
server.run(
|
||||||
|
transport="streamable-http",
|
||||||
|
host=settings.mcp_host,
|
||||||
|
port=settings.mcp_port,
|
||||||
|
streamable_http_path=settings.mcp_path,
|
||||||
|
json_response=True,
|
||||||
|
stateless_http=True,
|
||||||
|
max_request_body_size=settings.mcp_max_request_body_bytes,
|
||||||
|
transport_security=TransportSecuritySettings(
|
||||||
|
enable_dns_rebinding_protection=True,
|
||||||
|
allowed_hosts=settings.mcp_allowed_hosts,
|
||||||
|
allowed_origins=settings.mcp_allowed_origins,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__": # pragma: no cover - exercised through the console script.
|
||||||
|
main()
|
||||||
77
apps/mcp-server/src/ekp_mcp/search.py
Normal file
77
apps/mcp-server/src/ekp_mcp/search.py
Normal file
@ -0,0 +1,77 @@
|
|||||||
|
"""MCP-specific application boundary over the provider-neutral SearchService."""
|
||||||
|
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from typing import Protocol
|
||||||
|
from uuid import UUID
|
||||||
|
|
||||||
|
from pydantic import BaseModel
|
||||||
|
|
||||||
|
from ekp_core.search import SearchResult
|
||||||
|
from ekp_mcp.auth import McpIdentity
|
||||||
|
|
||||||
|
|
||||||
|
class KnowledgeSearcher(Protocol):
|
||||||
|
"""Subset of SearchService required by the MCP gateway."""
|
||||||
|
|
||||||
|
def search(self, query: str, top_k: int | None = None) -> SearchResult:
|
||||||
|
"""Return reranked canonical sources for a normalized query."""
|
||||||
|
|
||||||
|
|
||||||
|
class McpSourceResponse(BaseModel):
|
||||||
|
"""Provider-neutral source shape exposed by `search_knowledge`."""
|
||||||
|
|
||||||
|
document_id: UUID
|
||||||
|
filename: str
|
||||||
|
page: int | None
|
||||||
|
section: str | None
|
||||||
|
chunk_id: UUID
|
||||||
|
content: str
|
||||||
|
score: float
|
||||||
|
rerank_score: float
|
||||||
|
|
||||||
|
|
||||||
|
class McpSearchResponse(BaseModel):
|
||||||
|
"""Structured MCP result aligned with the REST search source fields."""
|
||||||
|
|
||||||
|
results: list[McpSourceResponse]
|
||||||
|
|
||||||
|
|
||||||
|
class McpKnowledgeSearch(Protocol):
|
||||||
|
"""Identity-aware search boundary consumed by the protocol server."""
|
||||||
|
|
||||||
|
def search(self, identity: McpIdentity, query: str, top_k: int) -> McpSearchResponse:
|
||||||
|
"""Return authorized, structured knowledge sources."""
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class McpSearchApplication:
|
||||||
|
"""Require an identity before delegating to the shared SearchService.
|
||||||
|
|
||||||
|
Document ACL is outside the current Demo scope. Keeping identity explicit
|
||||||
|
here prevents the gateway from becoming an anonymous bypass when the
|
||||||
|
permission resolver is added in the V1 tasks.
|
||||||
|
"""
|
||||||
|
|
||||||
|
searcher: KnowledgeSearcher
|
||||||
|
required_scope: str
|
||||||
|
|
||||||
|
def search(self, identity: McpIdentity, query: str, top_k: int) -> McpSearchResponse:
|
||||||
|
"""Authorize the Demo scope and serialize canonical search results."""
|
||||||
|
if self.required_scope not in identity.scopes:
|
||||||
|
raise PermissionError("MCP identity lacks the knowledge search scope")
|
||||||
|
result = self.searcher.search(query, top_k)
|
||||||
|
return McpSearchResponse(
|
||||||
|
results=[
|
||||||
|
McpSourceResponse(
|
||||||
|
document_id=source.document_id,
|
||||||
|
filename=source.filename,
|
||||||
|
page=source.page,
|
||||||
|
section=source.section,
|
||||||
|
chunk_id=source.chunk_id,
|
||||||
|
content=source.content,
|
||||||
|
score=source.score,
|
||||||
|
rerank_score=source.rerank_score,
|
||||||
|
)
|
||||||
|
for source in result.results
|
||||||
|
]
|
||||||
|
)
|
||||||
133
apps/mcp-server/src/ekp_mcp/server.py
Normal file
133
apps/mcp-server/src/ekp_mcp/server.py
Normal file
@ -0,0 +1,133 @@
|
|||||||
|
"""Authenticated Streamable HTTP MCP server exposing Demo business tools."""
|
||||||
|
|
||||||
|
import logging
|
||||||
|
from functools import partial
|
||||||
|
from typing import Annotated
|
||||||
|
|
||||||
|
import anyio
|
||||||
|
from mcp.server import MCPServer
|
||||||
|
from mcp.server.auth.settings import AuthSettings
|
||||||
|
from mcp.server.mcpserver.exceptions import ToolError
|
||||||
|
from mcp.server.transport_security import TransportSecuritySettings
|
||||||
|
from pydantic import Field
|
||||||
|
from starlette.applications import Starlette
|
||||||
|
|
||||||
|
from ekp_api.providers import ApiServices, create_runtime_search_service
|
||||||
|
from ekp_core.config import Settings, get_settings
|
||||||
|
from ekp_core.dense_retrieval import InvalidRetrievalQueryError
|
||||||
|
from ekp_mcp.auth import StaticServiceTokenVerifier, current_mcp_identity
|
||||||
|
from ekp_mcp.search import McpKnowledgeSearch, McpSearchApplication, McpSearchResponse
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
class McpConfigurationError(ValueError):
|
||||||
|
"""Raised when the gateway lacks its dedicated runtime credential."""
|
||||||
|
|
||||||
|
|
||||||
|
def _create_search_application(
|
||||||
|
settings: Settings,
|
||||||
|
api_services: ApiServices | None,
|
||||||
|
) -> McpSearchApplication:
|
||||||
|
searcher = (
|
||||||
|
api_services.search if api_services is not None else create_runtime_search_service(settings)
|
||||||
|
)
|
||||||
|
return McpSearchApplication(searcher, settings.mcp_required_scope)
|
||||||
|
|
||||||
|
|
||||||
|
def create_mcp_server(
|
||||||
|
settings: Settings | None = None,
|
||||||
|
*,
|
||||||
|
api_services: ApiServices | None = None,
|
||||||
|
search_application: McpKnowledgeSearch | None = None,
|
||||||
|
) -> MCPServer[dict[str, object]]:
|
||||||
|
"""Create the authenticated MCP gateway with only `search_knowledge`."""
|
||||||
|
resolved_settings = settings or get_settings()
|
||||||
|
if (
|
||||||
|
resolved_settings.mcp_auth_token is None
|
||||||
|
or len(resolved_settings.mcp_auth_token.get_secret_value()) < 32
|
||||||
|
):
|
||||||
|
raise McpConfigurationError("MCP_AUTH_TOKEN must contain at least 32 characters")
|
||||||
|
|
||||||
|
application = search_application or _create_search_application(
|
||||||
|
resolved_settings,
|
||||||
|
api_services,
|
||||||
|
)
|
||||||
|
token_verifier = StaticServiceTokenVerifier(
|
||||||
|
resolved_settings.mcp_auth_token,
|
||||||
|
client_id=resolved_settings.mcp_client_id,
|
||||||
|
subject=resolved_settings.mcp_service_identity,
|
||||||
|
scope=resolved_settings.mcp_required_scope,
|
||||||
|
issuer=str(resolved_settings.mcp_issuer_url),
|
||||||
|
)
|
||||||
|
server: MCPServer[dict[str, object]] = MCPServer(
|
||||||
|
name="enterprise-knowledge",
|
||||||
|
title="Enterprise Knowledge MCP Server",
|
||||||
|
description="Authenticated enterprise knowledge search backed by canonical sources.",
|
||||||
|
version="0.1.0",
|
||||||
|
token_verifier=token_verifier,
|
||||||
|
auth=AuthSettings(
|
||||||
|
issuer_url=resolved_settings.mcp_issuer_url,
|
||||||
|
resource_server_url=resolved_settings.mcp_public_url,
|
||||||
|
required_scopes=[resolved_settings.mcp_required_scope],
|
||||||
|
),
|
||||||
|
log_level=resolved_settings.log_level,
|
||||||
|
)
|
||||||
|
|
||||||
|
@server.tool(
|
||||||
|
name="search_knowledge",
|
||||||
|
description=(
|
||||||
|
"Search authorized enterprise knowledge and return traceable source passages. "
|
||||||
|
"Use for factual questions that should be grounded in indexed company documents."
|
||||||
|
),
|
||||||
|
structured_output=True,
|
||||||
|
)
|
||||||
|
async def search_knowledge(
|
||||||
|
query: Annotated[str, Field(min_length=1, max_length=1600)],
|
||||||
|
top_k: Annotated[int, Field(ge=1, le=64, strict=True)] = 5,
|
||||||
|
) -> McpSearchResponse:
|
||||||
|
"""Run the shared synchronous search pipeline outside the ASGI event loop."""
|
||||||
|
identity = current_mcp_identity()
|
||||||
|
try:
|
||||||
|
return await anyio.to_thread.run_sync(
|
||||||
|
partial(application.search, identity, query, top_k)
|
||||||
|
)
|
||||||
|
except InvalidRetrievalQueryError as error:
|
||||||
|
raise ToolError("search query must contain non-whitespace text") from error
|
||||||
|
except PermissionError as error:
|
||||||
|
raise ToolError("knowledge search is not authorized") from error
|
||||||
|
except Exception:
|
||||||
|
# Do not reflect provider, database, endpoint, or credential details
|
||||||
|
# into MCP content or logs. Operations can correlate this generic
|
||||||
|
# event with upstream sanitized service telemetry.
|
||||||
|
logger.error("MCP knowledge search failed")
|
||||||
|
raise ToolError("knowledge search failed") from None
|
||||||
|
|
||||||
|
return server
|
||||||
|
|
||||||
|
|
||||||
|
def create_mcp_http_app(
|
||||||
|
settings: Settings | None = None,
|
||||||
|
*,
|
||||||
|
api_services: ApiServices | None = None,
|
||||||
|
search_application: McpKnowledgeSearch | None = None,
|
||||||
|
) -> Starlette:
|
||||||
|
"""Build the stateless Streamable HTTP app used by WorkBuddy and tests."""
|
||||||
|
resolved_settings = settings or get_settings()
|
||||||
|
server = create_mcp_server(
|
||||||
|
resolved_settings,
|
||||||
|
api_services=api_services,
|
||||||
|
search_application=search_application,
|
||||||
|
)
|
||||||
|
return server.streamable_http_app(
|
||||||
|
streamable_http_path=resolved_settings.mcp_path,
|
||||||
|
json_response=True,
|
||||||
|
stateless_http=True,
|
||||||
|
max_request_body_size=resolved_settings.mcp_max_request_body_bytes,
|
||||||
|
host=resolved_settings.mcp_host,
|
||||||
|
transport_security=TransportSecuritySettings(
|
||||||
|
enable_dns_rebinding_protection=True,
|
||||||
|
allowed_hosts=resolved_settings.mcp_allowed_hosts,
|
||||||
|
allowed_origins=resolved_settings.mcp_allowed_origins,
|
||||||
|
),
|
||||||
|
)
|
||||||
61
deploy/docker/Dockerfile
Normal file
61
deploy/docker/Dockerfile
Normal file
@ -0,0 +1,61 @@
|
|||||||
|
# syntax=docker/dockerfile:1.7
|
||||||
|
# One locked runtime produces three independently deployable process images.
|
||||||
|
ARG PYTHON_IMAGE=python:3.12.14-slim-bookworm@sha256:782412e85d0f0984994c290652577d4018aff08145c85b262bb63dc0c7522254
|
||||||
|
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.12.5@sha256:e85be844203885286c60ffad8a858d48afb6c5a5c237ca0e67f12e74b8f174b1
|
||||||
|
|
||||||
|
FROM ${UV_IMAGE} AS uv-bin
|
||||||
|
|
||||||
|
FROM ${PYTHON_IMAGE} AS runtime
|
||||||
|
|
||||||
|
ENV PYTHONUNBUFFERED=1 \
|
||||||
|
PYTHONDONTWRITEBYTECODE=1 \
|
||||||
|
UV_COMPILE_BYTECODE=1 \
|
||||||
|
UV_LINK_MODE=copy \
|
||||||
|
VIRTUAL_ENV=/app/.venv \
|
||||||
|
PATH=/app/.venv/bin:${PATH} \
|
||||||
|
HOME=/tmp/ekp \
|
||||||
|
XDG_CACHE_HOME=/tmp/ekp/cache \
|
||||||
|
APP_ENV=production \
|
||||||
|
API_HOST=0.0.0.0 \
|
||||||
|
MCP_HOST=0.0.0.0 \
|
||||||
|
ADMIN_WEB_ROOT=/app/apps/admin-web
|
||||||
|
|
||||||
|
WORKDIR /app
|
||||||
|
|
||||||
|
COPY --from=uv-bin /uv /uvx /usr/local/bin/
|
||||||
|
COPY pyproject.toml uv.lock README.md alembic.ini ./
|
||||||
|
COPY apps ./apps
|
||||||
|
COPY packages ./packages
|
||||||
|
COPY migrations ./migrations
|
||||||
|
|
||||||
|
# --frozen prevents lock changes; --no-dev and --no-editable keep the runtime
|
||||||
|
# independent from source-tree import hooks while retaining migration sources.
|
||||||
|
RUN uv sync --frozen --no-dev --no-editable \
|
||||||
|
&& test -f /app/apps/admin-web/index.html \
|
||||||
|
&& groupadd --gid 10001 ekp \
|
||||||
|
&& useradd --uid 10001 --gid 10001 --no-create-home --shell /usr/sbin/nologin ekp \
|
||||||
|
&& mkdir -p /tmp/ekp/cache \
|
||||||
|
&& chown -R 10001:10001 /tmp/ekp \
|
||||||
|
&& rm -rf /root/.cache/uv
|
||||||
|
|
||||||
|
USER 10001:10001
|
||||||
|
|
||||||
|
FROM runtime AS api
|
||||||
|
EXPOSE 8000
|
||||||
|
# This endpoint is intentionally credential-free and suitable for container liveness.
|
||||||
|
HEALTHCHECK --interval=30s --timeout=3s --start-period=10s --retries=3 \
|
||||||
|
CMD ["python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/healthz', timeout=2)"]
|
||||||
|
CMD ["uvicorn", "ekp_api.main:app", "--host", "0.0.0.0", "--port", "8000", "--no-access-log"]
|
||||||
|
|
||||||
|
FROM runtime AS mcp
|
||||||
|
EXPOSE 8766
|
||||||
|
# MCP requests require authentication, so the image-level check only verifies PID 1.
|
||||||
|
HEALTHCHECK --interval=30s --timeout=3s --start-period=10s --retries=3 \
|
||||||
|
CMD ["python", "-c", "import os; os.kill(1, 0)"]
|
||||||
|
CMD ["ekp-mcp"]
|
||||||
|
|
||||||
|
FROM runtime AS worker
|
||||||
|
# Worker health is process-level here; durable Job recovery is validated by the service tests.
|
||||||
|
HEALTHCHECK --interval=30s --timeout=3s --start-period=10s --retries=3 \
|
||||||
|
CMD ["python", "-c", "import os; os.kill(1, 0)"]
|
||||||
|
CMD ["ekp-worker"]
|
||||||
36
deploy/docker/docker-bake.hcl
Normal file
36
deploy/docker/docker-bake.hcl
Normal file
@ -0,0 +1,36 @@
|
|||||||
|
# Reproducible target names consumed by local and CI buildx invocations.
|
||||||
|
variable "IMAGE_REGISTRY" {
|
||||||
|
default = "knowledge-platform"
|
||||||
|
}
|
||||||
|
|
||||||
|
variable "IMAGE_TAG" {
|
||||||
|
default = "dev"
|
||||||
|
}
|
||||||
|
|
||||||
|
group "default" {
|
||||||
|
targets = ["api", "mcp", "worker"]
|
||||||
|
}
|
||||||
|
|
||||||
|
target "common" {
|
||||||
|
context = "."
|
||||||
|
dockerfile = "deploy/docker/Dockerfile"
|
||||||
|
platforms = ["linux/amd64", "linux/arm64"]
|
||||||
|
}
|
||||||
|
|
||||||
|
target "api" {
|
||||||
|
inherits = ["common"]
|
||||||
|
target = "api"
|
||||||
|
tags = ["${IMAGE_REGISTRY}/enterprise-knowledge-api:${IMAGE_TAG}"]
|
||||||
|
}
|
||||||
|
|
||||||
|
target "mcp" {
|
||||||
|
inherits = ["common"]
|
||||||
|
target = "mcp"
|
||||||
|
tags = ["${IMAGE_REGISTRY}/enterprise-knowledge-mcp:${IMAGE_TAG}"]
|
||||||
|
}
|
||||||
|
|
||||||
|
target "worker" {
|
||||||
|
inherits = ["common"]
|
||||||
|
target = "worker"
|
||||||
|
tags = ["${IMAGE_REGISTRY}/enterprise-knowledge-worker:${IMAGE_TAG}"]
|
||||||
|
}
|
||||||
50
deploy/helm/environments/values.example.yaml
Normal file
50
deploy/helm/environments/values.example.yaml
Normal file
@ -0,0 +1,50 @@
|
|||||||
|
# Copy this file outside the repository for one environment. It contains only
|
||||||
|
# non-sensitive settings and Kubernetes Secret references; never add values for
|
||||||
|
# DATABASE_URL, RABBITMQ_URL, provider keys, passwords or MCP_AUTH_TOKEN here.
|
||||||
|
existingSecret: knowledge-platform-runtime
|
||||||
|
|
||||||
|
images:
|
||||||
|
api:
|
||||||
|
repository: registry.example.internal/knowledge/enterprise-knowledge-api
|
||||||
|
tag: "0.1.0"
|
||||||
|
digest: ""
|
||||||
|
mcp:
|
||||||
|
repository: registry.example.internal/knowledge/enterprise-knowledge-mcp
|
||||||
|
tag: "0.1.0"
|
||||||
|
digest: ""
|
||||||
|
worker:
|
||||||
|
repository: registry.example.internal/knowledge/enterprise-knowledge-worker
|
||||||
|
tag: "0.1.0"
|
||||||
|
digest: ""
|
||||||
|
|
||||||
|
ingress:
|
||||||
|
enabled: true
|
||||||
|
className: nginx
|
||||||
|
api:
|
||||||
|
host: knowledge.example.internal
|
||||||
|
tlsSecretName: knowledge-platform-api-tls
|
||||||
|
mcp:
|
||||||
|
host: mcp.example.internal
|
||||||
|
tlsSecretName: knowledge-platform-mcp-tls
|
||||||
|
|
||||||
|
config:
|
||||||
|
objectStorage:
|
||||||
|
endpoint: https://s3.example.internal
|
||||||
|
bucket: knowledge-platform
|
||||||
|
vectorStore:
|
||||||
|
uri: oceanbase.example.internal:2881
|
||||||
|
user: knowledge-platform
|
||||||
|
database: knowledge_platform
|
||||||
|
mcp:
|
||||||
|
publicUrl: https://mcp.example.internal/mcp
|
||||||
|
issuerUrl: https://mcp.example.internal
|
||||||
|
allowedHosts:
|
||||||
|
- mcp.example.internal
|
||||||
|
allowedOrigins:
|
||||||
|
- https://mcp.example.internal
|
||||||
|
|
||||||
|
# External RabbitMQ is the production default. If this local dependency is
|
||||||
|
# explicitly enabled for a demo cluster, create its Secret before deployment.
|
||||||
|
rabbitmq:
|
||||||
|
enabled: false
|
||||||
|
existingSecret: knowledge-platform-rabbitmq
|
||||||
15
deploy/helm/knowledge-platform/Chart.yaml
Normal file
15
deploy/helm/knowledge-platform/Chart.yaml
Normal file
@ -0,0 +1,15 @@
|
|||||||
|
apiVersion: v2
|
||||||
|
name: knowledge-platform
|
||||||
|
description: Enterprise Knowledge Platform public-knowledge deployment
|
||||||
|
type: application
|
||||||
|
version: 0.1.0
|
||||||
|
appVersion: "0.1.0"
|
||||||
|
|
||||||
|
# This local dependency is disabled by default. Production deployments should
|
||||||
|
# use an externally managed RabbitMQ service with an independently owned data
|
||||||
|
# lifecycle; the subchart exists only for explicit development/demo use.
|
||||||
|
dependencies:
|
||||||
|
- name: rabbitmq
|
||||||
|
version: 0.1.0
|
||||||
|
repository: file://charts/rabbitmq
|
||||||
|
condition: rabbitmq.enabled
|
||||||
@ -0,0 +1,6 @@
|
|||||||
|
apiVersion: v2
|
||||||
|
name: rabbitmq
|
||||||
|
description: Minimal single-node RabbitMQ dependency for local demo clusters only
|
||||||
|
type: application
|
||||||
|
version: 0.1.0
|
||||||
|
appVersion: "4.1.4"
|
||||||
@ -0,0 +1,11 @@
|
|||||||
|
{{- define "rabbitmq.fullname" -}}
|
||||||
|
{{- printf "%s-rabbitmq" .Release.Name | trunc 63 | trimSuffix "-" -}}
|
||||||
|
{{- end -}}
|
||||||
|
|
||||||
|
{{- define "rabbitmq.image" -}}
|
||||||
|
{{- if .Values.image.digest -}}
|
||||||
|
{{- printf "%s@%s" .Values.image.repository .Values.image.digest -}}
|
||||||
|
{{- else -}}
|
||||||
|
{{- printf "%s:%s" .Values.image.repository .Values.image.tag -}}
|
||||||
|
{{- end -}}
|
||||||
|
{{- end -}}
|
||||||
@ -0,0 +1,16 @@
|
|||||||
|
apiVersion: v1
|
||||||
|
kind: Service
|
||||||
|
metadata:
|
||||||
|
name: {{ include "rabbitmq.fullname" . }}
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: rabbitmq
|
||||||
|
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||||
|
spec:
|
||||||
|
clusterIP: None
|
||||||
|
ports:
|
||||||
|
- name: amqp
|
||||||
|
port: {{ .Values.service.port }}
|
||||||
|
targetPort: amqp
|
||||||
|
selector:
|
||||||
|
app.kubernetes.io/name: rabbitmq
|
||||||
|
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||||
@ -0,0 +1,8 @@
|
|||||||
|
apiVersion: v1
|
||||||
|
kind: ServiceAccount
|
||||||
|
metadata:
|
||||||
|
name: {{ include "rabbitmq.fullname" . }}
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: rabbitmq
|
||||||
|
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||||
|
automountServiceAccountToken: false
|
||||||
@ -0,0 +1,109 @@
|
|||||||
|
apiVersion: apps/v1
|
||||||
|
kind: StatefulSet
|
||||||
|
metadata:
|
||||||
|
name: {{ include "rabbitmq.fullname" . }}
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: rabbitmq
|
||||||
|
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||||
|
spec:
|
||||||
|
serviceName: {{ include "rabbitmq.fullname" . }}
|
||||||
|
replicas: 1
|
||||||
|
selector:
|
||||||
|
matchLabels:
|
||||||
|
app.kubernetes.io/name: rabbitmq
|
||||||
|
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: rabbitmq
|
||||||
|
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||||
|
spec:
|
||||||
|
serviceAccountName: {{ include "rabbitmq.fullname" . }}
|
||||||
|
automountServiceAccountToken: false
|
||||||
|
securityContext:
|
||||||
|
runAsNonRoot: true
|
||||||
|
runAsUser: 999
|
||||||
|
runAsGroup: 999
|
||||||
|
fsGroup: 999
|
||||||
|
seccompProfile:
|
||||||
|
type: RuntimeDefault
|
||||||
|
containers:
|
||||||
|
- name: rabbitmq
|
||||||
|
image: {{ include "rabbitmq.image" . | quote }}
|
||||||
|
imagePullPolicy: {{ .Values.image.pullPolicy }}
|
||||||
|
securityContext:
|
||||||
|
allowPrivilegeEscalation: false
|
||||||
|
readOnlyRootFilesystem: true
|
||||||
|
capabilities:
|
||||||
|
drop: ["ALL"]
|
||||||
|
ports:
|
||||||
|
- name: amqp
|
||||||
|
containerPort: 5672
|
||||||
|
env:
|
||||||
|
- name: RABBITMQ_DEFAULT_USER
|
||||||
|
valueFrom:
|
||||||
|
secretKeyRef:
|
||||||
|
name: {{ .Values.existingSecret }}
|
||||||
|
key: {{ .Values.secretKeys.username }}
|
||||||
|
- name: RABBITMQ_DEFAULT_PASS
|
||||||
|
valueFrom:
|
||||||
|
secretKeyRef:
|
||||||
|
name: {{ .Values.existingSecret }}
|
||||||
|
key: {{ .Values.secretKeys.password }}
|
||||||
|
- name: RABBITMQ_ERLANG_COOKIE
|
||||||
|
valueFrom:
|
||||||
|
secretKeyRef:
|
||||||
|
name: {{ .Values.existingSecret }}
|
||||||
|
key: {{ .Values.secretKeys.erlangCookie }}
|
||||||
|
startupProbe:
|
||||||
|
exec:
|
||||||
|
command: ["rabbitmq-diagnostics", "-q", "ping"]
|
||||||
|
periodSeconds: 5
|
||||||
|
failureThreshold: 24
|
||||||
|
readinessProbe:
|
||||||
|
exec:
|
||||||
|
command: ["rabbitmq-diagnostics", "-q", "check_running"]
|
||||||
|
periodSeconds: 10
|
||||||
|
timeoutSeconds: 5
|
||||||
|
failureThreshold: 3
|
||||||
|
livenessProbe:
|
||||||
|
exec:
|
||||||
|
command: ["rabbitmq-diagnostics", "-q", "ping"]
|
||||||
|
periodSeconds: 20
|
||||||
|
timeoutSeconds: 5
|
||||||
|
failureThreshold: 3
|
||||||
|
resources:
|
||||||
|
{{- toYaml .Values.resources | nindent 12 }}
|
||||||
|
volumeMounts:
|
||||||
|
- name: data
|
||||||
|
mountPath: /var/lib/rabbitmq
|
||||||
|
- name: config
|
||||||
|
mountPath: /etc/rabbitmq/conf.d
|
||||||
|
- name: log
|
||||||
|
mountPath: /var/log/rabbitmq
|
||||||
|
- name: tmp
|
||||||
|
mountPath: /tmp
|
||||||
|
volumes:
|
||||||
|
- name: config
|
||||||
|
emptyDir: {}
|
||||||
|
- name: log
|
||||||
|
emptyDir: {}
|
||||||
|
- name: tmp
|
||||||
|
emptyDir: {}
|
||||||
|
{{- if not .Values.persistence.enabled }}
|
||||||
|
- name: data
|
||||||
|
emptyDir: {}
|
||||||
|
{{- end }}
|
||||||
|
{{- if .Values.persistence.enabled }}
|
||||||
|
volumeClaimTemplates:
|
||||||
|
- metadata:
|
||||||
|
name: data
|
||||||
|
spec:
|
||||||
|
accessModes: [{{ .Values.persistence.accessMode | quote }}]
|
||||||
|
{{- if .Values.persistence.storageClass }}
|
||||||
|
storageClassName: {{ .Values.persistence.storageClass | quote }}
|
||||||
|
{{- end }}
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
storage: {{ .Values.persistence.size }}
|
||||||
|
{{- end }}
|
||||||
@ -0,0 +1,69 @@
|
|||||||
|
{
|
||||||
|
"$schema": "https://json-schema.org/draft-07/schema#",
|
||||||
|
"type": "object",
|
||||||
|
"additionalProperties": false,
|
||||||
|
"required": ["enabled", "existingSecret", "secretKeys", "image", "service", "persistence", "resources"],
|
||||||
|
"properties": {
|
||||||
|
"global": {"type": "object"},
|
||||||
|
"enabled": {"type": "boolean"},
|
||||||
|
"existingSecret": {"type": "string", "minLength": 1},
|
||||||
|
"secretKeys": {
|
||||||
|
"type": "object",
|
||||||
|
"additionalProperties": false,
|
||||||
|
"required": ["username", "password", "erlangCookie"],
|
||||||
|
"properties": {
|
||||||
|
"username": {"type": "string", "minLength": 1},
|
||||||
|
"password": {"type": "string", "minLength": 1},
|
||||||
|
"erlangCookie": {"type": "string", "minLength": 1}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"image": {
|
||||||
|
"type": "object",
|
||||||
|
"additionalProperties": false,
|
||||||
|
"required": ["repository", "tag", "digest", "pullPolicy"],
|
||||||
|
"properties": {
|
||||||
|
"repository": {"type": "string", "minLength": 1},
|
||||||
|
"tag": {"type": "string", "minLength": 1, "not": {"enum": ["latest"]}},
|
||||||
|
"digest": {"type": "string", "pattern": "^$|^sha256:[a-f0-9]{64}$"},
|
||||||
|
"pullPolicy": {"enum": ["Always", "IfNotPresent", "Never"]}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"service": {
|
||||||
|
"type": "object",
|
||||||
|
"additionalProperties": false,
|
||||||
|
"required": ["port"],
|
||||||
|
"properties": {"port": {"type": "integer", "minimum": 1, "maximum": 65535}}
|
||||||
|
},
|
||||||
|
"persistence": {
|
||||||
|
"type": "object",
|
||||||
|
"additionalProperties": false,
|
||||||
|
"required": ["enabled", "storageClass", "accessMode", "size"],
|
||||||
|
"properties": {
|
||||||
|
"enabled": {"type": "boolean"},
|
||||||
|
"storageClass": {"type": "string"},
|
||||||
|
"accessMode": {"enum": ["ReadWriteOnce", "ReadWriteMany"]},
|
||||||
|
"size": {"type": "string", "minLength": 2}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"resources": {
|
||||||
|
"type": "object",
|
||||||
|
"additionalProperties": false,
|
||||||
|
"required": ["requests", "limits"],
|
||||||
|
"properties": {
|
||||||
|
"requests": {"$ref": "#/definitions/resourcePair"},
|
||||||
|
"limits": {"$ref": "#/definitions/resourcePair"}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"definitions": {
|
||||||
|
"resourcePair": {
|
||||||
|
"type": "object",
|
||||||
|
"additionalProperties": false,
|
||||||
|
"required": ["cpu", "memory"],
|
||||||
|
"properties": {
|
||||||
|
"cpu": {"type": ["string", "number"]},
|
||||||
|
"memory": {"type": "string", "minLength": 2}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
25
deploy/helm/knowledge-platform/charts/rabbitmq/values.yaml
Normal file
25
deploy/helm/knowledge-platform/charts/rabbitmq/values.yaml
Normal file
@ -0,0 +1,25 @@
|
|||||||
|
enabled: false
|
||||||
|
existingSecret: knowledge-platform-rabbitmq
|
||||||
|
secretKeys:
|
||||||
|
username: username
|
||||||
|
password: password
|
||||||
|
erlangCookie: erlang-cookie
|
||||||
|
image:
|
||||||
|
repository: rabbitmq
|
||||||
|
tag: 4.1.4-alpine
|
||||||
|
digest: sha256:b736d649308e1b3e1a116c3f36986b605ee3d03e88f10166be2900083d2e63f2
|
||||||
|
pullPolicy: IfNotPresent
|
||||||
|
service:
|
||||||
|
port: 5672
|
||||||
|
persistence:
|
||||||
|
enabled: true
|
||||||
|
storageClass: ""
|
||||||
|
accessMode: ReadWriteOnce
|
||||||
|
size: 8Gi
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
cpu: 250m
|
||||||
|
memory: 512Mi
|
||||||
|
limits:
|
||||||
|
cpu: "1"
|
||||||
|
memory: 1Gi
|
||||||
12
deploy/helm/knowledge-platform/templates/NOTES.txt
Normal file
12
deploy/helm/knowledge-platform/templates/NOTES.txt
Normal file
@ -0,0 +1,12 @@
|
|||||||
|
Enterprise Knowledge Platform {{ .Chart.AppVersion }} is managed by Helm release {{ .Release.Name }}.
|
||||||
|
|
||||||
|
Runtime credentials are read only from Secret {{ .Values.existingSecret }}.
|
||||||
|
{{- if .Values.ingress.enabled }}
|
||||||
|
Admin/API: https://{{ .Values.ingress.api.host }}{{ .Values.ingress.api.path }}
|
||||||
|
MCP: https://{{ .Values.ingress.mcp.host }}{{ .Values.ingress.mcp.path }}
|
||||||
|
{{- else }}
|
||||||
|
Ingress is disabled. Enable it in the environment values file before remote use.
|
||||||
|
{{- end }}
|
||||||
|
|
||||||
|
Inspect rollout state without printing environment variables:
|
||||||
|
kubectl -n {{ .Release.Namespace }} get deploy,job,pod
|
||||||
106
deploy/helm/knowledge-platform/templates/_helpers.tpl
Normal file
106
deploy/helm/knowledge-platform/templates/_helpers.tpl
Normal file
@ -0,0 +1,106 @@
|
|||||||
|
{{/* Stable resource name helpers keep every workload under one release identity. */}}
|
||||||
|
{{- define "knowledge-platform.name" -}}
|
||||||
|
{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
|
||||||
|
{{- end -}}
|
||||||
|
|
||||||
|
{{- define "knowledge-platform.fullname" -}}
|
||||||
|
{{- if .Values.fullnameOverride -}}
|
||||||
|
{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
|
||||||
|
{{- else -}}
|
||||||
|
{{- $name := include "knowledge-platform.name" . -}}
|
||||||
|
{{- if contains $name .Release.Name -}}
|
||||||
|
{{- .Release.Name | trunc 63 | trimSuffix "-" -}}
|
||||||
|
{{- else -}}
|
||||||
|
{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
|
||||||
|
{{- end -}}
|
||||||
|
{{- end -}}
|
||||||
|
{{- end -}}
|
||||||
|
|
||||||
|
{{- define "knowledge-platform.labels" -}}
|
||||||
|
helm.sh/chart: {{ printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | quote }}
|
||||||
|
app.kubernetes.io/name: {{ include "knowledge-platform.name" . }}
|
||||||
|
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||||
|
app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
|
||||||
|
app.kubernetes.io/managed-by: {{ .Release.Service }}
|
||||||
|
{{- end -}}
|
||||||
|
|
||||||
|
{{- define "knowledge-platform.selectorLabels" -}}
|
||||||
|
app.kubernetes.io/name: {{ include "knowledge-platform.name" . }}
|
||||||
|
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||||
|
{{- end -}}
|
||||||
|
|
||||||
|
{{- define "knowledge-platform.serviceAccountName" -}}
|
||||||
|
{{- if .Values.serviceAccount.create -}}
|
||||||
|
{{- default (include "knowledge-platform.fullname" .) .Values.serviceAccount.name -}}
|
||||||
|
{{- else -}}
|
||||||
|
{{- required "serviceAccount.name is required when serviceAccount.create=false" .Values.serviceAccount.name -}}
|
||||||
|
{{- end -}}
|
||||||
|
{{- end -}}
|
||||||
|
|
||||||
|
{{/* A supplied digest takes precedence over the immutable release tag. */}}
|
||||||
|
{{- define "knowledge-platform.image" -}}
|
||||||
|
{{- if .digest -}}
|
||||||
|
{{- printf "%s@%s" .repository .digest -}}
|
||||||
|
{{- else -}}
|
||||||
|
{{- printf "%s:%s" .repository .tag -}}
|
||||||
|
{{- end -}}
|
||||||
|
{{- end -}}
|
||||||
|
|
||||||
|
{{- define "knowledge-platform.configMapName" -}}
|
||||||
|
{{- printf "%s-runtime" (include "knowledge-platform.fullname" .) -}}
|
||||||
|
{{- end -}}
|
||||||
|
|
||||||
|
{{/* Cross-field rules complement values.schema.json's structural checks. */}}
|
||||||
|
{{- define "knowledge-platform.validateValues" -}}
|
||||||
|
{{- if lt (float64 .Values.config.ingestion.retryBackoffMaxSeconds) (float64 .Values.config.ingestion.retryBackoffSeconds) -}}
|
||||||
|
{{- fail "config.ingestion.retryBackoffMaxSeconds must be >= retryBackoffSeconds" -}}
|
||||||
|
{{- end -}}
|
||||||
|
{{- if lt (float64 .Values.config.ingestion.batchMaxTotalBytes) (float64 .Values.config.ingestion.maxUploadBytes) -}}
|
||||||
|
{{- fail "config.ingestion.batchMaxTotalBytes must be >= maxUploadBytes" -}}
|
||||||
|
{{- end -}}
|
||||||
|
{{- if ge (float64 .Values.config.chunking.overlapTokens) (float64 .Values.config.chunking.maxTokens) -}}
|
||||||
|
{{- fail "config.chunking.overlapTokens must be < maxTokens" -}}
|
||||||
|
{{- end -}}
|
||||||
|
{{- if gt (float64 .Values.config.retrieval.searchTopK) (float64 .Values.config.retrieval.denseTopN) -}}
|
||||||
|
{{- fail "config.retrieval.searchTopK must be <= denseTopN" -}}
|
||||||
|
{{- end -}}
|
||||||
|
{{- end -}}
|
||||||
|
|
||||||
|
{{- define "knowledge-platform.podSecurityContext" -}}
|
||||||
|
runAsNonRoot: true
|
||||||
|
runAsUser: 10001
|
||||||
|
runAsGroup: 10001
|
||||||
|
fsGroup: 10001
|
||||||
|
fsGroupChangePolicy: OnRootMismatch
|
||||||
|
seccompProfile:
|
||||||
|
type: RuntimeDefault
|
||||||
|
{{- end -}}
|
||||||
|
|
||||||
|
{{- define "knowledge-platform.containerSecurityContext" -}}
|
||||||
|
allowPrivilegeEscalation: false
|
||||||
|
readOnlyRootFilesystem: true
|
||||||
|
runAsNonRoot: true
|
||||||
|
runAsUser: 10001
|
||||||
|
runAsGroup: 10001
|
||||||
|
capabilities:
|
||||||
|
drop:
|
||||||
|
- ALL
|
||||||
|
{{- end -}}
|
||||||
|
|
||||||
|
{{- define "knowledge-platform.envFrom" -}}
|
||||||
|
- configMapRef:
|
||||||
|
name: {{ include "knowledge-platform.configMapName" . }}
|
||||||
|
- secretRef:
|
||||||
|
name: {{ .Values.existingSecret }}
|
||||||
|
{{- end -}}
|
||||||
|
|
||||||
|
{{- define "knowledge-platform.tmpVolume" -}}
|
||||||
|
- name: tmp
|
||||||
|
emptyDir:
|
||||||
|
sizeLimit: 1Gi
|
||||||
|
{{- end -}}
|
||||||
|
|
||||||
|
{{- define "knowledge-platform.tmpMount" -}}
|
||||||
|
- name: tmp
|
||||||
|
mountPath: /tmp
|
||||||
|
{{- end -}}
|
||||||
75
deploy/helm/knowledge-platform/templates/api-deployment.yaml
Normal file
75
deploy/helm/knowledge-platform/templates/api-deployment.yaml
Normal file
@ -0,0 +1,75 @@
|
|||||||
|
apiVersion: apps/v1
|
||||||
|
kind: Deployment
|
||||||
|
metadata:
|
||||||
|
name: {{ include "knowledge-platform.fullname" . }}-api
|
||||||
|
labels:
|
||||||
|
{{- include "knowledge-platform.labels" . | nindent 4 }}
|
||||||
|
app.kubernetes.io/component: api
|
||||||
|
spec:
|
||||||
|
replicas: {{ .Values.api.replicaCount }}
|
||||||
|
minReadySeconds: 5
|
||||||
|
strategy:
|
||||||
|
type: RollingUpdate
|
||||||
|
rollingUpdate:
|
||||||
|
maxUnavailable: 0
|
||||||
|
maxSurge: 1
|
||||||
|
selector:
|
||||||
|
matchLabels:
|
||||||
|
{{- include "knowledge-platform.selectorLabels" . | nindent 6 }}
|
||||||
|
app.kubernetes.io/component: api
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
{{- include "knowledge-platform.selectorLabels" . | nindent 8 }}
|
||||||
|
app.kubernetes.io/component: api
|
||||||
|
{{- with .Values.podLabels }}
|
||||||
|
{{- toYaml . | nindent 8 }}
|
||||||
|
{{- end }}
|
||||||
|
annotations:
|
||||||
|
checksum/runtime-config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }}
|
||||||
|
checksum/runtime-secret-reference: {{ .Values.existingSecret | sha256sum }}
|
||||||
|
{{- with .Values.podAnnotations }}
|
||||||
|
{{- toYaml . | nindent 8 }}
|
||||||
|
{{- end }}
|
||||||
|
spec:
|
||||||
|
serviceAccountName: {{ include "knowledge-platform.serviceAccountName" . }}
|
||||||
|
automountServiceAccountToken: false
|
||||||
|
securityContext:
|
||||||
|
{{- include "knowledge-platform.podSecurityContext" . | nindent 8 }}
|
||||||
|
{{- with .Values.imagePullSecrets }}
|
||||||
|
imagePullSecrets:
|
||||||
|
{{- toYaml . | nindent 8 }}
|
||||||
|
{{- end }}
|
||||||
|
containers:
|
||||||
|
- name: api
|
||||||
|
image: {{ include "knowledge-platform.image" .Values.images.api | quote }}
|
||||||
|
imagePullPolicy: {{ .Values.images.api.pullPolicy }}
|
||||||
|
securityContext:
|
||||||
|
{{- include "knowledge-platform.containerSecurityContext" . | nindent 12 }}
|
||||||
|
ports:
|
||||||
|
- name: http
|
||||||
|
containerPort: 8000
|
||||||
|
protocol: TCP
|
||||||
|
envFrom:
|
||||||
|
{{- include "knowledge-platform.envFrom" . | nindent 12 }}
|
||||||
|
startupProbe:
|
||||||
|
httpGet:
|
||||||
|
path: /healthz
|
||||||
|
port: http
|
||||||
|
{{- toYaml .Values.api.probes.startup | nindent 12 }}
|
||||||
|
readinessProbe:
|
||||||
|
httpGet:
|
||||||
|
path: /healthz
|
||||||
|
port: http
|
||||||
|
{{- toYaml .Values.api.probes.readiness | nindent 12 }}
|
||||||
|
livenessProbe:
|
||||||
|
httpGet:
|
||||||
|
path: /healthz
|
||||||
|
port: http
|
||||||
|
{{- toYaml .Values.api.probes.liveness | nindent 12 }}
|
||||||
|
resources:
|
||||||
|
{{- toYaml .Values.api.resources | nindent 12 }}
|
||||||
|
volumeMounts:
|
||||||
|
{{- include "knowledge-platform.tmpMount" . | nindent 12 }}
|
||||||
|
volumes:
|
||||||
|
{{- include "knowledge-platform.tmpVolume" . | nindent 8 }}
|
||||||
17
deploy/helm/knowledge-platform/templates/api-service.yaml
Normal file
17
deploy/helm/knowledge-platform/templates/api-service.yaml
Normal file
@ -0,0 +1,17 @@
|
|||||||
|
apiVersion: v1
|
||||||
|
kind: Service
|
||||||
|
metadata:
|
||||||
|
name: {{ include "knowledge-platform.fullname" . }}-api
|
||||||
|
labels:
|
||||||
|
{{- include "knowledge-platform.labels" . | nindent 4 }}
|
||||||
|
app.kubernetes.io/component: api
|
||||||
|
spec:
|
||||||
|
type: {{ .Values.api.service.type }}
|
||||||
|
ports:
|
||||||
|
- name: http
|
||||||
|
port: {{ .Values.api.service.port }}
|
||||||
|
targetPort: http
|
||||||
|
protocol: TCP
|
||||||
|
selector:
|
||||||
|
{{- include "knowledge-platform.selectorLabels" . | nindent 4 }}
|
||||||
|
app.kubernetes.io/component: api
|
||||||
66
deploy/helm/knowledge-platform/templates/configmap.yaml
Normal file
66
deploy/helm/knowledge-platform/templates/configmap.yaml
Normal file
@ -0,0 +1,66 @@
|
|||||||
|
{{- include "knowledge-platform.validateValues" . }}
|
||||||
|
apiVersion: v1
|
||||||
|
kind: ConfigMap
|
||||||
|
metadata:
|
||||||
|
name: {{ include "knowledge-platform.configMapName" . }}
|
||||||
|
labels:
|
||||||
|
{{- include "knowledge-platform.labels" . | nindent 4 }}
|
||||||
|
data:
|
||||||
|
# Only the explicitly enumerated non-sensitive settings are rendered here.
|
||||||
|
APP_ENV: "production"
|
||||||
|
APP_NAME: {{ .Values.config.appName | quote }}
|
||||||
|
LOG_LEVEL: {{ .Values.config.logLevel | quote }}
|
||||||
|
API_HOST: "0.0.0.0"
|
||||||
|
API_PORT: "8000"
|
||||||
|
API_PREFIX: "/api/v1"
|
||||||
|
DATABASE_POOL_SIZE: {{ .Values.config.database.poolSize | quote }}
|
||||||
|
DATABASE_MAX_OVERFLOW: {{ .Values.config.database.maxOverflow | quote }}
|
||||||
|
DATABASE_POOL_RECYCLE_SECONDS: {{ .Values.config.database.poolRecycleSeconds | quote }}
|
||||||
|
DATABASE_POOL_TIMEOUT_SECONDS: {{ .Values.config.database.poolTimeoutSeconds | quote }}
|
||||||
|
DATABASE_CONNECT_TIMEOUT_SECONDS: {{ .Values.config.database.connectTimeoutSeconds | quote }}
|
||||||
|
INGESTION_MAX_ATTEMPTS: {{ .Values.config.ingestion.maxAttempts | quote }}
|
||||||
|
INGESTION_RETRY_BACKOFF_SECONDS: {{ .Values.config.ingestion.retryBackoffSeconds | quote }}
|
||||||
|
INGESTION_RETRY_BACKOFF_MAX_SECONDS: {{ .Values.config.ingestion.retryBackoffMaxSeconds | quote }}
|
||||||
|
INGESTION_WORKER_LEASE_SECONDS: {{ .Values.config.ingestion.workerLeaseSeconds | quote }}
|
||||||
|
INGESTION_RECOVERY_INTERVAL_SECONDS: {{ .Values.config.ingestion.recoveryIntervalSeconds | quote }}
|
||||||
|
BATCH_MAX_FILES: {{ .Values.config.ingestion.batchMaxFiles | quote }}
|
||||||
|
BATCH_MAX_TOTAL_BYTES: {{ printf "%.0f" .Values.config.ingestion.batchMaxTotalBytes | quote }}
|
||||||
|
MAX_UPLOAD_BYTES: {{ printf "%.0f" .Values.config.ingestion.maxUploadBytes | quote }}
|
||||||
|
OBJECT_STORAGE_PROVIDER: {{ .Values.config.objectStorage.provider | quote }}
|
||||||
|
BOS_ENDPOINT: {{ .Values.config.objectStorage.endpoint | quote }}
|
||||||
|
BOS_BUCKET: {{ .Values.config.objectStorage.bucket | quote }}
|
||||||
|
BOS_KEY_PREFIX: {{ .Values.config.objectStorage.keyPrefix | quote }}
|
||||||
|
BOS_TIMEOUT_SECONDS: {{ .Values.config.objectStorage.timeoutSeconds | quote }}
|
||||||
|
CHUNK_MAX_TOKENS: {{ .Values.config.chunking.maxTokens | quote }}
|
||||||
|
CHUNK_OVERLAP_TOKENS: {{ .Values.config.chunking.overlapTokens | quote }}
|
||||||
|
CHUNK_MAX_CHARACTERS: {{ .Values.config.chunking.maxCharacters | quote }}
|
||||||
|
EMBEDDING_PROVIDER: {{ .Values.config.embedding.provider | quote }}
|
||||||
|
EMBEDDING_MODEL: {{ .Values.config.embedding.model | quote }}
|
||||||
|
EMBEDDING_ENDPOINT: {{ .Values.config.embedding.endpoint | quote }}
|
||||||
|
EMBEDDING_TIMEOUT_SECONDS: {{ .Values.config.embedding.timeoutSeconds | quote }}
|
||||||
|
VECTOR_STORE_PROVIDER: {{ .Values.config.vectorStore.provider | quote }}
|
||||||
|
OCEANBASE_VECTOR_URI: {{ .Values.config.vectorStore.uri | quote }}
|
||||||
|
OCEANBASE_VECTOR_USER: {{ .Values.config.vectorStore.user | quote }}
|
||||||
|
OCEANBASE_VECTOR_DATABASE: {{ .Values.config.vectorStore.database | quote }}
|
||||||
|
OCEANBASE_VECTOR_TABLE: {{ .Values.config.vectorStore.table | quote }}
|
||||||
|
OCEANBASE_VECTOR_INDEX: {{ .Values.config.vectorStore.index | quote }}
|
||||||
|
OCEANBASE_VECTOR_BATCH_SIZE: {{ .Values.config.vectorStore.batchSize | quote }}
|
||||||
|
OCEANBASE_VECTOR_TIMEOUT_SECONDS: {{ .Values.config.vectorStore.timeoutSeconds | quote }}
|
||||||
|
INDEX_PROFILE_ID: {{ .Values.config.retrieval.indexProfileId | quote }}
|
||||||
|
DENSE_TOP_N: {{ .Values.config.retrieval.denseTopN | quote }}
|
||||||
|
SEARCH_TOP_K: {{ .Values.config.retrieval.searchTopK | quote }}
|
||||||
|
RERANKER_PROVIDER: {{ .Values.config.reranker.provider | quote }}
|
||||||
|
RERANKER_MODEL: {{ .Values.config.reranker.model | quote }}
|
||||||
|
RERANKER_ENDPOINT: {{ .Values.config.reranker.endpoint | quote }}
|
||||||
|
RERANKER_TIMEOUT_SECONDS: {{ .Values.config.reranker.timeoutSeconds | quote }}
|
||||||
|
MCP_HOST: "0.0.0.0"
|
||||||
|
MCP_PORT: "8766"
|
||||||
|
MCP_PATH: "/mcp"
|
||||||
|
MCP_PUBLIC_URL: {{ .Values.config.mcp.publicUrl | quote }}
|
||||||
|
MCP_ISSUER_URL: {{ .Values.config.mcp.issuerUrl | quote }}
|
||||||
|
MCP_CLIENT_ID: {{ .Values.config.mcp.clientId | quote }}
|
||||||
|
MCP_SERVICE_IDENTITY: {{ .Values.config.mcp.serviceIdentity | quote }}
|
||||||
|
MCP_REQUIRED_SCOPE: {{ .Values.config.mcp.requiredScope | quote }}
|
||||||
|
MCP_ALLOWED_HOSTS: {{ .Values.config.mcp.allowedHosts | toJson | quote }}
|
||||||
|
MCP_ALLOWED_ORIGINS: {{ .Values.config.mcp.allowedOrigins | toJson | quote }}
|
||||||
|
MCP_MAX_REQUEST_BODY_BYTES: {{ printf "%.0f" .Values.config.mcp.maxRequestBodyBytes | quote }}
|
||||||
42
deploy/helm/knowledge-platform/templates/ingress.yaml
Normal file
42
deploy/helm/knowledge-platform/templates/ingress.yaml
Normal file
@ -0,0 +1,42 @@
|
|||||||
|
{{- if .Values.ingress.enabled }}
|
||||||
|
apiVersion: networking.k8s.io/v1
|
||||||
|
kind: Ingress
|
||||||
|
metadata:
|
||||||
|
name: {{ include "knowledge-platform.fullname" . }}
|
||||||
|
labels:
|
||||||
|
{{- include "knowledge-platform.labels" . | nindent 4 }}
|
||||||
|
{{- with .Values.ingress.annotations }}
|
||||||
|
annotations:
|
||||||
|
{{- toYaml . | nindent 4 }}
|
||||||
|
{{- end }}
|
||||||
|
spec:
|
||||||
|
ingressClassName: {{ .Values.ingress.className }}
|
||||||
|
tls:
|
||||||
|
- hosts:
|
||||||
|
- {{ .Values.ingress.api.host | quote }}
|
||||||
|
secretName: {{ .Values.ingress.api.tlsSecretName }}
|
||||||
|
- hosts:
|
||||||
|
- {{ .Values.ingress.mcp.host | quote }}
|
||||||
|
secretName: {{ .Values.ingress.mcp.tlsSecretName }}
|
||||||
|
rules:
|
||||||
|
- host: {{ .Values.ingress.api.host | quote }}
|
||||||
|
http:
|
||||||
|
paths:
|
||||||
|
- path: {{ .Values.ingress.api.path }}
|
||||||
|
pathType: Prefix
|
||||||
|
backend:
|
||||||
|
service:
|
||||||
|
name: {{ include "knowledge-platform.fullname" . }}-api
|
||||||
|
port:
|
||||||
|
number: {{ .Values.api.service.port }}
|
||||||
|
- host: {{ .Values.ingress.mcp.host | quote }}
|
||||||
|
http:
|
||||||
|
paths:
|
||||||
|
- path: {{ .Values.ingress.mcp.path }}
|
||||||
|
pathType: Prefix
|
||||||
|
backend:
|
||||||
|
service:
|
||||||
|
name: {{ include "knowledge-platform.fullname" . }}-mcp
|
||||||
|
port:
|
||||||
|
number: {{ .Values.mcp.service.port }}
|
||||||
|
{{- end }}
|
||||||
72
deploy/helm/knowledge-platform/templates/mcp-deployment.yaml
Normal file
72
deploy/helm/knowledge-platform/templates/mcp-deployment.yaml
Normal file
@ -0,0 +1,72 @@
|
|||||||
|
apiVersion: apps/v1
|
||||||
|
kind: Deployment
|
||||||
|
metadata:
|
||||||
|
name: {{ include "knowledge-platform.fullname" . }}-mcp
|
||||||
|
labels:
|
||||||
|
{{- include "knowledge-platform.labels" . | nindent 4 }}
|
||||||
|
app.kubernetes.io/component: mcp
|
||||||
|
spec:
|
||||||
|
replicas: {{ .Values.mcp.replicaCount }}
|
||||||
|
minReadySeconds: 5
|
||||||
|
strategy:
|
||||||
|
type: RollingUpdate
|
||||||
|
rollingUpdate:
|
||||||
|
maxUnavailable: 0
|
||||||
|
maxSurge: 1
|
||||||
|
selector:
|
||||||
|
matchLabels:
|
||||||
|
{{- include "knowledge-platform.selectorLabels" . | nindent 6 }}
|
||||||
|
app.kubernetes.io/component: mcp
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
{{- include "knowledge-platform.selectorLabels" . | nindent 8 }}
|
||||||
|
app.kubernetes.io/component: mcp
|
||||||
|
{{- with .Values.podLabels }}
|
||||||
|
{{- toYaml . | nindent 8 }}
|
||||||
|
{{- end }}
|
||||||
|
annotations:
|
||||||
|
checksum/runtime-config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }}
|
||||||
|
checksum/runtime-secret-reference: {{ .Values.existingSecret | sha256sum }}
|
||||||
|
{{- with .Values.podAnnotations }}
|
||||||
|
{{- toYaml . | nindent 8 }}
|
||||||
|
{{- end }}
|
||||||
|
spec:
|
||||||
|
serviceAccountName: {{ include "knowledge-platform.serviceAccountName" . }}
|
||||||
|
automountServiceAccountToken: false
|
||||||
|
securityContext:
|
||||||
|
{{- include "knowledge-platform.podSecurityContext" . | nindent 8 }}
|
||||||
|
{{- with .Values.imagePullSecrets }}
|
||||||
|
imagePullSecrets:
|
||||||
|
{{- toYaml . | nindent 8 }}
|
||||||
|
{{- end }}
|
||||||
|
containers:
|
||||||
|
- name: mcp
|
||||||
|
image: {{ include "knowledge-platform.image" .Values.images.mcp | quote }}
|
||||||
|
imagePullPolicy: {{ .Values.images.mcp.pullPolicy }}
|
||||||
|
securityContext:
|
||||||
|
{{- include "knowledge-platform.containerSecurityContext" . | nindent 12 }}
|
||||||
|
ports:
|
||||||
|
- name: mcp
|
||||||
|
containerPort: 8766
|
||||||
|
protocol: TCP
|
||||||
|
envFrom:
|
||||||
|
{{- include "knowledge-platform.envFrom" . | nindent 12 }}
|
||||||
|
startupProbe:
|
||||||
|
tcpSocket:
|
||||||
|
port: mcp
|
||||||
|
{{- toYaml .Values.mcp.probes.startup | nindent 12 }}
|
||||||
|
readinessProbe:
|
||||||
|
tcpSocket:
|
||||||
|
port: mcp
|
||||||
|
{{- toYaml .Values.mcp.probes.readiness | nindent 12 }}
|
||||||
|
livenessProbe:
|
||||||
|
tcpSocket:
|
||||||
|
port: mcp
|
||||||
|
{{- toYaml .Values.mcp.probes.liveness | nindent 12 }}
|
||||||
|
resources:
|
||||||
|
{{- toYaml .Values.mcp.resources | nindent 12 }}
|
||||||
|
volumeMounts:
|
||||||
|
{{- include "knowledge-platform.tmpMount" . | nindent 12 }}
|
||||||
|
volumes:
|
||||||
|
{{- include "knowledge-platform.tmpVolume" . | nindent 8 }}
|
||||||
17
deploy/helm/knowledge-platform/templates/mcp-service.yaml
Normal file
17
deploy/helm/knowledge-platform/templates/mcp-service.yaml
Normal file
@ -0,0 +1,17 @@
|
|||||||
|
apiVersion: v1
|
||||||
|
kind: Service
|
||||||
|
metadata:
|
||||||
|
name: {{ include "knowledge-platform.fullname" . }}-mcp
|
||||||
|
labels:
|
||||||
|
{{- include "knowledge-platform.labels" . | nindent 4 }}
|
||||||
|
app.kubernetes.io/component: mcp
|
||||||
|
spec:
|
||||||
|
type: {{ .Values.mcp.service.type }}
|
||||||
|
ports:
|
||||||
|
- name: mcp
|
||||||
|
port: {{ .Values.mcp.service.port }}
|
||||||
|
targetPort: mcp
|
||||||
|
protocol: TCP
|
||||||
|
selector:
|
||||||
|
{{- include "knowledge-platform.selectorLabels" . | nindent 4 }}
|
||||||
|
app.kubernetes.io/component: mcp
|
||||||
59
deploy/helm/knowledge-platform/templates/migration-job.yaml
Normal file
59
deploy/helm/knowledge-platform/templates/migration-job.yaml
Normal file
@ -0,0 +1,59 @@
|
|||||||
|
{{- if .Values.migration.enabled }}
|
||||||
|
apiVersion: batch/v1
|
||||||
|
kind: Job
|
||||||
|
metadata:
|
||||||
|
name: {{ include "knowledge-platform.fullname" . }}-migration
|
||||||
|
labels:
|
||||||
|
{{- include "knowledge-platform.labels" . | nindent 4 }}
|
||||||
|
app.kubernetes.io/component: migration
|
||||||
|
annotations:
|
||||||
|
# Helm waits for pre-install/pre-upgrade hooks. A failed migration blocks
|
||||||
|
# the application rollout, and the previous Job remains until the next run.
|
||||||
|
helm.sh/hook: pre-install,pre-upgrade
|
||||||
|
helm.sh/hook-weight: "-10"
|
||||||
|
helm.sh/hook-delete-policy: before-hook-creation
|
||||||
|
spec:
|
||||||
|
backoffLimit: {{ .Values.migration.backoffLimit }}
|
||||||
|
activeDeadlineSeconds: {{ .Values.migration.activeDeadlineSeconds }}
|
||||||
|
ttlSecondsAfterFinished: {{ .Values.migration.ttlSecondsAfterFinished }}
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
{{- include "knowledge-platform.selectorLabels" . | nindent 8 }}
|
||||||
|
app.kubernetes.io/component: migration
|
||||||
|
spec:
|
||||||
|
restartPolicy: Never
|
||||||
|
# Pre-install hooks run before the release ServiceAccount exists. The
|
||||||
|
# namespace default account is used with token mounting disabled; the
|
||||||
|
# migration needs no Kubernetes API permissions.
|
||||||
|
serviceAccountName: default
|
||||||
|
automountServiceAccountToken: false
|
||||||
|
securityContext:
|
||||||
|
{{- include "knowledge-platform.podSecurityContext" . | nindent 8 }}
|
||||||
|
{{- with .Values.imagePullSecrets }}
|
||||||
|
imagePullSecrets:
|
||||||
|
{{- toYaml . | nindent 8 }}
|
||||||
|
{{- end }}
|
||||||
|
containers:
|
||||||
|
- name: migration
|
||||||
|
image: {{ include "knowledge-platform.image" .Values.images.api | quote }}
|
||||||
|
imagePullPolicy: {{ .Values.images.api.pullPolicy }}
|
||||||
|
command: ["alembic"]
|
||||||
|
args: ["upgrade", "head"]
|
||||||
|
securityContext:
|
||||||
|
{{- include "knowledge-platform.containerSecurityContext" . | nindent 12 }}
|
||||||
|
env:
|
||||||
|
- name: APP_ENV
|
||||||
|
value: production
|
||||||
|
- name: DATABASE_URL
|
||||||
|
valueFrom:
|
||||||
|
secretKeyRef:
|
||||||
|
name: {{ .Values.existingSecret }}
|
||||||
|
key: DATABASE_URL
|
||||||
|
resources:
|
||||||
|
{{- toYaml .Values.migration.resources | nindent 12 }}
|
||||||
|
volumeMounts:
|
||||||
|
{{- include "knowledge-platform.tmpMount" . | nindent 12 }}
|
||||||
|
volumes:
|
||||||
|
{{- include "knowledge-platform.tmpVolume" . | nindent 8 }}
|
||||||
|
{{- end }}
|
||||||
@ -0,0 +1,71 @@
|
|||||||
|
{{- if .Values.scheduler.enabled }}
|
||||||
|
apiVersion: apps/v1
|
||||||
|
kind: Deployment
|
||||||
|
metadata:
|
||||||
|
name: {{ include "knowledge-platform.fullname" . }}-scheduler
|
||||||
|
labels:
|
||||||
|
{{- include "knowledge-platform.labels" . | nindent 4 }}
|
||||||
|
app.kubernetes.io/component: scheduler
|
||||||
|
spec:
|
||||||
|
# A single Celery Beat owns periodic durable-Job recovery. Recreate prevents
|
||||||
|
# two schedulers from overlapping during upgrades.
|
||||||
|
replicas: 1
|
||||||
|
strategy:
|
||||||
|
type: Recreate
|
||||||
|
selector:
|
||||||
|
matchLabels:
|
||||||
|
{{- include "knowledge-platform.selectorLabels" . | nindent 6 }}
|
||||||
|
app.kubernetes.io/component: scheduler
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
{{- include "knowledge-platform.selectorLabels" . | nindent 8 }}
|
||||||
|
app.kubernetes.io/component: scheduler
|
||||||
|
{{- with .Values.podLabels }}
|
||||||
|
{{- toYaml . | nindent 8 }}
|
||||||
|
{{- end }}
|
||||||
|
annotations:
|
||||||
|
checksum/runtime-config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }}
|
||||||
|
checksum/runtime-secret-reference: {{ .Values.existingSecret | sha256sum }}
|
||||||
|
{{- with .Values.podAnnotations }}
|
||||||
|
{{- toYaml . | nindent 8 }}
|
||||||
|
{{- end }}
|
||||||
|
spec:
|
||||||
|
serviceAccountName: {{ include "knowledge-platform.serviceAccountName" . }}
|
||||||
|
automountServiceAccountToken: false
|
||||||
|
securityContext:
|
||||||
|
{{- include "knowledge-platform.podSecurityContext" . | nindent 8 }}
|
||||||
|
{{- with .Values.imagePullSecrets }}
|
||||||
|
imagePullSecrets:
|
||||||
|
{{- toYaml . | nindent 8 }}
|
||||||
|
{{- end }}
|
||||||
|
containers:
|
||||||
|
- name: scheduler
|
||||||
|
image: {{ include "knowledge-platform.image" .Values.images.worker | quote }}
|
||||||
|
imagePullPolicy: {{ .Values.images.worker.pullPolicy }}
|
||||||
|
workingDir: /tmp
|
||||||
|
command: ["celery"]
|
||||||
|
args:
|
||||||
|
- --app=ekp_api.worker:celery_app
|
||||||
|
- beat
|
||||||
|
- --loglevel=INFO
|
||||||
|
- --schedule=/tmp/celerybeat-schedule
|
||||||
|
securityContext:
|
||||||
|
{{- include "knowledge-platform.containerSecurityContext" . | nindent 12 }}
|
||||||
|
envFrom:
|
||||||
|
{{- include "knowledge-platform.envFrom" . | nindent 12 }}
|
||||||
|
startupProbe:
|
||||||
|
exec:
|
||||||
|
command: ["python", "-c", "import os; os.kill(1, 0)"]
|
||||||
|
{{- toYaml .Values.scheduler.probes.startup | nindent 12 }}
|
||||||
|
livenessProbe:
|
||||||
|
exec:
|
||||||
|
command: ["python", "-c", "import os; os.kill(1, 0)"]
|
||||||
|
{{- toYaml .Values.scheduler.probes.liveness | nindent 12 }}
|
||||||
|
resources:
|
||||||
|
{{- toYaml .Values.scheduler.resources | nindent 12 }}
|
||||||
|
volumeMounts:
|
||||||
|
{{- include "knowledge-platform.tmpMount" . | nindent 12 }}
|
||||||
|
volumes:
|
||||||
|
{{- include "knowledge-platform.tmpVolume" . | nindent 8 }}
|
||||||
|
{{- end }}
|
||||||
13
deploy/helm/knowledge-platform/templates/serviceaccount.yaml
Normal file
13
deploy/helm/knowledge-platform/templates/serviceaccount.yaml
Normal file
@ -0,0 +1,13 @@
|
|||||||
|
{{- if .Values.serviceAccount.create }}
|
||||||
|
apiVersion: v1
|
||||||
|
kind: ServiceAccount
|
||||||
|
metadata:
|
||||||
|
name: {{ include "knowledge-platform.serviceAccountName" . }}
|
||||||
|
labels:
|
||||||
|
{{- include "knowledge-platform.labels" . | nindent 4 }}
|
||||||
|
{{- with .Values.serviceAccount.annotations }}
|
||||||
|
annotations:
|
||||||
|
{{- toYaml . | nindent 4 }}
|
||||||
|
{{- end }}
|
||||||
|
automountServiceAccountToken: false
|
||||||
|
{{- end }}
|
||||||
@ -0,0 +1,17 @@
|
|||||||
|
apiVersion: v1
|
||||||
|
kind: ConfigMap
|
||||||
|
metadata:
|
||||||
|
name: {{ include "knowledge-platform.fullname" . }}-preflight-references
|
||||||
|
labels:
|
||||||
|
{{- include "knowledge-platform.labels" . | nindent 4 }}
|
||||||
|
annotations:
|
||||||
|
# The Make preflight renders this test hook to discover only resource
|
||||||
|
# names. It never renders or reads Secret data.
|
||||||
|
helm.sh/hook: test
|
||||||
|
data:
|
||||||
|
runtimeSecret: {{ .Values.existingSecret | quote }}
|
||||||
|
ingressEnabled: {{ .Values.ingress.enabled | quote }}
|
||||||
|
apiTlsSecret: {{ .Values.ingress.api.tlsSecretName | quote }}
|
||||||
|
mcpTlsSecret: {{ .Values.ingress.mcp.tlsSecretName | quote }}
|
||||||
|
rabbitmqEnabled: {{ .Values.rabbitmq.enabled | quote }}
|
||||||
|
rabbitmqSecret: {{ .Values.rabbitmq.existingSecret | quote }}
|
||||||
@ -0,0 +1,71 @@
|
|||||||
|
apiVersion: apps/v1
|
||||||
|
kind: Deployment
|
||||||
|
metadata:
|
||||||
|
name: {{ include "knowledge-platform.fullname" . }}-worker
|
||||||
|
labels:
|
||||||
|
{{- include "knowledge-platform.labels" . | nindent 4 }}
|
||||||
|
app.kubernetes.io/component: worker
|
||||||
|
spec:
|
||||||
|
replicas: {{ .Values.worker.replicaCount }}
|
||||||
|
strategy:
|
||||||
|
type: RollingUpdate
|
||||||
|
rollingUpdate:
|
||||||
|
maxUnavailable: 1
|
||||||
|
maxSurge: 1
|
||||||
|
selector:
|
||||||
|
matchLabels:
|
||||||
|
{{- include "knowledge-platform.selectorLabels" . | nindent 6 }}
|
||||||
|
app.kubernetes.io/component: worker
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
{{- include "knowledge-platform.selectorLabels" . | nindent 8 }}
|
||||||
|
app.kubernetes.io/component: worker
|
||||||
|
{{- with .Values.podLabels }}
|
||||||
|
{{- toYaml . | nindent 8 }}
|
||||||
|
{{- end }}
|
||||||
|
annotations:
|
||||||
|
checksum/runtime-config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }}
|
||||||
|
checksum/runtime-secret-reference: {{ .Values.existingSecret | sha256sum }}
|
||||||
|
{{- with .Values.podAnnotations }}
|
||||||
|
{{- toYaml . | nindent 8 }}
|
||||||
|
{{- end }}
|
||||||
|
spec:
|
||||||
|
serviceAccountName: {{ include "knowledge-platform.serviceAccountName" . }}
|
||||||
|
automountServiceAccountToken: false
|
||||||
|
terminationGracePeriodSeconds: {{ .Values.worker.terminationGracePeriodSeconds }}
|
||||||
|
securityContext:
|
||||||
|
{{- include "knowledge-platform.podSecurityContext" . | nindent 8 }}
|
||||||
|
{{- with .Values.imagePullSecrets }}
|
||||||
|
imagePullSecrets:
|
||||||
|
{{- toYaml . | nindent 8 }}
|
||||||
|
{{- end }}
|
||||||
|
containers:
|
||||||
|
- name: worker
|
||||||
|
image: {{ include "knowledge-platform.image" .Values.images.worker | quote }}
|
||||||
|
imagePullPolicy: {{ .Values.images.worker.pullPolicy }}
|
||||||
|
workingDir: /tmp
|
||||||
|
command: ["celery"]
|
||||||
|
args:
|
||||||
|
- --app=ekp_api.worker:celery_app
|
||||||
|
- worker
|
||||||
|
- --loglevel=INFO
|
||||||
|
- --queues=ingestion
|
||||||
|
securityContext:
|
||||||
|
{{- include "knowledge-platform.containerSecurityContext" . | nindent 12 }}
|
||||||
|
envFrom:
|
||||||
|
{{- include "knowledge-platform.envFrom" . | nindent 12 }}
|
||||||
|
startupProbe:
|
||||||
|
exec:
|
||||||
|
command: ["python", "-c", "import os; os.kill(1, 0)"]
|
||||||
|
{{- toYaml .Values.worker.probes.startup | nindent 12 }}
|
||||||
|
livenessProbe:
|
||||||
|
exec:
|
||||||
|
command: ["python", "-c", "import os; os.kill(1, 0)"]
|
||||||
|
{{- toYaml .Values.worker.probes.liveness | nindent 12 }}
|
||||||
|
resources:
|
||||||
|
{{- toYaml .Values.worker.resources | nindent 12 }}
|
||||||
|
volumeMounts:
|
||||||
|
{{- include "knowledge-platform.tmpMount" . | nindent 12 }}
|
||||||
|
volumes:
|
||||||
|
{{- include "knowledge-platform.tmpVolume" . | nindent 8 }}
|
||||||
338
deploy/helm/knowledge-platform/values.schema.json
Normal file
338
deploy/helm/knowledge-platform/values.schema.json
Normal file
@ -0,0 +1,338 @@
|
|||||||
|
{
|
||||||
|
"$schema": "https://json-schema.org/draft-07/schema#",
|
||||||
|
"title": "Enterprise Knowledge Platform Helm values",
|
||||||
|
"type": "object",
|
||||||
|
"additionalProperties": false,
|
||||||
|
"required": [
|
||||||
|
"existingSecret",
|
||||||
|
"images",
|
||||||
|
"serviceAccount",
|
||||||
|
"api",
|
||||||
|
"mcp",
|
||||||
|
"worker",
|
||||||
|
"scheduler",
|
||||||
|
"migration",
|
||||||
|
"ingress",
|
||||||
|
"config",
|
||||||
|
"rabbitmq"
|
||||||
|
],
|
||||||
|
"properties": {
|
||||||
|
"nameOverride": {"type": "string"},
|
||||||
|
"fullnameOverride": {"type": "string"},
|
||||||
|
"existingSecret": {"$ref": "#/definitions/kubernetesName"},
|
||||||
|
"imagePullSecrets": {
|
||||||
|
"type": "array",
|
||||||
|
"items": {
|
||||||
|
"type": "object",
|
||||||
|
"additionalProperties": false,
|
||||||
|
"required": ["name"],
|
||||||
|
"properties": {"name": {"$ref": "#/definitions/kubernetesName"}}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"images": {
|
||||||
|
"type": "object",
|
||||||
|
"additionalProperties": false,
|
||||||
|
"required": ["api", "mcp", "worker"],
|
||||||
|
"properties": {
|
||||||
|
"api": {"$ref": "#/definitions/image"},
|
||||||
|
"mcp": {"$ref": "#/definitions/image"},
|
||||||
|
"worker": {"$ref": "#/definitions/image"}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"serviceAccount": {
|
||||||
|
"type": "object",
|
||||||
|
"additionalProperties": false,
|
||||||
|
"required": ["create", "name", "annotations"],
|
||||||
|
"properties": {
|
||||||
|
"create": {"type": "boolean"},
|
||||||
|
"name": {"type": "string"},
|
||||||
|
"annotations": {"$ref": "#/definitions/stringMap"}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"podAnnotations": {"$ref": "#/definitions/stringMap"},
|
||||||
|
"podLabels": {"$ref": "#/definitions/stringMap"},
|
||||||
|
"api": {"$ref": "#/definitions/httpWorkload"},
|
||||||
|
"mcp": {"$ref": "#/definitions/httpWorkload"},
|
||||||
|
"worker": {
|
||||||
|
"type": "object",
|
||||||
|
"additionalProperties": false,
|
||||||
|
"required": ["replicaCount", "terminationGracePeriodSeconds", "resources", "probes"],
|
||||||
|
"properties": {
|
||||||
|
"replicaCount": {"type": "integer", "minimum": 1, "maximum": 100},
|
||||||
|
"terminationGracePeriodSeconds": {"type": "integer", "minimum": 30, "maximum": 3600},
|
||||||
|
"resources": {"$ref": "#/definitions/resources"},
|
||||||
|
"probes": {"$ref": "#/definitions/processProbes"}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"scheduler": {
|
||||||
|
"type": "object",
|
||||||
|
"additionalProperties": false,
|
||||||
|
"required": ["enabled", "resources", "probes"],
|
||||||
|
"properties": {
|
||||||
|
"enabled": {"type": "boolean"},
|
||||||
|
"resources": {"$ref": "#/definitions/resources"},
|
||||||
|
"probes": {"$ref": "#/definitions/processProbes"}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"migration": {
|
||||||
|
"type": "object",
|
||||||
|
"additionalProperties": false,
|
||||||
|
"required": ["enabled", "activeDeadlineSeconds", "backoffLimit", "ttlSecondsAfterFinished", "resources"],
|
||||||
|
"properties": {
|
||||||
|
"enabled": {"type": "boolean"},
|
||||||
|
"activeDeadlineSeconds": {"type": "integer", "minimum": 60, "maximum": 3600},
|
||||||
|
"backoffLimit": {"type": "integer", "minimum": 0, "maximum": 3},
|
||||||
|
"ttlSecondsAfterFinished": {"type": "integer", "minimum": 0},
|
||||||
|
"resources": {"$ref": "#/definitions/resources"}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"ingress": {
|
||||||
|
"type": "object",
|
||||||
|
"additionalProperties": false,
|
||||||
|
"required": ["enabled", "className", "annotations", "api", "mcp"],
|
||||||
|
"properties": {
|
||||||
|
"enabled": {"type": "boolean"},
|
||||||
|
"className": {"type": "string", "minLength": 1},
|
||||||
|
"annotations": {"$ref": "#/definitions/stringMap"},
|
||||||
|
"api": {"$ref": "#/definitions/ingressRoute"},
|
||||||
|
"mcp": {"$ref": "#/definitions/ingressRoute"}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"config": {
|
||||||
|
"type": "object",
|
||||||
|
"additionalProperties": false,
|
||||||
|
"required": ["appName", "logLevel", "database", "ingestion", "objectStorage", "chunking", "embedding", "vectorStore", "retrieval", "reranker", "mcp"],
|
||||||
|
"properties": {
|
||||||
|
"appName": {"type": "string", "minLength": 1},
|
||||||
|
"logLevel": {"enum": ["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"]},
|
||||||
|
"database": {"$ref": "#/definitions/databaseConfig"},
|
||||||
|
"ingestion": {"$ref": "#/definitions/ingestionConfig"},
|
||||||
|
"objectStorage": {"$ref": "#/definitions/objectStorageConfig"},
|
||||||
|
"chunking": {"$ref": "#/definitions/chunkingConfig"},
|
||||||
|
"embedding": {"$ref": "#/definitions/embeddingConfig"},
|
||||||
|
"vectorStore": {"$ref": "#/definitions/vectorStoreConfig"},
|
||||||
|
"retrieval": {"$ref": "#/definitions/retrievalConfig"},
|
||||||
|
"reranker": {"$ref": "#/definitions/rerankerConfig"},
|
||||||
|
"mcp": {"$ref": "#/definitions/mcpConfig"}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"rabbitmq": {
|
||||||
|
"type": "object",
|
||||||
|
"required": ["enabled", "existingSecret"],
|
||||||
|
"properties": {
|
||||||
|
"enabled": {"type": "boolean"},
|
||||||
|
"existingSecret": {"$ref": "#/definitions/kubernetesName"}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"definitions": {
|
||||||
|
"kubernetesName": {
|
||||||
|
"type": "string",
|
||||||
|
"minLength": 1,
|
||||||
|
"maxLength": 253,
|
||||||
|
"pattern": "^[a-z0-9]([-a-z0-9.]*[a-z0-9])?$"
|
||||||
|
},
|
||||||
|
"stringMap": {
|
||||||
|
"type": "object",
|
||||||
|
"additionalProperties": {"type": "string"}
|
||||||
|
},
|
||||||
|
"image": {
|
||||||
|
"type": "object",
|
||||||
|
"additionalProperties": false,
|
||||||
|
"required": ["repository", "tag", "digest", "pullPolicy"],
|
||||||
|
"properties": {
|
||||||
|
"repository": {"type": "string", "minLength": 1},
|
||||||
|
"tag": {"type": "string", "minLength": 1, "not": {"enum": ["latest"]}},
|
||||||
|
"digest": {"type": "string", "pattern": "^$|^sha256:[a-f0-9]{64}$"},
|
||||||
|
"pullPolicy": {"enum": ["Always", "IfNotPresent", "Never"]}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"probeTiming": {
|
||||||
|
"type": "object",
|
||||||
|
"additionalProperties": false,
|
||||||
|
"required": ["failureThreshold", "periodSeconds"],
|
||||||
|
"properties": {
|
||||||
|
"failureThreshold": {"type": "integer", "minimum": 1},
|
||||||
|
"periodSeconds": {"type": "integer", "minimum": 1},
|
||||||
|
"timeoutSeconds": {"type": "integer", "minimum": 1}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"httpProbes": {
|
||||||
|
"type": "object",
|
||||||
|
"additionalProperties": false,
|
||||||
|
"required": ["startup", "readiness", "liveness"],
|
||||||
|
"properties": {
|
||||||
|
"startup": {"$ref": "#/definitions/probeTiming"},
|
||||||
|
"readiness": {"$ref": "#/definitions/probeTiming"},
|
||||||
|
"liveness": {"$ref": "#/definitions/probeTiming"}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"processProbes": {
|
||||||
|
"type": "object",
|
||||||
|
"additionalProperties": false,
|
||||||
|
"required": ["startup", "liveness"],
|
||||||
|
"properties": {
|
||||||
|
"startup": {"$ref": "#/definitions/probeTiming"},
|
||||||
|
"liveness": {"$ref": "#/definitions/probeTiming"}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"resources": {
|
||||||
|
"type": "object",
|
||||||
|
"additionalProperties": false,
|
||||||
|
"required": ["requests", "limits"],
|
||||||
|
"properties": {
|
||||||
|
"requests": {"$ref": "#/definitions/resourcePair"},
|
||||||
|
"limits": {"$ref": "#/definitions/resourcePair"}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"resourcePair": {
|
||||||
|
"type": "object",
|
||||||
|
"additionalProperties": false,
|
||||||
|
"required": ["cpu", "memory"],
|
||||||
|
"properties": {
|
||||||
|
"cpu": {"type": ["string", "number"]},
|
||||||
|
"memory": {"type": "string", "minLength": 2}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"httpWorkload": {
|
||||||
|
"type": "object",
|
||||||
|
"additionalProperties": false,
|
||||||
|
"required": ["replicaCount", "service", "resources", "probes"],
|
||||||
|
"properties": {
|
||||||
|
"replicaCount": {"type": "integer", "minimum": 2, "maximum": 100},
|
||||||
|
"service": {
|
||||||
|
"type": "object",
|
||||||
|
"additionalProperties": false,
|
||||||
|
"required": ["type", "port"],
|
||||||
|
"properties": {
|
||||||
|
"type": {"enum": ["ClusterIP"]},
|
||||||
|
"port": {"type": "integer", "minimum": 1, "maximum": 65535}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"resources": {"$ref": "#/definitions/resources"},
|
||||||
|
"probes": {"$ref": "#/definitions/httpProbes"}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"ingressRoute": {
|
||||||
|
"type": "object",
|
||||||
|
"additionalProperties": false,
|
||||||
|
"required": ["host", "path", "tlsSecretName"],
|
||||||
|
"properties": {
|
||||||
|
"host": {"type": "string", "minLength": 1},
|
||||||
|
"path": {"type": "string", "pattern": "^/"},
|
||||||
|
"tlsSecretName": {"$ref": "#/definitions/kubernetesName"}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"databaseConfig": {
|
||||||
|
"type": "object",
|
||||||
|
"additionalProperties": false,
|
||||||
|
"required": ["poolSize", "maxOverflow", "poolRecycleSeconds", "poolTimeoutSeconds", "connectTimeoutSeconds"],
|
||||||
|
"properties": {
|
||||||
|
"poolSize": {"type": "integer", "minimum": 1, "maximum": 100},
|
||||||
|
"maxOverflow": {"type": "integer", "minimum": 0, "maximum": 100},
|
||||||
|
"poolRecycleSeconds": {"type": "integer", "minimum": 1},
|
||||||
|
"poolTimeoutSeconds": {"type": "number", "exclusiveMinimum": 0},
|
||||||
|
"connectTimeoutSeconds": {"type": "integer", "minimum": 1}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"ingestionConfig": {
|
||||||
|
"type": "object",
|
||||||
|
"additionalProperties": false,
|
||||||
|
"required": ["maxAttempts", "retryBackoffSeconds", "retryBackoffMaxSeconds", "workerLeaseSeconds", "recoveryIntervalSeconds", "batchMaxFiles", "batchMaxTotalBytes", "maxUploadBytes"],
|
||||||
|
"properties": {
|
||||||
|
"maxAttempts": {"type": "integer", "minimum": 1, "maximum": 10},
|
||||||
|
"retryBackoffSeconds": {"type": "integer", "minimum": 1},
|
||||||
|
"retryBackoffMaxSeconds": {"type": "integer", "minimum": 1},
|
||||||
|
"workerLeaseSeconds": {"type": "integer", "minimum": 30},
|
||||||
|
"recoveryIntervalSeconds": {"type": "integer", "minimum": 5},
|
||||||
|
"batchMaxFiles": {"type": "integer", "minimum": 1, "maximum": 100},
|
||||||
|
"batchMaxTotalBytes": {"type": "integer", "minimum": 1},
|
||||||
|
"maxUploadBytes": {"type": "integer", "minimum": 1}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"objectStorageConfig": {
|
||||||
|
"type": "object",
|
||||||
|
"additionalProperties": false,
|
||||||
|
"required": ["provider", "endpoint", "bucket", "keyPrefix", "timeoutSeconds"],
|
||||||
|
"properties": {
|
||||||
|
"provider": {"const": "bos"},
|
||||||
|
"endpoint": {"type": "string", "minLength": 1},
|
||||||
|
"bucket": {"type": "string", "minLength": 1},
|
||||||
|
"keyPrefix": {"type": "string", "minLength": 1},
|
||||||
|
"timeoutSeconds": {"type": "number", "exclusiveMinimum": 0}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"chunkingConfig": {
|
||||||
|
"type": "object",
|
||||||
|
"additionalProperties": false,
|
||||||
|
"required": ["maxTokens", "overlapTokens", "maxCharacters"],
|
||||||
|
"properties": {
|
||||||
|
"maxTokens": {"type": "integer", "minimum": 1},
|
||||||
|
"overlapTokens": {"type": "integer", "minimum": 0},
|
||||||
|
"maxCharacters": {"type": "integer", "minimum": 4}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"embeddingConfig": {
|
||||||
|
"type": "object",
|
||||||
|
"additionalProperties": false,
|
||||||
|
"required": ["provider", "model", "endpoint", "timeoutSeconds"],
|
||||||
|
"properties": {
|
||||||
|
"provider": {"const": "qianfan"},
|
||||||
|
"model": {"const": "bge-large-zh"},
|
||||||
|
"endpoint": {"type": "string", "minLength": 1},
|
||||||
|
"timeoutSeconds": {"type": "number", "exclusiveMinimum": 0}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"vectorStoreConfig": {
|
||||||
|
"type": "object",
|
||||||
|
"additionalProperties": false,
|
||||||
|
"required": ["provider", "uri", "user", "database", "table", "index", "batchSize", "timeoutSeconds"],
|
||||||
|
"properties": {
|
||||||
|
"provider": {"const": "oceanbase"},
|
||||||
|
"uri": {"type": "string", "minLength": 1},
|
||||||
|
"user": {"type": "string", "minLength": 1},
|
||||||
|
"database": {"type": "string", "minLength": 1},
|
||||||
|
"table": {"type": "string", "minLength": 1},
|
||||||
|
"index": {"type": "string", "minLength": 1},
|
||||||
|
"batchSize": {"type": "integer", "minimum": 1, "maximum": 1000},
|
||||||
|
"timeoutSeconds": {"type": "number", "exclusiveMinimum": 0}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"retrievalConfig": {
|
||||||
|
"type": "object",
|
||||||
|
"additionalProperties": false,
|
||||||
|
"required": ["indexProfileId", "denseTopN", "searchTopK"],
|
||||||
|
"properties": {
|
||||||
|
"indexProfileId": {"type": "string", "minLength": 1},
|
||||||
|
"denseTopN": {"type": "integer", "minimum": 1, "maximum": 64},
|
||||||
|
"searchTopK": {"type": "integer", "minimum": 1, "maximum": 64}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"rerankerConfig": {
|
||||||
|
"type": "object",
|
||||||
|
"additionalProperties": false,
|
||||||
|
"required": ["provider", "model", "endpoint", "timeoutSeconds"],
|
||||||
|
"properties": {
|
||||||
|
"provider": {"const": "qianfan"},
|
||||||
|
"model": {"const": "bce-reranker-base"},
|
||||||
|
"endpoint": {"type": "string", "minLength": 1},
|
||||||
|
"timeoutSeconds": {"type": "number", "exclusiveMinimum": 0}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"mcpConfig": {
|
||||||
|
"type": "object",
|
||||||
|
"additionalProperties": false,
|
||||||
|
"required": ["publicUrl", "issuerUrl", "clientId", "serviceIdentity", "requiredScope", "allowedHosts", "allowedOrigins", "maxRequestBodyBytes"],
|
||||||
|
"properties": {
|
||||||
|
"publicUrl": {"type": "string", "pattern": "^https://"},
|
||||||
|
"issuerUrl": {"type": "string", "pattern": "^https://"},
|
||||||
|
"clientId": {"type": "string", "minLength": 1},
|
||||||
|
"serviceIdentity": {"type": "string", "minLength": 1},
|
||||||
|
"requiredScope": {"const": "knowledge:search"},
|
||||||
|
"allowedHosts": {"type": "array", "minItems": 1, "items": {"type": "string", "minLength": 1}},
|
||||||
|
"allowedOrigins": {"type": "array", "minItems": 1, "items": {"type": "string", "pattern": "^https://"}},
|
||||||
|
"maxRequestBodyBytes": {"type": "integer", "minimum": 1024, "maximum": 4194304}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
218
deploy/helm/knowledge-platform/values.yaml
Normal file
218
deploy/helm/knowledge-platform/values.yaml
Normal file
@ -0,0 +1,218 @@
|
|||||||
|
# DEPLOY-002 safe defaults. Real endpoints, immutable image coordinates and
|
||||||
|
# Secret names are supplied in an environment-specific values file.
|
||||||
|
nameOverride: ""
|
||||||
|
fullnameOverride: ""
|
||||||
|
|
||||||
|
existingSecret: knowledge-platform-runtime
|
||||||
|
imagePullSecrets: []
|
||||||
|
|
||||||
|
images:
|
||||||
|
api:
|
||||||
|
repository: knowledge-platform/enterprise-knowledge-api
|
||||||
|
tag: "0.1.0"
|
||||||
|
digest: ""
|
||||||
|
pullPolicy: IfNotPresent
|
||||||
|
mcp:
|
||||||
|
repository: knowledge-platform/enterprise-knowledge-mcp
|
||||||
|
tag: "0.1.0"
|
||||||
|
digest: ""
|
||||||
|
pullPolicy: IfNotPresent
|
||||||
|
worker:
|
||||||
|
repository: knowledge-platform/enterprise-knowledge-worker
|
||||||
|
tag: "0.1.0"
|
||||||
|
digest: ""
|
||||||
|
pullPolicy: IfNotPresent
|
||||||
|
|
||||||
|
serviceAccount:
|
||||||
|
create: true
|
||||||
|
name: ""
|
||||||
|
annotations: {}
|
||||||
|
|
||||||
|
podAnnotations: {}
|
||||||
|
podLabels: {}
|
||||||
|
|
||||||
|
api:
|
||||||
|
replicaCount: 2
|
||||||
|
service:
|
||||||
|
type: ClusterIP
|
||||||
|
port: 80
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
cpu: 250m
|
||||||
|
memory: 512Mi
|
||||||
|
limits:
|
||||||
|
cpu: "1"
|
||||||
|
memory: 1Gi
|
||||||
|
probes:
|
||||||
|
startup:
|
||||||
|
failureThreshold: 30
|
||||||
|
periodSeconds: 2
|
||||||
|
readiness:
|
||||||
|
periodSeconds: 10
|
||||||
|
timeoutSeconds: 2
|
||||||
|
failureThreshold: 3
|
||||||
|
liveness:
|
||||||
|
periodSeconds: 20
|
||||||
|
timeoutSeconds: 2
|
||||||
|
failureThreshold: 3
|
||||||
|
|
||||||
|
mcp:
|
||||||
|
replicaCount: 2
|
||||||
|
service:
|
||||||
|
type: ClusterIP
|
||||||
|
port: 80
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
cpu: 250m
|
||||||
|
memory: 512Mi
|
||||||
|
limits:
|
||||||
|
cpu: "1"
|
||||||
|
memory: 1Gi
|
||||||
|
probes:
|
||||||
|
startup:
|
||||||
|
failureThreshold: 30
|
||||||
|
periodSeconds: 2
|
||||||
|
readiness:
|
||||||
|
periodSeconds: 10
|
||||||
|
timeoutSeconds: 2
|
||||||
|
failureThreshold: 3
|
||||||
|
liveness:
|
||||||
|
periodSeconds: 20
|
||||||
|
timeoutSeconds: 2
|
||||||
|
failureThreshold: 3
|
||||||
|
|
||||||
|
worker:
|
||||||
|
replicaCount: 2
|
||||||
|
terminationGracePeriodSeconds: 60
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
cpu: 500m
|
||||||
|
memory: 1Gi
|
||||||
|
limits:
|
||||||
|
cpu: "2"
|
||||||
|
memory: 2Gi
|
||||||
|
probes:
|
||||||
|
startup:
|
||||||
|
failureThreshold: 30
|
||||||
|
periodSeconds: 2
|
||||||
|
liveness:
|
||||||
|
periodSeconds: 20
|
||||||
|
timeoutSeconds: 2
|
||||||
|
failureThreshold: 3
|
||||||
|
|
||||||
|
# Celery Beat is separated from scalable workers so only one recovery
|
||||||
|
# scheduler publishes due durable Jobs at a time.
|
||||||
|
scheduler:
|
||||||
|
enabled: true
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
cpu: 100m
|
||||||
|
memory: 256Mi
|
||||||
|
limits:
|
||||||
|
cpu: 500m
|
||||||
|
memory: 512Mi
|
||||||
|
probes:
|
||||||
|
startup:
|
||||||
|
failureThreshold: 30
|
||||||
|
periodSeconds: 2
|
||||||
|
liveness:
|
||||||
|
periodSeconds: 20
|
||||||
|
timeoutSeconds: 2
|
||||||
|
failureThreshold: 3
|
||||||
|
|
||||||
|
migration:
|
||||||
|
enabled: true
|
||||||
|
activeDeadlineSeconds: 600
|
||||||
|
backoffLimit: 0
|
||||||
|
ttlSecondsAfterFinished: 86400
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
cpu: 100m
|
||||||
|
memory: 256Mi
|
||||||
|
limits:
|
||||||
|
cpu: 500m
|
||||||
|
memory: 512Mi
|
||||||
|
|
||||||
|
ingress:
|
||||||
|
enabled: false
|
||||||
|
className: nginx
|
||||||
|
annotations: {}
|
||||||
|
api:
|
||||||
|
host: knowledge.example.invalid
|
||||||
|
path: /
|
||||||
|
tlsSecretName: knowledge-platform-api-tls
|
||||||
|
mcp:
|
||||||
|
host: mcp.example.invalid
|
||||||
|
path: /mcp
|
||||||
|
tlsSecretName: knowledge-platform-mcp-tls
|
||||||
|
|
||||||
|
# Only non-sensitive runtime settings belong here. The templates enumerate
|
||||||
|
# keys explicitly so credentials cannot be added to this ConfigMap by mistake.
|
||||||
|
config:
|
||||||
|
appName: enterprise-knowledge-platform
|
||||||
|
logLevel: INFO
|
||||||
|
database:
|
||||||
|
poolSize: 10
|
||||||
|
maxOverflow: 10
|
||||||
|
poolRecycleSeconds: 600
|
||||||
|
poolTimeoutSeconds: 30
|
||||||
|
connectTimeoutSeconds: 30
|
||||||
|
ingestion:
|
||||||
|
maxAttempts: 3
|
||||||
|
retryBackoffSeconds: 2
|
||||||
|
retryBackoffMaxSeconds: 60
|
||||||
|
workerLeaseSeconds: 300
|
||||||
|
recoveryIntervalSeconds: 30
|
||||||
|
batchMaxFiles: 20
|
||||||
|
batchMaxTotalBytes: 209715200
|
||||||
|
maxUploadBytes: 52428800
|
||||||
|
objectStorage:
|
||||||
|
provider: bos
|
||||||
|
endpoint: https://s3.example.invalid
|
||||||
|
bucket: knowledge-platform
|
||||||
|
keyPrefix: knowledge-platform
|
||||||
|
timeoutSeconds: 30
|
||||||
|
chunking:
|
||||||
|
maxTokens: 450
|
||||||
|
overlapTokens: 75
|
||||||
|
maxCharacters: 2000
|
||||||
|
embedding:
|
||||||
|
provider: qianfan
|
||||||
|
model: bge-large-zh
|
||||||
|
endpoint: https://qianfan.baidubce.com/v2/embeddings
|
||||||
|
timeoutSeconds: 30
|
||||||
|
vectorStore:
|
||||||
|
provider: oceanbase
|
||||||
|
uri: oceanbase.example.invalid:2881
|
||||||
|
user: knowledge-platform
|
||||||
|
database: knowledge_platform
|
||||||
|
table: knowledge_chunks_v1
|
||||||
|
index: vidx_knowledge_chunks_embedding
|
||||||
|
batchSize: 100
|
||||||
|
timeoutSeconds: 30
|
||||||
|
retrieval:
|
||||||
|
indexProfileId: demo-bge-large-zh-structure-v1-oceanbase-v1
|
||||||
|
denseTopN: 20
|
||||||
|
searchTopK: 5
|
||||||
|
reranker:
|
||||||
|
provider: qianfan
|
||||||
|
model: bce-reranker-base
|
||||||
|
endpoint: https://qianfan.baidubce.com/v2/rerank
|
||||||
|
timeoutSeconds: 30
|
||||||
|
mcp:
|
||||||
|
publicUrl: https://mcp.example.invalid/mcp
|
||||||
|
issuerUrl: https://mcp.example.invalid
|
||||||
|
clientId: workbuddy-enterprise
|
||||||
|
serviceIdentity: workbuddy-public-knowledge
|
||||||
|
requiredScope: knowledge:search
|
||||||
|
allowedHosts:
|
||||||
|
- mcp.example.invalid
|
||||||
|
allowedOrigins:
|
||||||
|
- https://mcp.example.invalid
|
||||||
|
maxRequestBodyBytes: 1048576
|
||||||
|
|
||||||
|
# Optional local dependency. Keep disabled for production and provide
|
||||||
|
# RABBITMQ_URL through existingSecret for an externally managed broker.
|
||||||
|
rabbitmq:
|
||||||
|
enabled: false
|
||||||
|
existingSecret: knowledge-platform-rabbitmq
|
||||||
31
deploy/oceanbase/001_knowledge_chunks_v1.sql
Normal file
31
deploy/oceanbase/001_knowledge_chunks_v1.sql
Normal file
@ -0,0 +1,31 @@
|
|||||||
|
-- ARCH-003 rebuildable OceanBase Vector index for bge-large-zh (1024 dimensions).
|
||||||
|
-- Canonical metadata is co-located in OceanBase relational tables; this vector
|
||||||
|
-- table remains a derived index and may be rebuilt from canonical chunks.
|
||||||
|
-- Requires OceanBase 4.3.5.4+; ARCH-006 verifies PRE-FILTER in the target test cluster.
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS knowledge_chunks_v1 (
|
||||||
|
index_record_id VARCHAR(36) NOT NULL,
|
||||||
|
chunk_id VARCHAR(36) NOT NULL,
|
||||||
|
document_id VARCHAR(36) NOT NULL,
|
||||||
|
filename VARCHAR(255) NOT NULL,
|
||||||
|
page INT NULL,
|
||||||
|
page_end INT NULL,
|
||||||
|
section VARCHAR(1024) NULL,
|
||||||
|
chunk_index INT NOT NULL,
|
||||||
|
content MEDIUMTEXT NOT NULL,
|
||||||
|
embedding VECTOR(1024) NOT NULL,
|
||||||
|
embedding_model VARCHAR(128) NOT NULL,
|
||||||
|
index_profile_id VARCHAR(128) NOT NULL,
|
||||||
|
PRIMARY KEY (index_record_id),
|
||||||
|
KEY idx_knowledge_chunks_document_profile (document_id, index_profile_id),
|
||||||
|
KEY idx_knowledge_chunks_profile (index_profile_id),
|
||||||
|
VECTOR KEY vidx_knowledge_chunks_embedding (embedding)
|
||||||
|
WITH (
|
||||||
|
DISTANCE = COSINE,
|
||||||
|
TYPE = HNSW,
|
||||||
|
LIB = VSAG,
|
||||||
|
M = 16,
|
||||||
|
EF_CONSTRUCTION = 256,
|
||||||
|
EF_SEARCH = 200
|
||||||
|
) BLOCK_SIZE 16384
|
||||||
|
) ORGANIZATION = HEAP;
|
||||||
12
deploy/workbuddy-mcp.example.json
Normal file
12
deploy/workbuddy-mcp.example.json
Normal file
@ -0,0 +1,12 @@
|
|||||||
|
{
|
||||||
|
"mcpServers": {
|
||||||
|
"enterprise-knowledge": {
|
||||||
|
"type": "http",
|
||||||
|
"url": "${EKP_MCP_URL:-http://127.0.0.1:8766/mcp}",
|
||||||
|
"headers": {
|
||||||
|
"Authorization": "Bearer ${EKP_MCP_TOKEN}"
|
||||||
|
},
|
||||||
|
"description": "企业知识检索"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -5,3 +5,139 @@ Version all public APIs under `/api/v1`.
|
|||||||
Initial domains: auth, users/groups, knowledge-bases, documents, compiled-knowledge, search, evaluations, jobs, audit-logs, providers and MCP administration.
|
Initial domains: auth, users/groups, knowledge-bases, documents, compiled-knowledge, search, evaluations, jobs, audit-logs, providers and MCP administration.
|
||||||
|
|
||||||
Use OpenAPI as executable source of truth once implementation starts. Provider-specific concepts must not leak into public contracts.
|
Use OpenAPI as executable source of truth once implementation starts. Provider-specific concepts must not leak into public contracts.
|
||||||
|
|
||||||
|
## Demo v0.1 P0
|
||||||
|
|
||||||
|
DEMO-009 公开以下最小接口。`docs/demo/DEMO_SCOPE.md` 中的 `/api/*` 逻辑路径按本契约统一
|
||||||
|
版本化为 `/api/v1/*`。
|
||||||
|
|
||||||
|
`DEMO-015` 已将单文件上传切换为持久化异步 Job;`DEMO-016` 已增加批量提交和聚合查询。
|
||||||
|
|
||||||
|
| Method | Path | 行为 |
|
||||||
|
|---|---|---|
|
||||||
|
| `POST` | `/api/v1/documents` | multipart 单文件接收;持久化 Source、Document、Job 后返回 `202` |
|
||||||
|
| `POST` | `/api/v1/ingestion-batches` | multipart `files` 多文件接收;返回批次与逐文件状态 |
|
||||||
|
| `GET` | `/api/v1/ingestion-batches/{batch_id}` | 返回批次汇总和逐文件最新 Job 状态 |
|
||||||
|
| `GET` | `/api/v1/documents` | 按创建时间倒序返回文档列表 |
|
||||||
|
| `GET` | `/api/v1/documents/{document_id}` | 返回单个文档状态与安全错误信息 |
|
||||||
|
| `DELETE` | `/api/v1/documents/{document_id}` | 直接删除向量、Metadata、解析产物和原始文件 |
|
||||||
|
| `GET` | `/api/v1/jobs/{job_id}` | 返回阶段、状态、attempt、最大尝试次数和安全错误信息 |
|
||||||
|
| `POST` | `/api/v1/search` | Dense Top20、Reranker、TopK、来源补全 |
|
||||||
|
|
||||||
|
上传字段名为 `file`,支持 PDF、DOCX、TXT 和 Markdown。成功返回 `202`、Document 字段及
|
||||||
|
`job_id`、`job_status=queued`、`max_attempts`;响应不等待 Parse、Clean、Chunk、Embedding 或
|
||||||
|
Index。Document 响应不暴露 `storage_key`、Provider 配置或索引实现。
|
||||||
|
|
||||||
|
Search 请求:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"query": "产品A退款周期是多少?",
|
||||||
|
"top_k": 5
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Search 响应:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"results": [
|
||||||
|
{
|
||||||
|
"document_id": "6e5ec370-5a45-4eb7-95d7-e84267ad8752",
|
||||||
|
"filename": "产品A说明书.pdf",
|
||||||
|
"page": 12,
|
||||||
|
"section": "退款政策",
|
||||||
|
"chunk_id": "76ba1c30-bff7-44e5-9989-fb8372f54f46",
|
||||||
|
"content": "……",
|
||||||
|
"score": 0.82,
|
||||||
|
"rerank_score": 0.94
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
公开错误语义:请求校验失败返回 `422`,不支持的上传类型返回 `415`,文件过大返回 `413`,
|
||||||
|
文档、Job 或批次不存在返回 `404`,活动 Job 的文档删除返回 `409`,运行时 Provider 未配置返回
|
||||||
|
`503`。上游 Provider 或持久化异常只返回经过收敛的 `5xx` 信息,不回显凭证、请求正文或
|
||||||
|
厂商响应。
|
||||||
|
|
||||||
|
## Demo Admin 客户端路由
|
||||||
|
|
||||||
|
DEMO-010 提供 `/admin/upload`、`/admin/documents` 和 `/admin/search` 三个客户端路由,
|
||||||
|
`/admin` 默认展示上传页。它们是同源静态页面,不是新增业务 API,也不改变上述公开 REST
|
||||||
|
契约;所有业务操作仍调用 `/api/v1`。
|
||||||
|
|
||||||
|
## 当前公共知识库部署边界(PLAN-002)
|
||||||
|
|
||||||
|
当前阶段所有 Document 属于同一个公共知识可见域,不新增用户、知识库或 ACL 字段,也不把
|
||||||
|
这些完整未实现能力伪装为默认放行。Admin 写接口只允许部署在企业受控网络或统一接入网关后;
|
||||||
|
Ingress 是否进一步认证属于部署配置,但前端隐藏按钮或隐藏路由不构成安全控制。
|
||||||
|
|
||||||
|
集群重启、扩缩容和滚动部署不得改变本契约或要求浏览器重新注入 Provider 配置。数据库、BOS、
|
||||||
|
模型与队列凭据由服务端 Secret 提供,任何 API 响应都不得返回这些值。
|
||||||
|
|
||||||
|
## 异步入库契约(DEMO-015~016 已实现)
|
||||||
|
|
||||||
|
| Method | Path | 行为 |
|
||||||
|
|---|---|---|
|
||||||
|
| `POST` | `/api/v1/documents` | 已实现:单文件接收并返回 Document/Job 标识 |
|
||||||
|
| `POST` | `/api/v1/ingestion-batches` | multipart 多文件接收,返回批次及逐文件 Document/Job 标识 |
|
||||||
|
| `GET` | `/api/v1/ingestion-batches/{batch_id}` | 返回批次汇总和逐文件最新状态 |
|
||||||
|
| `GET` | `/api/v1/jobs/{job_id}` | 已实现:返回持久化 Job 状态 |
|
||||||
|
|
||||||
|
单文件响应示意:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"id": "6e5ec370-5a45-4eb7-95d7-e84267ad8752",
|
||||||
|
"filename": "退款政策.md",
|
||||||
|
"media_type": "text/markdown",
|
||||||
|
"size_bytes": 1024,
|
||||||
|
"status": "uploaded",
|
||||||
|
"error_message": null,
|
||||||
|
"created_at": "2026-09-02T12:00:00Z",
|
||||||
|
"updated_at": "2026-09-02T12:00:00Z",
|
||||||
|
"job_id": "7adc4278-1157-4c3d-95a4-6926ad77a04e",
|
||||||
|
"job_status": "queued",
|
||||||
|
"max_attempts": 3
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
批量字段名为 `files`,成功返回 `202`。响应示意:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"batch_id": "d21f8012-4c4e-4bb4-89af-e11c0f9f470e",
|
||||||
|
"status": "processing",
|
||||||
|
"total_files": 2,
|
||||||
|
"accepted_files": 1,
|
||||||
|
"rejected_files": 1,
|
||||||
|
"completed_files": 1,
|
||||||
|
"created_at": "2026-09-02T12:00:00Z",
|
||||||
|
"items": [
|
||||||
|
{
|
||||||
|
"item_index": 0,
|
||||||
|
"filename": "退款政策.md",
|
||||||
|
"acceptance_status": "accepted",
|
||||||
|
"document_id": "6e5ec370-5a45-4eb7-95d7-e84267ad8752",
|
||||||
|
"job_id": "7adc4278-1157-4c3d-95a4-6926ad77a04e",
|
||||||
|
"job_status": "queued",
|
||||||
|
"stage": null,
|
||||||
|
"attempt": 0,
|
||||||
|
"max_attempts": 3,
|
||||||
|
"error_message": null
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Job 公开状态为 `queued | running | retrying | succeeded | failed | cancelled`,并通过 `stage`
|
||||||
|
表示 `parsing | cleaning | chunking | embedding | indexing`。错误字段只返回有界、安全摘要。
|
||||||
|
批次状态为 `processing | completed | partial_failure | failed`;逐文件接收状态为
|
||||||
|
`pending | accepted | rejected`。批次允许逐文件部分接收/失败,不以一个文件的处理结果回滚
|
||||||
|
其他文件。文件数量、单文件大小和请求总大小分别由 `BATCH_MAX_FILES`、`MAX_UPLOAD_BYTES`
|
||||||
|
和 `BATCH_MAX_TOTAL_BYTES` 限制。数量超限返回 `413`;类型、大小或内容不合规作为该文件的
|
||||||
|
`rejected` 安全结果返回,其他合规文件继续接收。
|
||||||
|
|
||||||
|
自动重试是服务端 Job 行为,不要求客户端再次上传,也不新增公开手动重试接口。每次重试前
|
||||||
|
清理派生数据但保留原始文件和稳定的 Document ID;达到次数上限或清理失败后进入 `failed`。
|
||||||
|
|||||||
@ -8,4 +8,44 @@ P0 business tools:
|
|||||||
- `get_source`
|
- `get_source`
|
||||||
- `list_knowledge_bases`
|
- `list_knowledge_bases`
|
||||||
|
|
||||||
Every request maps to a user/service identity and uses the same authorization/application services as REST. Do not expose Mochow, Qwen, BGE, MinerU or other infrastructure choices.
|
Every request maps to a user/service identity and uses the same authorization/application services as REST. Do not expose OceanBase Vector, BOS, model names, parsers or other infrastructure choices.
|
||||||
|
|
||||||
|
## Demo priority slice(DEMO-012,已实现)
|
||||||
|
|
||||||
|
`DEMO-012` 已提升为当前最高实现优先级。首个可运行 Gateway 只注册
|
||||||
|
`search_knowledge`,其输入与 `/api/v1/search` 对齐:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"query": "产品A退款周期是多少?",
|
||||||
|
"top_k": 5
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
输出保留业务来源字段:`document_id`、`filename`、`page`、`section`、`chunk_id`、`content`
|
||||||
|
和排序分数。Gateway 必须复用现有 Search Service 或正式 REST API,不得复制检索、来源补全
|
||||||
|
或排序逻辑。
|
||||||
|
|
||||||
|
Demo 验证也必须建立 user/service identity 映射边界,禁止使用共享超级管理员 Token。由于
|
||||||
|
Document ACL 尚未进入 Demo Scope,`DEMO-012` 只能证明身份传递、认证拒绝和业务服务复用,
|
||||||
|
不能声称完成 V1 ACL 授权闭环。错误响应和日志不得包含 Provider Endpoint、凭据、数据库表名
|
||||||
|
或模型名。
|
||||||
|
|
||||||
|
验收至少覆盖:MCP 工具发现、参数校验、成功检索、无结果、认证失败、安全错误收敛,以及
|
||||||
|
WorkBuddy Enterprise 的一次端到端调用。其余 P0 tools 继续留在完整 V1 `EKP-071`。
|
||||||
|
|
||||||
|
实现使用独立 Streamable HTTP Gateway,默认端点为 `http://127.0.0.1:8766/mcp`。请求必须
|
||||||
|
携带 `Authorization: Bearer <token>`;Token 只从运行环境读取,并映射为显式 service identity
|
||||||
|
和 `knowledge:search` scope。当前静态 service credential 是 Demo 兼容层,不替代 V1 OAuth、
|
||||||
|
ACL 或用户委托授权。
|
||||||
|
|
||||||
|
公开错误边界:缺失/无效 Token 为 HTTP `401`,scope 不足为 `403`;Tool 参数不合法返回 MCP
|
||||||
|
参数错误;检索运行失败只返回 `knowledge search failed`。Host/Origin allowlist 默认只允许
|
||||||
|
本机,远程部署必须显式注入公开域名。
|
||||||
|
|
||||||
|
## 当前公共知识库部署边界(PLAN-002)
|
||||||
|
|
||||||
|
集群阶段仍只暴露只读 `search_knowledge`,并为 WorkBuddy 配置稳定的 HTTPS MCP 地址。
|
||||||
|
Credential 由 Kubernetes Secret 在服务端运行时注入,客户端只保存管理员分发的最小
|
||||||
|
`knowledge:search` Token;Pod 重启或滚动升级不得要求重新生成配置。公共知识域不等于匿名
|
||||||
|
访问或超级管理员身份,未来引入非公共知识前必须升级为用户委托身份与 ACL-aware Search。
|
||||||
|
|||||||
@ -2,9 +2,40 @@
|
|||||||
|
|
||||||
- ADR-001: API First + Admin Console — Accepted
|
- ADR-001: API First + Admin Console — Accepted
|
||||||
- ADR-002: Modular monolith + workers + MCP gateway for V1 — Accepted
|
- ADR-002: Modular monolith + workers + MCP gateway for V1 — Accepted
|
||||||
- ADR-003: PostgreSQL + object storage are canonical — Accepted
|
- ADR-003: PostgreSQL + object storage are canonical — Superseded by ADR-012
|
||||||
- ADR-004: Mochow is initial retrieval backend but replaceable — Accepted
|
- ADR-004: Mochow is initial retrieval backend but replaceable — Superseded by ADR-010
|
||||||
- ADR-005: Provider abstraction for embedding/reranker/parser/LLM — Accepted
|
- ADR-005: Provider abstraction for embedding/reranker/parser/LLM — Accepted
|
||||||
- ADR-006: RAG + Knowledge Compilation — Accepted
|
- ADR-006: RAG + Knowledge Compilation — Accepted
|
||||||
- ADR-007: ACL before/during retrieval — Accepted
|
- ADR-007: ACL before/during retrieval — Accepted
|
||||||
- ADR-008: MCP hides infrastructure details — Accepted
|
- ADR-008: MCP hides infrastructure details — Accepted
|
||||||
|
- ADR-009: Baidu BOS is the primary object-storage provider; local storage remains for tests and local development — Accepted
|
||||||
|
- ADR-010: OceanBase Vector is the primary rebuildable retrieval index behind `VectorStore`; PostgreSQL remains canonical — Superseded by ADR-012
|
||||||
|
- ADR-011: Qianfan `bge-large-zh` plus `bce-reranker-base` is the initial Demo model baseline — Accepted
|
||||||
|
- ADR-012: OceanBase MySQL relational tables + BOS are canonical; OceanBase vector tables remain rebuildable indexes — Accepted
|
||||||
|
|
||||||
|
## Architecture migration boundary
|
||||||
|
|
||||||
|
ARCH-006 的只读环境核验确认,目标平台没有独立 PostgreSQL,OceanBase MySQL 模式同时承载
|
||||||
|
关系数据和向量数据。因此 ADR-012 取代 ADR-003/010 中的 PostgreSQL 部署假设:OceanBase
|
||||||
|
关系表保存 canonical metadata,BOS 保存不可变源文件,OceanBase 专用向量表仍只保存可由
|
||||||
|
canonical Chunk 重建的检索行。共用数据库实例不改变两类数据的所有权边界,也不允许业务层
|
||||||
|
绕过 `VectorStore` 直接依赖 `pyobvector`。
|
||||||
|
|
||||||
|
目标 test 集群为 OceanBase 4.3.5.4 MySQL 模式。ARCH-006 已用只读 `EXPLAIN` 验证
|
||||||
|
`VECTOR_INDEX(... pre_filter)` 产生 `VECTOR INDEX ADAPTIVE SCAN (PRE-FILTER)`;版本下限与
|
||||||
|
运行检查据此调整为 4.3.5.4。生产升级和重新验收仍由运维流程控制。
|
||||||
|
|
||||||
|
## Public knowledge delivery boundary
|
||||||
|
|
||||||
|
PLAN-002 将当前交付阶段限定为企业受控网络内的单一公共知识可见域,并加入 Kubernetes/Helm
|
||||||
|
集群交付任务。这是部署范围和实施顺序调整,不是新的架构选型:继续采用 ADR-002 的模块化
|
||||||
|
单体 API、独立 Worker 和 MCP Gateway,以及架构既定的 Celery/RabbitMQ;OceanBase/BOS
|
||||||
|
边界继续遵循 ADR-012。
|
||||||
|
|
||||||
|
ADR-007 仍为 Accepted。当前阶段只是延后用户/组/角色和 Document ACL;开始接收任何非公共
|
||||||
|
知识前,必须先实现 Permission Resolver 和检索前/检索中 ACL 下推。公共知识库也不等于匿名
|
||||||
|
公网开放,Admin 写操作只允许经受控网络或统一接入网关访问,MCP 使用最小只读服务身份。
|
||||||
|
|
||||||
|
The migration retains the Demo's Dense Top20 → Reranker Top5 behavior and current lightweight
|
||||||
|
Parser scope. Top50 retrieval, distance thresholds, dual-query RRF, Marker/Pandoc and LLM-derived
|
||||||
|
index content require separate product and evaluation decisions.
|
||||||
|
|||||||
@ -63,10 +63,11 @@ Wiki / FAQ / SOP / Summary 等派生知识
|
|||||||
┌────────────────────────┼────────────────────────┐
|
┌────────────────────────┼────────────────────────┐
|
||||||
│ │ │
|
│ │ │
|
||||||
▼ ▼ ▼
|
▼ ▼ ▼
|
||||||
┌──────────────┐ ┌──────────────┐ ┌──────────────┐
|
┌────────────────────────┐ ┌──────────────┐
|
||||||
│ PostgreSQL │ │ Object Store │ │ Mochow │
|
│ OceanBase MySQL │ │ Object Store │
|
||||||
│ Canonical DB │ │ S3/MinIO │ │ Search Index │
|
│ Canonical + Vector │ │ BOS │
|
||||||
└──────────────┘ └──────────────┘ └──────────────┘
|
│ (logical separation) │ └──────────────┘
|
||||||
|
└────────────────────────┘
|
||||||
▲ ▲
|
▲ ▲
|
||||||
│ │
|
│ │
|
||||||
┌───────┴────────────────────────┘
|
┌───────┴────────────────────────┘
|
||||||
@ -88,7 +89,7 @@ Wiki / FAQ / SOP / Summary 等派生知识
|
|||||||
|
|
||||||
1. **API First**:后台、MCP、其他业务系统均调用统一 API/Service。
|
1. **API First**:后台、MCP、其他业务系统均调用统一 API/Service。
|
||||||
2. **模块化单体优先**:V1 不拆大量微服务,API 服务 + 异步 Worker + MCP Gateway 足够。
|
2. **模块化单体优先**:V1 不拆大量微服务,API 服务 + 异步 Worker + MCP Gateway 足够。
|
||||||
3. **Canonical Data 独立**:PostgreSQL + 对象存储保存事实数据,Vector DB 仅保存可重建索引。
|
3. **Canonical Data 逻辑独立**:OceanBase 关系表 + BOS 保存事实数据,同库向量表仅保存可重建索引。
|
||||||
4. **Provider 可替换**:Embedding、Reranker、Vector DB、Parser、LLM 均通过 Adapter 抽象。
|
4. **Provider 可替换**:Embedding、Reranker、Vector DB、Parser、LLM 均通过 Adapter 抽象。
|
||||||
5. **MCP 不暴露底层实现**:客户端只看到 `search_knowledge` 等业务工具。
|
5. **MCP 不暴露底层实现**:客户端只看到 `search_knowledge` 等业务工具。
|
||||||
|
|
||||||
@ -99,14 +100,14 @@ Wiki / FAQ / SOP / Summary 等派生知识
|
|||||||
| 模块 | V1 当前选型 | 说明 |
|
| 模块 | V1 当前选型 | 说明 |
|
||||||
|---|---|---|
|
|---|---|---|
|
||||||
| Backend | Python + FastAPI | AI / 数据处理生态成熟,适合 MCP、异步任务和模型调用 |
|
| Backend | Python + FastAPI | AI / 数据处理生态成熟,适合 MCP、异步任务和模型调用 |
|
||||||
| Metadata DB | PostgreSQL | 保存用户、权限、文档、版本、任务、Compiled Knowledge 等 Canonical Data |
|
| Metadata DB | OceanBase MySQL | 保存用户、权限、文档、版本、任务、Compiled Knowledge 等 Canonical Data |
|
||||||
| Object Storage | S3 Compatible / MinIO | 保存原始文件、解析产物、中间文件 |
|
| Object Storage | Baidu BOS | 保存不可变原始文件、解析产物和中间文件;Local Adapter 仅用于测试/本地开发 |
|
||||||
| Task Queue | Celery + RabbitMQ | 文件解析、Embedding、索引、Compilation 等异步任务 |
|
| Task Queue | Celery + RabbitMQ | 文件解析、Embedding、索引、Compilation 等异步任务 |
|
||||||
| PDF / OCR Parser | MinerU 优先 | 中文复杂 PDF、扫描件、表格、版面解析能力较强 |
|
| PDF / OCR Parser | MinerU 优先 | 中文复杂 PDF、扫描件、表格、版面解析能力较强 |
|
||||||
| Office Parser | Docling 补充 | DOCX/PPTX/XLSX/HTML 等格式统一解析 |
|
| Office Parser | Docling 补充 | DOCX/PPTX/XLSX/HTML 等格式统一解析 |
|
||||||
| Vector/Search DB | Mochow | 延续公司现有经验,支持向量、BM25、Hybrid、Filter |
|
| Vector/Search DB | OceanBase Vector | 与关系表同库部署但逻辑隔离,必须通过 `VectorStore` 并可从 canonical Chunk 重建 |
|
||||||
| Embedding | 多模型 Benchmark 决定 | 基线 bge-large-zh,对比 Qwen3 Embedding 等 |
|
| Embedding | Qianfan bge-large-zh | 当前 1024 维基线,后续模型变更使用版本化 Shadow Index |
|
||||||
| Reranker | Qwen3 Reranker 等候选 | V1 正式检索链必选能力 |
|
| Reranker | Qianfan bce-reranker-base | 当前 Dense Top20 → Reranker Top5 基线 |
|
||||||
| LLM | Provider 抽象,云端优先 | 用于 Knowledge Compilation、分类、摘要等 |
|
| LLM | Provider 抽象,云端优先 | 用于 Knowledge Compilation、分类、摘要等 |
|
||||||
| MCP | 独立 MCP Gateway | 面向 WorkBuddy 等客户端,隐藏内部技术栈 |
|
| MCP | 独立 MCP Gateway | 面向 WorkBuddy 等客户端,隐藏内部技术栈 |
|
||||||
| Frontend | Admin Console | 知识库、文档、权限、检索调试、Benchmark、任务管理 |
|
| Frontend | Admin Console | 知识库、文档、权限、检索调试、Benchmark、任务管理 |
|
||||||
@ -173,6 +174,46 @@ Queued / Parsing / Cleaning / Chunking / Embedding / Indexing / Completed / Fail
|
|||||||
- 软删除;
|
- 软删除;
|
||||||
- 来源页码和章节追踪。
|
- 来源页码和章节追踪。
|
||||||
|
|
||||||
|
### 4.2.1 批量异步提交与失败恢复
|
||||||
|
|
||||||
|
API 只负责校验并持久化不可变源文件、Document 和 Ingestion Job,然后返回 `202`。批次是
|
||||||
|
提交与聚合查询边界,不是跨文件数据库事务;每个文件独立排队、执行和失败。
|
||||||
|
|
||||||
|
DEMO-016 以 canonical `ingestion_batches` / `ingestion_batch_items` 保存输入顺序和接收引用;
|
||||||
|
处理阶段、Attempt 与最终结果继续以 `ingestion_jobs` / `ingestion_attempts` 为事实源,批次表
|
||||||
|
不复制 Worker 状态。
|
||||||
|
|
||||||
|
```text
|
||||||
|
Batch Request
|
||||||
|
├─ Source A + Document A + Job A ─→ Worker
|
||||||
|
├─ Source B + Document B + Job B ─→ Worker
|
||||||
|
└─ Source C + Document C + Job C ─→ Worker
|
||||||
|
```
|
||||||
|
|
||||||
|
Worker 对可重试错误执行有界退避。重新执行前必须通过独立的派生数据清理服务按以下顺序
|
||||||
|
恢复干净状态:
|
||||||
|
|
||||||
|
```text
|
||||||
|
Vector/Sparse rows
|
||||||
|
→ ChunkIndexState
|
||||||
|
→ Chunks
|
||||||
|
→ Parsed artifact + parsed_storage_key
|
||||||
|
→ reset processing state
|
||||||
|
→ next attempt from immutable source
|
||||||
|
```
|
||||||
|
|
||||||
|
清理服务与 Document 删除语义分离:重试清理保留原始 BOS 对象、Document 和 Job/Attempt
|
||||||
|
历史。清理失败时 Job 终止,不能在未知状态继续执行。Job 通过数据库状态、唯一有效 Attempt
|
||||||
|
和幂等键避免重复投递产生并发执行;具体队列实现保持可替换,不进入业务服务接口。
|
||||||
|
|
||||||
|
公共知识库集群交付阶段由 Celery/RabbitMQ 作为首个生产执行器。API 只调用队列执行器抽象,
|
||||||
|
Celery task 只负责恢复应用上下文并调用 Ingestion Application Service;重试分类、Attempt 状态、
|
||||||
|
清理顺序和业务幂等仍由应用层持有。Broker 消息不是 Job 的 source of truth,Worker 重启或消息
|
||||||
|
重复投递必须以 OceanBase 中的持久化 Job/Attempt 状态收敛。
|
||||||
|
|
||||||
|
DEPLOY-001 将 API/Admin、MCP Gateway 与 Async Worker 构建为三个共享锁定 Python 运行时的
|
||||||
|
非 root 镜像 target;进程边界与上述模块边界一致,不新增服务或把 Provider 配置烘焙进镜像。
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
### 4.3 文档结构与 Chunk
|
### 4.3 文档结构与 Chunk
|
||||||
@ -326,21 +367,28 @@ profile_v2
|
|||||||
|
|
||||||
避免直接原地覆盖现有生产索引。
|
避免直接原地覆盖现有生产索引。
|
||||||
|
|
||||||
|
ARCH-005/006 将 Profile 生命周期以 `shadow / active / retired` 记录在 OceanBase 关系表;任一时刻至多
|
||||||
|
一个 Active Profile。重建从 canonical Chunk 生成确定性 `chunk_id + profile_id` 索引行,
|
||||||
|
重复执行跳过已完整文档,激活前重新核对全部 ChunkIndexState,并在同一事务内锁定 Chunk
|
||||||
|
写入、退役旧 Profile、激活新 Profile。OceanBase 表本身仍不是 canonical source。
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## 8. Vector Store 设计
|
## 8. Vector Store 设计
|
||||||
|
|
||||||
### 当前默认:Mochow
|
### 当前默认:OceanBase Vector
|
||||||
|
|
||||||
选择理由:
|
选择理由:
|
||||||
|
|
||||||
- 公司已有使用经验;
|
- 公司现有知识平台主链已有使用经验;
|
||||||
- 已有百度云体系;
|
- 可复用现有 OceanBase 运维体系;
|
||||||
- 支持 Dense Vector;
|
- 支持 Dense Vector;
|
||||||
- 支持 BM25 / Full Text;
|
|
||||||
- 支持 Hybrid Search;
|
|
||||||
- 支持 Metadata / Scalar Filter;
|
- 支持 Metadata / Scalar Filter;
|
||||||
- 满足 V1 ACL 下推需求。
|
- 满足当前 Demo 的 Profile 过滤和后续 ACL 下推边界;
|
||||||
|
- `pyobvector` 仅允许出现在 Provider Adapter 内。
|
||||||
|
|
||||||
|
BM25/Hybrid 仍是 V1 产品能力,但不在本轮 Provider 迁移中提前扩展;相关实现必须经
|
||||||
|
独立 Task、契约测试和 RAG 回归后启用。
|
||||||
|
|
||||||
### 保留替换能力
|
### 保留替换能力
|
||||||
|
|
||||||
@ -348,15 +396,18 @@ profile_v2
|
|||||||
|
|
||||||
```text
|
```text
|
||||||
VectorStoreProvider
|
VectorStoreProvider
|
||||||
|
├─ OceanBase Vector (Primary)
|
||||||
├─ Mochow
|
├─ Mochow
|
||||||
├─ Qdrant
|
├─ Qdrant
|
||||||
├─ Milvus
|
├─ Milvus
|
||||||
└─ pgvector
|
└─ pgvector
|
||||||
```
|
```
|
||||||
|
|
||||||
业务 Domain Model 不依赖 Mochow SDK 数据结构。
|
业务 Domain Model 不依赖 OceanBase/`pyobvector` SDK 数据结构。
|
||||||
|
|
||||||
Mochow 中的数据全部视为 **可重建索引**,真实 Canonical Data 保存在 PostgreSQL / Object Storage。
|
OceanBase Vector 表中的数据全部视为 **可重建索引**。同一数据库的关系表保存文档、Chunk、
|
||||||
|
Profile 和索引状态等 canonical 数据,BOS 保存不可变原始文件和派生产物。物理共库不允许
|
||||||
|
向量表反向成为业务事实来源。
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
@ -441,7 +492,7 @@ Manage
|
|||||||
User Identity
|
User Identity
|
||||||
→ Permission Resolver
|
→ Permission Resolver
|
||||||
→ Allowed Scope / ACL Filter
|
→ Allowed Scope / ACL Filter
|
||||||
→ Mochow Hybrid Retrieval
|
→ Provider-neutral Retrieval(OceanBase Vector 当前实现)
|
||||||
```
|
```
|
||||||
|
|
||||||
MCP / REST / Admin Console 必须复用同一权限服务。
|
MCP / REST / Admin Console 必须复用同一权限服务。
|
||||||
@ -481,7 +532,8 @@ V1 建议工具:
|
|||||||
MCP Response 不出现:
|
MCP Response 不出现:
|
||||||
|
|
||||||
```text
|
```text
|
||||||
Mochow
|
OceanBase Vector
|
||||||
|
BOS
|
||||||
Qdrant
|
Qdrant
|
||||||
bge-large-zh
|
bge-large-zh
|
||||||
Qwen3
|
Qwen3
|
||||||
@ -492,6 +544,17 @@ MinerU
|
|||||||
|
|
||||||
首要验证客户端:**WorkBuddy Enterprise**。
|
首要验证客户端:**WorkBuddy Enterprise**。
|
||||||
|
|
||||||
|
### 11.1 DEMO-012 实现边界
|
||||||
|
|
||||||
|
当前已落地独立 Streamable HTTP Gateway,默认只绑定 `127.0.0.1:8766/mcp`,只注册
|
||||||
|
`search_knowledge`。Gateway 与 REST 共享 `SearchService`,同步检索通过工作线程执行,避免
|
||||||
|
阻塞 MCP ASGI 事件循环。
|
||||||
|
|
||||||
|
Demo 使用运行时专用 Bearer Token 映射到 client/service identity 和最小
|
||||||
|
`knowledge:search` scope,并启用 Host/Origin allowlist。该边界用于 WorkBuddy 接入验证,
|
||||||
|
后续 `EKP-070` 仍需用正式 OAuth/企业身份系统和 Permission Resolver 替换静态 Token;
|
||||||
|
`DEMO-012` 不改变 ADR-007 的 ACL 要求。
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## 12. REST API
|
## 12. REST API
|
||||||
@ -592,6 +655,11 @@ ReindexJob
|
|||||||
EvaluationJob
|
EvaluationJob
|
||||||
```
|
```
|
||||||
|
|
||||||
|
当前增量实现先提供单个 `IngestionJob` 编排既有 Parse/Clean/Chunk/Embedding/Indexing 阶段,
|
||||||
|
保留阶段级状态和 Attempt 历史;以后拆分为独立阶段 Job 时不得改变公开批次/Job 查询契约。
|
||||||
|
Demo 默认最多 3 次总尝试,只有瞬时 Provider、网络、存储或数据库错误进入自动重试;输入
|
||||||
|
校验和确定性内容错误直接终止。
|
||||||
|
|
||||||
GPU 节点优先承担:
|
GPU 节点优先承担:
|
||||||
|
|
||||||
- MinerU / OCR;
|
- MinerU / OCR;
|
||||||
@ -604,7 +672,7 @@ LLM V1 可继续使用云端 Provider,不要求部署本地大模型。
|
|||||||
|
|
||||||
## 15. 数据存储原则
|
## 15. 数据存储原则
|
||||||
|
|
||||||
### PostgreSQL
|
### OceanBase MySQL 关系表
|
||||||
|
|
||||||
保存:
|
保存:
|
||||||
|
|
||||||
@ -629,14 +697,14 @@ LLM V1 可继续使用云端 Provider,不要求部署本地大模型。
|
|||||||
- Markdown / JSON 中间产物;
|
- Markdown / JSON 中间产物;
|
||||||
- 需要持久化的大对象。
|
- 需要持久化的大对象。
|
||||||
|
|
||||||
### Mochow
|
### OceanBase Vector
|
||||||
|
|
||||||
仅保存:
|
仅保存:
|
||||||
|
|
||||||
- Dense Vector;
|
- Dense Vector;
|
||||||
- Sparse/BM25 Index;
|
|
||||||
- Retrieval metadata;
|
- Retrieval metadata;
|
||||||
- ACL 所需过滤字段。
|
- Profile/ACL 所需过滤字段;
|
||||||
|
- 经独立能力验证后启用的 Sparse/BM25 索引。
|
||||||
|
|
||||||
数据库损坏或更换时,可由 Canonical Data 全量重建。
|
数据库损坏或更换时,可由 Canonical Data 全量重建。
|
||||||
|
|
||||||
@ -653,15 +721,13 @@ Reverse Proxy / Ingress
|
|||||||
├─ Knowledge API
|
├─ Knowledge API
|
||||||
└─ MCP Gateway
|
└─ MCP Gateway
|
||||||
|
|
||||||
PostgreSQL
|
OceanBase MySQL(Canonical + Vector,逻辑隔离)
|
||||||
RabbitMQ
|
RabbitMQ
|
||||||
Object Storage
|
Baidu BOS
|
||||||
|
|
||||||
CPU Worker
|
CPU Worker
|
||||||
GPU Worker
|
GPU Worker
|
||||||
|
|
||||||
Mochow
|
|
||||||
|
|
||||||
Observability
|
Observability
|
||||||
```
|
```
|
||||||
|
|
||||||
@ -678,6 +744,30 @@ Observability
|
|||||||
|
|
||||||
未来根据负载再拆服务。
|
未来根据负载再拆服务。
|
||||||
|
|
||||||
|
### 16.1 当前公共知识库集群交付基线
|
||||||
|
|
||||||
|
PLAN-002 将当前交付范围收敛为企业受控网络内的单一公共知识可见域。该范围不重开 ADR-007:
|
||||||
|
所有现有文档共享同一可见性,因而当前检索不执行事后 ACL 过滤;接收任何非公共知识前,必须
|
||||||
|
先实现 Permission Resolver 和数据库/向量检索前置过滤。Admin 写接口不得直接暴露到匿名公网,
|
||||||
|
MCP 使用运行时 Secret 注入的最小 `knowledge:search` 服务身份。
|
||||||
|
|
||||||
|
集群中 API/Admin、MCP Gateway 和 Async Worker 分别部署并独立扩缩容,业务核心继续保持模块化
|
||||||
|
单体。当前 Parser 及云端 Embedding/Reranker 不需要 GPU Worker,因此它不进入最小交付,但角色
|
||||||
|
边界保留。OceanBase、BOS 和 AI Provider 使用外部服务;RabbitMQ 默认使用外部地址,可为开发/
|
||||||
|
演示显式启用 Chart 依赖。
|
||||||
|
|
||||||
|
Kubernetes/Helm 必须提供:生产镜像、`existingSecret`、串行 Migration Job、Ingress/TLS、
|
||||||
|
readiness/liveness、资源限制、非 root 与只读根文件系统、多副本和滚动更新。仓库的一键入口只
|
||||||
|
编排已有集群和外部服务,不创建云账号或把凭据写入 Values、镜像、命令行及日志。现有
|
||||||
|
`docker-compose.dev.yml` 包含已被 ADR-012 取代的 PostgreSQL/MinIO 假设,只能视为待清理的
|
||||||
|
本地开发资产,不能作为集群模板。详细依赖和验收见
|
||||||
|
[`PUBLIC_KNOWLEDGE_CLUSTER_PLAN.md`](../plans/PUBLIC_KNOWLEDGE_CLUSTER_PLAN.md)。
|
||||||
|
|
||||||
|
DEPLOY-002 的运行拓扑将 Celery Worker 与 Celery Beat 恢复调度器拆成独立 Pod:Worker 可横向
|
||||||
|
扩缩,Beat 固定单副本并使用 `Recreate`,避免每个 Worker 都发布周期恢复消息。二者仍调用同一
|
||||||
|
应用服务与持久化 Job/Attempt 状态,不形成新业务服务,也不改变 ADR-002。Migration 使用 Helm
|
||||||
|
`pre-install,pre-upgrade` Hook;失败会在业务 Deployment 更新前终止原子升级。
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## 17. V1 验收重点
|
## 17. V1 验收重点
|
||||||
@ -732,13 +822,13 @@ WorkBuddy 可以通过 MCP 完成知识搜索和 Source 获取。
|
|||||||
|
|
||||||
```text
|
```text
|
||||||
Backend Python + FastAPI
|
Backend Python + FastAPI
|
||||||
Metadata PostgreSQL
|
Metadata OceanBase MySQL relational tables
|
||||||
Object Storage S3 Compatible / MinIO
|
Object Storage Baidu BOS(Local Adapter 仅测试/本地开发)
|
||||||
Queue RabbitMQ + Celery
|
Queue RabbitMQ + Celery
|
||||||
Parser MinerU 主力 + Docling 补充
|
Parser MinerU 主力 + Docling 补充
|
||||||
Vector Store Mochow Primary,Adapter 可替换
|
Vector Store OceanBase Vector Primary,Adapter 可替换,索引可重建
|
||||||
Embedding bge-large-zh 基线,多模型 Benchmark 后决定生产模型
|
Embedding Qianfan bge-large-zh(1024 维)
|
||||||
Reranker Qwen3 Reranker 等候选,V1 必选
|
Reranker Qianfan bce-reranker-base,Dense Top20 → Top5
|
||||||
LLM Provider 抽象,允许云端
|
LLM Provider 抽象,允许云端
|
||||||
Frontend Admin Console
|
Frontend Admin Console
|
||||||
Protocol REST API + MCP
|
Protocol REST API + MCP
|
||||||
|
|||||||
@ -6,7 +6,7 @@
|
|||||||
|
|
||||||
该 Demo 的目标不是完成完整企业级知识平台,而是优先验证最核心的技术闭环:
|
该 Demo 的目标不是完成完整企业级知识平台,而是优先验证最核心的技术闭环:
|
||||||
|
|
||||||
> 文件上传 → 文档解析/清洗 → Chunk → Embedding → Mochow 入库 → 检索召回 → Reranker → 来源返回 → 简单管理页面
|
> 文件上传 → 文档解析/清洗 → Chunk → Embedding → OceanBase Vector 入库 → 检索召回 → Reranker → 来源返回 → 简单管理页面
|
||||||
|
|
||||||
当前 Demo Scope 的优先级高于完整 V1 PRD。
|
当前 Demo Scope 的优先级高于完整 V1 PRD。
|
||||||
|
|
||||||
@ -20,7 +20,7 @@
|
|||||||
|
|
||||||
1. 中文企业文档是否能够稳定上传和解析;
|
1. 中文企业文档是否能够稳定上传和解析;
|
||||||
2. 清洗后的内容是否可以形成质量可接受的 Chunk;
|
2. 清洗后的内容是否可以形成质量可接受的 Chunk;
|
||||||
3. 使用现有 Embedding + Mochow 能否完成稳定召回;
|
3. 使用现有 Embedding + OceanBase Vector 能否完成稳定召回;
|
||||||
4. Reranker 是否能提升最终 TopK 结果质量;
|
4. Reranker 是否能提升最终 TopK 结果质量;
|
||||||
5. 检索结果是否能够准确返回文件名、页码、章节和原文;
|
5. 检索结果是否能够准确返回文件名、页码、章节和原文;
|
||||||
6. 非研发人员是否能够通过简单页面完成上传和检索验证。
|
6. 非研发人员是否能够通过简单页面完成上传和检索验证。
|
||||||
@ -49,13 +49,13 @@ FastAPI
|
|||||||
│ ↓
|
│ ↓
|
||||||
│ Embedding
|
│ Embedding
|
||||||
│ ↓
|
│ ↓
|
||||||
│ Mochow
|
│ OceanBase Vector
|
||||||
│
|
│
|
||||||
└── Search
|
└── Search
|
||||||
↓
|
↓
|
||||||
Query Embedding
|
Query Embedding
|
||||||
↓
|
↓
|
||||||
Mochow TopN
|
OceanBase Vector TopN
|
||||||
↓
|
↓
|
||||||
Reranker
|
Reranker
|
||||||
↓
|
↓
|
||||||
@ -67,10 +67,10 @@ FastAPI
|
|||||||
辅助存储:
|
辅助存储:
|
||||||
|
|
||||||
```text
|
```text
|
||||||
SQLite / PostgreSQL
|
OceanBase MySQL
|
||||||
└── document / chunk metadata
|
└── canonical document / chunk metadata
|
||||||
|
|
||||||
Local Storage / MinIO
|
Local Storage / BOS
|
||||||
└── original files
|
└── original files
|
||||||
```
|
```
|
||||||
|
|
||||||
@ -83,16 +83,16 @@ Demo 阶段优先简单实现,不引入不必要的基础设施。
|
|||||||
| 模块 | Demo 推荐选型 | 说明 |
|
| 模块 | Demo 推荐选型 | 说明 |
|
||||||
|---|---|---|
|
|---|---|---|
|
||||||
| Backend | FastAPI | 简单、开发效率高 |
|
| Backend | FastAPI | 简单、开发效率高 |
|
||||||
| Metadata DB | SQLite 或 PostgreSQL | 单机 Demo 可 SQLite;已有环境可直接 PostgreSQL |
|
| Metadata DB | OceanBase MySQL | 目标环境使用 OceanBase;SQLite 仅用于离线测试 |
|
||||||
| File Storage | 本地目录 | Demo 阶段优先简单;后续可替换 MinIO/S3 |
|
| File Storage | 百度 BOS | Local Adapter 仅用于测试和本地开发,目标环境使用 BOS |
|
||||||
| Parser | PyMuPDF + python-docx 或 MinerU | 优先快速跑通;复杂 PDF 后续增强 |
|
| Parser | PyMuPDF + python-docx 或 MinerU | 优先快速跑通;复杂 PDF 后续增强 |
|
||||||
| Cleaner | 自定义基础清洗 | 处理空行、空白、异常换行等 |
|
| Cleaner | 自定义基础清洗 | 处理空行、空白、异常换行等 |
|
||||||
| Chunker | 标题/段落优先 + Token 限制 | 暂不设计复杂 Chunk 策略 |
|
| Chunker | 标题/段落优先 + Token 限制 | 暂不设计复杂 Chunk 策略 |
|
||||||
| Embedding | bge-large-zh | 延续公司现有百度千帆方案 |
|
| Embedding | bge-large-zh | 延续公司现有百度千帆方案 |
|
||||||
| Vector DB | Mochow | 当前默认,不做多数据库实现 |
|
| Vector DB | OceanBase Vector | 当前默认,通过 `VectorStore` 隔离 SDK,不做多数据库运行时切换 |
|
||||||
| Reranker | Qwen3 Reranker 或现有可调用 Reranker | Demo 中保留 |
|
| Reranker | `bce-reranker-base` | 延续当前可调用的千帆 Reranker,Demo 中保留 |
|
||||||
| Frontend | 简单 React/Vue 或 FastAPI HTML | 以最小可用为目标 |
|
| Frontend | 简单 React/Vue 或 FastAPI HTML | 以最小可用为目标 |
|
||||||
| Deployment | 本地 / 单机 Docker | 暂不考虑 K8s |
|
| Deployment | Kubernetes/Helm | PLAN-002 起面向企业受控网络交付;本地模式仅用于开发测试 |
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
@ -171,9 +171,9 @@ content
|
|||||||
|
|
||||||
Demo 阶段不实现多 Embedding Benchmark 平台和动态模型切换,但代码层建议保留简单接口,避免业务逻辑直接写死调用细节。
|
Demo 阶段不实现多 Embedding Benchmark 平台和动态模型切换,但代码层建议保留简单接口,避免业务逻辑直接写死调用细节。
|
||||||
|
|
||||||
### 5.6 Mochow 入库
|
### 5.6 OceanBase Vector 入库
|
||||||
|
|
||||||
将 Chunk 写入 Mochow,至少保存:
|
将 Chunk 写入 OceanBase Vector 专用索引表,至少保存:
|
||||||
|
|
||||||
```text
|
```text
|
||||||
chunk_id
|
chunk_id
|
||||||
@ -187,7 +187,9 @@ embedding
|
|||||||
|
|
||||||
支持:Upsert、按 document_id 删除相关 Chunk、Dense Search。
|
支持:Upsert、按 document_id 删除相关 Chunk、Dense Search。
|
||||||
|
|
||||||
Demo 阶段不实现 Qdrant/Milvus/pgvector,也不实现完整 VectorStore Adapter 体系,但避免 Mochow SDK 调用散落在整个项目中。
|
Demo 阶段不实现 Qdrant/Milvus/pgvector,也不实现多 Vector Store 动态切换,但避免
|
||||||
|
OceanBase/pyobvector 调用散落在整个项目中。OceanBase Vector 行仍是可重建索引,不能替代
|
||||||
|
同库关系表中的 canonical Chunk。
|
||||||
|
|
||||||
### 5.7 Dense Retrieval
|
### 5.7 Dense Retrieval
|
||||||
|
|
||||||
@ -196,7 +198,7 @@ Query
|
|||||||
↓
|
↓
|
||||||
bge-large-zh Embedding
|
bge-large-zh Embedding
|
||||||
↓
|
↓
|
||||||
Mochow Dense Search
|
OceanBase Vector Dense Search
|
||||||
↓
|
↓
|
||||||
Top 20 Candidates
|
Top 20 Candidates
|
||||||
```
|
```
|
||||||
@ -246,7 +248,8 @@ DELETE /api/documents/{id}
|
|||||||
POST /api/search
|
POST /api/search
|
||||||
```
|
```
|
||||||
|
|
||||||
Demo 阶段删除可以直接执行:删除原始文件 + 删除 Metadata + 删除 Mochow 对应 Chunk,暂不要求 Soft Delete。
|
Demo 阶段删除可以直接执行:删除 BOS/本地原始文件 + 删除 Metadata + 删除 OceanBase
|
||||||
|
Vector 对应 Chunk,暂不要求 Soft Delete。
|
||||||
|
|
||||||
Search Request 示例:
|
Search Request 示例:
|
||||||
|
|
||||||
@ -302,7 +305,7 @@ Search Response 示例:
|
|||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## 8. P1 可选增强
|
## 8. Demo 后续增强
|
||||||
|
|
||||||
### DEMO-011 Hybrid Search
|
### DEMO-011 Hybrid Search
|
||||||
|
|
||||||
@ -320,12 +323,26 @@ Reranker
|
|||||||
|
|
||||||
### DEMO-012 MCP
|
### DEMO-012 MCP
|
||||||
|
|
||||||
只实现一个最小 MCP Tool:`search_knowledge`,内部调用 `/api/search`,用于验证 WorkBuddy 接入。
|
**已完成。** 只实现一个最小 MCP Tool:`search_knowledge`,复用现有
|
||||||
|
Search Service 或 `/api/v1/search`,用于验证标准 MCP、identity 边界和 WorkBuddy 接入。
|
||||||
|
不在本 Task 扩展其他 P0 tools 或声称完成尚未实施的 Document ACL。
|
||||||
|
|
||||||
### DEMO-013 OCR / MinerU
|
### DEMO-013 OCR / MinerU
|
||||||
|
|
||||||
增加扫描 PDF、图片、OCR 和复杂中文 PDF 处理能力。
|
增加扫描 PDF、图片、OCR 和复杂中文 PDF 处理能力。
|
||||||
|
|
||||||
|
### DEMO-015 异步入库 Job 与自动重试清理
|
||||||
|
|
||||||
|
**已完成。** 将现有同步入库流水线放入持久化 Ingestion Job。可重试失败自动执行有界重试;每次重试前
|
||||||
|
删除旧解析产物、Chunk、索引状态和向量数据,但保留不可变原始文件、Document 和 Attempt
|
||||||
|
历史。清理失败时终止,不在未知脏状态继续执行。通过执行器抽象接入 Celery/RabbitMQ,使
|
||||||
|
API 与 Worker 可独立部署,并以持久化状态处理 Worker 重启和重复消息。
|
||||||
|
|
||||||
|
### DEMO-016 批量异步上传
|
||||||
|
|
||||||
|
**已完成。** 在 DEMO-015 基础上支持一次提交多个文件、逐文件 Document/Job 状态、批次聚合查询和
|
||||||
|
Admin Console 状态展示。批次内文件相互隔离,不包含目录上传、压缩包解包或断点续传。
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## 9. 明确后置能力
|
## 9. 明确后置能力
|
||||||
@ -345,37 +362,67 @@ Reranker
|
|||||||
- 多 Embedding Benchmark Platform;
|
- 多 Embedding Benchmark Platform;
|
||||||
- 多 Vector DB;
|
- 多 Vector DB;
|
||||||
- 动态 Provider;
|
- 动态 Provider;
|
||||||
- RabbitMQ / Celery;
|
|
||||||
- Kubernetes;
|
|
||||||
- 高可用;
|
|
||||||
- 灾备;
|
- 灾备;
|
||||||
- 完整 Observability。
|
- 完整 Observability。
|
||||||
|
|
||||||
|
用户/权限隔离属于完整 V1 后置能力,不因“公共知识库”而取消;本阶段所有已接收知识共享一个
|
||||||
|
可见域,且仅部署在企业受控网络或统一接入网关后。RabbitMQ/Celery 与最小 Kubernetes 可用性
|
||||||
|
已由 PLAN-002 纳入当前交付,跨区域容灾和完整平台级高可用仍不在范围内。
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## 10. Demo Task 清单
|
## 10. Demo Task 清单
|
||||||
|
|
||||||
```text
|
```text
|
||||||
DEMO-001 项目骨架 + FastAPI + 配置
|
DEMO-001 项目骨架 + FastAPI + 配置
|
||||||
DEMO-002 Document 数据模型 + 文件存储
|
DEMO-002 Document 数据模型 + 本地文件存储
|
||||||
DEMO-003 PDF/DOCX/TXT/MD 上传
|
DEMO-003 PDF/DOCX/TXT/MD 上传
|
||||||
DEMO-004 Parser + Cleaner
|
DEMO-004 Parser + Cleaner
|
||||||
DEMO-005 Chunk Pipeline
|
DEMO-005 Chunk Pipeline
|
||||||
DEMO-006 bge-large-zh Embedding + Mochow 入库
|
DEMO-006 bge-large-zh Embedding + 初始 Mochow 入库(ARCH-003~005 已迁移为 OceanBase)
|
||||||
DEMO-007 Dense Retrieval
|
DEMO-007 Dense Retrieval
|
||||||
DEMO-008 Reranker + Source Result
|
DEMO-008 Reranker + Source Result
|
||||||
DEMO-009 Document/Search REST API
|
DEMO-009 Document/Search REST API
|
||||||
DEMO-010 简单 Admin Console
|
DEMO-010 简单 Admin Console
|
||||||
|
DEMO-012 MCP search_knowledge(已提升优先级)
|
||||||
|
DEMO-014 地瓜机器人品牌化前端优化(用户新增)
|
||||||
|
DEMO-015 异步入库 Job + 自动重试前清理(用户新增)
|
||||||
|
DEMO-016 批量异步上传 + 状态展示(用户新增)
|
||||||
|
ARCH-001~006 Provider 架构迁移与真实验收
|
||||||
|
PLAN-002 公共知识库集群交付规划(已完成)
|
||||||
|
DEPLOY-001 API/Admin、MCP Gateway、Async Worker 生产镜像
|
||||||
|
DEPLOY-002 Kubernetes/Helm 一键部署
|
||||||
|
DEPLOY-003 可用性、滚动升级和远程 WorkBuddy 验收
|
||||||
```
|
```
|
||||||
|
|
||||||
|
实施状态(2026-09-02):`DEMO-001~DEMO-010`、用户新增的 `DEMO-014` 与
|
||||||
|
`ARCH-001~ARCH-006` 均已完成并通过当前 Scope 的闭环验收。`PLAN-001` 已完成现状审计和
|
||||||
|
任务拆分;`DEMO-012`、`DEMO-015~016` 已完成。`PLAN-002` 已将公共知识库集群交付纳入范围,
|
||||||
|
后续严格按 `DEPLOY-001~003` 顺序实施。`DEMO-011` 和 `DEMO-013` 仍为
|
||||||
|
可选项,不因本状态记录自动进入范围。
|
||||||
|
|
||||||
可选:
|
可选:
|
||||||
|
|
||||||
```text
|
```text
|
||||||
DEMO-011 Hybrid Search
|
DEMO-011 Hybrid Search
|
||||||
DEMO-012 MCP search_knowledge
|
|
||||||
DEMO-013 OCR / MinerU
|
DEMO-013 OCR / MinerU
|
||||||
```
|
```
|
||||||
|
|
||||||
|
已批准顺序:
|
||||||
|
|
||||||
|
```text
|
||||||
|
PLAN-002 Public Knowledge Cluster Plan
|
||||||
|
→ DEMO-015 Celery/RabbitMQ Async Job + Retry Cleanup
|
||||||
|
→ DEMO-016 Batch Async Upload + UI
|
||||||
|
→ DEPLOY-001 Production Images
|
||||||
|
→ DEPLOY-002 Kubernetes/Helm One-click Deploy
|
||||||
|
→ DEPLOY-003 Availability + Remote Acceptance
|
||||||
|
```
|
||||||
|
|
||||||
|
详细 Task 映射、依赖与验收边界见
|
||||||
|
[`docs/plans/ASYNC_INGESTION_MCP_PLAN.md`](../plans/ASYNC_INGESTION_MCP_PLAN.md) 和
|
||||||
|
[`docs/plans/PUBLIC_KNOWLEDGE_CLUSTER_PLAN.md`](../plans/PUBLIC_KNOWLEDGE_CLUSTER_PLAN.md)。
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## 11. Demo 测试数据
|
## 11. Demo 测试数据
|
||||||
@ -422,7 +469,7 @@ Upload
|
|||||||
→ Clean
|
→ Clean
|
||||||
→ Chunk
|
→ Chunk
|
||||||
→ Embedding
|
→ Embedding
|
||||||
→ Mochow
|
→ OceanBase Vector
|
||||||
```
|
```
|
||||||
|
|
||||||
完整跑通。
|
完整跑通。
|
||||||
@ -432,7 +479,7 @@ Upload
|
|||||||
```text
|
```text
|
||||||
Query
|
Query
|
||||||
→ Embedding
|
→ Embedding
|
||||||
→ Mochow Dense Top20
|
→ OceanBase Vector Dense Top20
|
||||||
→ Reranker
|
→ Reranker
|
||||||
→ Top5
|
→ Top5
|
||||||
```
|
```
|
||||||
@ -464,9 +511,13 @@ Demo 阶段不设严格生产指标。
|
|||||||
|
|
||||||
## 14. Codex 开发约束
|
## 14. Codex 开发约束
|
||||||
|
|
||||||
Codex 当前只能实现 `DEMO-001 ~ DEMO-010`。
|
Codex 当前只能实现 `DEMO-001 ~ DEMO-010`、`DEMO-012`、`DEMO-014 ~ DEMO-016`、
|
||||||
|
`ARCH-001 ~ ARCH-006`、`PLAN-001 ~ PLAN-002` 和 `DEPLOY-001 ~ DEPLOY-003`。
|
||||||
|
PLAN-002 只授权实现 Celery/RabbitMQ 异步执行器与最小 Kubernetes/Helm 集群交付,不授权
|
||||||
|
拆分额外微服务或扩大业务能力。
|
||||||
|
|
||||||
除非明确要求,否则禁止提前实现权限、审计、Knowledge Compilation、多数据库、多模型平台、微服务、Kubernetes 和企业级 Job Queue。
|
除非明确要求,否则禁止提前实现权限、审计、Knowledge Compilation、多数据库、多模型平台、
|
||||||
|
额外微服务、跨区域容灾和完整 Observability 平台。
|
||||||
|
|
||||||
当本文件与完整 PRD 的实现优先级冲突时:
|
当本文件与完整 PRD 的实现优先级冲突时:
|
||||||
|
|
||||||
@ -495,20 +546,29 @@ docs/demo/DEMO_SCOPE.md
|
|||||||
>
|
>
|
||||||
完整 V1 PRD
|
完整 V1 PRD
|
||||||
|
|
||||||
只允许实现 DEMO-001 ~ DEMO-010。
|
只允许实现本文件明确列出的 Demo、Architecture、Plan 与 Deploy Task,并且一次只执行一个
|
||||||
|
Task。当前严格顺序为 DEMO-016 → DEPLOY-001 → DEPLOY-002 → DEPLOY-003。
|
||||||
|
|
||||||
当前 Demo 的目标只有:
|
当前公共知识库阶段的业务目标保持为:
|
||||||
|
|
||||||
Upload
|
Upload
|
||||||
→ Parse/Clean
|
→ Parse/Clean
|
||||||
→ Chunk
|
→ Chunk
|
||||||
→ bge-large-zh
|
→ bge-large-zh
|
||||||
→ Mochow
|
→ OceanBase Vector
|
||||||
→ Dense Retrieval
|
→ Dense Retrieval
|
||||||
→ Reranker
|
→ Reranker
|
||||||
→ Source Result
|
→ Source Result
|
||||||
→ Search UI
|
→ Search UI
|
||||||
|
|
||||||
|
并补齐以下交付能力:
|
||||||
|
|
||||||
|
Async Job / Retry Cleanup
|
||||||
|
→ Batch Async Upload
|
||||||
|
→ Production Images
|
||||||
|
→ Kubernetes/Helm One-click Deploy
|
||||||
|
→ Availability + Remote WorkBuddy Acceptance
|
||||||
|
|
||||||
不要提前实现:
|
不要提前实现:
|
||||||
|
|
||||||
- ACL
|
- ACL
|
||||||
@ -518,9 +578,9 @@ Upload
|
|||||||
- Knowledge Compilation
|
- Knowledge Compilation
|
||||||
- Multi Vector Store
|
- Multi Vector Store
|
||||||
- Embedding Benchmark Platform
|
- Embedding Benchmark Platform
|
||||||
- Celery/RabbitMQ
|
- 额外微服务
|
||||||
- Kubernetes
|
- 跨区域容灾
|
||||||
- 企业级高可用
|
- 完整 Observability 平台
|
||||||
|
|
||||||
先不要修改代码。
|
先不要修改代码。
|
||||||
|
|
||||||
@ -529,7 +589,7 @@ Upload
|
|||||||
1. 当前仓库状态;
|
1. 当前仓库状态;
|
||||||
2. 为完成 Demo 需要保留哪些现有结构;
|
2. 为完成 Demo 需要保留哪些现有结构;
|
||||||
3. 哪些完整 V1 设计可以暂时不实现;
|
3. 哪些完整 V1 设计可以暂时不实现;
|
||||||
4. DEMO-001 ~ DEMO-010 的建议执行顺序;
|
4. 当前已批准 Task 的执行顺序;
|
||||||
5. 下一步应该实施的第一个 Task。
|
5. 下一步应该实施的第一个 Task。
|
||||||
```
|
```
|
||||||
|
|
||||||
@ -540,10 +600,20 @@ Upload
|
|||||||
```text
|
```text
|
||||||
Demo
|
Demo
|
||||||
↓
|
↓
|
||||||
Hybrid Search
|
|
||||||
↓
|
|
||||||
MCP
|
MCP
|
||||||
↓
|
↓
|
||||||
|
Async Job + Retry Cleanup
|
||||||
|
↓
|
||||||
|
Batch Async Upload
|
||||||
|
↓
|
||||||
|
Production Images
|
||||||
|
↓
|
||||||
|
Kubernetes/Helm One-click Deploy
|
||||||
|
↓
|
||||||
|
Availability + Remote WorkBuddy Acceptance
|
||||||
|
↓
|
||||||
|
Hybrid Search
|
||||||
|
↓
|
||||||
OCR / Complex Parsing
|
OCR / Complex Parsing
|
||||||
↓
|
↓
|
||||||
Document Version
|
Document Version
|
||||||
|
|||||||
59
docs/development/ADMIN_CONSOLE.md
Normal file
59
docs/development/ADMIN_CONSOLE.md
Normal file
@ -0,0 +1,59 @@
|
|||||||
|
# Demo Admin Console 使用说明
|
||||||
|
|
||||||
|
DEMO-010 在 FastAPI 服务中提供一个轻量、同源的管理客户端。页面只负责展示和交互,上传、
|
||||||
|
文档管理与检索仍全部通过 `/api/v1` 公开业务接口完成,不复制服务端业务逻辑。
|
||||||
|
|
||||||
|
## 启动与访问
|
||||||
|
|
||||||
|
完成运行时配置后启动 Worker 与 API:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
make run-worker
|
||||||
|
make run
|
||||||
|
```
|
||||||
|
|
||||||
|
页面地址:
|
||||||
|
|
||||||
|
| 页面 | 地址 | 对应业务接口 |
|
||||||
|
|---|---|---|
|
||||||
|
| 文档上传 | `/admin/upload` | `POST/GET /api/v1/ingestion-batches*` |
|
||||||
|
| 文档列表与详情 | `/admin/documents` | `GET/DELETE /api/v1/documents*` |
|
||||||
|
| Search Lab | `/admin/search` | `POST /api/v1/search` |
|
||||||
|
|
||||||
|
`/admin` 默认展示上传页。OpenAPI 文档仍位于 `/api/v1/docs`。
|
||||||
|
|
||||||
|
## 品牌视觉基线
|
||||||
|
|
||||||
|
DEMO-014 参考地瓜机器人公开官网的设计语言,使用品牌橙 `#FF3C00`、黑白高对比、中文
|
||||||
|
无衬线标题、紧凑数据字体、低圆角容器和宽松留白。贯穿页面标题的“计算信号轨”是唯一的
|
||||||
|
强调性图形,序号对应上传、文档、检索三步真实操作顺序。
|
||||||
|
|
||||||
|
品牌图形使用仓库内的代码原生 SVG/CSS 绘制,不复制或热链官网图片、字体和脚本;页面
|
||||||
|
运行时不依赖品牌官网。前端设计方法参考 GitHub `anthropics/skills` 的 Apache-2.0
|
||||||
|
`frontend-design` Skill,并以现有 Demo 信息架构、可访问性和业务正确性为优先约束。
|
||||||
|
|
||||||
|
## Demo 操作闭环
|
||||||
|
|
||||||
|
1. 在上传页选择一个或多个 PDF、DOCX、TXT 或 Markdown 文件并提交;
|
||||||
|
2. 接口返回 `202` 后,页面自动轮询批次,逐文件显示接收状态、处理阶段、尝试次数和安全错误;
|
||||||
|
3. 批次进入 `Completed`、`Partial failure` 或 `Failed` 后停止轮询,再在文档列表查看元数据;
|
||||||
|
4. 在 Search Lab 输入问题并查看 Top-K 的文件名、页码、章节、Chunk 内容、Dense Score 和
|
||||||
|
Rerank Score;
|
||||||
|
5. 如需清理 Demo 数据,在文档列表确认后直接删除。
|
||||||
|
|
||||||
|
## 范围与安全边界
|
||||||
|
|
||||||
|
- 页面静态文件位于 `apps/admin-web`,由同一个 FastAPI 进程提供,避免额外前端运行时;
|
||||||
|
- API 返回内容仅通过 DOM `textContent` 渲染,不作为 HTML 注入;
|
||||||
|
- 页面不保存或展示 Provider 凭据、内部 Endpoint、storage key 或 Index Profile;
|
||||||
|
- Console 只轮询当前批次的聚合接口,不读取 Provider/队列细节,也不提供人工重试按钮;
|
||||||
|
- 批量上传不包含目录上传、压缩包解包、断点续传或跨批次去重;
|
||||||
|
- 本任务不包含 ACL/RBAC、审计、Document Version、Compiled Knowledge、MCP、Hybrid/BM25
|
||||||
|
或完整 V1 Admin Console 导航;
|
||||||
|
- 删除是 Demo 的直接删除语义,确认后会清理 canonical Metadata、派生产物、源文件与向量行。
|
||||||
|
|
||||||
|
## 验证
|
||||||
|
|
||||||
|
静态路由和范围契约由 `tests/integration/test_admin_console.py` 覆盖;现有 REST 全链路测试覆盖
|
||||||
|
上传、列表、详情、搜索与删除。浏览器验收使用确定性本地 Provider,不读取真实企业文档,
|
||||||
|
并覆盖桌面与移动布局。
|
||||||
97
docs/development/ASYNC_INGESTION_SETUP.md
Normal file
97
docs/development/ASYNC_INGESTION_SETUP.md
Normal file
@ -0,0 +1,97 @@
|
|||||||
|
# 异步入库与失败恢复说明
|
||||||
|
|
||||||
|
## 范围
|
||||||
|
|
||||||
|
DEMO-015 将单文件上传从 API 请求线程迁移到独立 Celery Worker,DEMO-016 在其上增加逐文件
|
||||||
|
隔离的批量接收和状态聚合。OceanBase 中的
|
||||||
|
`ingestion_jobs` 和 `ingestion_attempts` 是执行状态事实源;RabbitMQ 消息只携带 Job ID,
|
||||||
|
不携带文件内容、Provider 凭据或业务状态。
|
||||||
|
|
||||||
|
批次只聚合接收与 Job 状态,不改变每个文件的独立执行和重试边界。
|
||||||
|
|
||||||
|
## 运行时配置
|
||||||
|
|
||||||
|
以下值必须通过环境变量或部署 Secret 注入,不得提交真实值:
|
||||||
|
|
||||||
|
```text
|
||||||
|
RABBITMQ_URL=amqp://<user>:<password>@<host>:5672/<vhost>
|
||||||
|
INGESTION_MAX_ATTEMPTS=3
|
||||||
|
INGESTION_RETRY_BACKOFF_SECONDS=2
|
||||||
|
INGESTION_RETRY_BACKOFF_MAX_SECONDS=60
|
||||||
|
INGESTION_WORKER_LEASE_SECONDS=300
|
||||||
|
INGESTION_RECOVERY_INTERVAL_SECONDS=30
|
||||||
|
BATCH_MAX_FILES=20
|
||||||
|
BATCH_MAX_TOTAL_BYTES=209715200
|
||||||
|
```
|
||||||
|
|
||||||
|
Worker 与 API 还需要相同的 OceanBase、BOS、Embedding、VectorStore 和 Index Profile 配置。
|
||||||
|
RabbitMQ 用户应只具备指定 vhost/queue 的最小权限。
|
||||||
|
|
||||||
|
## 启动顺序
|
||||||
|
|
||||||
|
```bash
|
||||||
|
uv run alembic upgrade head
|
||||||
|
make run-worker
|
||||||
|
make run
|
||||||
|
```
|
||||||
|
|
||||||
|
`make run-worker` 在当前 Demo 中同时启动 Worker 和恢复调度器。恢复调度器周期扫描:
|
||||||
|
|
||||||
|
- 尚未成功投递的 `queued` Job;
|
||||||
|
- 已到重试时间的 `retrying` Job;
|
||||||
|
- Worker 丢失后租约已过期的 `running` Job。
|
||||||
|
|
||||||
|
任务投递使用 late acknowledgement、worker-lost reject 和单任务 prefetch。重复消息会先锁定
|
||||||
|
Job 行;有效租约内不会创建第二个 Attempt。租约过期后,旧 Attempt 以安全错误关闭,再按重试
|
||||||
|
规则恢复。
|
||||||
|
|
||||||
|
## 重试与清理
|
||||||
|
|
||||||
|
确定性的内容错误(无法解析、清洗后为空、无 Chunk、派生数据身份不一致)直接进入 `failed`。
|
||||||
|
其余运行时/Provider 错误按指数退避重试,默认最多三次总 Attempt。
|
||||||
|
|
||||||
|
第二次及后续 Attempt 创建前严格执行:
|
||||||
|
|
||||||
|
```text
|
||||||
|
OceanBase Vector rows
|
||||||
|
→ ChunkIndexState
|
||||||
|
→ Chunk
|
||||||
|
→ parsed JSON object
|
||||||
|
→ reset Document processing state
|
||||||
|
```
|
||||||
|
|
||||||
|
不可变源文件、Document、Job 和已结束 Attempt 始终保留。任一清理步骤失败时不创建新 Attempt,
|
||||||
|
Job 直接以 `retry cleanup failed` 终止,禁止在未知脏状态继续执行。
|
||||||
|
|
||||||
|
## API 状态
|
||||||
|
|
||||||
|
`POST /api/v1/documents` 成功返回 `202` 和 `job_id`。查询:
|
||||||
|
|
||||||
|
```text
|
||||||
|
GET /api/v1/jobs/{job_id}
|
||||||
|
```
|
||||||
|
|
||||||
|
Job 状态为 `queued | running | retrying | succeeded | failed | cancelled`;阶段为
|
||||||
|
`parsing | cleaning | chunking | embedding | indexing`。公开错误只返回固定安全摘要,不保存或
|
||||||
|
返回 Broker/Provider 异常正文。活动 Job 的 Document 暂不允许删除,API 返回 `409`。
|
||||||
|
|
||||||
|
批量提交与查询:
|
||||||
|
|
||||||
|
```text
|
||||||
|
POST /api/v1/ingestion-batches # multipart 字段 files,可重复
|
||||||
|
GET /api/v1/ingestion-batches/{batch_id}
|
||||||
|
```
|
||||||
|
|
||||||
|
批次创建后逐文件返回 `accepted` 或 `rejected`;已接收文件继续通过上述 Job 状态异步推进。
|
||||||
|
批次聚合状态为 `processing | completed | partial_failure | failed`。请求文件数超过
|
||||||
|
`BATCH_MAX_FILES` 时整体返回 `413`;单文件或共享总字节预算超限只拒绝当前文件,不删除此前
|
||||||
|
已接收文件。默认总预算 200 MiB,且配置不得小于单文件 `MAX_UPLOAD_BYTES`。
|
||||||
|
|
||||||
|
## 故障检查
|
||||||
|
|
||||||
|
1. Job 长期为 `queued`:检查 RabbitMQ 连通性、恢复调度器以及 `ingestion` queue;
|
||||||
|
2. Job 长期为 `running`:检查 Worker,租约到期后恢复调度器会重新投递;
|
||||||
|
3. Job 为 `retrying`:等待 `next_run_at`,客户端不应重新上传同一文件;
|
||||||
|
4. Job 为 `failed` 且错误为 `retry cleanup failed`:停止人工重试,先核对向量、关系数据与 BOS
|
||||||
|
派生对象的一致性;
|
||||||
|
5. 不得通过日志打印 `RABBITMQ_URL`、数据库 URL、BOS/模型凭据或原始 Provider 响应。
|
||||||
63
docs/development/BOS_SETUP.md
Normal file
63
docs/development/BOS_SETUP.md
Normal file
@ -0,0 +1,63 @@
|
|||||||
|
# ARCH-002 BOS 配置与对象契约
|
||||||
|
|
||||||
|
ARCH-002 在既有 `ObjectStorage` 契约后新增百度 BOS Adapter。业务服务仍只使用逻辑 key,
|
||||||
|
不知道 Bucket、Prefix、SDK 类型或凭据。ARCH-005 已将 BOS 设为正式运行默认值;Local
|
||||||
|
Adapter 继续用于单元测试和显式配置的本地开发。
|
||||||
|
|
||||||
|
## 运行配置
|
||||||
|
|
||||||
|
配置只能由部署环境或 Secret Manager 注入,禁止把真实值写入 `.env`、镜像、日志、测试
|
||||||
|
快照或提交记录。
|
||||||
|
|
||||||
|
```text
|
||||||
|
OBJECT_STORAGE_PROVIDER=bos
|
||||||
|
BOS_ENDPOINT=<已授权的 BOS Host 或 HTTP(S) Origin>
|
||||||
|
BOS_ACCESS_KEY_ID=<运行时注入>
|
||||||
|
BOS_SECRET_ACCESS_KEY=<运行时注入>
|
||||||
|
BOS_BUCKET=<预创建的 Bucket>
|
||||||
|
BOS_KEY_PREFIX=knowledge-platform
|
||||||
|
BOS_TIMEOUT_SECONDS=30
|
||||||
|
```
|
||||||
|
|
||||||
|
- Endpoint 必须是无用户名、密码、Path、Query 和 Fragment 的 Host 或 HTTP(S) Origin;旧配置
|
||||||
|
的纯 Host 会在 Provider 边界强制规范化为 HTTPS,禁止沿用 SDK 的默认 HTTP;
|
||||||
|
- Bucket 名必须符合 BOS 的小写字母、数字和连字符规则;
|
||||||
|
- Prefix 可为空;非空时使用安全的 POSIX 相对 key,不允许空段、`.`、`..` 或反斜杠;
|
||||||
|
- 应用不会创建、删除或修改 Bucket 配置。
|
||||||
|
|
||||||
|
## 对象与不可变约束
|
||||||
|
|
||||||
|
canonical `Document.storage_key` 保存与 Provider 无关的逻辑 key,例如:
|
||||||
|
|
||||||
|
```text
|
||||||
|
documents/<document_id>/<filename>
|
||||||
|
```
|
||||||
|
|
||||||
|
BOS Adapter 发起请求时才映射为:
|
||||||
|
|
||||||
|
```text
|
||||||
|
<BOS_KEY_PREFIX>/documents/<document_id>/<filename>
|
||||||
|
```
|
||||||
|
|
||||||
|
上传先写入可回卷的临时流,以满足官方 SDK 对长度和 CRC 计算的要求;随后使用
|
||||||
|
`If-None-Match: *` 执行单对象 Put,服务端返回 412 时统一映射为
|
||||||
|
`ObjectAlreadyExistsError`。这样不会用“先 HEAD、再 PUT”的竞态检查破坏不可变来源约束。
|
||||||
|
|
||||||
|
下载返回 SDK 的流式 Response Body,由调用方负责关闭;存在性检查只发 HEAD,不下载正文。
|
||||||
|
404、412 和其他 SDK 异常映射为稳定的 Provider-neutral 错误,外层错误不包含 Endpoint、
|
||||||
|
SDK Response Body 或凭据。
|
||||||
|
|
||||||
|
## 权限与验收边界
|
||||||
|
|
||||||
|
运行身份只授予目标 Bucket/Prefix 所需的 Put、Get/Head 和 Delete 权限,不授予 Bucket 管理或
|
||||||
|
全局 List 权限。若生产策略要求来源不可物理删除,应在后续 Document Version/Retention
|
||||||
|
设计中单独处理;当前 Demo 的直接删除行为保持不变。
|
||||||
|
|
||||||
|
默认测试使用 Fake Client,不连接 BOS。真实连接、权限和临时对象清理已在 ARCH-006 的
|
||||||
|
受控 test 资源中验证;凭据只允许运行时注入,不得写入仓库、日志或测试快照。
|
||||||
|
|
||||||
|
官方参考:
|
||||||
|
|
||||||
|
- [BOS Python SDK 上传文件](https://cloud.baidu.com/doc/BOS/s/yl5jfscvi)
|
||||||
|
- [BOS 获取 Object](https://cloud.baidu.com/doc/BOS/s/Mk4xtzv8v)
|
||||||
|
- [BOS 访问控制](https://cloud.baidu.com/doc/BOS/s/Tjwvysda9)
|
||||||
91
docs/development/CONTAINER_IMAGES.md
Normal file
91
docs/development/CONTAINER_IMAGES.md
Normal file
@ -0,0 +1,91 @@
|
|||||||
|
# 生产容器镜像说明
|
||||||
|
|
||||||
|
## 范围
|
||||||
|
|
||||||
|
DEPLOY-001 使用同一个锁定运行时构建三个独立镜像目标:
|
||||||
|
|
||||||
|
| Target | 默认镜像名 | 入口 | 端口 | 镜像健康检查 |
|
||||||
|
|---|---|---|---|---|
|
||||||
|
| `api` | `enterprise-knowledge-api` | Uvicorn + API/Admin | `8000` | `GET /healthz` |
|
||||||
|
| `mcp` | `enterprise-knowledge-mcp` | `ekp-mcp` | `8766` | PID 1 存活 |
|
||||||
|
| `worker` | `enterprise-knowledge-worker` | `ekp-worker` | 无 | PID 1 存活 |
|
||||||
|
|
||||||
|
镜像没有内置数据库迁移、Helm、Ingress 或 Secret 对象;DEPLOY-002 已在独立 Chart 中提供这些
|
||||||
|
编排能力。API 镜像包含 Alembic 和 migrations,由一次性 Migration Job 覆盖命令后复用。详见
|
||||||
|
[`KUBERNETES_DEPLOYMENT.md`](KUBERNETES_DEPLOYMENT.md)。
|
||||||
|
|
||||||
|
## 可重复构建基线
|
||||||
|
|
||||||
|
- Python 基础镜像固定为 `3.12.14-slim-bookworm`,uv 固定为 `0.12.5`,两者同时锁定 manifest
|
||||||
|
digest;
|
||||||
|
- Python 依赖只使用提交的 `uv.lock`,以 `uv sync --frozen --no-dev --no-editable` 安装;
|
||||||
|
- 三个目标共享相同依赖层,不复制不同版本的业务运行时;
|
||||||
|
- 进程均以固定 UID/GID `10001:10001` 运行,不具备 root 权限;
|
||||||
|
- `.dockerignore` 排除 `.env`、Git、虚拟环境、缓存、本地存储、测试和开发产物;
|
||||||
|
- Bake 计划同时声明 `linux/amd64` 与 `linux/arm64`,不下载或打包平台专属模型。
|
||||||
|
|
||||||
|
基础镜像 tag/digest 与应用依赖均已固定;镜像发布系统仍应保存本项目镜像的构建 digest,并让
|
||||||
|
Helm 按 digest 部署。本项目镜像 digest 由实际 Registry 发布产生,不把某个环境的 Registry
|
||||||
|
地址写入仓库。
|
||||||
|
|
||||||
|
## 构建命令
|
||||||
|
|
||||||
|
先确认 Docker Buildx builder 支持两个目标平台:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
make images-print
|
||||||
|
```
|
||||||
|
|
||||||
|
构建全部目标:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
IMAGE_REGISTRY=registry.example.internal/knowledge \
|
||||||
|
IMAGE_TAG=<immutable-release-tag> \
|
||||||
|
make images
|
||||||
|
```
|
||||||
|
|
||||||
|
只有发布流水线才执行推送:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
IMAGE_REGISTRY=registry.example.internal/knowledge \
|
||||||
|
IMAGE_TAG=<immutable-release-tag> \
|
||||||
|
make images-push
|
||||||
|
```
|
||||||
|
|
||||||
|
不要把 Registry 密码放入变量或命令行;构建节点应提前通过其凭据助手登录。禁止使用
|
||||||
|
`latest` 作为集群部署版本。
|
||||||
|
|
||||||
|
## 运行时配置
|
||||||
|
|
||||||
|
镜像只内置以下非敏感进程默认值:`APP_ENV=production`、容器监听地址、静态页面绝对路径和
|
||||||
|
临时缓存目录。OceanBase、BOS、RabbitMQ、Embedding、Reranker 与 MCP Token 必须在容器启动
|
||||||
|
时由 Secret 注入。镜像构建参数、Layer、标签、健康检查和启动命令都不得包含这些值。
|
||||||
|
|
||||||
|
API 的 `/healthz` 只证明进程存活,不读取云端凭据或探测依赖。Kubernetes readiness、资源限制、
|
||||||
|
只读根文件系统所需临时卷及依赖就绪策略由 DEPLOY-002 Chart 配置。MCP endpoint 需要认证,Worker
|
||||||
|
没有 HTTP server,因此这两个镜像的 Docker 健康检查只验证主进程;业务可恢复性仍以持久化
|
||||||
|
Job/Attempt 和 DEPLOY-003 故障验收为准。
|
||||||
|
|
||||||
|
## 本地单目标检查
|
||||||
|
|
||||||
|
需要加载到本地 Docker 时,可直接构建当前平台的一个 target:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker build --target api -f deploy/docker/Dockerfile -t ekp-api:test .
|
||||||
|
docker build --target mcp -f deploy/docker/Dockerfile -t ekp-mcp:test .
|
||||||
|
docker build --target worker -f deploy/docker/Dockerfile -t ekp-worker:test .
|
||||||
|
```
|
||||||
|
|
||||||
|
启动时使用 `--env-file` 或编排系统的 Secret 引用,不在命令中展开敏感值。API/Admin smoke test
|
||||||
|
至少检查 `/healthz`、`/admin/upload` 和 `/api/v1/openapi.json`;MCP 与 Worker 的真实依赖启动
|
||||||
|
验收在具备 RabbitMQ 和目标 Provider 的受控环境执行。
|
||||||
|
|
||||||
|
## DEPLOY-001 构建验收
|
||||||
|
|
||||||
|
2026-09-02 已使用 Buildx 分别构建并加载三个 `linux/arm64` target 和三个 `linux/amd64`
|
||||||
|
target。API/Admin 以非 root 用户完成健康、静态页面和 OpenAPI smoke test;MCP/Worker 完成非
|
||||||
|
root 安装入口检查。构建历史与容器文件系统检查未发现 `.env`、Git、测试文件或敏感配置。
|
||||||
|
|
||||||
|
这次验收只生成本地 `deploy001-*` 测试 tag,没有登录或推送 Registry,因此不产生可供集群
|
||||||
|
引用的项目镜像 digest。DEPLOY-002 必须使用操作者指定的不可变 Registry tag/digest,不得把
|
||||||
|
本地测试 tag 当作发布版本。
|
||||||
@ -1,3 +1,19 @@
|
|||||||
# Definition of Done
|
# Definition of Done
|
||||||
|
|
||||||
A task is done only when scope is implemented, architecture invariants hold, format/lint/typecheck pass, relevant tests pass, migrations/contracts/docs are updated as needed, no secrets are committed, retrieval changes include RAG regression results, and known limitations are documented.
|
A task is done only when scope is implemented, architecture invariants hold, format/lint/typecheck pass, relevant tests pass, migrations/contracts/docs are updated as needed, no secrets are committed, retrieval changes include RAG regression results, and known limitations are documented.
|
||||||
|
|
||||||
|
For asynchronous ingestion or retry tasks, relevant tests must also prove durable Job/Attempt state,
|
||||||
|
bounded retry classification, idempotent duplicate delivery, cleanup-before-retry, preservation of the
|
||||||
|
immutable source, and terminal behavior when cleanup fails. Batch ingestion must cover per-file partial
|
||||||
|
failure and configured limits.
|
||||||
|
|
||||||
|
For MCP tasks, contract tests must cover tool discovery, identity propagation, authorization/error
|
||||||
|
boundaries, REST/application-service parity, infrastructure redaction, and the explicitly targeted MCP
|
||||||
|
client compatibility check.
|
||||||
|
|
||||||
|
For cluster delivery tasks, completion also requires reproducible production images, Secret-only
|
||||||
|
sensitive configuration, serialized migration, readiness/liveness probes, resource and pod-security
|
||||||
|
defaults, idempotent Helm install/upgrade, rolling-restart verification, durable worker recovery, stable
|
||||||
|
HTTPS Admin/MCP endpoints, and remote WorkBuddy acceptance. Manifests, rendered output, logs and images
|
||||||
|
must be checked for credential leakage. Public-knowledge delivery does not complete or waive V1
|
||||||
|
user/ACL tasks.
|
||||||
|
|||||||
53
docs/development/INDEX_REBUILD.md
Normal file
53
docs/development/INDEX_REBUILD.md
Normal file
@ -0,0 +1,53 @@
|
|||||||
|
# ARCH-005 Shadow Index 重建与切换
|
||||||
|
|
||||||
|
ARCH-005 将正式运行默认值切换为 BOS、OceanBase Vector 和千帆 BCE Reranker;ARCH-006
|
||||||
|
进一步确认 canonical 关系数据与向量索引同处 OceanBase MySQL 数据库。关系表保存 Document、
|
||||||
|
Chunk、ChunkIndexState 和 Index Profile;向量表只保存可以重建的检索行。旧 Mochow 运行配置、依赖和代码已移除,但本流程不会自动删除
|
||||||
|
旧外部索引,物理退役必须由受控运维流程另行确认。
|
||||||
|
|
||||||
|
## Profile 与迁移前提
|
||||||
|
|
||||||
|
默认新 Profile 为:
|
||||||
|
|
||||||
|
```text
|
||||||
|
demo-bge-large-zh-structure-v1-oceanbase-v1
|
||||||
|
```
|
||||||
|
|
||||||
|
它与旧 Demo Profile 使用不同 ID,避免已有 Mochow `ChunkIndexState` 被误认为 OceanBase
|
||||||
|
写入证据。自定义 ID 仍由 `INDEX_PROFILE_ID` 注入,并且必须是全新的、稳定的版本标识。
|
||||||
|
|
||||||
|
执行前必须:
|
||||||
|
|
||||||
|
1. 运行 `uv run alembic upgrade head`,创建 canonical `index_profiles` 表;
|
||||||
|
2. 使用 `deploy/oceanbase/001_knowledge_chunks_v1.sql` 预创建目标表和 HNSW 索引;
|
||||||
|
3. 仅在运行时注入已授权的 OceanBase 与千帆凭据,不写入仓库或日志;
|
||||||
|
4. 确认目标 OceanBase 为 4.3.5.4+,且应用账号只有目标数据库/表所需权限;
|
||||||
|
5. 在重建期间避免修改 Chunk 策略;激活事务会阻止 canonical Chunk 并发写入穿过最终核对。
|
||||||
|
|
||||||
|
## 顺序执行
|
||||||
|
|
||||||
|
每一步只输出 Profile ID 和计数,不输出 Endpoint、SQL、Chunk 内容或凭据。
|
||||||
|
|
||||||
|
```bash
|
||||||
|
uv run ekp-index-rebuild build
|
||||||
|
uv run ekp-index-rebuild verify
|
||||||
|
uv run ekp-index-rebuild activate
|
||||||
|
```
|
||||||
|
|
||||||
|
- `build` 注册 Shadow Profile,从 OceanBase canonical Chunk 分文档生成 Embedding 并写入
|
||||||
|
OceanBase。命令中断后可以原样重试;已完整文档会跳过,部分目标 Profile 数据会先定向
|
||||||
|
清理再重建,不触碰 Active Profile;
|
||||||
|
- `verify` 要求目标策略下至少存在一个 canonical Chunk,并核对全部 Chunk 与匹配模型、
|
||||||
|
维度的 ChunkIndexState。输出中的 `complete` 必须为 `true`;
|
||||||
|
- `activate` 在 OceanBase 事务内再次验证,锁定 Chunk 写入,将原 Active Profile 标为
|
||||||
|
`retired`,再将目标 Shadow Profile 标为 `active`。重复执行已成功的激活是幂等的。
|
||||||
|
|
||||||
|
激活后 API 从 OceanBase canonical 关系表解析 Active Profile,不再仅依赖进程环境中的 ID。新上传与查询
|
||||||
|
统一使用 OceanBase Active Profile。部署仍应保留前一版本制品和旧索引回滚窗口;本 Task
|
||||||
|
不自动销毁任何旧外部数据。
|
||||||
|
|
||||||
|
## 验证与已知边界
|
||||||
|
|
||||||
|
离线测试覆盖多文档构建、重试跳过、完整性失败关闭、单 Active 约束、原子激活、迁移和
|
||||||
|
CLI 输出脱敏。真实 BOS/OceanBase/Qianfan 连通性、DDL 权限、30~50 条脱敏 Golden Query、
|
||||||
|
召回质量和延迟属于 ARCH-006,且只能使用运行时授权凭据与受控测试资源。
|
||||||
143
docs/development/KUBERNETES_DEPLOYMENT.md
Normal file
143
docs/development/KUBERNETES_DEPLOYMENT.md
Normal file
@ -0,0 +1,143 @@
|
|||||||
|
# Kubernetes / Helm 部署说明
|
||||||
|
|
||||||
|
## 1. 交付范围
|
||||||
|
|
||||||
|
DEPLOY-002 提供单一 Helm Chart,将以下运行角色部署到已有 Kubernetes 集群:
|
||||||
|
|
||||||
|
- API/Admin:默认两个副本,提供 `/healthz`、`/admin/*` 和 `/api/v1/*`;
|
||||||
|
- MCP Gateway:默认两个副本,提供经 Bearer Token 保护的 `/mcp`;
|
||||||
|
- Async Worker:默认两个副本,只消费 `ingestion` 队列;
|
||||||
|
- Recovery Scheduler:固定单副本,以 `Recreate` 策略运行 Celery Beat;
|
||||||
|
- Migration Job:每次安装/升级前串行执行 `alembic upgrade head`;
|
||||||
|
- 可选 RabbitMQ:只供开发/演示集群显式启用,生产默认使用外部 Broker。
|
||||||
|
|
||||||
|
OceanBase MySQL、BOS、Embedding/Reranker 服务继续使用集群外已有资源。Chart 不创建数据库、
|
||||||
|
BOS Bucket、域名、证书或云账号,也没有扩大公共知识库阶段的业务范围。
|
||||||
|
|
||||||
|
## 2. 前置条件
|
||||||
|
|
||||||
|
操作者需要提前准备:
|
||||||
|
|
||||||
|
1. 可用的 Kubernetes Namespace 和对应 `kubectl` 上下文;
|
||||||
|
2. Helm 3/4 与 Kubernetes 客户端;
|
||||||
|
3. 已推送到目标 Registry 的 API、MCP、Worker 不可变 tag 或 digest;
|
||||||
|
4. Namespace 内的运行时 Secret、TLS Secret,以及私有 Registry 所需的 ImagePullSecret;
|
||||||
|
5. 集群到 OceanBase、BOS、RabbitMQ 和模型服务的网络连通性。
|
||||||
|
|
||||||
|
Chart 默认引用 `knowledge-platform-runtime`,但不会创建它。该 Secret 至少包含以下键:
|
||||||
|
|
||||||
|
```text
|
||||||
|
DATABASE_URL
|
||||||
|
RABBITMQ_URL
|
||||||
|
BOS_ACCESS_KEY_ID
|
||||||
|
BOS_SECRET_ACCESS_KEY
|
||||||
|
BAIDU_QIANFAN_API_KEY
|
||||||
|
OCEANBASE_VECTOR_PASSWORD
|
||||||
|
MCP_AUTH_TOKEN
|
||||||
|
```
|
||||||
|
|
||||||
|
`RERANKER_API_KEY` 可单独提供;未提供时应用按既有契约复用
|
||||||
|
`BAIDU_QIANFAN_API_KEY`。`MCP_AUTH_TOKEN` 至少 32 个字符,只具备
|
||||||
|
`knowledge:search` 服务范围。不要使用 `kubectl --from-literal`、Shell 参数或 Helm
|
||||||
|
`--set` 传递以上值;应由受控 Secret 管理系统或运维流程预先写入 Namespace。
|
||||||
|
|
||||||
|
## 3. 环境 Values
|
||||||
|
|
||||||
|
复制 [`values.example.yaml`](../../deploy/helm/environments/values.example.yaml) 到仓库外的受控
|
||||||
|
路径,填写以下非敏感信息:
|
||||||
|
|
||||||
|
- 三个镜像的 Registry 路径与不可变 tag/digest;
|
||||||
|
- `existingSecret` 和 `imagePullSecrets` 的资源名称;
|
||||||
|
- OceanBase/BOS 的非敏感 Endpoint、数据库标识和 Bucket;
|
||||||
|
- Admin/API 与 MCP 的 Ingress Host、TLS Secret 名称;
|
||||||
|
- 与外部地址一致的 MCP Public URL、Issuer、Allowed Host/Origin;
|
||||||
|
- 副本、资源、超时和批量上传限制。
|
||||||
|
|
||||||
|
`values.schema.json` 拒绝未知顶层字段、`latest` tag、无效 digest、单副本 API/MCP、错误类型和
|
||||||
|
空的关键引用;模板还会校验 Chunk、检索和入库限制之间的大小关系。Values 只允许非敏感配置,
|
||||||
|
模板逐项生成 ConfigMap,不提供任意 `extraEnv` 入口,避免把凭据误写进渲染产物。
|
||||||
|
|
||||||
|
## 4. 安装与幂等升级
|
||||||
|
|
||||||
|
先做离线校验和清单检查:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
make deploy-lint VALUES=/secure/path/environment-values.yaml
|
||||||
|
make deploy-render VALUES=/secure/path/environment-values.yaml
|
||||||
|
```
|
||||||
|
|
||||||
|
安装或升级:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
NAMESPACE=knowledge-platform \
|
||||||
|
RELEASE=knowledge-platform \
|
||||||
|
make deploy VALUES=/secure/path/environment-values.yaml
|
||||||
|
```
|
||||||
|
|
||||||
|
`make deploy` 只接收 Values 文件路径。脚本会先验证 Chart、Namespace、运行时 Secret 及必需
|
||||||
|
键名、可选 RabbitMQ Secret 键名,以及启用 Ingress 时的两个 TLS Secret 引用,然后执行带失败
|
||||||
|
回滚和等待语义的 `helm upgrade --install`;脚本对 Helm 3 使用 `--atomic`,对 Helm 4 使用等价
|
||||||
|
的 `--rollback-on-failure`。它只验证 Secret 资源和键是否存在,不读取或输出 Secret 值。
|
||||||
|
|
||||||
|
Migration 是 `pre-install,pre-upgrade` Hook,固定名称配合
|
||||||
|
`before-hook-creation` 保证同一 Release 内不会并发执行。Helm 等待 Migration 成功后才更新业务
|
||||||
|
工作负载;失败时 Helm 保留上一版本。完成的 Job 最长保留一天供日志追查,并在下次升级前删除。
|
||||||
|
安装命令可以重复执行;数据库迁移必须保持 forward-only 与幂等。
|
||||||
|
|
||||||
|
## 5. Ingress、探针与 Pod 安全
|
||||||
|
|
||||||
|
生产环境启用 Ingress 后,Admin/API 与 MCP 使用独立稳定 HTTPS Host,也可由同一 Ingress
|
||||||
|
Controller 托管。证书只通过预建 TLS Secret 引用,不进入 Values。
|
||||||
|
|
||||||
|
- API 的 startup/readiness/liveness 调用无凭据 `/healthz`;
|
||||||
|
- MCP 的三类探针使用 TCP 端口,不发送或记录 Token;
|
||||||
|
- Worker/Scheduler 使用 PID 1 存活探针,业务恢复正确性由持久化 Job/Attempt 保证;
|
||||||
|
- 全部应用容器固定 UID/GID `10001`、禁止提权、丢弃 Linux capabilities、使用
|
||||||
|
`RuntimeDefault` seccomp 和只读根文件系统;
|
||||||
|
- `/tmp` 使用有大小上限的 `emptyDir`,ServiceAccount 不挂载 API Token;
|
||||||
|
- API/MCP 滚动更新使用 `maxUnavailable: 0`,未 Ready Pod 不进入 Service Endpoint。
|
||||||
|
|
||||||
|
`/healthz` 当前只表示 API 进程存活,不探测所有外部 Provider;真实依赖故障、滚动重启和
|
||||||
|
Worker 恢复行为由 DEPLOY-003 在目标集群验收。
|
||||||
|
|
||||||
|
## 6. 可选 Demo RabbitMQ
|
||||||
|
|
||||||
|
生产默认 `rabbitmq.enabled=false`,`RABBITMQ_URL` 指向外部受管 Broker。仅开发/演示时可开启
|
||||||
|
本地子 Chart;它使用单节点 StatefulSet 和 PVC,并要求预建 Secret 包含:
|
||||||
|
|
||||||
|
```text
|
||||||
|
username
|
||||||
|
password
|
||||||
|
erlang-cookie
|
||||||
|
```
|
||||||
|
|
||||||
|
同时必须让运行时 Secret 中的 `RABBITMQ_URL` 指向该 Service。子 Chart 使用固定
|
||||||
|
`rabbitmq:4.1.4-alpine` tag 与 multi-architecture manifest digest;它不具备生产 RabbitMQ 的
|
||||||
|
高可用、备份或升级保证,禁止作为生产默认值。卸载 Release 时 StatefulSet PVC 默认保留,
|
||||||
|
删除前必须由数据责任人确认。
|
||||||
|
|
||||||
|
## 7. 回滚、卸载与 Secret 轮换
|
||||||
|
|
||||||
|
查看历史并回滚时不要输出 Pod 环境变量:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
helm -n knowledge-platform history knowledge-platform
|
||||||
|
helm -n knowledge-platform rollback knowledge-platform <revision> --wait
|
||||||
|
```
|
||||||
|
|
||||||
|
更新外部 Secret 后,Secret 名称不变;用户和 WorkBuddy 配置无需重新注入。当前应通过受控
|
||||||
|
滚动重启让 Pod 读取新值,目标集群的无中断验证属于 DEPLOY-003。
|
||||||
|
|
||||||
|
卸载只删除本 Release 管理的 Kubernetes 对象:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
helm -n knowledge-platform uninstall knowledge-platform
|
||||||
|
```
|
||||||
|
|
||||||
|
外部 OceanBase、BOS、RabbitMQ、TLS/运行时 Secret 和可选 RabbitMQ PVC 不由卸载命令删除。
|
||||||
|
|
||||||
|
## 8. DEPLOY-002 验收边界
|
||||||
|
|
||||||
|
本任务完成 Chart lint、双路径渲染、严格 Kubernetes Schema、Values 失败关闭、Secret 泄漏扫描
|
||||||
|
以及本地集群两次安装/升级验证。目标企业集群的稳定 HTTPS、滚动重启、Worker 故障恢复、真实
|
||||||
|
Provider 和远程 WorkBuddy 验收统一属于后续 `DEPLOY-003`,不得提前宣称完成。
|
||||||
67
docs/development/LIVE_ACCEPTANCE.md
Normal file
67
docs/development/LIVE_ACCEPTANCE.md
Normal file
@ -0,0 +1,67 @@
|
|||||||
|
# ARCH-006 受控真实链路验收
|
||||||
|
|
||||||
|
本验收覆盖 OceanBase canonical relational metadata → BOS immutable source → 千帆
|
||||||
|
`bge-large-zh` → OceanBase Vector → 千帆 `bce-reranker-base` → canonical Source Result。数据集包含
|
||||||
|
32 条完全合成、脱敏 Query,不含真实客户、产品、政策或内部知识。
|
||||||
|
|
||||||
|
## 强制安全门禁
|
||||||
|
|
||||||
|
只有同时满足以下条件才允许执行:
|
||||||
|
|
||||||
|
1. 数据库、BOS 和千帆凭据已经所有者明确授权;配置清单中的现有值只允许本次 test
|
||||||
|
验收临时注入,禁止保存为 `.env`、写入日志或用于 fat/prod,完成后仍必须轮换;
|
||||||
|
2. OceanBase Database/Table 和 BOS Bucket/Prefix 已明确授权用于测试;
|
||||||
|
3. `APP_ENV=test`;
|
||||||
|
4. `BOS_KEY_PREFIX` 以 `live-acceptance/` 开头,避免写入共享业务前缀;
|
||||||
|
5. 显式设置 `EKP_RUN_LIVE_TESTS=1`、`EKP_CREDENTIALS_AUTHORIZED=1` 和
|
||||||
|
`EKP_LIVE_RESOURCE_ACK=1`。
|
||||||
|
|
||||||
|
任一条件不满足时,命令在创建 Provider 或外部连接前失败关闭,并且不输出 Endpoint、用户
|
||||||
|
名、SQL、响应正文、Chunk 内容或 Secret。仓库 `.env.example` 中三个开关固定为 `0`。
|
||||||
|
|
||||||
|
## 执行顺序
|
||||||
|
|
||||||
|
先完成 ARCH-005 切换:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
uv run alembic upgrade head
|
||||||
|
uv run ekp-index-rebuild build
|
||||||
|
uv run ekp-index-rebuild verify
|
||||||
|
uv run ekp-index-rebuild activate
|
||||||
|
```
|
||||||
|
|
||||||
|
确认 `verify` 和 `activate` 输出的 `complete` 均为 `true` 后,再运行:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
make live-acceptance
|
||||||
|
```
|
||||||
|
|
||||||
|
命令会使用随机运行前缀上传两份合成 Markdown,执行全部 32 条 Query,并在 `finally` 清理
|
||||||
|
本次运行的 OceanBase Vector 行、canonical Document/Chunk/State 以及 BOS 对象。清理失败会使验收
|
||||||
|
失败,且不会尝试删除不带本次随机文件名前缀的数据。
|
||||||
|
|
||||||
|
## 验收指标
|
||||||
|
|
||||||
|
默认门槛:
|
||||||
|
|
||||||
|
```text
|
||||||
|
Recall@1 >= 0.90
|
||||||
|
Recall@5 = 1.00
|
||||||
|
MRR >= 0.90
|
||||||
|
NDCG@5 >= 0.90
|
||||||
|
Search P95 <= 10000 ms
|
||||||
|
```
|
||||||
|
|
||||||
|
输出只包含 Query 数、Recall@1/3/5、MRR、NDCG@5、P50/P95 和 `passed`。如受控环境需要调整
|
||||||
|
门槛,只能通过 `EKP_LIVE_MIN_*` / `EKP_LIVE_MAX_P95_MS` 显式注入,并把实际门槛与结果记录
|
||||||
|
到受保护的验收日志;不要把 Endpoint、账号、凭据或原始响应提交到仓库。
|
||||||
|
|
||||||
|
## 验收结果(2026-09-02)
|
||||||
|
|
||||||
|
目标 test OceanBase 已完成 migration、专用向量表/HNSW、PRE-FILTER、BOS、Embedding、
|
||||||
|
Reranker 和 canonical Source Result 验收。32 条查询结果:Recall@1/3/5、MRR、NDCG@5 均为
|
||||||
|
`1.0`,P50 `3787.61 ms`,P95 `4759.21 ms`,`passed=true`。正式命令退出码为 `0`,数据库
|
||||||
|
复核 `live-` Document/Chunk/Vector 残留为 `0`。
|
||||||
|
|
||||||
|
首次正式运行出现过一次无细节瞬时失败;分阶段 Provider/单文档/32 查询诊断以及随后正式
|
||||||
|
运行均通过。后续稳定性任务应补充有界重试、限流指标与告警;这不阻塞本次受控验收。
|
||||||
102
docs/development/MCP_SETUP.md
Normal file
102
docs/development/MCP_SETUP.md
Normal file
@ -0,0 +1,102 @@
|
|||||||
|
# DEMO-012 MCP Gateway 使用说明
|
||||||
|
|
||||||
|
## 范围
|
||||||
|
|
||||||
|
DEMO-012 提供独立的 Enterprise Knowledge MCP Gateway,通过标准 Streamable HTTP 暴露一个
|
||||||
|
只读工具:`search_knowledge`。Gateway 直接复用 API 进程使用的 `SearchService` 组合,不复制
|
||||||
|
Dense Retrieval、Reranker 或 Source Enrichment 逻辑。
|
||||||
|
|
||||||
|
当前 Demo 只建立认证和身份传递边界:一个运行时注入的专用 Bearer Token 映射到一个
|
||||||
|
user/service identity,并且只有 `knowledge:search` scope。它不是共享超级管理员 Token,也不
|
||||||
|
代表 V1 OAuth、Document ACL 或完整 MCP P0 工具已经完成。
|
||||||
|
|
||||||
|
## 配置
|
||||||
|
|
||||||
|
必须通过运行环境提供至少 32 字符的随机 Token:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
export MCP_AUTH_TOKEN="<runtime-secret-with-at-least-32-characters>"
|
||||||
|
make run-mcp
|
||||||
|
```
|
||||||
|
|
||||||
|
默认地址为 `http://127.0.0.1:8766/mcp`。除现有数据库、BOS、Embedding、Vector 和 Reranker
|
||||||
|
配置外,Gateway 支持:
|
||||||
|
|
||||||
|
| 变量 | 默认值 | 说明 |
|
||||||
|
|---|---|---|
|
||||||
|
| `MCP_HOST` | `127.0.0.1` | 监听地址;本地默认不暴露到其他网卡 |
|
||||||
|
| `MCP_PORT` | `8766` | Gateway 端口 |
|
||||||
|
| `MCP_PATH` | `/mcp` | Streamable HTTP 单一端点 |
|
||||||
|
| `MCP_PUBLIC_URL` | `http://127.0.0.1:8766/mcp` | MCP Resource Server URL |
|
||||||
|
| `MCP_ISSUER_URL` | `http://127.0.0.1:8766` | Demo Token 的身份颁发方标识 |
|
||||||
|
| `MCP_CLIENT_ID` | `workbuddy-enterprise` | MCP Client 类型标识 |
|
||||||
|
| `MCP_SERVICE_IDENTITY` | `workbuddy-demo` | 传入业务边界的服务身份 |
|
||||||
|
| `MCP_REQUIRED_SCOPE` | `knowledge:search` | Tool 所需最小 scope |
|
||||||
|
| `MCP_ALLOWED_HOSTS` | 仅本机/test | DNS rebinding Host allowlist,JSON 数组 |
|
||||||
|
| `MCP_ALLOWED_ORIGINS` | 仅本机 | Origin allowlist,JSON 数组 |
|
||||||
|
|
||||||
|
远程部署必须使用 TLS 反向代理,并把公开域名加入 `MCP_ALLOWED_HOSTS`;有 `Origin` 的客户端还
|
||||||
|
必须显式配置 `MCP_ALLOWED_ORIGINS`。Token、真实域名和内部 Endpoint 不得提交 Git。
|
||||||
|
|
||||||
|
## WorkBuddy / CodeBuddy 配置
|
||||||
|
|
||||||
|
WorkBuddy Enterprise 官方客户端支持 `type: "http"`、远程 URL、自定义 Header 和环境变量
|
||||||
|
展开。可复制 `deploy/workbuddy-mcp.example.json` 作为项目配置,内容如下:
|
||||||
|
|
||||||
|
```jsonc
|
||||||
|
{
|
||||||
|
"mcpServers": {
|
||||||
|
"enterprise-knowledge": {
|
||||||
|
"type": "http",
|
||||||
|
"url": "${EKP_MCP_URL:-http://127.0.0.1:8766/mcp}",
|
||||||
|
"headers": {
|
||||||
|
"Authorization": "Bearer ${EKP_MCP_TOKEN}"
|
||||||
|
},
|
||||||
|
"description": "企业知识检索"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
只提交包含环境变量占位符的配置;`EKP_MCP_TOKEN` 必须由每个运行环境安全注入。连接后应只
|
||||||
|
发现 `mcp__enterprise_knowledge__search_knowledge` 一个工具。
|
||||||
|
|
||||||
|
## 安全和错误语义
|
||||||
|
|
||||||
|
- 缺失或错误 Token 返回 `401`,缺少 scope 返回 `403`;
|
||||||
|
- 未批准 Host 返回 `421`,未批准 Origin 返回 `403`;
|
||||||
|
- Query 长度和 `top_k` 在 MCP schema 层校验;
|
||||||
|
- 预期业务错误返回安全 Tool Error;未知 Provider/数据库错误收敛为
|
||||||
|
`knowledge search failed`;
|
||||||
|
- MCP 输出和 Gateway 日志不得包含 Token、Provider Endpoint、数据库表名、模型名或
|
||||||
|
Index Profile。
|
||||||
|
|
||||||
|
## 验证
|
||||||
|
|
||||||
|
```bash
|
||||||
|
uv run pytest tests/unit/test_mcp_auth.py tests/unit/test_mcp_search.py
|
||||||
|
uv run pytest tests/integration/test_mcp_server.py
|
||||||
|
```
|
||||||
|
|
||||||
|
集成测试使用官方 MCP Python Client 通过 Streamable HTTP 完成 initialize、tools/list 和
|
||||||
|
tools/call,并使用与 WorkBuddy 相同的 Bearer Header 配置验证身份传递、无结果、参数错误、
|
||||||
|
认证失败、Origin 防护和错误脱敏。
|
||||||
|
|
||||||
|
### WorkBuddy 现场验收记录
|
||||||
|
|
||||||
|
2026-09-02 使用 WorkBuddy Desktop `5.3.14` 自带的 CodeBuddy CLI `2.115.0` 完成兼容性
|
||||||
|
验收。客户端直接加载 `deploy/workbuddy-mcp.example.json`,健康检查为 `Connected`;工具
|
||||||
|
发现结果只有 `mcp__enterprise-knowledge__search_knowledge`,随后以字符串 `query` 和
|
||||||
|
`top_k=1` 完成 Tool 调用并读取结构化 `filename`、`section`、`content` 等来源字段。
|
||||||
|
|
||||||
|
现场验收使用 DEMO-012 的真实 Gateway、Bearer 验证、service identity 传递、输入 Schema 和
|
||||||
|
Tool 注册代码。为避免外部数据库与模型波动掩盖客户端兼容性,检索应用使用确定性验收返回;
|
||||||
|
真实 Provider 检索链路由 `ARCH-006` 的受控验收负责。非交互 CLI 需要批准 MCP Tool 调用,
|
||||||
|
本次只在单次会话中允许 `DeferExecuteTool`,没有永久启用跳过权限。测试结束后已删除临时 MCP
|
||||||
|
配置并停止本地 Gateway,未写入真实凭据。
|
||||||
|
|
||||||
|
参考:
|
||||||
|
|
||||||
|
- [MCP Python SDK](https://github.com/modelcontextprotocol/python-sdk)
|
||||||
|
- [MCP Streamable HTTP 规范](https://modelcontextprotocol.io/specification/2025-11-25/basic/transports)
|
||||||
|
- [WorkBuddy Enterprise MCP 使用文档](https://cloud.tencent.cn/document/product/1831/137039)
|
||||||
72
docs/development/OCEANBASE_VECTOR_SETUP.md
Normal file
72
docs/development/OCEANBASE_VECTOR_SETUP.md
Normal file
@ -0,0 +1,72 @@
|
|||||||
|
# ARCH-003 OceanBase Vector 配置与索引契约
|
||||||
|
|
||||||
|
ARCH-003 在既有 `VectorStore` 契约后新增 OceanBase Vector Adapter。ARCH-006 确认 OceanBase
|
||||||
|
MySQL 关系表保存 canonical Document、Chunk、Index Profile 和 Chunk Index State;同库向量表
|
||||||
|
只保存可由这些数据重建的检索行。ARCH-005 已完成运行时默认切换,并移除 Mochow 运行配置与依赖。
|
||||||
|
|
||||||
|
## 数据库前提
|
||||||
|
|
||||||
|
- OceanBase MySQL 租户版本至少为 4.3.5.4:ARCH-006 已在目标 test 集群用只读
|
||||||
|
`EXPLAIN` 验证 `VECTOR_INDEX(... pre_filter)` 产生 PRE-FILTER 执行计划;
|
||||||
|
- 目标数据库和最小权限账号由受控部署流程预创建;
|
||||||
|
- Vector 内存、资源规格和 HNSW 参数由 DBA 在受控环境确认;
|
||||||
|
- 应用启动和请求路径不会创建、修改或删除表结构。
|
||||||
|
|
||||||
|
使用 [001_knowledge_chunks_v1.sql](../../deploy/oceanbase/001_knowledge_chunks_v1.sql)
|
||||||
|
在专用数据库中创建索引表。脚本固定:
|
||||||
|
|
||||||
|
```text
|
||||||
|
VECTOR(1024)
|
||||||
|
DISTANCE = COSINE
|
||||||
|
TYPE = HNSW
|
||||||
|
EF_CONSTRUCTION = 256
|
||||||
|
EF_SEARCH = 200
|
||||||
|
```
|
||||||
|
|
||||||
|
表的 `index_record_id` 是确定性的 Chunk/Profile 主键;`document_id + index_profile_id` 和
|
||||||
|
`index_profile_id` 是清理与 Profile 过滤的标量索引。表名或向量维度变化必须建立新表或
|
||||||
|
Shadow Profile,验证后再激活,禁止在生产索引上原地修改。
|
||||||
|
|
||||||
|
## 运行配置
|
||||||
|
|
||||||
|
配置只能由部署环境或 Secret Manager 注入,禁止把真实地址、用户名和密码写入 `.env`、
|
||||||
|
镜像、日志、测试快照或提交记录。
|
||||||
|
|
||||||
|
```text
|
||||||
|
VECTOR_STORE_PROVIDER=oceanbase
|
||||||
|
OCEANBASE_VECTOR_URI=<host:port>
|
||||||
|
OCEANBASE_VECTOR_USER=<运行时注入>
|
||||||
|
OCEANBASE_VECTOR_PASSWORD=<运行时注入>
|
||||||
|
OCEANBASE_VECTOR_DATABASE=<预创建数据库>
|
||||||
|
OCEANBASE_VECTOR_TABLE=knowledge_chunks_v1
|
||||||
|
OCEANBASE_VECTOR_INDEX=vidx_knowledge_chunks_embedding
|
||||||
|
OCEANBASE_VECTOR_BATCH_SIZE=100
|
||||||
|
OCEANBASE_VECTOR_TIMEOUT_SECONDS=30
|
||||||
|
```
|
||||||
|
|
||||||
|
- URI 只接受 `host:port`,不得包含 Scheme、用户信息、Path、Query 或 Fragment;
|
||||||
|
- Database/Table/Index 只接受安全标识符;
|
||||||
|
- Password 使用 `SecretStr`,Provider 异常只返回收敛后的错误,不回显 SQL、连接信息、
|
||||||
|
Chunk 内容或 SDK Response;
|
||||||
|
- `pyobvector` 只存在于 Adapter/Factory 内,业务服务不依赖其 Row、Engine 或距离函数类型。
|
||||||
|
|
||||||
|
## 写入、删除和检索语义
|
||||||
|
|
||||||
|
- Upsert 使用 `index_record_id` 主键,按可配置批次写入;
|
||||||
|
- 删除使用参数化的 `document_id + index_profile_id` 条件,不拼接用户输入;
|
||||||
|
- Dense Search 在同一 ANN 查询中使用 `VECTOR_INDEX(... pre_filter)`,保证
|
||||||
|
`index_profile_id` 在候选近邻搜索前过滤,并返回 Dense Top20;禁止依赖普通 `WHERE`
|
||||||
|
的自适应过滤语义,也禁止全局 TopK 后再过滤;
|
||||||
|
- OceanBase `cosine_distance` 是低值优先距离,Adapter 统一转换为
|
||||||
|
`score = 1 - cosine_distance`,保持 `VectorStore` 的高分优先契约;
|
||||||
|
- 返回 UUID、Profile、字段数、距离范围和排序不满足契约时失败关闭,不生成 Source Result。
|
||||||
|
|
||||||
|
默认测试使用 Fake Client,不连接 OceanBase。真实版本、DDL、权限、Cosine HNSW 召回、过滤
|
||||||
|
行为与 30~50 条脱敏 Golden Query 统一在 ARCH-006 的受控资源中验证。凭据只能在运行时
|
||||||
|
临时注入,禁止进入仓库、日志或测试快照。
|
||||||
|
|
||||||
|
官方参考:
|
||||||
|
|
||||||
|
- [pyobvector 官方仓库](https://github.com/oceanbase/pyobvector)
|
||||||
|
- [OceanBase 创建向量索引](https://en.oceanbase.com/docs/common-oceanbase-database-10000000003075357)
|
||||||
|
- [OceanBase 向量索引查询 Hint](https://en.oceanbase.com/docs/common-oceanbase-database-10000000004311163)
|
||||||
76
docs/development/RERANKER_SETUP.md
Normal file
76
docs/development/RERANKER_SETUP.md
Normal file
@ -0,0 +1,76 @@
|
|||||||
|
# ARCH-004 千帆 BCE Reranker 与 Source Result 配置
|
||||||
|
|
||||||
|
本文记录 Knowledge Platform Demo 的重排序连接方式、请求边界和 canonical Source Result
|
||||||
|
契约。真实 API Key 只允许由运行环境注入,不得写入代码、示例配置、日志或提交记录。
|
||||||
|
|
||||||
|
## 运行配置
|
||||||
|
|
||||||
|
```dotenv
|
||||||
|
RERANKER_PROVIDER=qianfan
|
||||||
|
RERANKER_MODEL=bce-reranker-base
|
||||||
|
RERANKER_ENDPOINT=https://qianfan.baidubce.com/v2/rerank
|
||||||
|
RERANKER_API_KEY=<运行环境注入,可空>
|
||||||
|
RERANKER_TIMEOUT_SECONDS=30
|
||||||
|
DENSE_TOP_N=20
|
||||||
|
SEARCH_TOP_K=5
|
||||||
|
```
|
||||||
|
|
||||||
|
`RERANKER_API_KEY` 未单独配置时,Demo 会复用 `BAIDU_QIANFAN_API_KEY`。两者都不存在时
|
||||||
|
Provider 创建失败。凭证使用 `SecretStr`,HTTP/JSON 异常不会包含请求头、请求正文或
|
||||||
|
响应正文。
|
||||||
|
|
||||||
|
## Reranker 请求契约
|
||||||
|
|
||||||
|
ARCH-004 固定使用千帆 `POST /v2/rerank`,模型基线为
|
||||||
|
`bce-reranker-base`:
|
||||||
|
|
||||||
|
- Query 去除首尾空白后不能为空,最多 1600 字符;
|
||||||
|
- documents 不能为空,最多 64 条,每条最多 4096 字符;
|
||||||
|
- 当前标准链路发送 Dense Top20,并请求 Reranker Top5;
|
||||||
|
- 响应必须恰好返回请求的 Top-N,`index` 唯一且指向原输入位置;
|
||||||
|
- `relevance_score` 必须是有限数值,并保持降序;
|
||||||
|
- 响应若回显 document,必须与对应输入文本完全一致。
|
||||||
|
|
||||||
|
上述边界与千帆重排序接口当前约束一致。BCE 和原 Demo Qwen3 模型共享请求/响应结构,
|
||||||
|
因此本次变更不修改 `RerankerProvider`、Search REST API 或最终 Top5 行为。Qwen3-Reranker
|
||||||
|
保留为 PRD 中的后续 Benchmark 候选,不作为当前运行默认值。
|
||||||
|
|
||||||
|
Adapter 只向业务层返回原候选索引和 Reranker 分数,不暴露千帆响应结构。默认测试使用
|
||||||
|
`FakeRerankerProvider` 的确定性字符覆盖率排序,不发起网络请求。
|
||||||
|
|
||||||
|
## Source Result 契约
|
||||||
|
|
||||||
|
Reranker 对 provider-neutral Dense Candidate 的 `content` 评分;选出 Top5 后,
|
||||||
|
`SearchService` 使用
|
||||||
|
`chunk_id` 一次性回查 OceanBase canonical `chunks` 关系表,并由 canonical row 生成:
|
||||||
|
|
||||||
|
```text
|
||||||
|
document_id
|
||||||
|
filename
|
||||||
|
page
|
||||||
|
section
|
||||||
|
chunk_id
|
||||||
|
content
|
||||||
|
score
|
||||||
|
rerank_score
|
||||||
|
```
|
||||||
|
|
||||||
|
其中 `score` 保留 Dense 分数,`rerank_score` 保存精排分数。若 Chunk 缺失、document_id
|
||||||
|
不一致或向量索引 content 已与 canonical content 不一致,搜索会失败关闭,避免返回来源
|
||||||
|
与评分文本不一致的结果。
|
||||||
|
|
||||||
|
DEMO-009 已通过 `POST /api/v1/search` 提供序列化和错误映射,参见
|
||||||
|
`docs/development/REST_API_SETUP.md`。Demo 仍不实现 RBAC/Document ACL,正式 V1 权限
|
||||||
|
范围必须在 Dense 召回阶段下推,不能在 Reranker 后过滤。
|
||||||
|
|
||||||
|
## 验证边界
|
||||||
|
|
||||||
|
默认测试离线覆盖 Dense Top20 → Reranker → Top5 → canonical Source Result,并使用合成
|
||||||
|
Golden Dataset 验证预期文档和章节升至首位。真实千帆联调前还需在受控运行环境确认:
|
||||||
|
|
||||||
|
- API Key 已开通目标 Reranker 模型权限;
|
||||||
|
- Endpoint 网络可达;
|
||||||
|
- 模型 API 名称与控制台服务一致;
|
||||||
|
- 真实中文 Query 的延迟、配额和 Top5 质量满足 Demo 演示需求。
|
||||||
|
|
||||||
|
官方接口参考:[千帆重排序 API](https://cloud.baidu.com/doc/qianfan-api/s/2m7u4zt74)。
|
||||||
76
docs/development/REST_API_SETUP.md
Normal file
76
docs/development/REST_API_SETUP.md
Normal file
@ -0,0 +1,76 @@
|
|||||||
|
# Demo REST API 使用说明
|
||||||
|
|
||||||
|
## 范围
|
||||||
|
|
||||||
|
DEMO-009 将入库和检索服务公开为版本化 REST API,DEMO-015 将单文件入库切换为持久化异步
|
||||||
|
Job,DEMO-016 增加批量接收与聚合查询。API 层只负责输入校验、
|
||||||
|
错误映射和响应序列化,不重复实现 Parser、Embedding、VectorStore 或 Reranker 业务逻辑。
|
||||||
|
|
||||||
|
当前不包含 ACL/RBAC、软删除、目录/压缩包上传、断点续传或 Hybrid Search。
|
||||||
|
|
||||||
|
## 启动前提
|
||||||
|
|
||||||
|
1. 按 `docs/development/BOS_SETUP.md` 配置 BOS;
|
||||||
|
2. 按 `docs/development/OCEANBASE_VECTOR_SETUP.md` 配置 OceanBase Vector 和千帆 Embedding;
|
||||||
|
3. 按 `docs/development/RERANKER_SETUP.md` 配置 Reranker;
|
||||||
|
4. 按 `docs/development/ASYNC_INGESTION_SETUP.md` 配置 RabbitMQ 并启动 Worker;
|
||||||
|
5. 执行 `uv run alembic upgrade head`;
|
||||||
|
6. 新旧索引迁移按 `docs/development/INDEX_REBUILD.md` 构建、验证并激活 Profile;
|
||||||
|
7. 执行 `make run`。
|
||||||
|
|
||||||
|
`/healthz` 和 OpenAPI 可以在没有云端凭证时启动和访问。业务依赖在首次访问 P0 业务接口时
|
||||||
|
延迟创建;配置不完整时返回 `503`,不会将凭证名以外的配置值暴露给客户端。
|
||||||
|
|
||||||
|
OpenAPI 地址:`/api/v1/openapi.json`
|
||||||
|
|
||||||
|
Swagger UI:`/api/v1/docs`
|
||||||
|
|
||||||
|
## 示例
|
||||||
|
|
||||||
|
上传并异步排队:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
curl -F 'file=@./policy.md;type=text/markdown' \
|
||||||
|
http://127.0.0.1:8000/api/v1/documents
|
||||||
|
```
|
||||||
|
|
||||||
|
使用响应中的 `job_id` 查询进度:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
curl http://127.0.0.1:8000/api/v1/jobs/<job-id>
|
||||||
|
```
|
||||||
|
|
||||||
|
批量上传并查询聚合进度:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
curl -F 'files=@./policy.md;type=text/markdown' \
|
||||||
|
-F 'files=@./manual.pdf;type=application/pdf' \
|
||||||
|
http://127.0.0.1:8000/api/v1/ingestion-batches
|
||||||
|
|
||||||
|
curl http://127.0.0.1:8000/api/v1/ingestion-batches/<batch-id>
|
||||||
|
```
|
||||||
|
|
||||||
|
检索:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
curl -H 'Content-Type: application/json' \
|
||||||
|
-d '{"query":"产品A退款周期是多少?","top_k":5}' \
|
||||||
|
http://127.0.0.1:8000/api/v1/search
|
||||||
|
```
|
||||||
|
|
||||||
|
上传只等待不可变源文件、Document 与 Job 持久化,成功返回 `202`。Worker 后台执行 Parse、
|
||||||
|
Clean、Chunk、Embedding 和 OceanBase Upsert;客户端通过 Job 状态判断何时可检索。
|
||||||
|
|
||||||
|
## 删除一致性
|
||||||
|
|
||||||
|
删除按以下顺序执行:
|
||||||
|
|
||||||
|
1. 删除所有已记录 Index Profile 以及当前 Profile 的 OceanBase 行;
|
||||||
|
2. 删除 OceanBase canonical 关系表中的 ChunkIndexState、Chunk 和 Document;
|
||||||
|
3. 删除解析 JSON 与不可变原始文件。
|
||||||
|
|
||||||
|
`queued`、`running` 或 `retrying` Job 对应的 Document 返回 `409`,避免删除与 Worker 并发。
|
||||||
|
|
||||||
|
向量清理失败时保留 canonical Metadata 和原始文件。Metadata 提交后若文件清理失败,接口
|
||||||
|
返回明确的 `500`,可能留下不可访问的孤立文件;这是 Demo 直接删除的已知边界,正式 V1
|
||||||
|
应通过可重试清理任务和审计记录强化。
|
||||||
@ -1,6 +1,6 @@
|
|||||||
# Test Strategy
|
# Test Strategy
|
||||||
|
|
||||||
Unit tests must run without Mochow, Qianfan, internet, GPU or production credentials.
|
Unit tests must run without OceanBase, BOS, Qianfan, internet, GPU or production credentials.
|
||||||
|
|
||||||
Provide deterministic fakes:
|
Provide deterministic fakes:
|
||||||
- FakeEmbeddingProvider
|
- FakeEmbeddingProvider
|
||||||
@ -11,6 +11,11 @@ Provide deterministic fakes:
|
|||||||
|
|
||||||
External integrations use separate integration suites/markers.
|
External integrations use separate integration suites/markers.
|
||||||
|
|
||||||
|
Async ingestion tests run with an in-process recording dispatcher and deterministic providers. They
|
||||||
|
must cover persisted Job/Attempt history, safe error summaries, bounded retry, cleanup-before-retry,
|
||||||
|
active-lease duplicate delivery, expired-lease recovery, immutable-source preservation and terminal
|
||||||
|
cleanup failure without requiring a live RabbitMQ broker.
|
||||||
|
|
||||||
E2E covers upload -> parse -> chunk -> embed -> index -> retrieve -> source trace, and source -> compilation draft -> review -> publish -> retrieval.
|
E2E covers upload -> parse -> chunk -> embed -> index -> retrieve -> source trace, and source -> compilation draft -> review -> publish -> retrieval.
|
||||||
|
|
||||||
Any parsing/chunking/embedding/fusion/retrieval/reranking/query-routing change must run the Golden Dataset evaluation.
|
Any parsing/chunking/embedding/fusion/retrieval/reranking/query-routing change must run the Golden Dataset evaluation.
|
||||||
|
|||||||
@ -6,4 +6,20 @@ Benchmark the full pipeline: Embedding × Retrieval Strategy × Fusion × Rerank
|
|||||||
|
|
||||||
Initial embedding candidates: bge-large-zh, Qwen3-Embedding-0.6B, Qwen3-Embedding-4B, optional 8B.
|
Initial embedding candidates: bge-large-zh, Qwen3-Embedding-0.6B, Qwen3-Embedding-4B, optional 8B.
|
||||||
|
|
||||||
Use synthetic/public/anonymized fixtures only.
|
Use synthetic/public/anonymized fixtures only. ARCH-006 expands the checked-in dataset to 32
|
||||||
|
fully synthetic queries across refund, technical support, lodging, and reimbursement-material
|
||||||
|
sources. Offline Fake Provider regression must remain deterministic; target-provider acceptance
|
||||||
|
uses the same records through `make live-acceptance` and reports aggregate metrics only.
|
||||||
|
|
||||||
|
Current live gates:
|
||||||
|
|
||||||
|
```text
|
||||||
|
Recall@1 >= 0.90
|
||||||
|
Recall@5 = 1.00
|
||||||
|
MRR >= 0.90
|
||||||
|
NDCG@5 >= 0.90
|
||||||
|
Latency P95 <= 10000 ms
|
||||||
|
```
|
||||||
|
|
||||||
|
Threshold overrides are allowed only as explicit test-run environment inputs and must be recorded
|
||||||
|
with the run result. See `docs/development/LIVE_ACCEPTANCE.md` for safety and cleanup requirements.
|
||||||
|
|||||||
85
docs/plans/ARCHITECTURE_MIGRATION_PLAN.md
Normal file
85
docs/plans/ARCHITECTURE_MIGRATION_PLAN.md
Normal file
@ -0,0 +1,85 @@
|
|||||||
|
# Provider 架构迁移计划
|
||||||
|
|
||||||
|
## 目标基线
|
||||||
|
|
||||||
|
本计划依据 2026-09-02 对最新内部配置与 test 环境的只读核验调整 EKP Provider 和数据库
|
||||||
|
选型,同时保留 canonical-data 和 API-first 原则。
|
||||||
|
|
||||||
|
```text
|
||||||
|
OceanBase MySQL canonical Metadata / Chunk / Index State
|
||||||
|
Baidu BOS immutable source and derived artifacts
|
||||||
|
OceanBase Vector co-located rebuildable Dense retrieval index
|
||||||
|
Qianfan bge-large-zh 1024-dimensional Embedding
|
||||||
|
Qianfan BCE Reranker bce-reranker-base
|
||||||
|
```
|
||||||
|
|
||||||
|
Provider、内部地址、数据库账户和凭据不得出现在 API/MCP Contract、日志、镜像或提交中。
|
||||||
|
本地测试继续使用 LocalObjectStorage、FakeVectorStore、FakeEmbeddingProvider 和
|
||||||
|
FakeRerankerProvider。
|
||||||
|
|
||||||
|
## 明确保留的行为
|
||||||
|
|
||||||
|
- OceanBase 关系表保持 canonical database,专用向量表只保存可重建索引;
|
||||||
|
- 原始文件不可变;
|
||||||
|
- Dense Top20 → Reranker Top5;
|
||||||
|
- 当前 PDF/DOCX/TXT/Markdown Parser 和 `structure_v1` Chunk;
|
||||||
|
- `/api/v1` 和 MCP 继续只暴露业务语义;
|
||||||
|
- `index_profile_id` 过滤在召回阶段下推;
|
||||||
|
- 新索引通过 shadow 表/Profile 构建和验证,不原地覆盖旧索引。
|
||||||
|
|
||||||
|
## 明确不随迁移引入
|
||||||
|
|
||||||
|
- 将 OceanBase 向量表当作 canonical source;
|
||||||
|
- Top50、距离阈值、标准答案捷径、双查询和 RRF;
|
||||||
|
- Marker/Pandoc、LLM 摘要/问题/关键词索引;
|
||||||
|
- RBAC/ACL、异步队列、MCP、Hybrid/BM25 或 Admin Console 扩展;
|
||||||
|
- 旧系统 `.env`/镜像配置加载方式或任何现有敏感凭据。
|
||||||
|
|
||||||
|
## Task 与验收
|
||||||
|
|
||||||
|
### ARCH-001 架构和执行基线
|
||||||
|
|
||||||
|
更新 ADR、PRD、Architecture、Demo Scope、Tasks 和 Progress,消除目标 Provider 歧义。
|
||||||
|
|
||||||
|
### ARCH-002 BOS ObjectStorage
|
||||||
|
|
||||||
|
新增 provider-neutral BOS Adapter、配置、脱敏错误和离线单元测试。Local Adapter 保留为
|
||||||
|
测试/开发实现。
|
||||||
|
|
||||||
|
依赖:ARCH-001。
|
||||||
|
|
||||||
|
### ARCH-003 OceanBase VectorStore
|
||||||
|
|
||||||
|
新增 OceanBase Vector Adapter 和专用索引 DDL,覆盖 Upsert、文档/Profile 删除、Dense
|
||||||
|
TopN、Cosine Distance 到统一高分优先 `score` 的转换,以及 profile 过滤。
|
||||||
|
|
||||||
|
依赖:ARCH-002。
|
||||||
|
|
||||||
|
### ARCH-004 BCE Reranker
|
||||||
|
|
||||||
|
将运行基线改为 `bce-reranker-base`,按真实接口约束更新配置、Adapter、单元测试和 RAG
|
||||||
|
Evaluation;保持 Top5 API 行为。
|
||||||
|
|
||||||
|
依赖:ARCH-003。
|
||||||
|
|
||||||
|
### ARCH-005 Runtime Cutover
|
||||||
|
|
||||||
|
将 API 运行时切换到 BOS、OceanBase Vector 和 BCE Reranker;提供从 canonical
|
||||||
|
Chunk 构建新 shadow index 的可重试命令,验证后激活新 Profile,再清理 Mochow 运行配置。
|
||||||
|
|
||||||
|
依赖:ARCH-004。
|
||||||
|
|
||||||
|
### ARCH-006 真实验收
|
||||||
|
|
||||||
|
修正先前错误的 PostgreSQL 前提:将 canonical SQLAlchemy/Alembic 迁移到 OceanBase MySQL,
|
||||||
|
同库保留逻辑隔离的可重建向量表;在受控 test 资源中执行 OceanBase canonical → BOS →
|
||||||
|
Qianfan Embedding → OceanBase Vector → BCE Reranker → canonical Source Result 全链路,并完成
|
||||||
|
30~50 条脱敏 Golden Query 验收。
|
||||||
|
|
||||||
|
依赖:ARCH-005、已授权的 test 运行凭据、数据库/表和 BOS 专用前缀。
|
||||||
|
|
||||||
|
## 安全边界
|
||||||
|
|
||||||
|
内部配置清单中的现有凭据按已暴露处理。经所有者明确要求,本次只允许把 test 凭据临时
|
||||||
|
注入 ARCH-006 进程;不得生成 `.env`、测试快照或调试日志,不得连接 fat/prod,验收后仍应
|
||||||
|
轮换。真实集成测试必须使用专用表/BOS 前缀并清理本次临时对象。
|
||||||
123
docs/plans/ASYNC_INGESTION_MCP_PLAN.md
Normal file
123
docs/plans/ASYNC_INGESTION_MCP_PLAN.md
Normal file
@ -0,0 +1,123 @@
|
|||||||
|
# MCP 与异步批量入库优先级方案
|
||||||
|
|
||||||
|
**Task:** PLAN-001
|
||||||
|
**Status:** Accepted planning baseline
|
||||||
|
**Date:** 2026-09-02
|
||||||
|
|
||||||
|
## 1. 现状判断
|
||||||
|
|
||||||
|
| 能力 | 当前实现 | 结论 |
|
||||||
|
|---|---|---|
|
||||||
|
| 单文件上传 | `POST /api/v1/documents` 持久化 Source/Document/Job 后返回 `202`,Worker 后台处理 | DEMO-015 已实现 |
|
||||||
|
| 批量上传 | 批次 API、逐文件 Job 状态和 Admin Console 自动轮询 | DEMO-016 已实现 |
|
||||||
|
| 失败补偿 | 阶段内补偿 + 重试前清理全部派生数据,保留不可变源和执行历史 | DEMO-015 已实现 |
|
||||||
|
| 自动重试 | OceanBase Job/Attempt + Celery/RabbitMQ + 有界退避 + Worker 租约恢复 | DEMO-015 已实现 |
|
||||||
|
| 重试前清理 | 重试前删除向量、索引状态、Chunk、解析产物,保留源文件与历史 | DEMO-015 已实现 |
|
||||||
|
| MCP | 独立 Gateway、认证边界、`search_knowledge` 与 WorkBuddy 协议级验收 | DEMO-012 已实现 |
|
||||||
|
|
||||||
|
PLAN-001 的异步执行缺口已由 DEMO-015 补齐,批量提交与状态 UI 已由 DEMO-016 补齐。
|
||||||
|
|
||||||
|
## 2. Task 归并结论
|
||||||
|
|
||||||
|
### DEMO-012:MCP `search_knowledge`(提升为最高优先级)
|
||||||
|
|
||||||
|
保留既有 Task ID,不新建重复任务。范围限定为:
|
||||||
|
|
||||||
|
- 提供可运行的标准 MCP Gateway;
|
||||||
|
- 只暴露 `search_knowledge`;
|
||||||
|
- 复用现有 Search Service 或正式 `/api/v1/search`,不复制检索逻辑;
|
||||||
|
- 建立 WorkBuddy 所需的 user/service identity 边界,不使用共享超级管理员凭据;
|
||||||
|
- 完成协议契约、错误收敛、凭据脱敏和 WorkBuddy 兼容性验证。
|
||||||
|
|
||||||
|
该 Task 是 `EKP-070`、`EKP-071`、`EKP-072` 的最小垂直切片,不能据此把三个完整 V1 Task 标记为完成。
|
||||||
|
|
||||||
|
### DEMO-015:异步入库 Job 与自动重试清理(新增)
|
||||||
|
|
||||||
|
从 `EKP-017` 和 `EKP-023` 抽取 Demo 所需的最小基础能力,并复用 DEMO-003~006 的处理服务:
|
||||||
|
|
||||||
|
- 新增持久化 Ingestion Job/Attempt,记录阶段、状态、尝试次数、安全错误和时间戳;
|
||||||
|
- 源文件成功写入 BOS 且 Document 已持久化后立即排队,API 不等待解析和索引;
|
||||||
|
- 同一 Document 同时最多一个有效执行,任务投递和消费具备幂等保护;
|
||||||
|
- 对可重试的瞬时错误自动执行有界重试和退避;Demo 默认最多 3 次总尝试,允许通过非敏感配置调整;
|
||||||
|
- 格式不支持、文件为空、无可提取内容等确定性错误直接失败,不进行无意义重试;
|
||||||
|
- 每次重新执行前,按顺序清理该 Document 的向量行、`ChunkIndexState`、Chunk 和解析产物,再从不可变原始文件重新处理;
|
||||||
|
- 原始文件、Document 标识和审计所需的 Job/Attempt 历史不得被重试清理删除;
|
||||||
|
- 任一清理步骤失败时禁止启动下一次尝试,并记录为终止失败,避免在未知脏状态上重复执行。
|
||||||
|
- 通过执行器抽象接入 Celery/RabbitMQ,使 API 和 Worker 可独立部署;业务应用服务不得直接
|
||||||
|
依赖 Celery API,确保执行器仍可替换;
|
||||||
|
- Worker 重启、消息重复投递和确认超时必须保持持久化 Job 状态一致,不能产生双重有效
|
||||||
|
Attempt 或重复派生数据。
|
||||||
|
|
||||||
|
该 Task 只完成公共知识库入库所需的 Job/Retry 与队列基线,不扩展到完整审计系统、
|
||||||
|
Compilation、OCR、跨区域高可用或通用企业级 Job 平台。
|
||||||
|
|
||||||
|
### DEMO-016:批量异步上传与状态展示(已完成)
|
||||||
|
|
||||||
|
建立在 DEMO-015 之上,不与 Job 基础合并:
|
||||||
|
|
||||||
|
- 新增批量上传 API,一次接收多个 Demo 支持类型文件;
|
||||||
|
- 每个文件独立生成 Document 和 Ingestion Job,单个文件失败不回滚已成功接收的其他文件;
|
||||||
|
- 返回批次 ID、逐文件 Document ID、Job ID 和接收状态,不等待后台处理完成;
|
||||||
|
- 提供批次/Job 查询能力,并在 Admin Console 展示逐文件阶段、重试次数和最终结果;
|
||||||
|
- 文件数量、单文件大小和请求总大小均受可配置限制;
|
||||||
|
- 不在本 Task 引入目录上传、压缩包解包、断点续传或跨批次去重。
|
||||||
|
|
||||||
|
该 Task 对应 `EKP-021` 的批量异步扩展及 `EKP-080`/`EKP-081` 的最小状态 UI 切片,不能据此把完整 V1 Task 标记为完成。
|
||||||
|
|
||||||
|
## 3. 实施顺序与依赖
|
||||||
|
|
||||||
|
```text
|
||||||
|
DEMO-009 + ARCH-006 → DEMO-012(已完成)
|
||||||
|
|
||||||
|
PLAN-002(已完成)
|
||||||
|
└──→ DEMO-003 → DEMO-004 → DEMO-005 → DEMO-006
|
||||||
|
└──→ DEMO-015 Async Job + Celery/RabbitMQ + Retry Cleanup
|
||||||
|
└──→ DEMO-016 Batch Async Upload + UI
|
||||||
|
└──→ DEPLOY-001 → DEPLOY-002 → DEPLOY-003
|
||||||
|
```
|
||||||
|
|
||||||
|
推荐严格按以下顺序执行并逐项闭环:
|
||||||
|
|
||||||
|
1. `PLAN-002`(已完成)
|
||||||
|
2. `DEMO-015`(已完成)
|
||||||
|
3. `DEMO-016`(已完成)
|
||||||
|
4. `DEPLOY-001`
|
||||||
|
5. `DEPLOY-002`
|
||||||
|
6. `DEPLOY-003`
|
||||||
|
|
||||||
|
MCP 与异步入库没有代码级前置依赖,因此提高 MCP 优先级不会阻塞后续批量上传。`DEMO-016` 必须等待 `DEMO-015`,否则会把同步流水线简单并发放大,缺少可靠的状态、隔离和恢复边界。
|
||||||
|
|
||||||
|
## 4. 验收边界
|
||||||
|
|
||||||
|
### DEMO-012
|
||||||
|
|
||||||
|
- MCP Inspector/标准客户端能发现并调用 `search_knowledge`;
|
||||||
|
- WorkBuddy 使用已映射的 user/service identity 完成一次端到端检索;
|
||||||
|
- REST 与 MCP 对同一查询复用同一业务逻辑并返回一致的来源字段;
|
||||||
|
- 响应与日志不泄露 OceanBase、BOS、模型名、Endpoint 或凭据。
|
||||||
|
|
||||||
|
### DEMO-015
|
||||||
|
|
||||||
|
- 上传接口在源文件和 Job 持久化后返回 `202`,后台完成现有全流水线;
|
||||||
|
- 注入一次可重试故障后可观察到自动重试,并最终成功;
|
||||||
|
- 第二次尝试开始前,旧解析产物、Chunk、索引状态和向量均已清理,原始源文件仍存在;
|
||||||
|
- 永久错误不重试,达到次数上限后稳定进入 `failed`;
|
||||||
|
- 并发或重复投递不会产生双重有效执行和重复派生数据。
|
||||||
|
- RabbitMQ 消息确认超时或 Worker 重启后,Job 可恢复到终态且不会重复生成派生数据。
|
||||||
|
|
||||||
|
### DEMO-016
|
||||||
|
|
||||||
|
- 一次提交多个文件后立即得到逐文件 Document/Job 标识;
|
||||||
|
- 批次内成功、处理中、重试中和失败状态互不影响且可查询;
|
||||||
|
- Admin Console 无需刷新即可轮询展示逐文件进度和最终状态;
|
||||||
|
- 批次边界、大小限制、部分失败和重试清理均有集成测试。
|
||||||
|
|
||||||
|
## 5. 非目标与 ADR 影响
|
||||||
|
|
||||||
|
- PLAN-002 已把 Celery/RabbitMQ 纳入 `DEMO-015`,并以独立 `DEPLOY-*` Task 纳入最小
|
||||||
|
Kubernetes、多副本和滚动升级能力;跨区域容灾、自动云资源编排和完整 Observability
|
||||||
|
仍为非目标。
|
||||||
|
- 不改变 OceanBase/BOS canonical、向量索引可重建、原始文档不可变、Provider 抽象和 MCP 隐藏基础设施等 Accepted ADR。
|
||||||
|
- 公共知识库只是阶段性单一可见域;ADR-007 保持 Accepted,用户/ACL 实现后置且不得用全局
|
||||||
|
Top-K 后过滤替代。
|
||||||
|
- 本次任务拆分和交付边界不需要新增或重开 Accepted ADR。
|
||||||
@ -1,15 +1,15 @@
|
|||||||
# 项目进度
|
# 项目进度
|
||||||
|
|
||||||
当前阶段:Knowledge Retrieval Demo v0.1
|
当前阶段:生产容器镜像闭环完成,准备实施 Kubernetes/Helm 一键部署
|
||||||
|
|
||||||
已完成:DEMO-001、DEMO-002
|
已完成:DEMO-001~DEMO-010、DEMO-012、DEMO-014~DEMO-016、ARCH-001~ARCH-006、PLAN-001~PLAN-002、DEPLOY-001
|
||||||
进行中:无
|
进行中:无
|
||||||
阻塞:无
|
阻塞:无
|
||||||
|
|
||||||
下一任务:DEMO-003
|
下一任务:`DEPLOY-002`,完成并合并后执行 `DEPLOY-003`。
|
||||||
|
|
||||||
完整 V1 待办将在 Demo Scope 完成后继续执行。当前任务顺序以
|
Provider 架构迁移按 `ARCH-001 → ARCH-002 → ARCH-003 → ARCH-004 → ARCH-005 → ARCH-006`
|
||||||
`docs/demo/DEMO_SCOPE.md` 为准。
|
执行并已完成。DEMO-010 在迁移闭环验收后实施,未扩大为完整 V1 Admin Console。
|
||||||
|
|
||||||
## 任务记录
|
## 任务记录
|
||||||
|
|
||||||
@ -29,3 +29,398 @@
|
|||||||
不安全路径并清理写入中断产生的半文件。
|
不安全路径并清理写入中断产生的半文件。
|
||||||
- 新增模型、数据库、存储、迁移及 Schema Drift 测试;无需连接生产数据库即可
|
- 新增模型、数据库、存储、迁移及 Schema Drift 测试;无需连接生产数据库即可
|
||||||
验证 PostgreSQL 方言的迁移 SQL。
|
验证 PostgreSQL 方言的迁移 SQL。
|
||||||
|
|
||||||
|
### DEMO-003
|
||||||
|
|
||||||
|
- 新增 PDF、DOCX、TXT、MD 单文件上传应用服务,完成文件名、扩展名、媒体类型、
|
||||||
|
空文件和流式大小限制校验。
|
||||||
|
- 上传使用 UUID 隔离的不可覆盖 storage key,并将原始文件与 `uploaded` 状态的
|
||||||
|
Document 元数据关联保存。
|
||||||
|
- 数据库写入失败时自动回滚并删除已保存文件;新增四类文件、拒绝规则、大小限制
|
||||||
|
和失败补偿集成测试。
|
||||||
|
- Parser 触发与公开上传 REST API 分别留在 DEMO-004 和 DEMO-009,不在本任务
|
||||||
|
提前实现。
|
||||||
|
|
||||||
|
### DEMO-004
|
||||||
|
|
||||||
|
- 新增统一的 `ParsedDocument` / `ParsedBlock` 领域模型和 Parser Provider 接口,
|
||||||
|
使用 PyMuPDF、python-docx 及轻量文本解析器支持 PDF、DOCX、UTF-8 TXT、Markdown。
|
||||||
|
- PDF 保留一基页码,DOCX/Markdown 保留基础标题层级;基础 Cleaner 统一处理多余
|
||||||
|
空白、空行、异常换行和空块。
|
||||||
|
- 新增上传后同步触发解析的应用层编排,将版本化解析 JSON 作为可重建派生文件保存,
|
||||||
|
并通过数据库迁移增加 `parsed_storage_key` 追踪结果;原始文件保持不可变。
|
||||||
|
- 解析完成后文档保持 `processing`,等待 DEMO-005/006 完成后续流水线;解析失败记录
|
||||||
|
`failed` 和有界错误信息,派生结果写库失败时执行文件补偿。
|
||||||
|
- 新增四类格式、页码/章节、清洗、解析持久化、禁止覆盖和失败状态测试。公开 REST API
|
||||||
|
仍按计划留在 DEMO-009。
|
||||||
|
|
||||||
|
### DEMO-005
|
||||||
|
|
||||||
|
- 新增可注入 `TokenCounter` 的 `structure_v1` Chunk 策略,默认使用 450 tokens 上限和
|
||||||
|
75 tokens overlap;标题或章节变化形成硬边界,同章节段落优先组合,超长内容按 token
|
||||||
|
窗口切分。
|
||||||
|
- 默认轻量计数器按 CJK 字符、标点和保守 ASCII 子词估算 token,避免引入模型运行时;
|
||||||
|
精确模型 tokenizer 可通过既有接口注入,留待 DEMO-006 与 bge-large-zh 联调。
|
||||||
|
- 新增 canonical Chunk 元数据与只向前迁移,原子保存确定性 `chunk_id`、文档/文件、
|
||||||
|
一基页码范围、章节路径、顺序、token 数和策略版本;禁止静默覆盖同策略结果。
|
||||||
|
- 新增上传 → 解析/清洗 → Chunk 的同步流水线编排。Chunk 完成后仍保持 `processing`,
|
||||||
|
等待 DEMO-006 Embedding/Mochow;失败时不产生半套 Chunk,并保留原始文件。
|
||||||
|
- 新增完全合成的 Markdown 测试文档和 Golden Dataset Chunk 回归,覆盖章节追踪、
|
||||||
|
token 上限、overlap、页码范围、迁移、持久化、重复执行和失败状态。
|
||||||
|
|
||||||
|
### DEMO-006
|
||||||
|
|
||||||
|
- 新增与厂商无关的 `EmbeddingProvider` / `VectorStore` 契约及确定性 Fake,业务编排
|
||||||
|
不依赖 Mochow SDK 数据结构。
|
||||||
|
- 新增千帆 v2 `bge-large-zh` Adapter,使用 Bearer API Key,支持最多 16 条批量请求,
|
||||||
|
Chunk 同时限制 450 tokens/2000 字符,并严格校验响应顺序、1024 维度和有限数值;
|
||||||
|
凭证使用 `SecretStr`,
|
||||||
|
不进入错误消息或提交记录。
|
||||||
|
- 新增官方 `pymochow` Adapter,支持分批 Upsert 和按 `document_id + index_profile_id`
|
||||||
|
删除;Mochow 只保存向量与检索元数据,PostgreSQL 继续保存 canonical Chunk 和索引状态。
|
||||||
|
- 新增版本化 `IndexProfile` 与 `chunk_index_states` 向前迁移,防止模型/Chunk 策略原地
|
||||||
|
覆盖;所有 Chunk 成功入库后文档才进入 `ready`。
|
||||||
|
- 新增上传 → 解析/清洗 → Chunk → Embedding → Mochow Upsert 同步编排。Mochow 批次
|
||||||
|
或 canonical 状态写入失败时按文档/profile 清理可能的半批索引并记录 `failed`。
|
||||||
|
- 新增 Qianfan、Mochow 映射、配置脱敏、索引状态、失败补偿和 Golden Dataset 全链路
|
||||||
|
回归测试。真实凭证/网络集成未在默认测试中执行;专用 Mochow 表需按
|
||||||
|
`docs/development/MOCHOW_SETUP.md` 预创建。
|
||||||
|
|
||||||
|
### DEMO-007
|
||||||
|
|
||||||
|
- 新增 `DenseSearchRequest` / `DenseSearchCandidate` 与 `DenseRetrievalService`,完成
|
||||||
|
Query 清洗、同模型 Embedding、版本化 `index_profile_id` 过滤和默认 Top20 候选召回;
|
||||||
|
此阶段不加入 Reranker、最终 Source Result 或 REST API。
|
||||||
|
- `FakeVectorStore` 新增离线精确余弦检索,先按 profile 过滤再排序,并使用稳定的 UUID
|
||||||
|
tie-break,支持无网络的单元/集成/Evaluation 回归。
|
||||||
|
- `MochowVectorStore` 新增官方 `VectorTopkSearchRequest` 调用,固定检索 `embedding` 字段,
|
||||||
|
下推 profile 标量过滤,支持可配置 HNSW `ef`,严格校验响应字段、UUID、有限分数和
|
||||||
|
Top-N 数量,异常不泄露 SDK 请求或凭证。
|
||||||
|
- 新增 Golden Dataset Dense Retrieval 回归,完整覆盖上传 → 解析/清洗 → Chunk →
|
||||||
|
Embedding/入库 → Query Embedding → Dense Top20,确认两个合成 Query 均召回预期文件和
|
||||||
|
章节。
|
||||||
|
- 默认自动化测试不连接真实千帆/Mochow;真实环境仍需按
|
||||||
|
`docs/development/MOCHOW_SETUP.md` 注入凭证并验证网络、表 Schema 与索引状态。
|
||||||
|
|
||||||
|
### DEMO-008
|
||||||
|
|
||||||
|
- 新增与厂商无关的 `RerankerProvider` / `RerankResult` 契约及确定性 Fake,使用原始
|
||||||
|
candidate index 映射精排结果,业务服务不依赖千帆响应结构。
|
||||||
|
- 新增千帆 v2 Reranker Adapter(初始为 `qwen3-reranker-0.6b`,ARCH-004 已迁移至
|
||||||
|
`bce-reranker-base`),调用 `/v2/rerank`,严格限制 Query、
|
||||||
|
documents 和 Top-N,校验响应数量、唯一索引、回显文本、有限分数与降序关系;错误不会
|
||||||
|
回显请求内容或凭证。
|
||||||
|
- 新增 `SearchService`,完成 Dense Top20 → Reranker → Top5。Reranker 选中候选后按
|
||||||
|
`chunk_id` 回查 PostgreSQL canonical Chunk,再生成包含 document、filename、page、
|
||||||
|
section、chunk、content、Dense score 和 rerank score 的最终 Source Result。
|
||||||
|
- Source Enrichment 对缺失 Chunk、document_id 不一致和 stale vector content 采用失败
|
||||||
|
关闭,防止 Mochow 副本与 canonical 来源不一致时返回错误证据。
|
||||||
|
- 新增 Provider、配置脱敏、排序映射、canonical 回查、一致性失败和 Golden Dataset
|
||||||
|
全链路回归。默认测试不调用真实千帆;真实联调边界记录在
|
||||||
|
`docs/development/RERANKER_SETUP.md`。公开 Search REST API 留在 DEMO-009。
|
||||||
|
|
||||||
|
### DEMO-009
|
||||||
|
|
||||||
|
- 新增 `/api/v1/documents` 上传/列表、`/api/v1/documents/{document_id}` 详情/直接删除和
|
||||||
|
`/api/v1/search`,通过 OpenAPI 固化 multipart 上传、Document 状态和最小 Source Result
|
||||||
|
契约;API 响应不暴露 storage key、Index Profile 或厂商实现。
|
||||||
|
- 上传接口同步复用 DEMO-003~006 的完整入库流水线,并在完成后回读 PostgreSQL canonical
|
||||||
|
Document,确保返回最终 `ready` 状态和数据库生成时间戳。
|
||||||
|
- 新增文档查询/删除应用服务。删除覆盖当前及历史已记录 Index Profile 的向量行、
|
||||||
|
ChunkIndexState、Chunk、Document、解析 JSON 和原始文件;向量清理失败时保留 canonical
|
||||||
|
数据,来源清理失败返回明确错误并记录 Demo 一致性边界。
|
||||||
|
- 业务依赖延迟创建,因此健康检查和 OpenAPI 启动不要求云端凭证;Provider 配置不完整时
|
||||||
|
业务接口返回 `503`,上游异常使用收敛后的错误响应,避免泄露凭证或厂商正文。
|
||||||
|
- 新增完全离线的 REST 全链路测试,覆盖上传、列表、详情、Dense/Reranker 检索、来源字段、
|
||||||
|
直接删除、OpenAPI 路径、输入校验、404、Provider 延迟加载及删除失败保护。
|
||||||
|
|
||||||
|
### ARCH-001
|
||||||
|
|
||||||
|
- 接受 BOS、OceanBase Vector、千帆 `bge-large-zh` 和 `bce-reranker-base` 作为新的 V1
|
||||||
|
Provider 基线;PostgreSQL 继续保存 canonical 数据,OceanBase 只保存可重建索引。
|
||||||
|
- 新增 ADR-009~ADR-011 和独立迁移计划,将 ARCH-002~ARCH-006 的范围、依赖、验收顺序
|
||||||
|
与安全边界写入任务清单。
|
||||||
|
- 保留 Demo 的 Dense Top20 → Reranker Top5、现有 Parser/Chunk、API/MCP 业务契约;不随迁移
|
||||||
|
引入 Top50、RRF、Marker/Pandoc、LLM 派生索引、ACL 或 Hybrid/BM25 扩展。
|
||||||
|
- 本任务只调整架构和产品文档,不修改运行时代码;旧系统地址、账号和凭据均未写入仓库,
|
||||||
|
真实验收只允许使用轮换后的 Secret 与受控测试资源。
|
||||||
|
|
||||||
|
### ARCH-002
|
||||||
|
|
||||||
|
- 新增官方 `bce-python-sdk` 驱动的 `BosObjectStorage`,保持逻辑 key 与 Bucket/Prefix 分离,
|
||||||
|
覆盖 Put、流式 Open、HEAD Exists 和 Delete。
|
||||||
|
- 上传先使用可回卷临时流确定长度,再通过 `If-None-Match: *` 原子拒绝覆盖;404/412 和
|
||||||
|
其他 SDK 异常映射为不泄露 Endpoint、响应正文或凭据的稳定存储错误。
|
||||||
|
- 新增 `local|bos` 类型化配置和运行时工厂;BOS AK/SK 使用 `SecretStr`,Bucket、Endpoint、
|
||||||
|
Prefix 和超时均由环境注入。Local 仍是测试/开发及 ARCH-005 前的默认实现。
|
||||||
|
- 新增完全离线的 BOS Adapter、配置、Provider wiring 测试和中文配置说明;未使用内部配置
|
||||||
|
清单中的真实地址或凭据,真实 BOS 联调留在 ARCH-006。
|
||||||
|
|
||||||
|
### ARCH-003
|
||||||
|
|
||||||
|
- 新增官方 `pyobvector` 驱动的 `OceanBaseVectorStore`,覆盖批量 Upsert、按 Document/Profile
|
||||||
|
参数化删除和 Profile 条件下推的 Cosine ANN TopN;SDK 类型不进入业务服务。检索固定使用
|
||||||
|
OceanBase 4.6 的 `VECTOR_INDEX(... pre_filter)`,不依赖普通 `WHERE` 的自适应过滤,也不做
|
||||||
|
全局 TopK 后过滤。
|
||||||
|
- 将低值优先的 `cosine_distance` 转为 `score = 1 - distance`,并对向量维度、有限值、
|
||||||
|
非零范数、返回字段、UUID、Profile、距离范围和排序执行失败关闭校验。
|
||||||
|
- 新增 `VECTOR(1024)`、Cosine HNSW、Profile/Document 标量索引的预置 DDL;应用不自动执行
|
||||||
|
外部索引 DDL,PostgreSQL 继续保存 canonical 数据,OceanBase 表可重建。
|
||||||
|
- 新增 `mochow|oceanbase` 类型化配置、SecretStr 密码和运行时工厂;Mochow 保持 ARCH-005
|
||||||
|
前的默认实现。全部 Adapter、配置、Schema 和 wiring 测试均离线运行,真实数据库验收留在
|
||||||
|
ARCH-006。
|
||||||
|
|
||||||
|
### ARCH-004
|
||||||
|
|
||||||
|
- 将千帆 Reranker 运行基线从 `qwen3-reranker-0.6b` 切换为已接受的
|
||||||
|
`bce-reranker-base`;配置使用受限 Literal,业务 API、Provider 抽象和 Dense Top20 →
|
||||||
|
Reranker Top5 行为保持不变。
|
||||||
|
- 按千帆 `POST /v2/rerank` 官方约束复核并保留 Query 1600 字符、64 个候选、单文档
|
||||||
|
4096 字符、Top-N 及响应排序/索引/分数失败关闭校验;真实 API Key 不进入代码和测试。
|
||||||
|
- 更新 Adapter、环境示例、Provider wiring、配置与请求契约测试;Golden Dataset 继续使用
|
||||||
|
确定性 Fake 离线评估,但显式绑定 `bce-reranker-base` 当前模型标识。真实 BCE 质量与延迟
|
||||||
|
统一留在 ARCH-006 使用轮换凭据验收。
|
||||||
|
|
||||||
|
### ARCH-005
|
||||||
|
|
||||||
|
- 将正式运行默认值切换为 BOS、OceanBase Vector 和 `bce-reranker-base`,新 Profile ID 为
|
||||||
|
`demo-bge-large-zh-structure-v1-oceanbase-v1`;Local/Fake 仍用于显式本地和离线测试。
|
||||||
|
- 新增 PostgreSQL canonical `index_profiles` 迁移与 `shadow / active / retired` 生命周期,
|
||||||
|
通过局部唯一索引保证至多一个 Active Profile;API 启动时优先解析数据库 Active Profile,
|
||||||
|
无记录的新部署才使用配置基线。
|
||||||
|
- 新增 `ekp-index-rebuild build|verify|activate`:从 canonical Chunk 分文档重建,确定性 Upsert,
|
||||||
|
重试时跳过完整文档并清理目标 Profile 的部分写入;激活前核对全部 ChunkIndexState,并在
|
||||||
|
PostgreSQL 事务内锁定 Chunk 写入、退役旧 Profile、激活新 Profile。
|
||||||
|
- 移除 Mochow 运行配置、公开 Adapter、测试、配置文档及 `pymochow` 依赖;不自动删除旧
|
||||||
|
外部索引,保留受控回滚和运维退役边界。
|
||||||
|
- 新增重建/重试/完整性失败/单 Active/激活/迁移/CLI 脱敏测试及中文操作文档。真实资源
|
||||||
|
构建、BOS/OceanBase/Qianfan 连通性和 30~50 条脱敏查询验收仍属于 ARCH-006。
|
||||||
|
|
||||||
|
### ARCH-006(已完成)
|
||||||
|
|
||||||
|
- Golden Dataset 已扩展为 32 条完全合成 Query,覆盖退款、技术支持、住宿标准和报销材料;
|
||||||
|
Dense、Reranker、canonical Source Result 的离线回归全部通过。
|
||||||
|
- 新增 `ekp-live-acceptance` / `make live-acceptance`,实际覆盖上传、解析、Chunk、Embedding、
|
||||||
|
OceanBase Dense Top20、BCE Reranker Top5、canonical 来源核对及本次运行数据清理。
|
||||||
|
- live 命令强制要求 `APP_ENV=test`、专用 `live-acceptance/` BOS Prefix、凭据授权确认、资源
|
||||||
|
授权确认和显式运行开关;当前环境下已验证会在创建 Provider 前失败关闭且不泄露细节。
|
||||||
|
- 根据最新配置清单和 test 环境只读探测,确认没有独立 PostgreSQL;目标为 OceanBase
|
||||||
|
4.3.5.4 MySQL 模式,同一数据库承载 canonical 关系数据和逻辑隔离的向量表。ADR-012 已
|
||||||
|
取代先前 PostgreSQL 部署假设,业务事实来源与可重建索引的逻辑边界保持不变。
|
||||||
|
- 已在目标 test 集群只读验证数据库连通、版本与兼容模式,并通过 `EXPLAIN` 确认
|
||||||
|
`VECTOR_INDEX(... pre_filter)` 生成 `VECTOR INDEX ADAPTIVE SCAN (PRE-FILTER)`;未读取
|
||||||
|
业务正文或输出任何连接信息/凭据。
|
||||||
|
- canonical SQLAlchemy 默认已切换为 `mysql+pymysql`,数据库 URL 使用 `SecretStr`,新增
|
||||||
|
有界连接池/超时配置;新增 forward-only migration,以生成列唯一索引保证 OceanBase 与
|
||||||
|
SQLite 下至多一个 Active Profile;PostgreSQL 驱动已移除。
|
||||||
|
- 配置清单中的 test 凭据仅按用户明确授权临时注入本次验收,不保存为 `.env`、不提交、
|
||||||
|
不连接 fat/prod;完成后仍记录为待轮换风险。
|
||||||
|
- 已在 test OceanBase 执行全部 forward-only migration,并对本项目表完成列、类型、默认值、
|
||||||
|
主键的 scoped schema drift 核对;OceanBase 扩展语法导致标准 MySQL 反射器不能可靠比较的
|
||||||
|
二级索引/唯一约束/外键,由 SQLite 全量 drift、真实唯一约束事务和 live 级联删除补充验证。创建
|
||||||
|
`knowledge_chunks_v1` 12 列专用向量表和 Cosine HNSW,确认 PRE-FILTER 执行计划。迁移前
|
||||||
|
已确认所有目标表均不存在,没有覆盖原有 31 张业务表;事务内双 Active 写入实测被唯一
|
||||||
|
索引拒绝并完整回滚。
|
||||||
|
- 已完成 OceanBase canonical → BOS → `bge-large-zh` → OceanBase Vector →
|
||||||
|
`bce-reranker-base` → canonical Source Result 正式验收:32 条查询的 Recall@1/3/5、MRR、
|
||||||
|
NDCG@5 均为 `1.0`,P50 `3787.61 ms`、P95 `4759.21 ms`,通过 P95 `10000 ms` 门槛。
|
||||||
|
- 正式命令退出码为 `0`;两份合成文档、canonical Chunk/State、向量行与 BOS 对象均走随机
|
||||||
|
前缀清理,数据库复核 `live-` Document/Chunk/Vector 残留均为 `0`。首次 live 运行曾出现
|
||||||
|
一次瞬时失败,分阶段诊断及随后完整 32 查询均成功,仍需在后续稳定性任务关注 Provider
|
||||||
|
重试、限流和告警。
|
||||||
|
|
||||||
|
### DEMO-010(已完成)
|
||||||
|
|
||||||
|
- 新增同源轻量 Admin Console,提供 `/admin/upload`、`/admin/documents` 和
|
||||||
|
`/admin/search` 三个直接可访问页面;FastAPI 只负责静态文件路由,页面业务操作全部复用
|
||||||
|
`/api/v1`,没有复制上传、文档管理或检索逻辑。
|
||||||
|
- 上传页支持 PDF、DOCX、TXT、Markdown 文件选择、处理状态和结果摘要;文档页展示文件名、
|
||||||
|
类型、上传时间、状态,支持详情与确认删除;Search Lab 显式展示 Rank、Filename、Page、
|
||||||
|
Section、Chunk Content、Dense Score 和 Rerank Score。
|
||||||
|
- 前端使用安全 DOM 文本节点渲染 API 数据,不注入 HTML,不展示 storage key、Index Profile、
|
||||||
|
Provider Endpoint 或凭据;只实现 Demo 规定的三个页面,没有引入 ACL/RBAC、任务、审计、
|
||||||
|
Compiled Knowledge、MCP、Hybrid/BM25 或额外前端构建工具链。
|
||||||
|
- 新增静态路由、资产、范围和 API First 契约测试;浏览器使用本地确定性 Provider 完成
|
||||||
|
`README.md` 上传 → Ready → 列表/详情 → Top5 检索,并验证桌面和 390px 移动布局无横向
|
||||||
|
溢出、无前端警告或错误。删除的 UI 确认和服务端清理语义由现有 REST 全链路测试覆盖。
|
||||||
|
- 完整验证结果:Ruff Format/Lint、Mypy、JavaScript 语法、Git diff、依赖锁、Alembic Head/
|
||||||
|
离线升级均通过;Unit + Integration `150 passed`,RAG Evaluation `5 passed`。本任务没有
|
||||||
|
修改检索算法或 Provider,不重复调用真实资源验收;ARCH-006 的 32 条真实查询结果继续
|
||||||
|
作为底层链路基线。
|
||||||
|
- 已知非阻塞风险保持不变:配置清单中的历史 test 凭据仍应轮换;真实 Provider 瞬时失败的
|
||||||
|
重试、限流与告警留待后续明确 Task。Demo Console 本身没有新增凭据或生产 Endpoint。
|
||||||
|
|
||||||
|
### DEMO-014(已完成)
|
||||||
|
|
||||||
|
- 将 DEMO-010 通用绿色界面升级为地瓜机器人品牌化 Admin Console,使用唯一品牌强调色
|
||||||
|
`#FF3C00`、黑白高对比、无衬线中文排版、低圆角工业组件和与三步操作对应的计算信号轨;
|
||||||
|
没有增加页面、接口或业务能力。
|
||||||
|
- 品牌标记由内联 SVG 与 CSS 原生绘制,不下载、复制或热链官网资产。页面没有新增外部
|
||||||
|
Runtime、字体请求或第三方脚本,继续保持原生 HTML/CSS/JS 和同源 `/api/v1` 调用。
|
||||||
|
- GitHub 调研确认官方 `openai/skills` 当前没有通用前端设计 Skill;选择并安装了 Apache-2.0
|
||||||
|
的 `anthropics/skills/skills/frontend-design`。该 Skill 只影响视觉策划和自检方法,不进入
|
||||||
|
项目运行依赖或仓库产物。
|
||||||
|
- 浏览器使用本地确定性 Provider 再次完成 README 上传 → Ready → 文档列表 → Top5 检索;
|
||||||
|
1440px 桌面端和移动端均无根页面横向溢出、无前端警告或错误。复验中发现隐藏表头将移动
|
||||||
|
页面撑宽至 503px,已改为 `aria-label`,最终根宽度与 463px 视口一致,表格只在容器内滚动。
|
||||||
|
- 完整验证结果:Ruff Format/Lint、Mypy、JavaScript 语法与 Git diff 通过;Unit、Integration
|
||||||
|
和 RAG Evaluation 合计 `155 passed`;Alembic Head 与离线升级通过。本任务不修改 API、
|
||||||
|
检索算法、Provider 或数据库,不重复调用真实资源验收。
|
||||||
|
|
||||||
|
### PLAN-001(已完成)
|
||||||
|
|
||||||
|
- 审计确认当前 `/api/v1/documents` 仅接受一个文件,并在请求线程同步执行解析、Chunk、
|
||||||
|
Embedding 和索引;API 与 Admin Console 均没有批量提交、Job 查询或后台状态轮询。
|
||||||
|
- 当前上传、解析和向量索引具备阶段内失败补偿,影子索引重建具备目标 Profile 的重试安全
|
||||||
|
清理,但正常上传流水线没有持久化 Job/Attempt、自动调度退避或统一的重试前派生数据清理,
|
||||||
|
因而不能视为已实现自动重试。
|
||||||
|
- 保留既有 `DEMO-012` 并提升为最高优先级;新增 `DEMO-015` 承担 Async Job、自动重试和
|
||||||
|
重试前清理,新增 `DEMO-016` 承担批量异步上传和逐文件状态 UI。严格执行顺序为
|
||||||
|
`DEMO-012 → DEMO-015 → DEMO-016`。
|
||||||
|
- 已更新 PRD、Architecture、API/MCP Contract、Execution Plan、Task List、Demo Scope 与 DoD;
|
||||||
|
本任务只形成规划和待实现契约,没有修改业务代码、数据库 Migration 或 Accepted ADR。
|
||||||
|
- 验证结果:Ruff Format/Lint、Mypy、Git diff 与 Alembic Head/离线升级通过;Unit + Integration
|
||||||
|
`150 passed`,RAG Evaluation `5 passed`。首次受限网络运行因无法解析 PyPI 失败,获准联网
|
||||||
|
获取已锁定的构建依赖后全套复验通过。
|
||||||
|
- 下一任务:`DEMO-012`。详细依赖、范围和验收标准见
|
||||||
|
`docs/plans/ASYNC_INGESTION_MCP_PLAN.md`。
|
||||||
|
|
||||||
|
### DEMO-012(已完成)
|
||||||
|
|
||||||
|
- 新增独立 `ekp-mcp` Streamable HTTP Gateway 和 `make run-mcp` 入口,使用官方 MCP Python
|
||||||
|
SDK v2;默认只监听 `127.0.0.1:8766/mcp`,且只注册 `search_knowledge` 一个业务 Tool。
|
||||||
|
- Gateway 直接复用现有 `SearchService`,没有复制 Dense Retrieval、Reranker 或 Source
|
||||||
|
Enrichment;结构化输出与 REST 保持相同来源字段,不返回 Index Profile 或 Provider 信息。
|
||||||
|
- 新增运行时专用 Bearer Token 验证,将凭据映射为显式 client/service identity 和最小
|
||||||
|
`knowledge:search` scope;Token 至少 32 字符并通过 `SecretStr` 管理,不允许写入仓库。
|
||||||
|
当前身份层是 Demo 边界,不宣称完成 V1 OAuth、Document ACL 或用户委托授权。
|
||||||
|
- 启用 Host/Origin allowlist 和请求体限制;缺失/错误认证、scope、参数和检索异常均按安全
|
||||||
|
边界收敛,响应和日志不回显 Token、内部 Endpoint、数据库、模型或索引实现。
|
||||||
|
- 新增官方 MCP Client 的协议级测试,覆盖 initialize、tools/list、tools/call、唯一 Tool、
|
||||||
|
identity 传递、REST 字段一致性、无结果、参数拒绝、认证失败、Origin 防护和错误脱敏;
|
||||||
|
WorkBuddy 的 HTTP + Bearer + 环境变量配置记录于 `docs/development/MCP_SETUP.md`。
|
||||||
|
- 验证结果:Ruff Format/Lint、Mypy、Git diff、Alembic Head/离线升级全部通过;Unit +
|
||||||
|
Integration `165 passed`,RAG Evaluation `5 passed`。本任务未改变检索算法或数据库结构,
|
||||||
|
不重复消耗真实 Embedding/OceanBase/Reranker 资源。
|
||||||
|
- 2026-09-02 补充完成 WorkBuddy 现场验收:本机 WorkBuddy Desktop `5.3.14` 自带的
|
||||||
|
CodeBuddy CLI `2.115.0` 通过仓库交付的 `deploy/workbuddy-mcp.example.json` 建立
|
||||||
|
Streamable HTTP + Bearer 连接,健康检查为 `Connected`,仅发现
|
||||||
|
`mcp__enterprise-knowledge__search_knowledge`,并以 `query` + `top_k=1` 完成真实 Tool
|
||||||
|
调用、读取结构化来源字段。验收使用真实 Gateway、认证、身份传递、Schema 和 Tool
|
||||||
|
注册代码,检索应用返回确定性数据,以隔离 WorkBuddy 兼容性与外部 Provider 波动。
|
||||||
|
- 非交互客户端调用使用仅对当前会话放行 `DeferExecuteTool` 的权限设置,没有持久化放宽权限;
|
||||||
|
临时客户端配置已删除、Gateway 已停止,仓库和用户配置中均未留下测试 Token。至此
|
||||||
|
`DEMO-012` 约定的 WorkBuddy 客户端验收项闭环;公网 HTTPS 部署验证属于部署环境验收,
|
||||||
|
不阻塞本地 Demo Task 完成。
|
||||||
|
- 下一任务:`DEMO-015`。
|
||||||
|
|
||||||
|
### PLAN-002(已完成)
|
||||||
|
|
||||||
|
- 将当前交付目标明确为企业受控网络内的单一公共知识库:本阶段所有知识共享一个可见域,
|
||||||
|
暂缓用户、角色、知识库空间和 Document ACL,但不废弃 ADR-007,也不把未实现项标记完成。
|
||||||
|
- 保持 Accepted ADR 的模块化单体边界,确定 API/Admin、MCP Gateway 和 Async Worker 独立
|
||||||
|
部署;OceanBase/BOS/AI Provider 使用外部服务,当前云端模型链路不启用 GPU Worker。
|
||||||
|
- 将 Celery/RabbitMQ 正式并入 `DEMO-015`,并新增 `DEPLOY-001~003`,严格顺序为
|
||||||
|
`DEMO-015 → DEMO-016 → DEPLOY-001 → DEPLOY-002 → DEPLOY-003`。
|
||||||
|
- 定义 `make deploy VALUES=...` 的一键部署边界:操作者预先提供集群访问与 Secret,仓库负责
|
||||||
|
生产镜像、串行 Migration、Helm 安装/升级、Ingress/TLS、探针、资源与 Pod 安全配置;不得
|
||||||
|
将凭据写入 Git、镜像、命令行或日志。
|
||||||
|
- 明确现有 `docker-compose.dev.yml` 的 PostgreSQL/MinIO 假设已被 ADR-012 取代,不作为集群
|
||||||
|
交付模板;新增 `docs/plans/PUBLIC_KNOWLEDGE_CLUSTER_PLAN.md` 作为任务范围和验收基线。
|
||||||
|
- 本任务仅修改规划、产品、架构、API/MCP 契约、Demo Scope 和 DoD 文档,没有修改业务代码、
|
||||||
|
数据库 Migration 或 Accepted ADR 状态。
|
||||||
|
- 验证结果:Markdown 相对链接、敏感值模式和 Git diff 检查通过;Ruff Format/Lint、Mypy、
|
||||||
|
Alembic Head/离线升级通过;Unit + Integration `165 passed`,RAG Evaluation `5 passed`。
|
||||||
|
- 下一任务:`DEMO-015`。
|
||||||
|
|
||||||
|
### DEMO-015(已完成)
|
||||||
|
|
||||||
|
- 新增 OceanBase canonical `ingestion_jobs` / `ingestion_attempts` 与 forward-only migration;
|
||||||
|
Document 唯一 Job、Attempt 编号上限和单一运行中 Attempt 均由数据库约束保护。
|
||||||
|
- `POST /api/v1/documents` 改为持久化不可变 Source、Document、Job 后返回 `202`,新增
|
||||||
|
`GET /api/v1/jobs/{job_id}`;活动 Job 的 Document 删除返回 `409`,避免与 Worker 竞态。
|
||||||
|
- 新增队列无关的 Dispatcher 与 Job Runner,首个适配器使用 Celery/RabbitMQ;消息只包含 Job
|
||||||
|
ID,启用 late ack、worker-lost reject、单任务 prefetch 与周期性持久化 Job 恢复扫描。
|
||||||
|
- 瞬时错误按配置执行指数退避和最多三次总 Attempt;Worker 租约有效时重复投递不创建新
|
||||||
|
Attempt,租约过期后关闭旧 Attempt 并恢复。错误记录为固定安全摘要,不保存 Provider 正文。
|
||||||
|
- 第二次及后续 Attempt 前依次清理向量、ChunkIndexState、Chunk 与 parsed JSON,再从不可变
|
||||||
|
源文件重建;清理失败直接终止且不创建新 Attempt。Job 创建失败会补偿 Document 与源对象。
|
||||||
|
- Admin Console 仅将上传提示调整为“已接收并进入队列”;批量选择、Job 轮询和逐文件状态 UI
|
||||||
|
没有提前实现,继续属于 `DEMO-016`。
|
||||||
|
- 验证结果:Ruff Format/Lint、Mypy、JavaScript 语法、依赖锁、Alembic Head/离线 MySQL SQL
|
||||||
|
与 SQLite schema drift 均通过;Unit + Integration `179 passed`,RAG Evaluation `5 passed`;
|
||||||
|
Worker 入口确认注册 `ekp.ingestion.run` 与 `ekp.ingestion.recover`。
|
||||||
|
- 环境限制:当前工作站没有 Docker/RabbitMQ,也没有持久化注入目标 OceanBase URL,因此未做
|
||||||
|
真实 Broker 断连/确认超时和目标库 migration;这些真实环境项必须在 DEPLOY-003 集群验收
|
||||||
|
重跑。本任务以 Celery 配置测试、SQLite 持久化/唯一约束和故障注入集成测试闭环代码语义。
|
||||||
|
- 下一任务:`DEMO-016`。
|
||||||
|
|
||||||
|
### DEMO-016(已完成)
|
||||||
|
|
||||||
|
- 新增 canonical `ingestion_batches` / `ingestion_batch_items` 及 forward-only migration,保存
|
||||||
|
输入顺序、逐文件接收结果与 Document/Job 引用;异步阶段和最终结果仍以 Job 为事实源,
|
||||||
|
批次不是跨文件事务。
|
||||||
|
- 新增 `POST /api/v1/ingestion-batches` 与 `GET /api/v1/ingestion-batches/{batch_id}`;批量
|
||||||
|
请求字段为 `files`,成功返回 `202`、聚合状态和逐文件接收/Job 状态。类型、大小或内容错误
|
||||||
|
只拒绝当前文件,错误映射为固定安全摘要,其他文件继续接收。
|
||||||
|
- 新增 `BATCH_MAX_FILES` 与 `BATCH_MAX_TOTAL_BYTES`,复用 `MAX_UPLOAD_BYTES` 约束单文件;
|
||||||
|
上传流共享总字节预算,不把整批文件读入内存。数量超限在创建批次前返回 `413`。
|
||||||
|
- Admin Console 支持多选提交,显示当前批次的接收数量、完成数量、逐文件阶段、Attempt 与安全
|
||||||
|
错误,并每 1.5 秒自动轮询;进入 `completed`、`partial_failure` 或 `failed` 后停止轮询。
|
||||||
|
DOM 数据继续只通过文本节点渲染。
|
||||||
|
- 集成测试覆盖顺序稳定、部分拒绝不回滚、数量/单文件/批次总大小、全成功/部分失败聚合、
|
||||||
|
未知批次、OpenAPI 和前端契约;迁移 drift 增加两张新表。Ruff Format/Lint、Mypy、
|
||||||
|
JavaScript 语法、依赖锁、Alembic Head/离线升级均通过;Unit + Integration `183 passed`,
|
||||||
|
RAG Evaluation `5 passed`。
|
||||||
|
- 浏览器验收确认桌面布局和 390px 窄屏布局正常、无根页面横向溢出、无控制台警告或错误;
|
||||||
|
本地页面不注入云端凭据,批次结果行为由确定性 API 集成测试验证。
|
||||||
|
- 未扩展目录上传、压缩包解包、断点续传、手动重试、去重、用户或 ACL;自动重试和清理继续
|
||||||
|
复用 DEMO-015,Accepted ADR 未变更。
|
||||||
|
- 下一任务:`DEPLOY-001`。
|
||||||
|
|
||||||
|
### DEPLOY-001(已完成)
|
||||||
|
|
||||||
|
- 新增共享生产 Dockerfile 的 `api`、`mcp`、`worker` 三个 target,使用精确 Python/uv 版本、
|
||||||
|
frozen production dependencies、固定非 root UID/GID、exec-form 启动命令和逐 target 健康检查。
|
||||||
|
- 新增 amd64/arm64 Buildx Bake 计划、镜像构建/渲染/推送 Make 入口和 `.dockerignore`;构建上下文
|
||||||
|
排除凭据、本地状态、缓存、测试与开发产物,镜像不包含环境 Endpoint 或 Secret。
|
||||||
|
- Admin 静态目录新增类型化运行时覆盖,使非 editable 安装后的 API 镜像仍能提供同源页面;
|
||||||
|
源码运行继续使用既有默认目录。
|
||||||
|
- 新增不依赖 Docker daemon 的镜像契约测试与中文运行文档;基础 Python/uv 镜像同时固定精确
|
||||||
|
tag 与 manifest digest,Registry 地址和本项目发布 digest 继续由部署环境提供。
|
||||||
|
- 使用 Docker Buildx `0.36.1` + Colima 分别实际构建并加载全部三个 `linux/arm64` 和三个
|
||||||
|
`linux/amd64` target;最终 digest 固定版本再次构建命中相同业务 layer。没有执行 Registry
|
||||||
|
push,也没有在构建参数、history 或镜像文件系统中发现凭据。
|
||||||
|
- API 镜像以 `10001:10001` 启动并进入 healthy,`/healthz`、`/admin/upload`、OpenAPI 及
|
||||||
|
DEMO-016 批量路径 smoke test 通过;MCP/Worker 镜像以同一非 root 用户完成安装入口导入,
|
||||||
|
默认 CMD 与 HEALTHCHECK 元数据核对通过。真实 Provider/Broker 启动仍属于 DEPLOY-003。
|
||||||
|
- Ruff Format/Lint、Mypy、Docker Bake 渲染、JavaScript/依赖锁/Alembic 与完整测试门禁通过;
|
||||||
|
Unit + Integration `187 passed`,RAG Evaluation `5 passed`。下一任务:`DEPLOY-002`。
|
||||||
|
|
||||||
|
### DEPLOY-002(已完成)
|
||||||
|
|
||||||
|
- 新增 `deploy/helm/knowledge-platform` Helm Chart,部署 API/Admin、MCP Gateway、Async Worker、
|
||||||
|
单实例 Recovery Scheduler、串行 Migration Job、Service、Ingress/TLS 和可选 Demo RabbitMQ;
|
||||||
|
API/MCP 默认各两个副本,Worker 可独立扩缩,Beat 使用 `Recreate` 避免重复调度。
|
||||||
|
- 新增严格 `values.schema.json` 与仓库外环境 Values 示例,拒绝未知顶层字段、`latest` tag、
|
||||||
|
非法 digest、API/MCP 单副本和错误类型;跨字段模板校验继续约束批量大小、重试、Chunk 和
|
||||||
|
Top-K 关系。应用和可选 RabbitMQ 镜像均支持 digest,Demo RabbitMQ 已固定 manifest digest。
|
||||||
|
- 敏感配置只通过预建 `existingSecret` 注入;ConfigMap 仅渲染显式白名单内的非敏感字段,
|
||||||
|
没有任意 `extraEnv` 或 Secret 对象。`scripts/deploy.sh` 在部署前校验 Values、Namespace、运行时
|
||||||
|
Secret/必需键、可选 RabbitMQ Secret/键和启用 Ingress 时的 TLS Secret 引用,不读取 Secret
|
||||||
|
值;Helm 3/4 分别使用等价的原子失败回滚参数。
|
||||||
|
- Migration 使用 `pre-install,pre-upgrade` Hook、固定名称和 `before-hook-creation`,安装/升级在
|
||||||
|
Migration 成功前不更新业务 Pod。应用容器固定非 root UID/GID、只读根文件系统、丢弃全部
|
||||||
|
capabilities、禁用 ServiceAccount Token,并为临时目录、RabbitMQ 日志和 Beat 状态提供
|
||||||
|
`emptyDir`;API/MCP/Worker/Scheduler 均配置启动和存活/就绪探针及资源 requests/limits。
|
||||||
|
- 本机使用 K3s `v1.35.0+k3s1`、Helm `4.2.4` 和 DEPLOY-001 ARM64 镜像完成真实安装;Migration
|
||||||
|
成功,API/MCP 为 `2/2`,Worker、Scheduler 与可选 RabbitMQ 全部 Ready。首次安装暴露的
|
||||||
|
RabbitMQ 只读日志目录和 Beat 调度文件权限问题已修复,随后相同 Values 连续升级至 Revision
|
||||||
|
3~5 均成功;API `/healthz` 和 Admin 上传页 smoke test 通过。
|
||||||
|
- 默认、环境示例和 RabbitMQ 三种 Helm lint 均通过;含 Ingress/TLS/RabbitMQ 的 14 个最终
|
||||||
|
资源通过 kubeconform strict 校验。仓库、Helm manifest、Pod 日志三路 Secret 值扫描均为 `0`;
|
||||||
|
Ruff Format/Lint、Mypy、Alembic、Bash 语法与完整门禁通过,Unit + Integration `195 passed`,
|
||||||
|
RAG Evaluation `5 passed`。
|
||||||
|
- 目标企业集群的稳定 HTTPS、滚动重启、Worker 故障恢复、真实外部依赖监测与远程 WorkBuddy
|
||||||
|
仍属于 `DEPLOY-003`;本任务没有把这些项提前标记完成,也没有改变 Accepted ADR 或公共知识库
|
||||||
|
范围。下一任务:`DEPLOY-003`。
|
||||||
|
|||||||
123
docs/plans/PUBLIC_KNOWLEDGE_CLUSTER_PLAN.md
Normal file
123
docs/plans/PUBLIC_KNOWLEDGE_CLUSTER_PLAN.md
Normal file
@ -0,0 +1,123 @@
|
|||||||
|
# 公共知识库集群交付计划
|
||||||
|
|
||||||
|
**Task:** PLAN-002
|
||||||
|
**Status:** Accepted planning baseline
|
||||||
|
**Date:** 2026-09-02
|
||||||
|
|
||||||
|
## 1. 阶段目标与边界
|
||||||
|
|
||||||
|
当前阶段只交付一个面向企业内部受控网络的公共知识库,先保证服务持续可用、可重复部署,
|
||||||
|
并让使用者打开 Admin Console 或连接 WorkBuddy 后即可使用,无需感知后端进程启停。
|
||||||
|
|
||||||
|
“公共”表示本阶段所有已接收知识处于同一个可见域,不代表面向公网匿名开放,也不代表
|
||||||
|
ADR-007 被废弃。用户、用户组、角色、知识库多空间、Document ACL、SSO、OAuth 用户委托等
|
||||||
|
完整 V1 能力继续后置;一旦需要接收非公共知识,必须先实现 Permission Resolver 和检索前/
|
||||||
|
检索中 ACL 下推,禁止先做全局 Top-K 再过滤。
|
||||||
|
|
||||||
|
Admin 写操作只部署在企业受控网络或统一接入网关后。MCP 继续使用运行时注入、只具备
|
||||||
|
`knowledge:search` scope 的服务凭据;静态 Demo credential 不得演变为共享超级管理员 Token。
|
||||||
|
|
||||||
|
## 2. 目标部署拓扑
|
||||||
|
|
||||||
|
```text
|
||||||
|
Ingress / TLS
|
||||||
|
│
|
||||||
|
┌──────────────┴──────────────┐
|
||||||
|
│ │
|
||||||
|
Admin Console + Knowledge API MCP Gateway
|
||||||
|
replicas replicas
|
||||||
|
│ │
|
||||||
|
└──────── Application Services ────────┐
|
||||||
|
│
|
||||||
|
┌─────────────────────────────┼─────────────┐
|
||||||
|
│ │ │
|
||||||
|
OceanBase MySQL BOS RabbitMQ
|
||||||
|
canonical + vector │
|
||||||
|
▼
|
||||||
|
Async Worker(s)
|
||||||
|
```
|
||||||
|
|
||||||
|
沿用 ADR-002:业务核心仍为模块化单体,不拆分额外微服务;API、MCP Gateway 和异步 Worker
|
||||||
|
分别作为可独立扩缩容的进程/容器交付。当前 Parser 与云端 Embedding/Reranker 不需要本地 GPU,
|
||||||
|
因此 GPU Worker 不进入本轮最小部署,未来引入本地 OCR/模型时再按既有架构角色启用。
|
||||||
|
|
||||||
|
OceanBase、BOS 和模型服务使用集群外已有服务。RabbitMQ 默认使用外部地址;Helm Chart 可以
|
||||||
|
提供显式开关安装开发/演示依赖,但生产值不得把队列数据生命周期绑定到应用 Pod。
|
||||||
|
|
||||||
|
## 3. “一键部署”的定义
|
||||||
|
|
||||||
|
在操作者已具备 Kubernetes 访问权限、目标 Namespace 和 Secret 的前提下,仓库提供单一入口:
|
||||||
|
|
||||||
|
```text
|
||||||
|
make deploy VALUES=<environment-values.yaml>
|
||||||
|
```
|
||||||
|
|
||||||
|
该入口负责:
|
||||||
|
|
||||||
|
- 校验非敏感配置和 `existingSecret` 引用,不把凭据写入 Git、镜像、命令行或日志;
|
||||||
|
- 运行一次性、可追踪且串行的数据库 Migration Job;
|
||||||
|
- 安装或升级 API/Admin、MCP Gateway、Async Worker、Service 和可选 RabbitMQ 依赖;
|
||||||
|
- 配置 Ingress/TLS、资源 requests/limits、非 root、只读根文件系统和最小 ServiceAccount;
|
||||||
|
- 配置 readiness/liveness probe、副本数和滚动升级策略;
|
||||||
|
- 支持重复执行和幂等升级,并提供对应卸载/回滚说明。
|
||||||
|
|
||||||
|
本定义不承诺自动创建 OceanBase、BOS、域名证书或云厂商账号,也不允许在仓库内保存这些
|
||||||
|
资源的敏感凭据。
|
||||||
|
|
||||||
|
## 4. Task 顺序与依赖
|
||||||
|
|
||||||
|
```text
|
||||||
|
PLAN-002
|
||||||
|
└──→ DEMO-015 Celery/RabbitMQ Async Job + Retry Cleanup
|
||||||
|
└──→ DEMO-016 Batch Async Upload + Status UI
|
||||||
|
└──→ DEPLOY-001 Production Images
|
||||||
|
└──→ DEPLOY-002 Kubernetes/Helm One-click Deploy
|
||||||
|
└──→ DEPLOY-003 Availability + Remote Acceptance
|
||||||
|
```
|
||||||
|
|
||||||
|
### DEMO-015
|
||||||
|
|
||||||
|
实现持久化 Job/Attempt、重试前清理、幂等消费和有界自动重试,并把执行器正式接入
|
||||||
|
Celery/RabbitMQ。队列调用保持在抽象边界后,不允许业务服务直接依赖 Celery API。
|
||||||
|
|
||||||
|
### DEMO-016
|
||||||
|
|
||||||
|
**已完成。** 基于 DEMO-015 提供逐文件隔离的批量异步提交、批次/Job 查询和 Admin Console
|
||||||
|
状态展示;下一交付门槛为 DEPLOY-001 的生产镜像。
|
||||||
|
|
||||||
|
### DEPLOY-001
|
||||||
|
|
||||||
|
**已完成。** 交付 API/Admin、MCP Gateway、Async Worker 的生产容器镜像与启动约定;镜像使用固定依赖、
|
||||||
|
非 root 用户、健康检查和多架构可构建基线,不内置凭据。
|
||||||
|
|
||||||
|
### DEPLOY-002
|
||||||
|
|
||||||
|
**已完成。** 交付 Kubernetes/Helm Chart、严格环境 Values Schema、`existingSecret`、串行
|
||||||
|
Migration Hook、Ingress/TLS、Pod 安全、资源限制、探针和 `make deploy` 入口;Worker 与单实例
|
||||||
|
恢复调度器已分离。现有 `docker-compose.dev.yml` 仍仅作为旧的本地开发文件,因其
|
||||||
|
PostgreSQL/MinIO 假设与 ADR-012 不一致,不得作为集群交付模板。下一交付门槛为 DEPLOY-003
|
||||||
|
的目标集群可用性与远程验收。
|
||||||
|
|
||||||
|
### DEPLOY-003
|
||||||
|
|
||||||
|
在目标集群完成安装、幂等升级、滚动重启、Worker 故障恢复和远程 WorkBuddy 验收;形成运行
|
||||||
|
手册、备份/恢复责任边界和最小日志/指标检查。灾备演练与完整 Observability 平台仍属后置项。
|
||||||
|
|
||||||
|
## 5. 最小验收标准
|
||||||
|
|
||||||
|
- 新 Namespace 可由单一命令完成安装,重复执行不会破坏已有数据;
|
||||||
|
- Migration 在应用升级前串行成功,失败时新版本不继续接管流量;
|
||||||
|
- Admin/API 与 MCP 使用稳定 HTTPS 地址,重启或滚动升级不要求最终用户重新注入配置;
|
||||||
|
- API/MCP 支持至少两个副本,readiness 未通过的 Pod 不接收流量;
|
||||||
|
- Worker 重启后持久化 Job 可继续恢复,重复消息不产生双重有效执行或重复派生数据;
|
||||||
|
- 远程 WorkBuddy 仅发现批准的 `search_knowledge` Tool,并完成一次真实检索;
|
||||||
|
- 清单、镜像、日志和验收产物不包含凭据,敏感值均来自 Secret;
|
||||||
|
- 关键请求、Job 和外部依赖失败具备可定位的结构化日志和最小指标。
|
||||||
|
|
||||||
|
## 6. 明确后置能力
|
||||||
|
|
||||||
|
- 用户、用户组、角色、多租户、知识库空间隔离和 Document ACL;
|
||||||
|
- LDAP/SSO、OAuth 用户委托、公共互联网匿名访问;
|
||||||
|
- GPU Worker、本地模型、OCR/MinerU;
|
||||||
|
- 跨区域容灾、自动云资源编排和完整 Observability 平台;
|
||||||
|
- 未被既有 Demo/V1 明确批准的业务能力。
|
||||||
@ -1,5 +1,52 @@
|
|||||||
# V1 Tasks
|
# V1 Tasks
|
||||||
|
|
||||||
|
## Architecture provider migration
|
||||||
|
|
||||||
|
- [x] ARCH-001 Architecture decision, PRD, Scope and execution baseline
|
||||||
|
- [x] ARCH-002 BOS ObjectStorage adapter and runtime configuration
|
||||||
|
- [x] ARCH-003 OceanBase VectorStore adapter and index schema
|
||||||
|
- [x] ARCH-004 Qianfan `bce-reranker-base` alignment
|
||||||
|
- [x] ARCH-005 Runtime cutover, shadow rebuild workflow and legacy configuration cleanup
|
||||||
|
- [x] ARCH-006 OceanBase canonical cutover, controlled live integration and RAG acceptance
|
||||||
|
|
||||||
|
Dependencies:
|
||||||
|
|
||||||
|
```text
|
||||||
|
ARCH-001
|
||||||
|
→ ARCH-002
|
||||||
|
→ ARCH-003
|
||||||
|
→ ARCH-004
|
||||||
|
→ ARCH-005
|
||||||
|
→ ARCH-006
|
||||||
|
```
|
||||||
|
|
||||||
|
Each architecture task uses an independent branch and is merged into `main` only after its own
|
||||||
|
validation. Provider credentials and internal endpoints are runtime inputs and never task artifacts.
|
||||||
|
|
||||||
|
## Post-Demo priority plan
|
||||||
|
|
||||||
|
- [x] PLAN-001 MCP 与异步批量入库现状审计、任务拆分和契约基线
|
||||||
|
- [x] DEMO-012 MCP `search_knowledge` Gateway + identity + WorkBuddy verification
|
||||||
|
- [x] PLAN-002 公共知识库集群交付范围、任务拆分和一键部署基线
|
||||||
|
- [x] DEMO-015 Async ingestion Job + Celery/RabbitMQ + bounded retry + pre-retry cleanup
|
||||||
|
- [x] DEMO-016 Batch async upload API + per-file Job status + Admin Console
|
||||||
|
- [x] DEPLOY-001 Production images for API/Admin, MCP Gateway and Async Worker
|
||||||
|
- [x] DEPLOY-002 Kubernetes/Helm one-click deployment + Secret/Migration/Ingress baseline
|
||||||
|
- [ ] DEPLOY-003 Availability, rolling upgrade and remote WorkBuddy acceptance
|
||||||
|
|
||||||
|
Dependencies and acceptance boundaries are defined in
|
||||||
|
[`ASYNC_INGESTION_MCP_PLAN.md`](ASYNC_INGESTION_MCP_PLAN.md) and
|
||||||
|
[`PUBLIC_KNOWLEDGE_CLUSTER_PLAN.md`](PUBLIC_KNOWLEDGE_CLUSTER_PLAN.md). These Demo/Deploy tasks are
|
||||||
|
vertical slices of the V1 backlog below; completing a slice does not automatically complete its
|
||||||
|
corresponding `EKP-*` task. The current public-knowledge phase defers, but does not complete or
|
||||||
|
supersede, the unchecked user/ACL tasks.
|
||||||
|
|
||||||
|
```text
|
||||||
|
DEMO-009 + ARCH-006 → DEMO-012
|
||||||
|
PLAN-002
|
||||||
|
└→ DEMO-003~006 → DEMO-015 → DEMO-016 → DEPLOY-001 → DEPLOY-002 → DEPLOY-003
|
||||||
|
```
|
||||||
|
|
||||||
## Phase 0
|
## Phase 0
|
||||||
- [ ] EKP-001 Repository bootstrap
|
- [ ] EKP-001 Repository bootstrap
|
||||||
- [ ] EKP-002 Backend Python toolchain
|
- [ ] EKP-002 Backend Python toolchain
|
||||||
@ -11,14 +58,14 @@
|
|||||||
|
|
||||||
## Canonical domain / ingestion
|
## Canonical domain / ingestion
|
||||||
- [ ] EKP-010 Core domain package
|
- [ ] EKP-010 Core domain package
|
||||||
- [ ] EKP-011 PostgreSQL schema and migrations
|
- [ ] EKP-011 OceanBase MySQL schema and migrations
|
||||||
- [ ] EKP-012 User/group/role model
|
- [ ] EKP-012 User/group/role model
|
||||||
- [ ] EKP-013 Knowledge Base model
|
- [ ] EKP-013 Knowledge Base model
|
||||||
- [ ] EKP-014 Folder model
|
- [ ] EKP-014 Folder model
|
||||||
- [ ] EKP-015 Document + DocumentVersion model
|
- [ ] EKP-015 Document + DocumentVersion model
|
||||||
- [ ] EKP-016 ACL model
|
- [ ] EKP-016 ACL model
|
||||||
- [ ] EKP-017 Job + audit models
|
- [ ] EKP-017 Job + audit models
|
||||||
- [ ] EKP-020 ObjectStorage abstraction + S3 adapter
|
- [ ] EKP-020 ObjectStorage abstraction + BOS adapter
|
||||||
- [ ] EKP-021 Document upload API
|
- [ ] EKP-021 Document upload API
|
||||||
- [ ] EKP-022 Source hash/dedup/version activation
|
- [ ] EKP-022 Source hash/dedup/version activation
|
||||||
- [ ] EKP-023 Async job framework
|
- [ ] EKP-023 Async job framework
|
||||||
@ -35,13 +82,13 @@
|
|||||||
- [ ] EKP-042 Qwen embedding adapter
|
- [ ] EKP-042 Qwen embedding adapter
|
||||||
- [ ] EKP-043 IndexProfile model
|
- [ ] EKP-043 IndexProfile model
|
||||||
- [ ] EKP-044 VectorStore + fake
|
- [ ] EKP-044 VectorStore + fake
|
||||||
- [ ] EKP-045 Mochow adapter
|
- [ ] EKP-045 OceanBase Vector adapter
|
||||||
- [ ] EKP-046 Index build + shadow index lifecycle
|
- [ ] EKP-046 Index build + shadow index lifecycle
|
||||||
- [ ] EKP-050 Permission resolver + ACL retrieval filters
|
- [ ] EKP-050 Permission resolver + ACL retrieval filters
|
||||||
- [ ] EKP-051 Dense retrieval
|
- [ ] EKP-051 Dense retrieval
|
||||||
- [ ] EKP-052 BM25/sparse retrieval
|
- [ ] EKP-052 BM25/sparse retrieval
|
||||||
- [ ] EKP-053 Hybrid fusion
|
- [ ] EKP-053 Hybrid fusion
|
||||||
- [ ] EKP-054 RerankerProvider + fake + Qwen adapter
|
- [ ] EKP-054 RerankerProvider + fake + Qianfan BCE adapter
|
||||||
- [ ] EKP-055 Source enrichment + Search service/API
|
- [ ] EKP-055 Source enrichment + Search service/API
|
||||||
- [ ] EKP-056 Retrieval diagnostics / Search Lab backend
|
- [ ] EKP-056 Retrieval diagnostics / Search Lab backend
|
||||||
|
|
||||||
|
|||||||
@ -3,9 +3,32 @@
|
|||||||
1. Repository / developer harness
|
1. Repository / developer harness
|
||||||
2. Canonical domain and ACL
|
2. Canonical domain and ACL
|
||||||
3. Document ingestion and parsing
|
3. Document ingestion and parsing
|
||||||
4. Embedding / Index Profile / Mochow adapter
|
4. Embedding / Index Profile / OceanBase Vector adapter
|
||||||
5. ACL-aware hybrid retrieval + reranker
|
5. ACL-aware hybrid retrieval + reranker
|
||||||
6. MCP gateway and WorkBuddy verification
|
6. MCP gateway and WorkBuddy verification
|
||||||
7. Admin Console
|
7. Admin Console
|
||||||
8. Knowledge Compilation
|
8. Knowledge Compilation
|
||||||
9. Evaluation, security and release hardening
|
9. Evaluation, security and release hardening
|
||||||
|
|
||||||
|
## 当前增量实施顺序(2026-09-02)
|
||||||
|
|
||||||
|
`DEMO-012` 已完成 MCP `search_knowledge` 与本地 WorkBuddy 验收。根据 PLAN-002,当前阶段按
|
||||||
|
“单一公共知识可见域、企业受控网络、持续可用和一键集群部署”推进,不提前实现用户/权限隔离:
|
||||||
|
|
||||||
|
1. `PLAN-002`:确定公共知识库交付边界、目标拓扑、Task 拆分与验收门槛;
|
||||||
|
2. `DEMO-015`(已完成):持久化异步 Job、Celery/RabbitMQ 执行器、自动重试和重试前清理;
|
||||||
|
3. `DEMO-016`(已完成):批量异步上传、批次/Job 查询和 Admin Console 状态展示;
|
||||||
|
4. `DEPLOY-001`(已完成):API/Admin、MCP Gateway、Async Worker 生产镜像;
|
||||||
|
5. `DEPLOY-002`(已完成):Kubernetes/Helm、Secret、Migration、Ingress/TLS 和一键部署入口;
|
||||||
|
6. `DEPLOY-003`:多副本可用性、滚动升级、Worker 恢复和远程 WorkBuddy 验收。
|
||||||
|
|
||||||
|
严格依赖为:
|
||||||
|
|
||||||
|
```text
|
||||||
|
PLAN-002 → DEMO-015 → DEMO-016 → DEPLOY-001 → DEPLOY-002 → DEPLOY-003
|
||||||
|
```
|
||||||
|
|
||||||
|
`DEMO-016` 不得在缺少持久化状态、幂等消费和恢复边界时直接并发调用同步流水线;集群交付也
|
||||||
|
不得继续使用 `docker-compose.dev.yml` 中已被 ADR-012 取代的 PostgreSQL/MinIO 假设。详细
|
||||||
|
范围和验收标准见 [`ASYNC_INGESTION_MCP_PLAN.md`](ASYNC_INGESTION_MCP_PLAN.md) 与
|
||||||
|
[`PUBLIC_KNOWLEDGE_CLUSTER_PLAN.md`](PUBLIC_KNOWLEDGE_CLUSTER_PLAN.md)。
|
||||||
|
|||||||
@ -10,6 +10,21 @@
|
|||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
|
## 当前阶段交付覆盖(PLAN-002)
|
||||||
|
|
||||||
|
为优先达成服务可用和一键集群部署,当前阶段只交付企业受控网络内的单一公共知识库:所有
|
||||||
|
已接收知识处于同一个可见域,用户打开 Admin Console 或连接 WorkBuddy 即可使用,无需感知
|
||||||
|
后端进程启停。用户、用户组、角色、知识库多空间、Document ACL、LDAP/SSO 和 OAuth 用户
|
||||||
|
委托仍是本 PRD 的后续 V1 要求,本阶段不实现,也不得标记为完成。
|
||||||
|
|
||||||
|
该阶段覆盖不会取代安全原则:公共知识库不等于匿名公网开放;Admin 写操作必须位于受控网络
|
||||||
|
或统一接入网关后,MCP 使用运行时注入的最小 `knowledge:search` 服务凭据。一旦平台需要接收
|
||||||
|
非公共知识,必须先完成 Permission Resolver 与检索前/检索中 ACL 下推,禁止全局 Top-K 后置
|
||||||
|
过滤。实施顺序和集群验收边界见
|
||||||
|
[`PUBLIC_KNOWLEDGE_CLUSTER_PLAN.md`](../plans/PUBLIC_KNOWLEDGE_CLUSTER_PLAN.md)。
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
# 1. 项目背景
|
# 1. 项目背景
|
||||||
|
|
||||||
企业内部存在大量分散于 PDF、Word、PPT、Excel、Markdown、图片、扫描件及其他文件中的知识资产。
|
企业内部存在大量分散于 PDF、Word、PPT、Excel、Markdown、图片、扫描件及其他文件中的知识资产。
|
||||||
@ -112,7 +127,8 @@ Admin Console 是官方管理客户端,主要解决:
|
|||||||
|
|
||||||
WorkBuddy、Cursor、Claude Code 或其他 Agent 不应该知道:
|
WorkBuddy、Cursor、Claude Code 或其他 Agent 不应该知道:
|
||||||
|
|
||||||
- Mochow;
|
- OceanBase Vector;
|
||||||
|
- BOS;
|
||||||
- Qdrant;
|
- Qdrant;
|
||||||
- Milvus;
|
- Milvus;
|
||||||
- bge-large-zh;
|
- bge-large-zh;
|
||||||
@ -446,6 +462,19 @@ V1 必须支持:
|
|||||||
|
|
||||||
Document processing 必须异步执行。
|
Document processing 必须异步执行。
|
||||||
|
|
||||||
|
源文件与 Document/Job 成功持久化后返回 `202 Accepted`。批量上传中的每个文件拥有独立的
|
||||||
|
Document ID 和 Ingestion Job;单个文件处理失败不得阻塞或回滚批次内其他已接收文件。
|
||||||
|
|
||||||
|
当前 Demo 已实现多文件 `202` 接收、批次聚合查询和逐文件 Job 状态;目录上传、压缩包解包、
|
||||||
|
断点续传以及完整 V1 的 Hash/Version/ACL 能力仍按各自 Task 后置。
|
||||||
|
|
||||||
|
公共知识库集群交付使用分别可扩缩的 API/Admin、MCP Gateway 和 Async Worker 生产镜像;
|
||||||
|
镜像只包含锁定代码与非敏感默认值,环境凭据必须在部署时注入。
|
||||||
|
|
||||||
|
集群编排由 Helm Chart 统一管理:API/MCP 默认多副本,Worker 与单实例恢复调度器分离;敏感值
|
||||||
|
只引用预建 Kubernetes Secret,数据库 Migration 在安装/升级前串行完成。Ingress/TLS、探针、
|
||||||
|
资源限制和 Pod 安全默认值属于部署契约,不改变 REST/MCP 业务接口。
|
||||||
|
|
||||||
**REQ-INGEST-004**
|
**REQ-INGEST-004**
|
||||||
|
|
||||||
展示处理阶段:
|
展示处理阶段:
|
||||||
@ -467,10 +496,18 @@ Failed
|
|||||||
失败必须展示:
|
失败必须展示:
|
||||||
|
|
||||||
- Failed Stage;
|
- Failed Stage;
|
||||||
|
- Attempt Count;
|
||||||
|
- 是否仍会自动重试;
|
||||||
- Error Code;
|
- Error Code;
|
||||||
- Error Message;
|
- Error Message;
|
||||||
- Retry。
|
- Retry。
|
||||||
|
|
||||||
|
**REQ-INGEST-006**
|
||||||
|
|
||||||
|
可重试的瞬时执行失败必须自动进行有界重试。每次重新执行前必须清理该 Document 的旧解析
|
||||||
|
产物、Chunk、Chunk Index State 和向量/稀疏索引数据,再从不可变原始文件开始处理。清理不得
|
||||||
|
删除原始文件、Document 标识或 Job/Attempt 历史;清理失败时不得继续重试。
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
# 11. 文件版本和去重
|
# 11. 文件版本和去重
|
||||||
@ -800,21 +837,24 @@ VectorStoreProvider
|
|||||||
|
|
||||||
抽象。
|
抽象。
|
||||||
|
|
||||||
默认首选:
|
当前默认:
|
||||||
|
|
||||||
```text
|
```text
|
||||||
Mochow
|
OceanBase Vector
|
||||||
```
|
```
|
||||||
|
|
||||||
原因:
|
原因:
|
||||||
|
|
||||||
公司已有使用经验,同时 Mochow 当前官方能力已经支持向量检索、BM25、Hybrid Search 和标量过滤。
|
公司现有知识平台主检索链已经使用 OceanBase Vector,并可与既有 OceanBase 运维体系复用。
|
||||||
|
V1 通过标量条件下推满足 Profile 与 ACL 过滤;BM25/Hybrid 能力必须继续通过统一检索契约
|
||||||
|
演进,不把特定数据库查询结构写入业务层。
|
||||||
|
|
||||||
但系统不得使用 Mochow-specific data model 污染业务 Domain Model。
|
系统不得使用 OceanBase/`pyobvector`-specific data model 污染业务 Domain Model。
|
||||||
|
|
||||||
未来可以新增:
|
未来可以新增:
|
||||||
|
|
||||||
```text
|
```text
|
||||||
|
Mochow
|
||||||
Qdrant
|
Qdrant
|
||||||
Milvus
|
Milvus
|
||||||
pgvector
|
pgvector
|
||||||
@ -913,15 +953,13 @@ Provider:
|
|||||||
RerankerProvider
|
RerankerProvider
|
||||||
```
|
```
|
||||||
|
|
||||||
初始候选:
|
当前基线与后续候选:
|
||||||
|
|
||||||
- Qwen3-Reranker-0.6B;
|
- bce-reranker-base(当前基线);
|
||||||
- Qwen3-Reranker-4B;
|
- Qwen3-Reranker 系列(后续 Benchmark 候选);
|
||||||
- Qwen3-Reranker-8B;
|
|
||||||
- bce-reranker-base;
|
|
||||||
- Local Reranker。
|
- Local Reranker。
|
||||||
|
|
||||||
千帆当前已经提供 Qwen3 Reranker 系列,最大输入支持 32K。
|
模型与参数由版本化 Index/Profile 配置记录,业务 API 不暴露模型名。
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
@ -1812,6 +1850,9 @@ Cancelled
|
|||||||
Retry
|
Retry
|
||||||
```
|
```
|
||||||
|
|
||||||
|
Retry 必须记录 Attempt 历史并区分可重试的瞬时错误与确定性永久错误;重试执行前遵循
|
||||||
|
`REQ-INGEST-006` 的派生数据清理规则。
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
# 57. 删除语义
|
# 57. 删除语义
|
||||||
@ -1871,15 +1912,15 @@ Embedding × Retrieval × Reranker
|
|||||||
|
|
||||||
```text
|
```text
|
||||||
bge-large-zh
|
bge-large-zh
|
||||||
+ Mochow Dense
|
+ OceanBase Vector Dense
|
||||||
+ No Reranker
|
+ No Reranker
|
||||||
|
|
||||||
bge-large-zh
|
bge-large-zh
|
||||||
+ Mochow Hybrid
|
+ OceanBase Vector Dense
|
||||||
+ Qwen3 Reranker
|
+ bce-reranker-base
|
||||||
|
|
||||||
Qwen3 Embedding
|
Qwen3 Embedding
|
||||||
+ Mochow Hybrid
|
+ OceanBase Vector Hybrid(能力验证后)
|
||||||
+ Qwen3 Reranker
|
+ Qwen3 Reranker
|
||||||
```
|
```
|
||||||
|
|
||||||
@ -2168,7 +2209,7 @@ MCP Error Rate
|
|||||||
10. Document Version;
|
10. Document Version;
|
||||||
11. Hash 去重;
|
11. Hash 去重;
|
||||||
12. Embedding Provider;
|
12. Embedding Provider;
|
||||||
13. Mochow Provider;
|
13. OceanBase Vector Provider;
|
||||||
14. Dense Search;
|
14. Dense Search;
|
||||||
15. BM25;
|
15. BM25;
|
||||||
16. Hybrid Retrieval;
|
16. Hybrid Retrieval;
|
||||||
@ -2291,9 +2332,9 @@ Embedding 和 Reranker 可以通过 Provider 配置替换。
|
|||||||
|
|
||||||
## Vector DB
|
## Vector DB
|
||||||
|
|
||||||
Mochow 可以作为默认生产 Vector Store。
|
OceanBase Vector 作为当前默认生产 Vector Store。
|
||||||
|
|
||||||
业务代码不存在必须依赖 Mochow SDK 数据结构的 API Contract。
|
业务代码不存在必须依赖 OceanBase/`pyobvector` SDK 数据结构的 API Contract。
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
@ -2338,11 +2379,14 @@ get_source
|
|||||||
|
|
||||||
# 73. 当前技术基线
|
# 73. 当前技术基线
|
||||||
|
|
||||||
PRD 阶段暂定但不最终锁定:
|
当前 V1 架构基线:
|
||||||
|
|
||||||
```text
|
```text
|
||||||
Primary Vector Store:
|
Primary Vector Store:
|
||||||
Mochow
|
OceanBase Vector
|
||||||
|
|
||||||
|
Primary Object Storage:
|
||||||
|
Baidu BOS
|
||||||
|
|
||||||
Embedding Baseline:
|
Embedding Baseline:
|
||||||
bge-large-zh
|
bge-large-zh
|
||||||
@ -2351,7 +2395,7 @@ Embedding Benchmark:
|
|||||||
Qwen3 Embedding
|
Qwen3 Embedding
|
||||||
|
|
||||||
Reranker:
|
Reranker:
|
||||||
Qwen3 Reranker / equivalent
|
bce-reranker-base
|
||||||
|
|
||||||
Document Parsing:
|
Document Parsing:
|
||||||
Benchmark MinerU / Docling
|
Benchmark MinerU / Docling
|
||||||
@ -2369,6 +2413,10 @@ Backend:
|
|||||||
API First
|
API First
|
||||||
```
|
```
|
||||||
|
|
||||||
|
运行时切换约束:索引 Provider 或 Profile 变更必须以新 Profile 构建 Shadow Index;Profile
|
||||||
|
生命周期和激活状态保存在 OceanBase canonical 关系表。只有 canonical Chunk 与 ChunkIndexState 完整核对
|
||||||
|
通过后才能原子激活,禁止复用旧 Profile ID 或原地覆盖当前 Active Index。
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
# 74. 已确认产品决策
|
# 74. 已确认产品决策
|
||||||
@ -2387,7 +2435,8 @@ API First + Admin Console
|
|||||||
|
|
||||||
**DEC-002**
|
**DEC-002**
|
||||||
|
|
||||||
Mochow 继续作为生产首选候选,同时必须保留 Vector Store 替换能力。
|
OceanBase Vector 作为当前生产主检索索引,同时必须保留 Vector Store 替换能力。
|
||||||
|
OceanBase 关系表保存 canonical 数据,同库 Vector Store 表只保存可重建索引。
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
@ -2425,7 +2474,7 @@ RAG + Knowledge Compilation
|
|||||||
|
|
||||||
**DEC-007**
|
**DEC-007**
|
||||||
|
|
||||||
MCP 只能暴露业务能力,禁止泄露 Mochow、Embedding Model 等底层实现。
|
MCP 只能暴露业务能力,禁止泄露 OceanBase Vector、BOS、Embedding Model 等底层实现。
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
@ -2443,13 +2492,13 @@ PRD 评审通过后进入《总体技术架构设计》。
|
|||||||
|
|
||||||
1. Backend 技术栈;
|
1. Backend 技术栈;
|
||||||
2. Admin Console 技术栈;
|
2. Admin Console 技术栈;
|
||||||
3. PostgreSQL 数据模型;
|
3. OceanBase MySQL canonical 数据模型;
|
||||||
4. Object Storage;
|
4. Object Storage;
|
||||||
5. Task Queue;
|
5. Task Queue;
|
||||||
6. MinerU vs Docling;
|
6. MinerU vs Docling;
|
||||||
7. OCR Pipeline;
|
7. OCR Pipeline;
|
||||||
8. Chunk Strategy;
|
8. Chunk Strategy;
|
||||||
9. Mochow Schema;
|
9. OceanBase Vector Index Schema;
|
||||||
10. VectorStore Adapter;
|
10. VectorStore Adapter;
|
||||||
11. Embedding Provider;
|
11. Embedding Provider;
|
||||||
12. Reranker Serving;
|
12. Reranker Serving;
|
||||||
@ -2473,7 +2522,9 @@ PRD 评审通过后进入《总体技术架构设计》。
|
|||||||
```text
|
```text
|
||||||
Qianfan
|
Qianfan
|
||||||
+
|
+
|
||||||
Mochow
|
BOS
|
||||||
|
+
|
||||||
|
OceanBase Vector
|
||||||
+
|
+
|
||||||
Local Parsing
|
Local Parsing
|
||||||
```
|
```
|
||||||
@ -2511,7 +2562,9 @@ Replaceable Embedding
|
|||||||
+
|
+
|
||||||
Replaceable Reranker
|
Replaceable Reranker
|
||||||
+
|
+
|
||||||
Mochow Primary
|
BOS Object Storage
|
||||||
|
+
|
||||||
|
OceanBase Vector Primary
|
||||||
+
|
+
|
||||||
Cloud LLM
|
Cloud LLM
|
||||||
```
|
```
|
||||||
@ -2524,9 +2577,12 @@ Cloud LLM
|
|||||||
|
|
||||||
# 77. 调研依据备注
|
# 77. 调研依据备注
|
||||||
|
|
||||||
当前百度千帆模型列表表明 bge-large-zh 与 Qwen3 Embedding、Qwen3 Reranker 可以作为同平台 Benchmark 候选,因此早期无需为模型实验同时引入多个云厂商。
|
内部最新配置基线确认现有知识平台主链使用千帆 `bge-large-zh`、`bce-reranker-base`、
|
||||||
|
BOS 与 OceanBase MySQL 模式;OceanBase 同时承载 canonical 关系表和逻辑隔离的可重建
|
||||||
|
向量表。连接地址、账号和密钥仍只能作为运行时 Secret,不进入产品文档或提交记录。
|
||||||
|
|
||||||
Mochow 当前官方 API 已支持向量检索与 BM25 全文检索组合的 Hybrid Search,同时可施加标量 Filter,与企业知识库的 Dense + Sparse + ACL Retrieval 需求相匹配。
|
OceanBase Vector 的生产接入必须验证向量检索、标量 Filter、索引参数和版本兼容性;
|
||||||
|
BM25/Hybrid 在相应任务验收前仍按产品能力要求保留,不在本轮 Provider 迁移中提前扩展。
|
||||||
|
|
||||||
WorkBuddy Enterprise 当前公开 MCP 文档支持自定义 MCP Server,并提供传输、权限、安全、远程 API Server 以及 iWiki 配置示例,因此“企业知识服务 → MCP → WorkBuddy”的产品路线具备正式产品支持基础。
|
WorkBuddy Enterprise 当前公开 MCP 文档支持自定义 MCP Server,并提供传输、权限、安全、远程 API Server 以及 iWiki 配置示例,因此“企业知识服务 → MCP → WorkBuddy”的产品路线具备正式产品支持基础。
|
||||||
|
|
||||||
@ -2534,4 +2590,4 @@ Karpathy 的 LLM Wiki 原始思路强调 immutable raw sources 与由 LLM 持续
|
|||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
**PRD v0.1 END**
|
**PRD v0.1 END**
|
||||||
|
|||||||
@ -1,3 +1,6 @@
|
|||||||
# Test Documents
|
# Test Documents
|
||||||
|
|
||||||
Only commit synthetic, public, or properly anonymized fixtures. Never commit real internal enterprise documents, HR data, customer data, contracts, credentials, or confidential material.
|
Only commit synthetic, public, or properly anonymized fixtures. Never commit real internal enterprise documents, HR data, customer data, contracts, credentials, or confidential material.
|
||||||
|
|
||||||
|
The `sample-*.md` files are entirely synthetic and provide stable parser/chunker
|
||||||
|
inputs for the Golden Dataset regression checks.
|
||||||
|
|||||||
11
fixtures/documents/sample-product-a-policy.md
Normal file
11
fixtures/documents/sample-product-a-policy.md
Normal file
@ -0,0 +1,11 @@
|
|||||||
|
# 示例产品 A 服务政策
|
||||||
|
|
||||||
|
> 本文件完全由测试内容构造,不描述任何真实产品或客户承诺。
|
||||||
|
|
||||||
|
## 退款
|
||||||
|
|
||||||
|
示例产品 A 的测试退款周期为提交申请后的七个自然日。本规则只用于检索效果验证。
|
||||||
|
|
||||||
|
## 技术支持
|
||||||
|
|
||||||
|
测试用户可以通过虚构的支持入口提交问题,本文不包含真实联系方式。
|
||||||
12
fixtures/documents/sample-travel-policy.md
Normal file
12
fixtures/documents/sample-travel-policy.md
Normal file
@ -0,0 +1,12 @@
|
|||||||
|
# 示例出差管理制度
|
||||||
|
|
||||||
|
> 本文件完全由测试内容构造,不代表任何真实企业政策。
|
||||||
|
|
||||||
|
## 住宿标准
|
||||||
|
|
||||||
|
员工出差时应在批准的预算范围内选择安全、交通便利的住宿。示例标准仅用于验证知识检索,
|
||||||
|
不作为实际报销依据。
|
||||||
|
|
||||||
|
## 报销材料
|
||||||
|
|
||||||
|
提交报销时需附上示例行程单和合规票据。本段内容同样只用于自动化测试。
|
||||||
@ -1,2 +1,32 @@
|
|||||||
{"id":"sample-001","query":"员工出差住宿标准是多少?","expected_documents":["sample-travel-policy.md"],"expected_sections":["住宿标准"],"tags":["制度","自然语言"]}
|
{"id":"refund-001","query":"产品A退款周期","expected_documents":["sample-product-a-policy.md"],"expected_sections":["退款"],"tags":["产品","短查询"]}
|
||||||
{"id":"sample-002","query":"产品A退款周期","expected_documents":["sample-product-a-policy.md"],"expected_sections":["退款"],"tags":["产品","短查询"]}
|
{"id":"refund-002","query":"示例产品 A 的退款需要几个自然日?","expected_documents":["sample-product-a-policy.md"],"expected_sections":["退款"],"tags":["产品","自然语言"]}
|
||||||
|
{"id":"refund-003","query":"申请产品A退款后多久完成?","expected_documents":["sample-product-a-policy.md"],"expected_sections":["退款"],"tags":["产品","同义表达"]}
|
||||||
|
{"id":"refund-004","query":"退款规则里写的处理周期是什么?","expected_documents":["sample-product-a-policy.md"],"expected_sections":["退款"],"tags":["产品","规则"]}
|
||||||
|
{"id":"refund-005","query":"产品 A 提交退款申请后的等待时间","expected_documents":["sample-product-a-policy.md"],"expected_sections":["退款"],"tags":["产品","长查询"]}
|
||||||
|
{"id":"refund-006","query":"退款是否按自然日计算?","expected_documents":["sample-product-a-policy.md"],"expected_sections":["退款"],"tags":["产品","精确术语"]}
|
||||||
|
{"id":"refund-007","query":"请查找产品A有关退款时限的说明","expected_documents":["sample-product-a-policy.md"],"expected_sections":["退款"],"tags":["产品","指令式"]}
|
||||||
|
{"id":"refund-008","query":"示例服务政策中的退款周期是多少天?","expected_documents":["sample-product-a-policy.md"],"expected_sections":["退款"],"tags":["产品","文档定位"]}
|
||||||
|
{"id":"support-001","query":"产品A遇到问题如何联系技术支持?","expected_documents":["sample-product-a-policy.md"],"expected_sections":["技术支持"],"tags":["支持","自然语言"]}
|
||||||
|
{"id":"support-002","query":"测试用户在哪里提交支持问题?","expected_documents":["sample-product-a-policy.md"],"expected_sections":["技术支持"],"tags":["支持","入口"]}
|
||||||
|
{"id":"support-003","query":"技术支持入口是否提供真实联系方式?","expected_documents":["sample-product-a-policy.md"],"expected_sections":["技术支持"],"tags":["支持","边界"]}
|
||||||
|
{"id":"support-004","query":"我想提交产品问题,支持说明是什么?","expected_documents":["sample-product-a-policy.md"],"expected_sections":["技术支持"],"tags":["支持","同义表达"]}
|
||||||
|
{"id":"support-005","query":"测试用户的虚构支持入口","expected_documents":["sample-product-a-policy.md"],"expected_sections":["技术支持"],"tags":["支持","短查询"]}
|
||||||
|
{"id":"support-006","query":"支持入口能否用于测试用户反馈问题?","expected_documents":["sample-product-a-policy.md"],"expected_sections":["技术支持"],"tags":["支持","多条件"]}
|
||||||
|
{"id":"support-007","query":"请定位服务政策里关于提交问题的段落","expected_documents":["sample-product-a-policy.md"],"expected_sections":["技术支持"],"tags":["支持","文档定位"]}
|
||||||
|
{"id":"support-008","query":"产品技术支持部分有没有联系电话?","expected_documents":["sample-product-a-policy.md"],"expected_sections":["技术支持"],"tags":["支持","否定事实"]}
|
||||||
|
{"id":"lodging-001","query":"员工出差住宿标准是多少?","expected_documents":["sample-travel-policy.md"],"expected_sections":["住宿标准"],"tags":["制度","自然语言"]}
|
||||||
|
{"id":"lodging-002","query":"出差时应如何选择住宿?","expected_documents":["sample-travel-policy.md"],"expected_sections":["住宿标准"],"tags":["制度","行为"]}
|
||||||
|
{"id":"lodging-003","query":"住宿地点需要满足哪些安全和交通条件?","expected_documents":["sample-travel-policy.md"],"expected_sections":["住宿标准"],"tags":["制度","多条件"]}
|
||||||
|
{"id":"lodging-004","query":"出差住宿是否必须在批准预算内?","expected_documents":["sample-travel-policy.md"],"expected_sections":["住宿标准"],"tags":["制度","预算"]}
|
||||||
|
{"id":"lodging-005","query":"批准预算范围内的安全住宿","expected_documents":["sample-travel-policy.md"],"expected_sections":["住宿标准"],"tags":["制度","短查询"]}
|
||||||
|
{"id":"lodging-006","query":"员工出差选酒店时要考虑什么?","expected_documents":["sample-travel-policy.md"],"expected_sections":["住宿标准"],"tags":["制度","同义表达"]}
|
||||||
|
{"id":"lodging-007","query":"请找出关于安全且交通便利住宿的说明","expected_documents":["sample-travel-policy.md"],"expected_sections":["住宿标准"],"tags":["制度","文档定位"]}
|
||||||
|
{"id":"lodging-008","query":"出差住宿示例标准能否作为实际报销依据?","expected_documents":["sample-travel-policy.md"],"expected_sections":["住宿标准"],"tags":["制度","边界"]}
|
||||||
|
{"id":"expense-001","query":"出差报销需要提交哪些材料?","expected_documents":["sample-travel-policy.md"],"expected_sections":["报销材料"],"tags":["报销","自然语言"]}
|
||||||
|
{"id":"expense-002","query":"报销时是否需要行程单?","expected_documents":["sample-travel-policy.md"],"expected_sections":["报销材料"],"tags":["报销","材料"]}
|
||||||
|
{"id":"expense-003","query":"合规票据是不是报销必需材料?","expected_documents":["sample-travel-policy.md"],"expected_sections":["报销材料"],"tags":["报销","票据"]}
|
||||||
|
{"id":"expense-004","query":"提交报销要附上什么凭证?","expected_documents":["sample-travel-policy.md"],"expected_sections":["报销材料"],"tags":["报销","同义表达"]}
|
||||||
|
{"id":"expense-005","query":"提交报销的示例行程单和合规票据","expected_documents":["sample-travel-policy.md"],"expected_sections":["报销材料"],"tags":["报销","短查询"]}
|
||||||
|
{"id":"expense-006","query":"行程单和票据应在什么时候提交?","expected_documents":["sample-travel-policy.md"],"expected_sections":["报销材料"],"tags":["报销","多条件"]}
|
||||||
|
{"id":"expense-007","query":"请定位管理制度中有关合规票据的段落","expected_documents":["sample-travel-policy.md"],"expected_sections":["报销材料"],"tags":["报销","文档定位"]}
|
||||||
|
{"id":"expense-008","query":"报销材料段落是否只用于自动化测试?","expected_documents":["sample-travel-policy.md"],"expected_sections":["报销材料"],"tags":["报销","边界"]}
|
||||||
|
|||||||
@ -1,6 +1,27 @@
|
|||||||
# Database migrations
|
# Database migrations
|
||||||
|
|
||||||
Migrations are forward-only and operate on PostgreSQL canonical metadata. Run
|
Migrations are forward-only and operate on OceanBase MySQL canonical metadata. Run
|
||||||
`make migration-check` to render the full upgrade path without changing a
|
`make migration-check` to render the full upgrade path without changing a
|
||||||
database. Integration tests also apply the migration to a temporary database
|
database. Integration tests also apply the migration to a temporary database
|
||||||
and compare it with SQLAlchemy metadata.
|
and compare it with SQLAlchemy metadata.
|
||||||
|
|
||||||
|
The target OceanBase database is shared. Alembic autogenerate and drift checks
|
||||||
|
are intentionally scoped to tables present in EKP `Base.metadata`; unrelated
|
||||||
|
application tables and the separately provisioned vector-index table are never
|
||||||
|
reflected or treated as migration targets.
|
||||||
|
|
||||||
|
OceanBase adds `BLOCK_SIZE ... LOCAL` syntax that SQLAlchemy's stock MySQL
|
||||||
|
reflector does not recognize for secondary indexes, unique constraints, and
|
||||||
|
foreign keys. Real OceanBase drift checks therefore compare columns, types,
|
||||||
|
defaults, and primary keys while excluding those three falsely reported object
|
||||||
|
types. Full constraint comparison still runs against SQLite in integration
|
||||||
|
tests; target-cluster checks separately verify HNSW/PRE-FILTER and the
|
||||||
|
single-active generated unique index.
|
||||||
|
|
||||||
|
DEMO-015 adds `ingestion_jobs` and `ingestion_attempts`. The former is the canonical execution state;
|
||||||
|
the latter preserves numbered history and uses a generated nullable active marker plus a unique index
|
||||||
|
to reject two running Attempts for the same Job on both OceanBase MySQL and SQLite.
|
||||||
|
|
||||||
|
DEMO-016 adds `ingestion_batches` and `ingestion_batch_items`. They retain input order and per-file
|
||||||
|
acceptance references while Job rows remain the processing state source of truth; a batch is an aggregate,
|
||||||
|
not a cross-file transaction.
|
||||||
|
|||||||
@ -1,6 +1,7 @@
|
|||||||
"""Alembic environment for canonical document metadata migrations."""
|
"""Alembic environment for canonical document metadata migrations."""
|
||||||
|
|
||||||
from logging.config import fileConfig
|
from logging.config import fileConfig
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
from alembic import context
|
from alembic import context
|
||||||
from sqlalchemy import engine_from_config, pool
|
from sqlalchemy import engine_from_config, pool
|
||||||
@ -13,12 +14,47 @@ if config.config_file_name is not None:
|
|||||||
fileConfig(config.config_file_name)
|
fileConfig(config.config_file_name)
|
||||||
|
|
||||||
target_metadata = Base.metadata
|
target_metadata = Base.metadata
|
||||||
|
TARGET_TABLE_NAMES = frozenset(target_metadata.tables)
|
||||||
|
|
||||||
|
|
||||||
|
def _include_name(
|
||||||
|
name: str | None,
|
||||||
|
type_: str,
|
||||||
|
parent_names: dict[str, str | None],
|
||||||
|
) -> bool:
|
||||||
|
"""Keep Alembic reflection inside EKP-owned canonical tables.
|
||||||
|
|
||||||
|
The target OceanBase database is shared with another application and
|
||||||
|
contains provider-specific VECTOR columns. Excluding unrelated tables
|
||||||
|
before reflection prevents schema checks from inspecting or proposing
|
||||||
|
changes to objects outside this repository's ownership boundary.
|
||||||
|
"""
|
||||||
|
del parent_names
|
||||||
|
return type_ != "table" or name in TARGET_TABLE_NAMES
|
||||||
|
|
||||||
|
|
||||||
|
def _include_oceanbase_object(
|
||||||
|
object_: Any,
|
||||||
|
name: str | None,
|
||||||
|
type_: str,
|
||||||
|
reflected: bool,
|
||||||
|
compare_to: Any,
|
||||||
|
) -> bool:
|
||||||
|
"""Skip constraints the stock MySQL reflector cannot parse from OceanBase.
|
||||||
|
|
||||||
|
OceanBase appends `BLOCK_SIZE ... LOCAL` to secondary constraints. SQLAlchemy
|
||||||
|
consequently reports model constraints as missing even when they exist.
|
||||||
|
Columns, types, defaults and primary keys remain in the real drift check;
|
||||||
|
constraints are covered by migration tests and targeted live invariants.
|
||||||
|
"""
|
||||||
|
del object_, name, reflected, compare_to
|
||||||
|
return type_ not in {"index", "unique_constraint", "foreign_key_constraint"}
|
||||||
|
|
||||||
|
|
||||||
def _database_url() -> str:
|
def _database_url() -> str:
|
||||||
"""Allow tests to override the URL without changing runtime settings."""
|
"""Allow tests to override the URL without changing runtime settings."""
|
||||||
override = config.attributes.get("database_url")
|
override = config.attributes.get("database_url")
|
||||||
return override if isinstance(override, str) else get_settings().database_url
|
return override if isinstance(override, str) else get_settings().database_url.get_secret_value()
|
||||||
|
|
||||||
|
|
||||||
def run_migrations_offline() -> None:
|
def run_migrations_offline() -> None:
|
||||||
@ -29,6 +65,7 @@ def run_migrations_offline() -> None:
|
|||||||
literal_binds=True,
|
literal_binds=True,
|
||||||
dialect_opts={"paramstyle": "named"},
|
dialect_opts={"paramstyle": "named"},
|
||||||
compare_type=True,
|
compare_type=True,
|
||||||
|
include_name=_include_name,
|
||||||
)
|
)
|
||||||
|
|
||||||
with context.begin_transaction():
|
with context.begin_transaction():
|
||||||
@ -48,6 +85,10 @@ def run_migrations_online() -> None:
|
|||||||
connection=connection,
|
connection=connection,
|
||||||
target_metadata=target_metadata,
|
target_metadata=target_metadata,
|
||||||
compare_type=True,
|
compare_type=True,
|
||||||
|
include_name=_include_name,
|
||||||
|
include_object=(
|
||||||
|
_include_oceanbase_object if connection.dialect.name == "mysql" else None
|
||||||
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
with context.begin_transaction():
|
with context.begin_transaction():
|
||||||
|
|||||||
29
migrations/versions/20260901_0002_add_parsed_storage_key.py
Normal file
29
migrations/versions/20260901_0002_add_parsed_storage_key.py
Normal file
@ -0,0 +1,29 @@
|
|||||||
|
"""Add parsed-result object reference to documents.
|
||||||
|
|
||||||
|
Revision ID: 20260901_0002
|
||||||
|
Revises: 20260901_0001
|
||||||
|
Create Date: 2026-09-01
|
||||||
|
"""
|
||||||
|
|
||||||
|
from collections.abc import Sequence
|
||||||
|
|
||||||
|
import sqlalchemy as sa
|
||||||
|
from alembic import op
|
||||||
|
|
||||||
|
revision: str = "20260901_0002"
|
||||||
|
down_revision: str | None = "20260901_0001"
|
||||||
|
branch_labels: str | Sequence[str] | None = None
|
||||||
|
depends_on: str | Sequence[str] | None = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
"""Reference the rebuildable parsed JSON object after parsing succeeds."""
|
||||||
|
op.add_column(
|
||||||
|
"documents",
|
||||||
|
sa.Column("parsed_storage_key", sa.String(length=512), nullable=True),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
"""Reject downgrade because project migrations are forward-only."""
|
||||||
|
raise NotImplementedError("Database migrations are forward-only")
|
||||||
73
migrations/versions/20260901_0003_create_chunks.py
Normal file
73
migrations/versions/20260901_0003_create_chunks.py
Normal file
@ -0,0 +1,73 @@
|
|||||||
|
"""Create canonical chunk metadata table.
|
||||||
|
|
||||||
|
Revision ID: 20260901_0003
|
||||||
|
Revises: 20260901_0002
|
||||||
|
Create Date: 2026-09-01
|
||||||
|
"""
|
||||||
|
|
||||||
|
from collections.abc import Sequence
|
||||||
|
|
||||||
|
import sqlalchemy as sa
|
||||||
|
from alembic import op
|
||||||
|
|
||||||
|
revision: str = "20260901_0003"
|
||||||
|
down_revision: str | None = "20260901_0002"
|
||||||
|
branch_labels: str | Sequence[str] | None = None
|
||||||
|
depends_on: str | Sequence[str] | None = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
"""Persist traceable chunks without introducing embedding/index fields."""
|
||||||
|
op.create_table(
|
||||||
|
"chunks",
|
||||||
|
sa.Column("chunk_id", sa.Uuid(), nullable=False),
|
||||||
|
sa.Column("document_id", sa.Uuid(), nullable=False),
|
||||||
|
sa.Column("filename", sa.String(length=255), nullable=False),
|
||||||
|
sa.Column("page", sa.Integer(), nullable=True),
|
||||||
|
sa.Column("page_end", sa.Integer(), nullable=True),
|
||||||
|
sa.Column("section", sa.Text(), nullable=True),
|
||||||
|
sa.Column("section_path", sa.JSON(), nullable=False),
|
||||||
|
sa.Column("chunk_index", sa.Integer(), nullable=False),
|
||||||
|
sa.Column("content", sa.Text(), nullable=False),
|
||||||
|
sa.Column("token_count", sa.Integer(), nullable=False),
|
||||||
|
sa.Column("strategy_version", sa.String(length=64), nullable=False),
|
||||||
|
sa.Column(
|
||||||
|
"created_at",
|
||||||
|
sa.DateTime(timezone=True),
|
||||||
|
server_default=sa.text("CURRENT_TIMESTAMP"),
|
||||||
|
nullable=False,
|
||||||
|
),
|
||||||
|
sa.CheckConstraint("length(filename) > 0", name="filename_not_empty"),
|
||||||
|
sa.CheckConstraint("length(content) > 0", name="content_not_empty"),
|
||||||
|
sa.CheckConstraint(
|
||||||
|
"length(strategy_version) > 0",
|
||||||
|
name="strategy_version_not_empty",
|
||||||
|
),
|
||||||
|
sa.CheckConstraint("chunk_index >= 0", name="chunk_index_non_negative"),
|
||||||
|
sa.CheckConstraint("token_count > 0", name="token_count_positive"),
|
||||||
|
sa.CheckConstraint("page IS NULL OR page > 0", name="page_positive"),
|
||||||
|
sa.CheckConstraint("page_end IS NULL OR page_end > 0", name="page_end_positive"),
|
||||||
|
sa.CheckConstraint(
|
||||||
|
"page IS NULL OR page_end IS NULL OR page_end >= page",
|
||||||
|
name="page_range_valid",
|
||||||
|
),
|
||||||
|
sa.ForeignKeyConstraint(
|
||||||
|
["document_id"],
|
||||||
|
["documents.id"],
|
||||||
|
name="fk_chunks_document_id_documents",
|
||||||
|
ondelete="CASCADE",
|
||||||
|
),
|
||||||
|
sa.PrimaryKeyConstraint("chunk_id", name="pk_chunks"),
|
||||||
|
sa.UniqueConstraint(
|
||||||
|
"document_id",
|
||||||
|
"strategy_version",
|
||||||
|
"chunk_index",
|
||||||
|
name="uq_chunks_document_strategy_index",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
op.create_index("ix_chunks_document_id", "chunks", ["document_id"], unique=False)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
"""Reject downgrade because project migrations are forward-only."""
|
||||||
|
raise NotImplementedError("Database migrations are forward-only")
|
||||||
@ -0,0 +1,58 @@
|
|||||||
|
"""Create versioned canonical chunk-index state.
|
||||||
|
|
||||||
|
Revision ID: 20260901_0004
|
||||||
|
Revises: 20260901_0003
|
||||||
|
Create Date: 2026-09-01
|
||||||
|
"""
|
||||||
|
|
||||||
|
from collections.abc import Sequence
|
||||||
|
|
||||||
|
import sqlalchemy as sa
|
||||||
|
from alembic import op
|
||||||
|
|
||||||
|
revision: str = "20260901_0004"
|
||||||
|
down_revision: str | None = "20260901_0003"
|
||||||
|
branch_labels: str | Sequence[str] | None = None
|
||||||
|
depends_on: str | Sequence[str] | None = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
"""Track successful index writes without storing canonical vectors in Mochow."""
|
||||||
|
op.create_table(
|
||||||
|
"chunk_index_states",
|
||||||
|
sa.Column("chunk_id", sa.Uuid(), nullable=False),
|
||||||
|
sa.Column("index_profile_id", sa.String(length=128), nullable=False),
|
||||||
|
sa.Column("embedding_model", sa.String(length=128), nullable=False),
|
||||||
|
sa.Column("vector_dimension", sa.Integer(), nullable=False),
|
||||||
|
sa.Column(
|
||||||
|
"indexed_at",
|
||||||
|
sa.DateTime(timezone=True),
|
||||||
|
server_default=sa.text("CURRENT_TIMESTAMP"),
|
||||||
|
nullable=False,
|
||||||
|
),
|
||||||
|
sa.CheckConstraint(
|
||||||
|
"length(index_profile_id) > 0",
|
||||||
|
name="profile_not_empty",
|
||||||
|
),
|
||||||
|
sa.CheckConstraint(
|
||||||
|
"length(embedding_model) > 0",
|
||||||
|
name="model_not_empty",
|
||||||
|
),
|
||||||
|
sa.CheckConstraint("vector_dimension > 0", name="dimension_positive"),
|
||||||
|
sa.ForeignKeyConstraint(
|
||||||
|
["chunk_id"],
|
||||||
|
["chunks.chunk_id"],
|
||||||
|
name="fk_chunk_index_states_chunk_id_chunks",
|
||||||
|
ondelete="CASCADE",
|
||||||
|
),
|
||||||
|
sa.PrimaryKeyConstraint(
|
||||||
|
"chunk_id",
|
||||||
|
"index_profile_id",
|
||||||
|
name="pk_chunk_index_states",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
"""Reject downgrade because project migrations are forward-only."""
|
||||||
|
raise NotImplementedError("Database migrations are forward-only")
|
||||||
77
migrations/versions/20260902_0005_create_index_profiles.py
Normal file
77
migrations/versions/20260902_0005_create_index_profiles.py
Normal file
@ -0,0 +1,77 @@
|
|||||||
|
"""Create canonical index-profile lifecycle records.
|
||||||
|
|
||||||
|
Revision ID: 20260902_0005
|
||||||
|
Revises: 20260901_0004
|
||||||
|
Create Date: 2026-09-02
|
||||||
|
"""
|
||||||
|
|
||||||
|
from collections.abc import Sequence
|
||||||
|
|
||||||
|
import sqlalchemy as sa
|
||||||
|
from alembic import op
|
||||||
|
|
||||||
|
revision: str = "20260902_0005"
|
||||||
|
down_revision: str | None = "20260901_0004"
|
||||||
|
branch_labels: str | Sequence[str] | None = None
|
||||||
|
depends_on: str | Sequence[str] | None = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
"""Persist shadow/active/retired profiles without storing vector data."""
|
||||||
|
op.create_table(
|
||||||
|
"index_profiles",
|
||||||
|
sa.Column("profile_id", sa.String(length=128), nullable=False),
|
||||||
|
sa.Column("chunk_strategy_version", sa.String(length=64), nullable=False),
|
||||||
|
sa.Column("embedding_model", sa.String(length=128), nullable=False),
|
||||||
|
sa.Column("vector_dimension", sa.Integer(), nullable=False),
|
||||||
|
sa.Column(
|
||||||
|
"status",
|
||||||
|
sa.Enum(
|
||||||
|
"shadow",
|
||||||
|
"active",
|
||||||
|
"retired",
|
||||||
|
name="index_profile_status",
|
||||||
|
native_enum=False,
|
||||||
|
create_constraint=False,
|
||||||
|
),
|
||||||
|
nullable=False,
|
||||||
|
),
|
||||||
|
sa.Column(
|
||||||
|
"created_at",
|
||||||
|
sa.DateTime(timezone=True),
|
||||||
|
server_default=sa.text("CURRENT_TIMESTAMP"),
|
||||||
|
nullable=False,
|
||||||
|
),
|
||||||
|
sa.Column("activated_at", sa.DateTime(timezone=True), nullable=True),
|
||||||
|
sa.CheckConstraint("length(profile_id) > 0", name="profile_id_not_empty"),
|
||||||
|
sa.CheckConstraint(
|
||||||
|
"length(chunk_strategy_version) > 0",
|
||||||
|
name="profile_chunk_strategy_not_empty",
|
||||||
|
),
|
||||||
|
sa.CheckConstraint(
|
||||||
|
"length(embedding_model) > 0",
|
||||||
|
name="profile_embedding_model_not_empty",
|
||||||
|
),
|
||||||
|
sa.CheckConstraint(
|
||||||
|
"vector_dimension > 0",
|
||||||
|
name="profile_dimension_positive",
|
||||||
|
),
|
||||||
|
sa.CheckConstraint(
|
||||||
|
"status IN ('shadow', 'active', 'retired')",
|
||||||
|
name="profile_status_valid",
|
||||||
|
),
|
||||||
|
sa.PrimaryKeyConstraint("profile_id", name="pk_index_profiles"),
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
"uq_index_profiles_single_active",
|
||||||
|
"index_profiles",
|
||||||
|
["status"],
|
||||||
|
unique=True,
|
||||||
|
postgresql_where=sa.text("status = 'active'"),
|
||||||
|
sqlite_where=sa.text("status = 'active'"),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
"""Reject downgrade because project migrations are forward-only."""
|
||||||
|
raise NotImplementedError("Database migrations are forward-only")
|
||||||
@ -0,0 +1,41 @@
|
|||||||
|
"""Make single-active profile enforcement portable to OceanBase MySQL.
|
||||||
|
|
||||||
|
Revision ID: 20260902_0006
|
||||||
|
Revises: 20260902_0005
|
||||||
|
Create Date: 2026-09-02
|
||||||
|
"""
|
||||||
|
|
||||||
|
from collections.abc import Sequence
|
||||||
|
|
||||||
|
import sqlalchemy as sa
|
||||||
|
from alembic import op
|
||||||
|
|
||||||
|
revision: str = "20260902_0006"
|
||||||
|
down_revision: str | None = "20260902_0005"
|
||||||
|
branch_labels: str | Sequence[str] | None = None
|
||||||
|
depends_on: str | Sequence[str] | None = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
"""Replace dialect-specific partial uniqueness with a generated marker."""
|
||||||
|
op.drop_index("uq_index_profiles_single_active", table_name="index_profiles")
|
||||||
|
op.add_column(
|
||||||
|
"index_profiles",
|
||||||
|
sa.Column(
|
||||||
|
"active_marker",
|
||||||
|
sa.String(length=6),
|
||||||
|
sa.Computed("CASE WHEN status = 'active' THEN 'active' ELSE NULL END"),
|
||||||
|
nullable=True,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
"uq_index_profiles_single_active",
|
||||||
|
"index_profiles",
|
||||||
|
["active_marker"],
|
||||||
|
unique=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
"""Reject downgrade because project migrations are forward-only."""
|
||||||
|
raise NotImplementedError("Database migrations are forward-only")
|
||||||
174
migrations/versions/20260902_0007_create_ingestion_jobs.py
Normal file
174
migrations/versions/20260902_0007_create_ingestion_jobs.py
Normal file
@ -0,0 +1,174 @@
|
|||||||
|
"""Create durable ingestion jobs and attempt history.
|
||||||
|
|
||||||
|
Revision ID: 20260902_0007
|
||||||
|
Revises: 20260902_0006
|
||||||
|
Create Date: 2026-09-02
|
||||||
|
"""
|
||||||
|
|
||||||
|
from collections.abc import Sequence
|
||||||
|
|
||||||
|
import sqlalchemy as sa
|
||||||
|
from alembic import op
|
||||||
|
|
||||||
|
revision: str = "20260902_0007"
|
||||||
|
down_revision: str | None = "20260902_0006"
|
||||||
|
branch_labels: str | Sequence[str] | None = None
|
||||||
|
depends_on: str | Sequence[str] | None = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
"""Persist queue-independent Job state and bounded execution attempts."""
|
||||||
|
op.create_table(
|
||||||
|
"ingestion_jobs",
|
||||||
|
sa.Column("id", sa.Uuid(), nullable=False),
|
||||||
|
sa.Column("document_id", sa.Uuid(), nullable=False),
|
||||||
|
sa.Column(
|
||||||
|
"status",
|
||||||
|
sa.Enum(
|
||||||
|
"queued",
|
||||||
|
"running",
|
||||||
|
"retrying",
|
||||||
|
"succeeded",
|
||||||
|
"failed",
|
||||||
|
"cancelled",
|
||||||
|
name="ingestion_job_status",
|
||||||
|
native_enum=False,
|
||||||
|
create_constraint=False,
|
||||||
|
),
|
||||||
|
nullable=False,
|
||||||
|
),
|
||||||
|
sa.Column(
|
||||||
|
"stage",
|
||||||
|
sa.Enum(
|
||||||
|
"parsing",
|
||||||
|
"cleaning",
|
||||||
|
"chunking",
|
||||||
|
"embedding",
|
||||||
|
"indexing",
|
||||||
|
name="ingestion_stage",
|
||||||
|
native_enum=False,
|
||||||
|
create_constraint=False,
|
||||||
|
),
|
||||||
|
nullable=True,
|
||||||
|
),
|
||||||
|
sa.Column("attempt_count", sa.Integer(), nullable=False),
|
||||||
|
sa.Column("max_attempts", sa.Integer(), nullable=False),
|
||||||
|
sa.Column("error_message", sa.Text(), nullable=True),
|
||||||
|
sa.Column("next_run_at", sa.DateTime(timezone=True), nullable=True),
|
||||||
|
sa.Column("lease_expires_at", sa.DateTime(timezone=True), nullable=True),
|
||||||
|
sa.Column(
|
||||||
|
"created_at",
|
||||||
|
sa.DateTime(timezone=True),
|
||||||
|
server_default=sa.text("CURRENT_TIMESTAMP"),
|
||||||
|
nullable=False,
|
||||||
|
),
|
||||||
|
sa.Column(
|
||||||
|
"updated_at",
|
||||||
|
sa.DateTime(timezone=True),
|
||||||
|
server_default=sa.text("CURRENT_TIMESTAMP"),
|
||||||
|
nullable=False,
|
||||||
|
),
|
||||||
|
sa.Column("completed_at", sa.DateTime(timezone=True), nullable=True),
|
||||||
|
sa.CheckConstraint("attempt_count >= 0", name="attempt_count_non_negative"),
|
||||||
|
sa.CheckConstraint("max_attempts > 0", name="max_attempts_positive"),
|
||||||
|
sa.CheckConstraint("attempt_count <= max_attempts", name="attempt_count_bounded"),
|
||||||
|
sa.CheckConstraint(
|
||||||
|
"status IN ('queued', 'running', 'retrying', 'succeeded', 'failed', 'cancelled')",
|
||||||
|
name="status_valid",
|
||||||
|
),
|
||||||
|
sa.CheckConstraint(
|
||||||
|
"stage IS NULL OR stage IN "
|
||||||
|
"('parsing', 'cleaning', 'chunking', 'embedding', 'indexing')",
|
||||||
|
name="stage_valid",
|
||||||
|
),
|
||||||
|
sa.ForeignKeyConstraint(
|
||||||
|
["document_id"],
|
||||||
|
["documents.id"],
|
||||||
|
name="fk_ingestion_jobs_document_id_documents",
|
||||||
|
ondelete="CASCADE",
|
||||||
|
),
|
||||||
|
sa.PrimaryKeyConstraint("id", name="pk_ingestion_jobs"),
|
||||||
|
sa.UniqueConstraint("document_id", name="uq_ingestion_jobs_document_id"),
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
"ix_ingestion_jobs_status_next_run",
|
||||||
|
"ingestion_jobs",
|
||||||
|
["status", "next_run_at"],
|
||||||
|
unique=False,
|
||||||
|
)
|
||||||
|
op.create_table(
|
||||||
|
"ingestion_attempts",
|
||||||
|
sa.Column("id", sa.Uuid(), nullable=False),
|
||||||
|
sa.Column("job_id", sa.Uuid(), nullable=False),
|
||||||
|
sa.Column("attempt_number", sa.Integer(), nullable=False),
|
||||||
|
sa.Column(
|
||||||
|
"status",
|
||||||
|
sa.Enum(
|
||||||
|
"running",
|
||||||
|
"succeeded",
|
||||||
|
"failed",
|
||||||
|
name="ingestion_attempt_status",
|
||||||
|
native_enum=False,
|
||||||
|
create_constraint=False,
|
||||||
|
),
|
||||||
|
nullable=False,
|
||||||
|
),
|
||||||
|
sa.Column(
|
||||||
|
"stage",
|
||||||
|
sa.Enum(
|
||||||
|
"parsing",
|
||||||
|
"cleaning",
|
||||||
|
"chunking",
|
||||||
|
"embedding",
|
||||||
|
"indexing",
|
||||||
|
name="ingestion_attempt_stage",
|
||||||
|
native_enum=False,
|
||||||
|
create_constraint=False,
|
||||||
|
),
|
||||||
|
nullable=True,
|
||||||
|
),
|
||||||
|
sa.Column("retryable", sa.Boolean(), nullable=True),
|
||||||
|
sa.Column("error_message", sa.Text(), nullable=True),
|
||||||
|
sa.Column(
|
||||||
|
"active_marker",
|
||||||
|
sa.Integer(),
|
||||||
|
sa.Computed("CASE WHEN status = 'running' THEN 1 ELSE NULL END"),
|
||||||
|
nullable=True,
|
||||||
|
),
|
||||||
|
sa.Column(
|
||||||
|
"started_at",
|
||||||
|
sa.DateTime(timezone=True),
|
||||||
|
server_default=sa.text("CURRENT_TIMESTAMP"),
|
||||||
|
nullable=False,
|
||||||
|
),
|
||||||
|
sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True),
|
||||||
|
sa.CheckConstraint("attempt_number > 0", name="attempt_number_positive"),
|
||||||
|
sa.CheckConstraint(
|
||||||
|
"status IN ('running', 'succeeded', 'failed')",
|
||||||
|
name="status_valid",
|
||||||
|
),
|
||||||
|
sa.CheckConstraint(
|
||||||
|
"stage IS NULL OR stage IN "
|
||||||
|
"('parsing', 'cleaning', 'chunking', 'embedding', 'indexing')",
|
||||||
|
name="stage_valid",
|
||||||
|
),
|
||||||
|
sa.ForeignKeyConstraint(
|
||||||
|
["job_id"],
|
||||||
|
["ingestion_jobs.id"],
|
||||||
|
name="fk_ingestion_attempts_job_id_ingestion_jobs",
|
||||||
|
ondelete="CASCADE",
|
||||||
|
),
|
||||||
|
sa.PrimaryKeyConstraint("id", name="pk_ingestion_attempts"),
|
||||||
|
sa.UniqueConstraint("job_id", "attempt_number", name="uq_ingestion_attempts_job_number"),
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
"uq_ingestion_attempts_active",
|
||||||
|
"ingestion_attempts",
|
||||||
|
["job_id", "active_marker"],
|
||||||
|
unique=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
"""Reject downgrade because project migrations are forward-only."""
|
||||||
|
raise NotImplementedError("Database migrations are forward-only")
|
||||||
109
migrations/versions/20260902_0008_create_ingestion_batches.py
Normal file
109
migrations/versions/20260902_0008_create_ingestion_batches.py
Normal file
@ -0,0 +1,109 @@
|
|||||||
|
"""Create ingestion batch aggregation metadata.
|
||||||
|
|
||||||
|
Revision ID: 20260902_0008
|
||||||
|
Revises: 20260902_0007
|
||||||
|
Create Date: 2026-09-02
|
||||||
|
"""
|
||||||
|
|
||||||
|
from collections.abc import Sequence
|
||||||
|
|
||||||
|
import sqlalchemy as sa
|
||||||
|
from alembic import op
|
||||||
|
|
||||||
|
revision: str = "20260902_0008"
|
||||||
|
down_revision: str | None = "20260902_0007"
|
||||||
|
branch_labels: str | Sequence[str] | None = None
|
||||||
|
depends_on: str | Sequence[str] | None = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
"""Persist ordered per-file acceptance without making a batch transactional."""
|
||||||
|
op.create_table(
|
||||||
|
"ingestion_batches",
|
||||||
|
sa.Column("id", sa.Uuid(), nullable=False),
|
||||||
|
sa.Column("total_files", sa.Integer(), nullable=False),
|
||||||
|
sa.Column(
|
||||||
|
"created_at",
|
||||||
|
sa.DateTime(timezone=True),
|
||||||
|
server_default=sa.text("CURRENT_TIMESTAMP"),
|
||||||
|
nullable=False,
|
||||||
|
),
|
||||||
|
sa.CheckConstraint("total_files > 0", name="total_files_positive"),
|
||||||
|
sa.PrimaryKeyConstraint("id", name="pk_ingestion_batches"),
|
||||||
|
)
|
||||||
|
op.create_table(
|
||||||
|
"ingestion_batch_items",
|
||||||
|
sa.Column("id", sa.Uuid(), nullable=False),
|
||||||
|
sa.Column("batch_id", sa.Uuid(), nullable=False),
|
||||||
|
sa.Column("item_index", sa.Integer(), nullable=False),
|
||||||
|
sa.Column("filename", sa.String(length=255), nullable=False),
|
||||||
|
sa.Column(
|
||||||
|
"acceptance_status",
|
||||||
|
sa.Enum(
|
||||||
|
"pending",
|
||||||
|
"accepted",
|
||||||
|
"rejected",
|
||||||
|
name="batch_item_acceptance_status",
|
||||||
|
native_enum=False,
|
||||||
|
create_constraint=False,
|
||||||
|
),
|
||||||
|
nullable=False,
|
||||||
|
),
|
||||||
|
sa.Column("document_id", sa.Uuid(), nullable=True),
|
||||||
|
sa.Column("job_id", sa.Uuid(), nullable=True),
|
||||||
|
sa.Column("error_message", sa.Text(), nullable=True),
|
||||||
|
sa.Column(
|
||||||
|
"created_at",
|
||||||
|
sa.DateTime(timezone=True),
|
||||||
|
server_default=sa.text("CURRENT_TIMESTAMP"),
|
||||||
|
nullable=False,
|
||||||
|
),
|
||||||
|
sa.Column(
|
||||||
|
"updated_at",
|
||||||
|
sa.DateTime(timezone=True),
|
||||||
|
server_default=sa.text("CURRENT_TIMESTAMP"),
|
||||||
|
nullable=False,
|
||||||
|
),
|
||||||
|
sa.CheckConstraint("item_index >= 0", name="item_index_non_negative"),
|
||||||
|
sa.CheckConstraint("length(filename) > 0", name="filename_not_empty"),
|
||||||
|
sa.CheckConstraint(
|
||||||
|
"acceptance_status IN ('pending', 'accepted', 'rejected')",
|
||||||
|
name="acceptance_status_valid",
|
||||||
|
),
|
||||||
|
sa.ForeignKeyConstraint(
|
||||||
|
["batch_id"],
|
||||||
|
["ingestion_batches.id"],
|
||||||
|
name="fk_ingestion_batch_items_batch_id_ingestion_batches",
|
||||||
|
ondelete="CASCADE",
|
||||||
|
),
|
||||||
|
sa.ForeignKeyConstraint(
|
||||||
|
["document_id"],
|
||||||
|
["documents.id"],
|
||||||
|
name="fk_ingestion_batch_items_document_id_documents",
|
||||||
|
ondelete="SET NULL",
|
||||||
|
),
|
||||||
|
sa.ForeignKeyConstraint(
|
||||||
|
["job_id"],
|
||||||
|
["ingestion_jobs.id"],
|
||||||
|
name="fk_ingestion_batch_items_job_id_ingestion_jobs",
|
||||||
|
ondelete="SET NULL",
|
||||||
|
),
|
||||||
|
sa.PrimaryKeyConstraint("id", name="pk_ingestion_batch_items"),
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
"uq_ingestion_batch_items_order",
|
||||||
|
"ingestion_batch_items",
|
||||||
|
["batch_id", "item_index"],
|
||||||
|
unique=True,
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
"uq_ingestion_batch_items_job_id",
|
||||||
|
"ingestion_batch_items",
|
||||||
|
["job_id"],
|
||||||
|
unique=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
"""Reject downgrade because project migrations are forward-only."""
|
||||||
|
raise NotImplementedError("Database migrations are forward-only")
|
||||||
@ -1 +1,26 @@
|
|||||||
"""Shared core utilities for the Enterprise Knowledge Platform."""
|
"""Shared core utilities for the Enterprise Knowledge Platform."""
|
||||||
|
|
||||||
|
from ekp_core.dense_retrieval import DenseRetrievalResult, DenseRetrievalService
|
||||||
|
from ekp_core.document_chunking import DocumentChunkingService, DocumentPipelineService
|
||||||
|
from ekp_core.document_cleaner import clean_parsed_document
|
||||||
|
from ekp_core.document_indexing import (
|
||||||
|
DocumentIndexingPipelineService,
|
||||||
|
DocumentIndexingService,
|
||||||
|
)
|
||||||
|
from ekp_core.document_parsing import DocumentIngestionService, DocumentParsingService
|
||||||
|
from ekp_core.search import SearchResult, SearchService, SourceResult
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
"DocumentChunkingService",
|
||||||
|
"DenseRetrievalResult",
|
||||||
|
"DenseRetrievalService",
|
||||||
|
"DocumentIngestionService",
|
||||||
|
"DocumentIndexingPipelineService",
|
||||||
|
"DocumentIndexingService",
|
||||||
|
"DocumentPipelineService",
|
||||||
|
"DocumentParsingService",
|
||||||
|
"SearchResult",
|
||||||
|
"SearchService",
|
||||||
|
"SourceResult",
|
||||||
|
"clean_parsed_document",
|
||||||
|
]
|
||||||
|
|||||||
223
packages/core/src/ekp_core/chunking.py
Normal file
223
packages/core/src/ekp_core/chunking.py
Normal file
@ -0,0 +1,223 @@
|
|||||||
|
"""Structure-aware, token-window chunking for the retrieval Demo."""
|
||||||
|
|
||||||
|
import re
|
||||||
|
from collections.abc import Iterable
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from typing import Protocol
|
||||||
|
|
||||||
|
from ekp_domain import ParsedBlock, ParsedBlockType, ParsedDocument
|
||||||
|
|
||||||
|
DEFAULT_CHUNK_STRATEGY_VERSION = "structure_v1"
|
||||||
|
TOKEN_PATTERN = re.compile(
|
||||||
|
r"[\u3400-\u4dbf\u4e00-\u9fff\u3040-\u30ff\uac00-\ud7af]"
|
||||||
|
r"|[A-Za-z0-9_]+"
|
||||||
|
r"|[^\s]"
|
||||||
|
)
|
||||||
|
ASCII_SUBTOKEN_CHARACTERS = 4
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class TokenSpan:
|
||||||
|
"""Character offsets for one conservatively estimated model token."""
|
||||||
|
|
||||||
|
start: int
|
||||||
|
end: int
|
||||||
|
|
||||||
|
|
||||||
|
class TokenCounter(Protocol):
|
||||||
|
"""Injectable token boundary contract for future model-specific counters."""
|
||||||
|
|
||||||
|
def spans(self, text: str) -> tuple[TokenSpan, ...]:
|
||||||
|
"""Return ordered, non-overlapping token character spans."""
|
||||||
|
|
||||||
|
def count(self, text: str) -> int:
|
||||||
|
"""Return the token estimate for text."""
|
||||||
|
|
||||||
|
|
||||||
|
class ApproximateTokenCounter:
|
||||||
|
"""Conservatively estimate Chinese/BGE-sized token windows without a model SDK.
|
||||||
|
|
||||||
|
CJK characters and punctuation count individually. ASCII words are divided
|
||||||
|
into four-character pieces so unusually long terms cannot bypass the limit.
|
||||||
|
DEMO-006 may inject a model-specific counter without changing chunk policy.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def spans(self, text: str) -> tuple[TokenSpan, ...]:
|
||||||
|
token_spans: list[TokenSpan] = []
|
||||||
|
for match in TOKEN_PATTERN.finditer(text):
|
||||||
|
value = match.group()
|
||||||
|
if value.isascii() and (value.isalnum() or "_" in value):
|
||||||
|
token_spans.extend(
|
||||||
|
TokenSpan(start, min(start + ASCII_SUBTOKEN_CHARACTERS, match.end()))
|
||||||
|
for start in range(match.start(), match.end(), ASCII_SUBTOKEN_CHARACTERS)
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
token_spans.append(TokenSpan(match.start(), match.end()))
|
||||||
|
return tuple(token_spans)
|
||||||
|
|
||||||
|
def count(self, text: str) -> int:
|
||||||
|
"""Count the same boundaries used to create chunk windows."""
|
||||||
|
return len(self.spans(text))
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class ChunkDraft:
|
||||||
|
"""Persistence-neutral chunk content produced by one strategy version."""
|
||||||
|
|
||||||
|
chunk_index: int
|
||||||
|
content: str
|
||||||
|
token_count: int
|
||||||
|
page: int | None
|
||||||
|
page_end: int | None
|
||||||
|
section_path: tuple[str, ...]
|
||||||
|
|
||||||
|
@property
|
||||||
|
def section(self) -> str | None:
|
||||||
|
"""Return a compact section label for Demo retrieval results."""
|
||||||
|
return " / ".join(self.section_path) or None
|
||||||
|
|
||||||
|
|
||||||
|
class ChunkingStrategy(Protocol):
|
||||||
|
"""Provider-neutral strategy contract consumed by the pipeline service."""
|
||||||
|
|
||||||
|
@property
|
||||||
|
def version(self) -> str:
|
||||||
|
"""Return a stable strategy identifier for rebuilds and shadow indexes."""
|
||||||
|
|
||||||
|
def chunk(self, document: ParsedDocument) -> tuple[ChunkDraft, ...]:
|
||||||
|
"""Build ordered chunks from a cleaned parsed document."""
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class _BlockRange:
|
||||||
|
start: int
|
||||||
|
end: int
|
||||||
|
page_number: int | None
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class _StructuralGroup:
|
||||||
|
content: str
|
||||||
|
section_path: tuple[str, ...]
|
||||||
|
block_ranges: tuple[_BlockRange, ...]
|
||||||
|
|
||||||
|
|
||||||
|
class StructureAwareChunker:
|
||||||
|
"""Combine paragraphs inside structural boundaries, then apply token overlap."""
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
max_tokens: int = 450,
|
||||||
|
overlap_tokens: int = 75,
|
||||||
|
max_characters: int = 2000,
|
||||||
|
token_counter: TokenCounter | None = None,
|
||||||
|
strategy_version: str = DEFAULT_CHUNK_STRATEGY_VERSION,
|
||||||
|
) -> None:
|
||||||
|
if max_tokens <= 0:
|
||||||
|
raise ValueError("max_tokens must be positive")
|
||||||
|
if overlap_tokens < 0 or overlap_tokens >= max_tokens:
|
||||||
|
raise ValueError("overlap_tokens must be non-negative and less than max_tokens")
|
||||||
|
if max_characters < ASCII_SUBTOKEN_CHARACTERS:
|
||||||
|
raise ValueError("max_characters must be at least four")
|
||||||
|
if not strategy_version:
|
||||||
|
raise ValueError("strategy_version must not be empty")
|
||||||
|
self._max_tokens = max_tokens
|
||||||
|
self._overlap_tokens = overlap_tokens
|
||||||
|
self._max_characters = max_characters
|
||||||
|
self._token_counter = token_counter or ApproximateTokenCounter()
|
||||||
|
self._version = strategy_version
|
||||||
|
|
||||||
|
@property
|
||||||
|
def version(self) -> str:
|
||||||
|
"""Identify the chunk policy used by persisted rows."""
|
||||||
|
return self._version
|
||||||
|
|
||||||
|
def chunk(self, document: ParsedDocument) -> tuple[ChunkDraft, ...]:
|
||||||
|
"""Create deterministic chunks without crossing heading/section boundaries."""
|
||||||
|
drafts: list[ChunkDraft] = []
|
||||||
|
for group in _structural_groups(document.blocks):
|
||||||
|
for content, token_count, page, page_end in self._windows(group):
|
||||||
|
drafts.append(
|
||||||
|
ChunkDraft(
|
||||||
|
chunk_index=len(drafts),
|
||||||
|
content=content,
|
||||||
|
token_count=token_count,
|
||||||
|
page=page,
|
||||||
|
page_end=page_end,
|
||||||
|
section_path=group.section_path,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return tuple(drafts)
|
||||||
|
|
||||||
|
def _windows(
|
||||||
|
self,
|
||||||
|
group: _StructuralGroup,
|
||||||
|
) -> Iterable[tuple[str, int, int | None, int | None]]:
|
||||||
|
spans = self._token_counter.spans(group.content)
|
||||||
|
start_token = 0
|
||||||
|
while start_token < len(spans):
|
||||||
|
end_token = min(start_token + self._max_tokens, len(spans))
|
||||||
|
char_start = spans[start_token].start
|
||||||
|
while (
|
||||||
|
end_token > start_token + 1
|
||||||
|
and spans[end_token - 1].end - char_start > self._max_characters
|
||||||
|
):
|
||||||
|
end_token -= 1
|
||||||
|
char_end = spans[end_token - 1].end
|
||||||
|
content = group.content[char_start:char_end].strip()
|
||||||
|
pages = [
|
||||||
|
block_range.page_number
|
||||||
|
for block_range in group.block_ranges
|
||||||
|
if block_range.page_number is not None
|
||||||
|
and block_range.end > char_start
|
||||||
|
and block_range.start < char_end
|
||||||
|
]
|
||||||
|
yield (
|
||||||
|
content,
|
||||||
|
end_token - start_token,
|
||||||
|
min(pages, default=None),
|
||||||
|
max(pages, default=None),
|
||||||
|
)
|
||||||
|
if end_token == len(spans):
|
||||||
|
break
|
||||||
|
start_token = end_token - self._overlap_tokens
|
||||||
|
|
||||||
|
|
||||||
|
def _structural_groups(blocks: tuple[ParsedBlock, ...]) -> Iterable[_StructuralGroup]:
|
||||||
|
"""Group content without crossing explicit headings or section changes."""
|
||||||
|
current: list[ParsedBlock] = []
|
||||||
|
current_section: tuple[str, ...] | None = None
|
||||||
|
|
||||||
|
for block in blocks:
|
||||||
|
is_new_boundary = bool(current) and (
|
||||||
|
block.block_type is ParsedBlockType.HEADING or block.section_path != current_section
|
||||||
|
)
|
||||||
|
if is_new_boundary:
|
||||||
|
yield _build_group(current)
|
||||||
|
current = []
|
||||||
|
if not current:
|
||||||
|
current_section = block.section_path
|
||||||
|
current.append(block)
|
||||||
|
|
||||||
|
if current:
|
||||||
|
yield _build_group(current)
|
||||||
|
|
||||||
|
|
||||||
|
def _build_group(blocks: list[ParsedBlock]) -> _StructuralGroup:
|
||||||
|
"""Join blocks with readable separators and retain their character ranges."""
|
||||||
|
parts: list[str] = []
|
||||||
|
block_ranges: list[_BlockRange] = []
|
||||||
|
offset = 0
|
||||||
|
for block in blocks:
|
||||||
|
if parts:
|
||||||
|
parts.append("\n\n")
|
||||||
|
offset += 2
|
||||||
|
start = offset
|
||||||
|
parts.append(block.text)
|
||||||
|
offset += len(block.text)
|
||||||
|
block_ranges.append(_BlockRange(start=start, end=offset, page_number=block.page_number))
|
||||||
|
return _StructuralGroup(
|
||||||
|
content="".join(parts),
|
||||||
|
section_path=blocks[0].section_path,
|
||||||
|
block_ranges=tuple(block_ranges),
|
||||||
|
)
|
||||||
@ -4,8 +4,9 @@ from functools import lru_cache
|
|||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Literal
|
from typing import Literal
|
||||||
|
|
||||||
from pydantic import Field
|
from pydantic import AnyHttpUrl, Field, SecretStr, model_validator
|
||||||
from pydantic_settings import BaseSettings, SettingsConfigDict
|
from pydantic_settings import BaseSettings, SettingsConfigDict
|
||||||
|
from sqlalchemy.engine import make_url
|
||||||
|
|
||||||
|
|
||||||
class Settings(BaseSettings):
|
class Settings(BaseSettings):
|
||||||
@ -24,9 +25,105 @@ class Settings(BaseSettings):
|
|||||||
api_host: str = "0.0.0.0"
|
api_host: str = "0.0.0.0"
|
||||||
api_port: int = Field(default=8000, ge=1, le=65535)
|
api_port: int = Field(default=8000, ge=1, le=65535)
|
||||||
api_prefix: str = "/api/v1"
|
api_prefix: str = "/api/v1"
|
||||||
database_url: str = "postgresql+psycopg://localhost:5432/ekp"
|
admin_web_root: Path | None = None
|
||||||
object_storage_provider: Literal["local"] = "local"
|
# SQLAlchemy URLs commonly embed a password, so configuration rendering
|
||||||
|
# must mask this value and unwrap it only at the database driver boundary.
|
||||||
|
database_url: SecretStr = SecretStr("mysql+pymysql://localhost:2881/ekp")
|
||||||
|
database_pool_size: int = Field(default=10, ge=1, le=100)
|
||||||
|
database_max_overflow: int = Field(default=10, ge=0, le=100)
|
||||||
|
database_pool_recycle_seconds: int = Field(default=600, ge=1)
|
||||||
|
database_pool_timeout_seconds: float = Field(default=30.0, gt=0)
|
||||||
|
database_connect_timeout_seconds: int = Field(default=30, ge=1)
|
||||||
|
rabbitmq_url: SecretStr | None = None
|
||||||
|
ingestion_max_attempts: int = Field(default=3, ge=1, le=10)
|
||||||
|
ingestion_retry_backoff_seconds: int = Field(default=2, ge=1, le=3600)
|
||||||
|
ingestion_retry_backoff_max_seconds: int = Field(default=60, ge=1, le=86400)
|
||||||
|
ingestion_worker_lease_seconds: int = Field(default=300, ge=30, le=86400)
|
||||||
|
ingestion_recovery_interval_seconds: int = Field(default=30, ge=5, le=3600)
|
||||||
|
batch_max_files: int = Field(default=20, ge=1, le=100)
|
||||||
|
batch_max_total_bytes: int = Field(default=200 * 1024 * 1024, gt=0)
|
||||||
|
object_storage_provider: Literal["local", "bos"] = "bos"
|
||||||
object_storage_path: Path = Path("storage")
|
object_storage_path: Path = Path("storage")
|
||||||
|
bos_endpoint: str | None = None
|
||||||
|
bos_access_key_id: SecretStr | None = None
|
||||||
|
bos_secret_access_key: SecretStr | None = None
|
||||||
|
bos_bucket: str | None = None
|
||||||
|
bos_key_prefix: str = "knowledge-platform"
|
||||||
|
bos_timeout_seconds: float = Field(default=30.0, gt=0)
|
||||||
|
max_upload_bytes: int = Field(default=50 * 1024 * 1024, gt=0)
|
||||||
|
chunk_max_tokens: int = Field(default=450, gt=0)
|
||||||
|
chunk_overlap_tokens: int = Field(default=75, ge=0)
|
||||||
|
chunk_max_characters: int = Field(default=2000, ge=4)
|
||||||
|
embedding_provider: Literal["qianfan"] = "qianfan"
|
||||||
|
embedding_model: Literal["bge-large-zh"] = "bge-large-zh"
|
||||||
|
embedding_endpoint: str = "https://qianfan.baidubce.com/v2/embeddings"
|
||||||
|
baidu_qianfan_api_key: SecretStr | None = None
|
||||||
|
embedding_timeout_seconds: float = Field(default=30.0, gt=0)
|
||||||
|
vector_store_provider: Literal["oceanbase"] = "oceanbase"
|
||||||
|
oceanbase_vector_uri: str | None = None
|
||||||
|
oceanbase_vector_user: str | None = None
|
||||||
|
oceanbase_vector_password: SecretStr | None = None
|
||||||
|
oceanbase_vector_database: str | None = None
|
||||||
|
oceanbase_vector_table: str = "knowledge_chunks_v1"
|
||||||
|
oceanbase_vector_index: str = "vidx_knowledge_chunks_embedding"
|
||||||
|
oceanbase_vector_batch_size: int = Field(default=100, ge=1, le=1000)
|
||||||
|
oceanbase_vector_timeout_seconds: float = Field(default=30.0, gt=0)
|
||||||
|
index_profile_id: str = "demo-bge-large-zh-structure-v1-oceanbase-v1"
|
||||||
|
dense_top_n: int = Field(default=20, ge=1, le=64)
|
||||||
|
reranker_provider: Literal["qianfan"] = "qianfan"
|
||||||
|
reranker_model: Literal["bce-reranker-base"] = "bce-reranker-base"
|
||||||
|
reranker_endpoint: str = "https://qianfan.baidubce.com/v2/rerank"
|
||||||
|
reranker_api_key: SecretStr | None = None
|
||||||
|
reranker_timeout_seconds: float = Field(default=30.0, gt=0)
|
||||||
|
search_top_k: int = Field(default=5, ge=1, le=64)
|
||||||
|
mcp_host: str = "127.0.0.1"
|
||||||
|
mcp_port: int = Field(default=8766, ge=1, le=65535)
|
||||||
|
mcp_path: str = "/mcp"
|
||||||
|
mcp_public_url: AnyHttpUrl = AnyHttpUrl("http://127.0.0.1:8766/mcp")
|
||||||
|
mcp_issuer_url: AnyHttpUrl = AnyHttpUrl("http://127.0.0.1:8766")
|
||||||
|
mcp_auth_token: SecretStr | None = None
|
||||||
|
mcp_client_id: str = Field(default="workbuddy-enterprise", min_length=1, max_length=200)
|
||||||
|
mcp_service_identity: str = Field(default="workbuddy-demo", min_length=1, max_length=200)
|
||||||
|
mcp_required_scope: str = Field(default="knowledge:search", min_length=1, max_length=200)
|
||||||
|
mcp_allowed_hosts: list[str] = ["127.0.0.1:*", "localhost:*", "[::1]:*", "testserver"]
|
||||||
|
mcp_allowed_origins: list[str] = [
|
||||||
|
"http://127.0.0.1:*",
|
||||||
|
"http://localhost:*",
|
||||||
|
"http://[::1]:*",
|
||||||
|
]
|
||||||
|
mcp_max_request_body_bytes: int = Field(default=1024 * 1024, ge=1024, le=4 * 1024 * 1024)
|
||||||
|
|
||||||
|
@model_validator(mode="after")
|
||||||
|
def validate_chunk_window(self) -> "Settings":
|
||||||
|
"""Require forward progress for every overlapping chunk window."""
|
||||||
|
if self.chunk_overlap_tokens >= self.chunk_max_tokens:
|
||||||
|
raise ValueError("chunk_overlap_tokens must be less than chunk_max_tokens")
|
||||||
|
if self.search_top_k > self.dense_top_n:
|
||||||
|
raise ValueError("search_top_k must not exceed dense_top_n")
|
||||||
|
if self.ingestion_retry_backoff_max_seconds < self.ingestion_retry_backoff_seconds:
|
||||||
|
raise ValueError(
|
||||||
|
"ingestion_retry_backoff_max_seconds must be at least the initial backoff"
|
||||||
|
)
|
||||||
|
longest_provider_timeout = max(
|
||||||
|
self.bos_timeout_seconds,
|
||||||
|
self.embedding_timeout_seconds,
|
||||||
|
self.oceanbase_vector_timeout_seconds,
|
||||||
|
)
|
||||||
|
if self.ingestion_worker_lease_seconds <= longest_provider_timeout:
|
||||||
|
raise ValueError("ingestion_worker_lease_seconds must exceed provider timeouts")
|
||||||
|
if self.batch_max_total_bytes < self.max_upload_bytes:
|
||||||
|
raise ValueError("batch_max_total_bytes must be at least max_upload_bytes")
|
||||||
|
if not self.mcp_path.startswith("/") or self.mcp_path.endswith("/"):
|
||||||
|
raise ValueError("mcp_path must start with '/' and must not end with '/'")
|
||||||
|
if not self.mcp_allowed_hosts:
|
||||||
|
raise ValueError("mcp_allowed_hosts must not be empty")
|
||||||
|
database_url = make_url(self.database_url.get_secret_value())
|
||||||
|
if database_url.get_backend_name() == "sqlite":
|
||||||
|
if self.app_env != "test":
|
||||||
|
raise ValueError("SQLite database URLs are restricted to APP_ENV=test")
|
||||||
|
elif database_url.drivername != "mysql+pymysql":
|
||||||
|
raise ValueError("canonical database must use OceanBase through mysql+pymysql")
|
||||||
|
return self
|
||||||
|
|
||||||
|
|
||||||
@lru_cache
|
@lru_cache
|
||||||
|
|||||||
@ -1,14 +1,35 @@
|
|||||||
"""SQLAlchemy engine and session construction for canonical metadata."""
|
"""SQLAlchemy engine and session construction for canonical metadata."""
|
||||||
|
|
||||||
from sqlalchemy import Engine, create_engine
|
from sqlalchemy import Engine, create_engine
|
||||||
|
from sqlalchemy.engine import make_url
|
||||||
from sqlalchemy.orm import Session, sessionmaker
|
from sqlalchemy.orm import Session, sessionmaker
|
||||||
|
|
||||||
from ekp_core.config import Settings
|
from ekp_core.config import Settings
|
||||||
|
|
||||||
|
|
||||||
def create_database_engine(settings: Settings) -> Engine:
|
def create_database_engine(settings: Settings) -> Engine:
|
||||||
"""Create the metadata engine without opening a connection eagerly."""
|
"""Create the canonical engine without opening a connection eagerly.
|
||||||
return create_engine(settings.database_url, pool_pre_ping=True)
|
|
||||||
|
OceanBase uses the synchronous PyMySQL driver in this Demo. Pool and
|
||||||
|
socket timeouts are explicit so an outage cannot hold API workers
|
||||||
|
indefinitely. SQLite remains available only for isolated tests.
|
||||||
|
"""
|
||||||
|
database_url = settings.database_url.get_secret_value()
|
||||||
|
url = make_url(database_url)
|
||||||
|
options: dict[str, object] = {"pool_pre_ping": True}
|
||||||
|
if url.get_backend_name() == "mysql":
|
||||||
|
options.update(
|
||||||
|
pool_size=settings.database_pool_size,
|
||||||
|
max_overflow=settings.database_max_overflow,
|
||||||
|
pool_recycle=settings.database_pool_recycle_seconds,
|
||||||
|
pool_timeout=settings.database_pool_timeout_seconds,
|
||||||
|
connect_args={
|
||||||
|
"connect_timeout": settings.database_connect_timeout_seconds,
|
||||||
|
"read_timeout": settings.database_connect_timeout_seconds,
|
||||||
|
"write_timeout": settings.database_connect_timeout_seconds,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
return create_engine(database_url, **options)
|
||||||
|
|
||||||
|
|
||||||
def create_session_factory(engine: Engine) -> sessionmaker[Session]:
|
def create_session_factory(engine: Engine) -> sessionmaker[Session]:
|
||||||
|
|||||||
90
packages/core/src/ekp_core/dense_retrieval.py
Normal file
90
packages/core/src/ekp_core/dense_retrieval.py
Normal file
@ -0,0 +1,90 @@
|
|||||||
|
"""Query embedding and profile-scoped dense candidate retrieval."""
|
||||||
|
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from math import isfinite
|
||||||
|
|
||||||
|
from ekp_domain import IndexProfile
|
||||||
|
from ekp_embedding import EmbeddingProvider
|
||||||
|
from ekp_vector_store import DenseSearchCandidate, DenseSearchRequest, VectorStore
|
||||||
|
|
||||||
|
DEFAULT_DENSE_TOP_N = 20
|
||||||
|
|
||||||
|
|
||||||
|
class InvalidRetrievalQueryError(ValueError):
|
||||||
|
"""Raised when a query cannot be embedded safely."""
|
||||||
|
|
||||||
|
|
||||||
|
class DenseRetrievalContractError(RuntimeError):
|
||||||
|
"""Raised when an embedding or vector provider violates its contract."""
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class DenseRetrievalResult:
|
||||||
|
"""Dense candidates before DEMO-008 reranking and source-result assembly."""
|
||||||
|
|
||||||
|
query: str
|
||||||
|
index_profile_id: str
|
||||||
|
candidates: tuple[DenseSearchCandidate, ...]
|
||||||
|
|
||||||
|
|
||||||
|
class DenseRetrievalService:
|
||||||
|
"""Embed one query and request Top-N from the active versioned index profile."""
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
embedding_provider: EmbeddingProvider,
|
||||||
|
vector_store: VectorStore,
|
||||||
|
index_profile: IndexProfile,
|
||||||
|
default_top_n: int = DEFAULT_DENSE_TOP_N,
|
||||||
|
) -> None:
|
||||||
|
if embedding_provider.model_name != index_profile.embedding_model:
|
||||||
|
raise ValueError("embedding provider model does not match index profile")
|
||||||
|
if embedding_provider.dimensions != index_profile.vector_dimension:
|
||||||
|
raise ValueError("embedding dimensions do not match index profile")
|
||||||
|
if not 1 <= default_top_n <= 1000:
|
||||||
|
raise ValueError("default_top_n must be between 1 and 1000")
|
||||||
|
self._embedding_provider = embedding_provider
|
||||||
|
self._vector_store = vector_store
|
||||||
|
self._index_profile = index_profile
|
||||||
|
self._default_top_n = default_top_n
|
||||||
|
|
||||||
|
def search(self, query: str, top_n: int | None = None) -> DenseRetrievalResult:
|
||||||
|
"""Return raw dense candidates without reranking or final API shaping."""
|
||||||
|
normalized_query = query.strip()
|
||||||
|
if not normalized_query:
|
||||||
|
raise InvalidRetrievalQueryError("retrieval query must not be empty")
|
||||||
|
requested_top_n = self._default_top_n if top_n is None else top_n
|
||||||
|
if not 1 <= requested_top_n <= 1000:
|
||||||
|
raise InvalidRetrievalQueryError("top_n must be between 1 and 1000")
|
||||||
|
|
||||||
|
embedded = self._embedding_provider.embed((normalized_query,))
|
||||||
|
if len(embedded) != 1:
|
||||||
|
raise DenseRetrievalContractError("embedding provider returned the wrong vector count")
|
||||||
|
query_vector = embedded[0]
|
||||||
|
if len(query_vector) != self._index_profile.vector_dimension:
|
||||||
|
raise DenseRetrievalContractError("embedding provider returned the wrong dimensions")
|
||||||
|
if any(not isfinite(value) for value in query_vector):
|
||||||
|
raise DenseRetrievalContractError("embedding provider returned non-finite values")
|
||||||
|
if not any(value != 0 for value in query_vector):
|
||||||
|
raise DenseRetrievalContractError("embedding provider returned a zero vector")
|
||||||
|
|
||||||
|
candidates = self._vector_store.dense_search(
|
||||||
|
DenseSearchRequest(
|
||||||
|
query_vector=query_vector,
|
||||||
|
index_profile_id=self._index_profile.profile_id,
|
||||||
|
top_n=requested_top_n,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
if len(candidates) > requested_top_n:
|
||||||
|
raise DenseRetrievalContractError("vector store returned too many candidates")
|
||||||
|
if any(
|
||||||
|
candidate.index_profile_id != self._index_profile.profile_id for candidate in candidates
|
||||||
|
):
|
||||||
|
raise DenseRetrievalContractError("vector store crossed the active index profile")
|
||||||
|
if any(not isfinite(candidate.score) for candidate in candidates):
|
||||||
|
raise DenseRetrievalContractError("vector store returned a non-finite score")
|
||||||
|
return DenseRetrievalResult(
|
||||||
|
query=normalized_query,
|
||||||
|
index_profile_id=self._index_profile.profile_id,
|
||||||
|
candidates=candidates,
|
||||||
|
)
|
||||||
194
packages/core/src/ekp_core/document_chunking.py
Normal file
194
packages/core/src/ekp_core/document_chunking.py
Normal file
@ -0,0 +1,194 @@
|
|||||||
|
"""Application services for building and persisting document chunks."""
|
||||||
|
|
||||||
|
from collections.abc import Callable
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from hashlib import sha256
|
||||||
|
from uuid import UUID, uuid5
|
||||||
|
|
||||||
|
from sqlalchemy import select
|
||||||
|
from sqlalchemy.orm import Session, sessionmaker
|
||||||
|
|
||||||
|
from ekp_core.chunking import ChunkDraft, ChunkingStrategy
|
||||||
|
from ekp_core.document_parsing import DocumentIngestionService, DocumentParsingService
|
||||||
|
from ekp_core.document_upload import DocumentUploadService
|
||||||
|
from ekp_core.parsed_document_artifact import deserialize_parsed_document
|
||||||
|
from ekp_domain import Chunk, Document, DocumentStatus, IngestionStage, ParsedDocument
|
||||||
|
from ekp_object_storage import BinarySource, ObjectStorage
|
||||||
|
|
||||||
|
MAX_CHUNKING_ERROR_LENGTH = 2000
|
||||||
|
|
||||||
|
|
||||||
|
class DocumentNotFoundError(LookupError):
|
||||||
|
"""Raised when chunking is requested for an unknown document id."""
|
||||||
|
|
||||||
|
|
||||||
|
class DocumentNotParsedError(RuntimeError):
|
||||||
|
"""Raised when chunking is requested before a parsed artifact exists."""
|
||||||
|
|
||||||
|
|
||||||
|
class DocumentAlreadyChunkedError(RuntimeError):
|
||||||
|
"""Raised instead of silently replacing an existing strategy result."""
|
||||||
|
|
||||||
|
|
||||||
|
class ParsedDocumentMismatchError(ValueError):
|
||||||
|
"""Raised when a parsed artifact does not belong to its Document metadata."""
|
||||||
|
|
||||||
|
|
||||||
|
class EmptyChunkSetError(ValueError):
|
||||||
|
"""Raised when a parsed document produces no usable chunks."""
|
||||||
|
|
||||||
|
|
||||||
|
class ChunkingStatePersistenceError(RuntimeError):
|
||||||
|
"""Raised when a chunking failure cannot be recorded in metadata."""
|
||||||
|
|
||||||
|
|
||||||
|
class DocumentChunkingService:
|
||||||
|
"""Load a parsed artifact, build chunks, and atomically persist their metadata."""
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
session_factory: sessionmaker[Session],
|
||||||
|
storage: ObjectStorage,
|
||||||
|
strategy: ChunkingStrategy,
|
||||||
|
) -> None:
|
||||||
|
self._session_factory = session_factory
|
||||||
|
self._storage = storage
|
||||||
|
self._strategy = strategy
|
||||||
|
|
||||||
|
def process(
|
||||||
|
self,
|
||||||
|
document_id: UUID,
|
||||||
|
stage_callback: Callable[[IngestionStage], None] | None = None,
|
||||||
|
) -> tuple[Chunk, ...]:
|
||||||
|
"""Build chunks while optionally reporting the durable Job stage."""
|
||||||
|
filename, parsed_key = self._start_chunking(document_id)
|
||||||
|
try:
|
||||||
|
if stage_callback is not None:
|
||||||
|
stage_callback(IngestionStage.CHUNKING)
|
||||||
|
with self._storage.open(parsed_key) as source:
|
||||||
|
parsed = deserialize_parsed_document(source.read())
|
||||||
|
if parsed.document_id != document_id or parsed.filename != filename:
|
||||||
|
raise ParsedDocumentMismatchError(
|
||||||
|
f"parsed artifact identity does not match document {document_id}"
|
||||||
|
)
|
||||||
|
|
||||||
|
drafts = self._strategy.chunk(parsed)
|
||||||
|
if not drafts:
|
||||||
|
raise EmptyChunkSetError(f"{filename!r} produced no chunks")
|
||||||
|
return self._persist_chunks(document_id, filename, drafts)
|
||||||
|
except Exception as error:
|
||||||
|
self._record_failure(document_id, error)
|
||||||
|
raise
|
||||||
|
|
||||||
|
def _start_chunking(self, document_id: UUID) -> tuple[str, str]:
|
||||||
|
"""Validate prerequisites and guard against in-place strategy replacement."""
|
||||||
|
with self._session_factory() as session:
|
||||||
|
document = session.get(Document, document_id)
|
||||||
|
if document is None:
|
||||||
|
raise DocumentNotFoundError(str(document_id))
|
||||||
|
if document.parsed_storage_key is None:
|
||||||
|
raise DocumentNotParsedError(str(document_id))
|
||||||
|
existing_chunk_id = session.scalar(
|
||||||
|
select(Chunk.chunk_id)
|
||||||
|
.where(
|
||||||
|
Chunk.document_id == document_id,
|
||||||
|
Chunk.strategy_version == self._strategy.version,
|
||||||
|
)
|
||||||
|
.limit(1)
|
||||||
|
)
|
||||||
|
if existing_chunk_id is not None:
|
||||||
|
raise DocumentAlreadyChunkedError(str(document_id))
|
||||||
|
|
||||||
|
document.status = DocumentStatus.PROCESSING
|
||||||
|
document.error_message = None
|
||||||
|
session.commit()
|
||||||
|
return document.filename, document.parsed_storage_key
|
||||||
|
|
||||||
|
def _persist_chunks(
|
||||||
|
self,
|
||||||
|
document_id: UUID,
|
||||||
|
filename: str,
|
||||||
|
drafts: tuple[ChunkDraft, ...],
|
||||||
|
) -> tuple[Chunk, ...]:
|
||||||
|
"""Persist all chunks in one transaction so partial sets are never visible."""
|
||||||
|
chunks: list[Chunk] = []
|
||||||
|
for draft in drafts:
|
||||||
|
content_digest = sha256(draft.content.encode("utf-8")).hexdigest()
|
||||||
|
chunks.append(
|
||||||
|
Chunk(
|
||||||
|
chunk_id=uuid5(
|
||||||
|
document_id,
|
||||||
|
f"{self._strategy.version}:{draft.chunk_index}:{content_digest}",
|
||||||
|
),
|
||||||
|
document_id=document_id,
|
||||||
|
filename=filename,
|
||||||
|
page=draft.page,
|
||||||
|
page_end=draft.page_end,
|
||||||
|
section=draft.section,
|
||||||
|
section_path=list(draft.section_path),
|
||||||
|
chunk_index=draft.chunk_index,
|
||||||
|
content=draft.content,
|
||||||
|
token_count=draft.token_count,
|
||||||
|
strategy_version=self._strategy.version,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
with self._session_factory() as session:
|
||||||
|
document = session.get(Document, document_id)
|
||||||
|
if document is None:
|
||||||
|
raise DocumentNotFoundError(str(document_id))
|
||||||
|
session.add_all(chunks)
|
||||||
|
document.status = DocumentStatus.PROCESSING
|
||||||
|
document.error_message = None
|
||||||
|
session.commit()
|
||||||
|
return tuple(chunks)
|
||||||
|
|
||||||
|
def _record_failure(self, document_id: UUID, error: Exception) -> None:
|
||||||
|
"""Persist a bounded chunking diagnostic after transaction rollback."""
|
||||||
|
try:
|
||||||
|
with self._session_factory() as session:
|
||||||
|
document = session.get(Document, document_id)
|
||||||
|
if document is None:
|
||||||
|
return
|
||||||
|
document.status = DocumentStatus.FAILED
|
||||||
|
message = str(error).strip() or type(error).__name__
|
||||||
|
document.error_message = message[:MAX_CHUNKING_ERROR_LENGTH]
|
||||||
|
session.commit()
|
||||||
|
except Exception as state_error:
|
||||||
|
raise ChunkingStatePersistenceError(
|
||||||
|
f"could not record chunking failure for {document_id}"
|
||||||
|
) from state_error
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class DocumentPipelineResult:
|
||||||
|
"""Current synchronous upload-to-chunk result before embedding is added."""
|
||||||
|
|
||||||
|
document: Document
|
||||||
|
parsed_document: ParsedDocument
|
||||||
|
chunks: tuple[Chunk, ...]
|
||||||
|
|
||||||
|
|
||||||
|
class DocumentPipelineService:
|
||||||
|
"""Compose upload, parse/clean, and chunk stages for the current Demo scope."""
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
upload_service: DocumentUploadService,
|
||||||
|
parsing_service: DocumentParsingService,
|
||||||
|
chunking_service: DocumentChunkingService,
|
||||||
|
) -> None:
|
||||||
|
self._ingestion_service = DocumentIngestionService(upload_service, parsing_service)
|
||||||
|
self._chunking_service = chunking_service
|
||||||
|
|
||||||
|
def ingest(
|
||||||
|
self,
|
||||||
|
filename: str,
|
||||||
|
media_type: str | None,
|
||||||
|
source: BinarySource,
|
||||||
|
) -> DocumentPipelineResult:
|
||||||
|
"""Run every completed Demo ingestion stage in dependency order."""
|
||||||
|
document, parsed = self._ingestion_service.ingest(filename, media_type, source)
|
||||||
|
chunks = self._chunking_service.process(document.id)
|
||||||
|
document.status = DocumentStatus.PROCESSING
|
||||||
|
return DocumentPipelineResult(document=document, parsed_document=parsed, chunks=chunks)
|
||||||
34
packages/core/src/ekp_core/document_cleaner.py
Normal file
34
packages/core/src/ekp_core/document_cleaner.py
Normal file
@ -0,0 +1,34 @@
|
|||||||
|
"""Basic deterministic cleanup for provider-neutral parsed documents."""
|
||||||
|
|
||||||
|
from ekp_domain import ParsedBlock, ParsedDocument
|
||||||
|
|
||||||
|
|
||||||
|
def clean_parsed_document(document: ParsedDocument) -> ParsedDocument:
|
||||||
|
"""Normalize whitespace, remove empty blocks, and clean section labels."""
|
||||||
|
cleaned_blocks: list[ParsedBlock] = []
|
||||||
|
for block in document.blocks:
|
||||||
|
text = _normalize_whitespace(block.text)
|
||||||
|
if not text:
|
||||||
|
continue
|
||||||
|
section_path = tuple(
|
||||||
|
cleaned for section in block.section_path if (cleaned := _normalize_whitespace(section))
|
||||||
|
)
|
||||||
|
cleaned_blocks.append(
|
||||||
|
ParsedBlock(
|
||||||
|
text=text,
|
||||||
|
block_type=block.block_type,
|
||||||
|
page_number=block.page_number,
|
||||||
|
section_path=section_path,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
return ParsedDocument(
|
||||||
|
document_id=document.document_id,
|
||||||
|
filename=document.filename,
|
||||||
|
blocks=tuple(cleaned_blocks),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _normalize_whitespace(value: str) -> str:
|
||||||
|
"""Collapse Unicode whitespace, including abnormal intra-block line breaks."""
|
||||||
|
return " ".join(value.split())
|
||||||
258
packages/core/src/ekp_core/document_indexing.py
Normal file
258
packages/core/src/ekp_core/document_indexing.py
Normal file
@ -0,0 +1,258 @@
|
|||||||
|
"""Embedding and vector-index orchestration for canonical document chunks."""
|
||||||
|
|
||||||
|
from collections.abc import Callable
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from uuid import UUID, uuid5
|
||||||
|
|
||||||
|
from sqlalchemy import select
|
||||||
|
from sqlalchemy.orm import Session, sessionmaker
|
||||||
|
|
||||||
|
from ekp_core.document_chunking import DocumentPipelineResult, DocumentPipelineService
|
||||||
|
from ekp_domain import (
|
||||||
|
Chunk,
|
||||||
|
ChunkIndexState,
|
||||||
|
Document,
|
||||||
|
DocumentStatus,
|
||||||
|
IndexProfile,
|
||||||
|
IngestionStage,
|
||||||
|
)
|
||||||
|
from ekp_embedding import EmbeddingProvider
|
||||||
|
from ekp_object_storage import BinarySource
|
||||||
|
from ekp_vector_store import VectorRecord, VectorStore
|
||||||
|
|
||||||
|
MAX_INDEXING_ERROR_LENGTH = 2000
|
||||||
|
|
||||||
|
|
||||||
|
class DocumentNotFoundError(LookupError):
|
||||||
|
"""Raised when indexing is requested for an unknown document id."""
|
||||||
|
|
||||||
|
|
||||||
|
class DocumentHasNoChunksError(RuntimeError):
|
||||||
|
"""Raised when the selected chunk strategy has no canonical rows."""
|
||||||
|
|
||||||
|
|
||||||
|
class DocumentAlreadyIndexedError(RuntimeError):
|
||||||
|
"""Raised instead of overwriting an existing versioned index profile."""
|
||||||
|
|
||||||
|
|
||||||
|
class EmbeddingResultError(RuntimeError):
|
||||||
|
"""Raised when provider output violates count or dimension contracts."""
|
||||||
|
|
||||||
|
|
||||||
|
class VectorIndexRollbackError(RuntimeError):
|
||||||
|
"""Raised when failed indexing also cannot delete partial vector rows."""
|
||||||
|
|
||||||
|
|
||||||
|
class IndexingStatePersistenceError(RuntimeError):
|
||||||
|
"""Raised when an indexing failure cannot be stored in canonical metadata."""
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class DocumentIndexingResult:
|
||||||
|
"""Summary of one completed versioned index build."""
|
||||||
|
|
||||||
|
document_id: UUID
|
||||||
|
index_profile_id: str
|
||||||
|
indexed_chunks: int
|
||||||
|
|
||||||
|
|
||||||
|
class DocumentIndexingService:
|
||||||
|
"""Embed canonical chunks and upsert a rebuildable VectorStore index."""
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
session_factory: sessionmaker[Session],
|
||||||
|
embedding_provider: EmbeddingProvider,
|
||||||
|
vector_store: VectorStore,
|
||||||
|
index_profile: IndexProfile,
|
||||||
|
) -> None:
|
||||||
|
if embedding_provider.model_name != index_profile.embedding_model:
|
||||||
|
raise ValueError("embedding provider model does not match index profile")
|
||||||
|
if embedding_provider.dimensions != index_profile.vector_dimension:
|
||||||
|
raise ValueError("embedding dimensions do not match index profile")
|
||||||
|
self._session_factory = session_factory
|
||||||
|
self._embedding_provider = embedding_provider
|
||||||
|
self._vector_store = vector_store
|
||||||
|
self._index_profile = index_profile
|
||||||
|
|
||||||
|
def process(
|
||||||
|
self,
|
||||||
|
document_id: UUID,
|
||||||
|
stage_callback: Callable[[IngestionStage], None] | None = None,
|
||||||
|
) -> DocumentIndexingResult:
|
||||||
|
"""Embed/index chunks while optionally reporting durable Job stages."""
|
||||||
|
chunks = self._start_indexing(document_id)
|
||||||
|
upsert_attempted = False
|
||||||
|
try:
|
||||||
|
vectors = self._embed_chunks(chunks, stage_callback)
|
||||||
|
records = tuple(
|
||||||
|
VectorRecord(
|
||||||
|
index_record_id=uuid5(
|
||||||
|
chunk.chunk_id,
|
||||||
|
self._index_profile.profile_id,
|
||||||
|
),
|
||||||
|
chunk_id=chunk.chunk_id,
|
||||||
|
document_id=chunk.document_id,
|
||||||
|
filename=chunk.filename,
|
||||||
|
page=chunk.page,
|
||||||
|
page_end=chunk.page_end,
|
||||||
|
section=chunk.section,
|
||||||
|
chunk_index=chunk.chunk_index,
|
||||||
|
content=chunk.content,
|
||||||
|
embedding=vector,
|
||||||
|
embedding_model=self._index_profile.embedding_model,
|
||||||
|
index_profile_id=self._index_profile.profile_id,
|
||||||
|
)
|
||||||
|
for chunk, vector in zip(chunks, vectors, strict=True)
|
||||||
|
)
|
||||||
|
if stage_callback is not None:
|
||||||
|
stage_callback(IngestionStage.INDEXING)
|
||||||
|
upsert_attempted = True
|
||||||
|
self._vector_store.upsert(records)
|
||||||
|
self._record_success(document_id, chunks)
|
||||||
|
return DocumentIndexingResult(
|
||||||
|
document_id=document_id,
|
||||||
|
index_profile_id=self._index_profile.profile_id,
|
||||||
|
indexed_chunks=len(chunks),
|
||||||
|
)
|
||||||
|
except Exception as error:
|
||||||
|
cleanup_error: Exception | None = None
|
||||||
|
if upsert_attempted:
|
||||||
|
try:
|
||||||
|
self._vector_store.delete_document(
|
||||||
|
document_id,
|
||||||
|
self._index_profile.profile_id,
|
||||||
|
)
|
||||||
|
except Exception as caught_cleanup_error:
|
||||||
|
cleanup_error = caught_cleanup_error
|
||||||
|
self._record_failure(document_id, error)
|
||||||
|
if cleanup_error is not None:
|
||||||
|
raise VectorIndexRollbackError(
|
||||||
|
f"indexing failed and vector cleanup failed for {document_id}"
|
||||||
|
) from cleanup_error
|
||||||
|
raise
|
||||||
|
|
||||||
|
def _start_indexing(self, document_id: UUID) -> tuple[Chunk, ...]:
|
||||||
|
"""Load the selected chunk version and reject in-place profile replacement."""
|
||||||
|
with self._session_factory() as session:
|
||||||
|
document = session.get(Document, document_id)
|
||||||
|
if document is None:
|
||||||
|
raise DocumentNotFoundError(str(document_id))
|
||||||
|
chunks = tuple(
|
||||||
|
session.scalars(
|
||||||
|
select(Chunk)
|
||||||
|
.where(
|
||||||
|
Chunk.document_id == document_id,
|
||||||
|
Chunk.strategy_version == self._index_profile.chunk_strategy_version,
|
||||||
|
)
|
||||||
|
.order_by(Chunk.chunk_index)
|
||||||
|
).all()
|
||||||
|
)
|
||||||
|
if not chunks:
|
||||||
|
raise DocumentHasNoChunksError(str(document_id))
|
||||||
|
chunk_ids = [chunk.chunk_id for chunk in chunks]
|
||||||
|
existing_state = session.scalar(
|
||||||
|
select(ChunkIndexState.chunk_id)
|
||||||
|
.where(
|
||||||
|
ChunkIndexState.index_profile_id == self._index_profile.profile_id,
|
||||||
|
ChunkIndexState.chunk_id.in_(chunk_ids),
|
||||||
|
)
|
||||||
|
.limit(1)
|
||||||
|
)
|
||||||
|
if existing_state is not None:
|
||||||
|
raise DocumentAlreadyIndexedError(str(document_id))
|
||||||
|
document.status = DocumentStatus.PROCESSING
|
||||||
|
document.error_message = None
|
||||||
|
session.commit()
|
||||||
|
return chunks
|
||||||
|
|
||||||
|
def _embed_chunks(
|
||||||
|
self,
|
||||||
|
chunks: tuple[Chunk, ...],
|
||||||
|
stage_callback: Callable[[IngestionStage], None] | None,
|
||||||
|
) -> tuple[tuple[float, ...], ...]:
|
||||||
|
"""Embed bounded batches and renew an optional Job lease before each call."""
|
||||||
|
vectors: list[tuple[float, ...]] = []
|
||||||
|
batch_size = self._embedding_provider.max_batch_size
|
||||||
|
if batch_size <= 0:
|
||||||
|
raise EmbeddingResultError("embedding provider has an invalid batch size")
|
||||||
|
for start in range(0, len(chunks), batch_size):
|
||||||
|
if stage_callback is not None:
|
||||||
|
stage_callback(IngestionStage.EMBEDDING)
|
||||||
|
batch = chunks[start : start + batch_size]
|
||||||
|
embedded = self._embedding_provider.embed([chunk.content for chunk in batch])
|
||||||
|
if len(embedded) != len(batch):
|
||||||
|
raise EmbeddingResultError("embedding provider returned the wrong vector count")
|
||||||
|
if any(len(vector) != self._index_profile.vector_dimension for vector in embedded):
|
||||||
|
raise EmbeddingResultError("embedding provider returned the wrong dimensions")
|
||||||
|
vectors.extend(embedded)
|
||||||
|
return tuple(vectors)
|
||||||
|
|
||||||
|
def _record_success(self, document_id: UUID, chunks: tuple[Chunk, ...]) -> None:
|
||||||
|
"""Atomically record every indexed chunk before marking the document ready."""
|
||||||
|
with self._session_factory() as session:
|
||||||
|
document = session.get(Document, document_id)
|
||||||
|
if document is None:
|
||||||
|
raise DocumentNotFoundError(str(document_id))
|
||||||
|
session.add_all(
|
||||||
|
[
|
||||||
|
ChunkIndexState(
|
||||||
|
chunk_id=chunk.chunk_id,
|
||||||
|
index_profile_id=self._index_profile.profile_id,
|
||||||
|
embedding_model=self._index_profile.embedding_model,
|
||||||
|
vector_dimension=self._index_profile.vector_dimension,
|
||||||
|
)
|
||||||
|
for chunk in chunks
|
||||||
|
]
|
||||||
|
)
|
||||||
|
document.status = DocumentStatus.READY
|
||||||
|
document.error_message = None
|
||||||
|
session.commit()
|
||||||
|
|
||||||
|
def _record_failure(self, document_id: UUID, error: Exception) -> None:
|
||||||
|
"""Store a bounded error without exposing credentials or request bodies."""
|
||||||
|
try:
|
||||||
|
with self._session_factory() as session:
|
||||||
|
document = session.get(Document, document_id)
|
||||||
|
if document is None:
|
||||||
|
return
|
||||||
|
document.status = DocumentStatus.FAILED
|
||||||
|
message = str(error).strip() or type(error).__name__
|
||||||
|
document.error_message = message[:MAX_INDEXING_ERROR_LENGTH]
|
||||||
|
session.commit()
|
||||||
|
except Exception as state_error:
|
||||||
|
raise IndexingStatePersistenceError(
|
||||||
|
f"could not record indexing failure for {document_id}"
|
||||||
|
) from state_error
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class IndexedDocumentPipelineResult:
|
||||||
|
"""Upload-to-index result for all Demo stages completed through DEMO-006."""
|
||||||
|
|
||||||
|
ingestion: DocumentPipelineResult
|
||||||
|
indexing: DocumentIndexingResult
|
||||||
|
|
||||||
|
|
||||||
|
class DocumentIndexingPipelineService:
|
||||||
|
"""Compose upload, parsing, chunking, embedding, and vector upsert."""
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
document_pipeline: DocumentPipelineService,
|
||||||
|
indexing_service: DocumentIndexingService,
|
||||||
|
) -> None:
|
||||||
|
self._document_pipeline = document_pipeline
|
||||||
|
self._indexing_service = indexing_service
|
||||||
|
|
||||||
|
def ingest(
|
||||||
|
self,
|
||||||
|
filename: str,
|
||||||
|
media_type: str | None,
|
||||||
|
source: BinarySource,
|
||||||
|
) -> IndexedDocumentPipelineResult:
|
||||||
|
"""Run the complete current ingestion pipeline and return trace metadata."""
|
||||||
|
ingestion = self._document_pipeline.ingest(filename, media_type, source)
|
||||||
|
indexing = self._indexing_service.process(ingestion.document.id)
|
||||||
|
ingestion.document.status = DocumentStatus.READY
|
||||||
|
return IndexedDocumentPipelineResult(ingestion=ingestion, indexing=indexing)
|
||||||
119
packages/core/src/ekp_core/document_management.py
Normal file
119
packages/core/src/ekp_core/document_management.py
Normal file
@ -0,0 +1,119 @@
|
|||||||
|
"""Document query and direct-delete services for the Demo REST API."""
|
||||||
|
|
||||||
|
from uuid import UUID
|
||||||
|
|
||||||
|
from sqlalchemy import delete, select
|
||||||
|
from sqlalchemy.orm import Session, sessionmaker
|
||||||
|
|
||||||
|
from ekp_domain import Chunk, ChunkIndexState, Document, IngestionJob, IngestionJobStatus
|
||||||
|
from ekp_object_storage import ObjectStorage
|
||||||
|
from ekp_vector_store import VectorStore
|
||||||
|
|
||||||
|
|
||||||
|
class DocumentNotFoundError(LookupError):
|
||||||
|
"""Raised when a requested document does not exist."""
|
||||||
|
|
||||||
|
|
||||||
|
class DocumentArtifactCleanupError(RuntimeError):
|
||||||
|
"""Raised when deleted metadata leaves an external storage artifact behind."""
|
||||||
|
|
||||||
|
|
||||||
|
class DocumentIngestionActiveError(RuntimeError):
|
||||||
|
"""Raised when direct deletion would race an active asynchronous Job."""
|
||||||
|
|
||||||
|
|
||||||
|
class DocumentManagementService:
|
||||||
|
"""Read canonical documents and coordinate the Demo's direct deletion.
|
||||||
|
|
||||||
|
Vector rows are removed before canonical metadata. Object bytes are removed
|
||||||
|
after the metadata commit, so a storage failure can only leave an unreachable
|
||||||
|
orphan rather than expose a searchable row without canonical provenance.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
session_factory: sessionmaker[Session],
|
||||||
|
storage: ObjectStorage,
|
||||||
|
vector_store: VectorStore,
|
||||||
|
active_index_profile_id: str,
|
||||||
|
) -> None:
|
||||||
|
if not active_index_profile_id:
|
||||||
|
raise ValueError("active_index_profile_id must not be empty")
|
||||||
|
self._session_factory = session_factory
|
||||||
|
self._storage = storage
|
||||||
|
self._vector_store = vector_store
|
||||||
|
self._active_index_profile_id = active_index_profile_id
|
||||||
|
|
||||||
|
def list_documents(self) -> tuple[Document, ...]:
|
||||||
|
"""Return documents newest first with a deterministic UUID tie-break."""
|
||||||
|
with self._session_factory() as session:
|
||||||
|
return tuple(
|
||||||
|
session.scalars(
|
||||||
|
select(Document).order_by(Document.created_at.desc(), Document.id.desc())
|
||||||
|
).all()
|
||||||
|
)
|
||||||
|
|
||||||
|
def get_document(self, document_id: UUID) -> Document:
|
||||||
|
"""Return one canonical document or raise a stable not-found error."""
|
||||||
|
with self._session_factory() as session:
|
||||||
|
document = session.get(Document, document_id)
|
||||||
|
if document is None:
|
||||||
|
raise DocumentNotFoundError(str(document_id))
|
||||||
|
return document
|
||||||
|
|
||||||
|
def delete_document(self, document_id: UUID) -> None:
|
||||||
|
"""Delete vector rows, canonical metadata, and immutable source artifacts."""
|
||||||
|
with self._session_factory() as session:
|
||||||
|
document = session.get(Document, document_id)
|
||||||
|
if document is None:
|
||||||
|
raise DocumentNotFoundError(str(document_id))
|
||||||
|
active_job_id = session.scalar(
|
||||||
|
select(IngestionJob.id)
|
||||||
|
.where(
|
||||||
|
IngestionJob.document_id == document_id,
|
||||||
|
IngestionJob.status.in_(
|
||||||
|
(
|
||||||
|
IngestionJobStatus.QUEUED,
|
||||||
|
IngestionJobStatus.RUNNING,
|
||||||
|
IngestionJobStatus.RETRYING,
|
||||||
|
)
|
||||||
|
),
|
||||||
|
)
|
||||||
|
.limit(1)
|
||||||
|
)
|
||||||
|
if active_job_id is not None:
|
||||||
|
raise DocumentIngestionActiveError(str(document_id))
|
||||||
|
|
||||||
|
source_keys = tuple(
|
||||||
|
key
|
||||||
|
for key in (document.parsed_storage_key, document.storage_key)
|
||||||
|
if key is not None
|
||||||
|
)
|
||||||
|
profile_ids = set(
|
||||||
|
session.scalars(
|
||||||
|
select(ChunkIndexState.index_profile_id)
|
||||||
|
.join(Chunk, Chunk.chunk_id == ChunkIndexState.chunk_id)
|
||||||
|
.where(Chunk.document_id == document_id)
|
||||||
|
.distinct()
|
||||||
|
).all()
|
||||||
|
)
|
||||||
|
# Also clean the active profile when an interrupted index write did
|
||||||
|
# not reach canonical ChunkIndexState persistence.
|
||||||
|
profile_ids.add(self._active_index_profile_id)
|
||||||
|
for profile_id in sorted(profile_ids):
|
||||||
|
self._vector_store.delete_document(document_id, profile_id)
|
||||||
|
|
||||||
|
chunk_ids = select(Chunk.chunk_id).where(Chunk.document_id == document_id)
|
||||||
|
session.execute(delete(ChunkIndexState).where(ChunkIndexState.chunk_id.in_(chunk_ids)))
|
||||||
|
session.execute(delete(Chunk).where(Chunk.document_id == document_id))
|
||||||
|
session.execute(delete(Document).where(Document.id == document_id))
|
||||||
|
session.commit()
|
||||||
|
|
||||||
|
try:
|
||||||
|
for source_key in source_keys:
|
||||||
|
if self._storage.exists(source_key):
|
||||||
|
self._storage.delete(source_key)
|
||||||
|
except Exception as error:
|
||||||
|
raise DocumentArtifactCleanupError(
|
||||||
|
f"external artifact cleanup failed for document {document_id}"
|
||||||
|
) from error
|
||||||
159
packages/core/src/ekp_core/document_parsing.py
Normal file
159
packages/core/src/ekp_core/document_parsing.py
Normal file
@ -0,0 +1,159 @@
|
|||||||
|
"""Application services for parsing and persisting derived document content."""
|
||||||
|
|
||||||
|
from collections.abc import Callable
|
||||||
|
from io import BytesIO
|
||||||
|
from uuid import UUID
|
||||||
|
|
||||||
|
from sqlalchemy.orm import Session, sessionmaker
|
||||||
|
|
||||||
|
from ekp_core.document_cleaner import clean_parsed_document
|
||||||
|
from ekp_core.document_upload import DocumentUploadService
|
||||||
|
from ekp_core.parsed_document_artifact import serialize_parsed_document
|
||||||
|
from ekp_domain import Document, DocumentStatus, IngestionStage, ParsedDocument
|
||||||
|
from ekp_object_storage import BinarySource, ObjectStorage
|
||||||
|
from ekp_parser import DocumentParseError, Parser
|
||||||
|
|
||||||
|
MAX_ERROR_MESSAGE_LENGTH = 2000
|
||||||
|
|
||||||
|
|
||||||
|
class DocumentNotFoundError(LookupError):
|
||||||
|
"""Raised when parsing is requested for an unknown document id."""
|
||||||
|
|
||||||
|
|
||||||
|
class DocumentAlreadyParsedError(RuntimeError):
|
||||||
|
"""Raised when immutable parsed output already exists for a document."""
|
||||||
|
|
||||||
|
|
||||||
|
class EmptyParsedDocumentError(DocumentParseError):
|
||||||
|
"""Raised when a source contains no extractable text after cleanup."""
|
||||||
|
|
||||||
|
|
||||||
|
class ParsedResultRollbackError(RuntimeError):
|
||||||
|
"""Raised when failed metadata persistence also leaves an undeletable object."""
|
||||||
|
|
||||||
|
|
||||||
|
class ParsingStatePersistenceError(RuntimeError):
|
||||||
|
"""Raised when a parsing failure cannot be reflected in document metadata."""
|
||||||
|
|
||||||
|
|
||||||
|
class DocumentParsingService:
|
||||||
|
"""Parse one uploaded source and persist a traceable, rebuildable JSON result."""
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
session_factory: sessionmaker[Session],
|
||||||
|
storage: ObjectStorage,
|
||||||
|
parser: Parser,
|
||||||
|
) -> None:
|
||||||
|
self._session_factory = session_factory
|
||||||
|
self._storage = storage
|
||||||
|
self._parser = parser
|
||||||
|
|
||||||
|
def process(
|
||||||
|
self,
|
||||||
|
document_id: UUID,
|
||||||
|
stage_callback: Callable[[IngestionStage], None] | None = None,
|
||||||
|
) -> ParsedDocument:
|
||||||
|
"""Parse and clean while optionally reporting durable Job stage boundaries."""
|
||||||
|
filename, source_key = self._start_processing(document_id)
|
||||||
|
parsed_key = f"parsed/{document_id}/document.json"
|
||||||
|
parsed_object_stored = False
|
||||||
|
|
||||||
|
try:
|
||||||
|
if stage_callback is not None:
|
||||||
|
stage_callback(IngestionStage.PARSING)
|
||||||
|
with self._storage.open(source_key) as source:
|
||||||
|
parsed = self._parser.parse(document_id, filename, source)
|
||||||
|
if stage_callback is not None:
|
||||||
|
stage_callback(IngestionStage.CLEANING)
|
||||||
|
cleaned = clean_parsed_document(parsed)
|
||||||
|
if not cleaned.blocks:
|
||||||
|
raise EmptyParsedDocumentError(
|
||||||
|
f"{filename!r} contains no extractable text after cleanup"
|
||||||
|
)
|
||||||
|
|
||||||
|
payload = serialize_parsed_document(cleaned)
|
||||||
|
self._storage.put(parsed_key, BytesIO(payload))
|
||||||
|
parsed_object_stored = True
|
||||||
|
self._record_success(document_id, parsed_key)
|
||||||
|
return cleaned
|
||||||
|
except Exception as error:
|
||||||
|
cleanup_error: Exception | None = None
|
||||||
|
if parsed_object_stored:
|
||||||
|
try:
|
||||||
|
self._storage.delete(parsed_key)
|
||||||
|
except Exception as caught_cleanup_error:
|
||||||
|
cleanup_error = caught_cleanup_error
|
||||||
|
self._record_failure(document_id, error)
|
||||||
|
if cleanup_error is not None:
|
||||||
|
raise ParsedResultRollbackError(
|
||||||
|
f"parsing failed and parsed-result cleanup failed for {parsed_key}"
|
||||||
|
) from cleanup_error
|
||||||
|
raise
|
||||||
|
|
||||||
|
def _start_processing(self, document_id: UUID) -> tuple[str, str]:
|
||||||
|
"""Validate state and durably mark a document as being processed."""
|
||||||
|
with self._session_factory() as session:
|
||||||
|
document = session.get(Document, document_id)
|
||||||
|
if document is None:
|
||||||
|
raise DocumentNotFoundError(str(document_id))
|
||||||
|
if document.parsed_storage_key is not None:
|
||||||
|
raise DocumentAlreadyParsedError(str(document_id))
|
||||||
|
|
||||||
|
document.status = DocumentStatus.PROCESSING
|
||||||
|
document.error_message = None
|
||||||
|
session.commit()
|
||||||
|
return document.filename, document.storage_key
|
||||||
|
|
||||||
|
def _record_success(self, document_id: UUID, parsed_key: str) -> None:
|
||||||
|
"""Attach parsed output while later pipeline stages remain processing."""
|
||||||
|
with self._session_factory() as session:
|
||||||
|
document = session.get(Document, document_id)
|
||||||
|
if document is None:
|
||||||
|
raise DocumentNotFoundError(str(document_id))
|
||||||
|
document.parsed_storage_key = parsed_key
|
||||||
|
# READY is reserved for completion of chunking and indexing.
|
||||||
|
document.status = DocumentStatus.PROCESSING
|
||||||
|
document.error_message = None
|
||||||
|
session.commit()
|
||||||
|
|
||||||
|
def _record_failure(self, document_id: UUID, error: Exception) -> None:
|
||||||
|
"""Persist a bounded diagnostic while preserving the original exception."""
|
||||||
|
try:
|
||||||
|
with self._session_factory() as session:
|
||||||
|
document = session.get(Document, document_id)
|
||||||
|
if document is None:
|
||||||
|
return
|
||||||
|
document.status = DocumentStatus.FAILED
|
||||||
|
message = str(error).strip() or type(error).__name__
|
||||||
|
document.error_message = message[:MAX_ERROR_MESSAGE_LENGTH]
|
||||||
|
session.commit()
|
||||||
|
except Exception as state_error:
|
||||||
|
raise ParsingStatePersistenceError(
|
||||||
|
f"could not record parsing failure for {document_id}"
|
||||||
|
) from state_error
|
||||||
|
|
||||||
|
|
||||||
|
class DocumentIngestionService:
|
||||||
|
"""Compose upload and synchronous parsing for the Demo ingestion trigger."""
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
upload_service: DocumentUploadService,
|
||||||
|
parsing_service: DocumentParsingService,
|
||||||
|
) -> None:
|
||||||
|
self._upload_service = upload_service
|
||||||
|
self._parsing_service = parsing_service
|
||||||
|
|
||||||
|
def ingest(
|
||||||
|
self,
|
||||||
|
filename: str,
|
||||||
|
media_type: str | None,
|
||||||
|
source: BinarySource,
|
||||||
|
) -> tuple[Document, ParsedDocument]:
|
||||||
|
"""Persist an upload, then immediately trigger the parsing stage."""
|
||||||
|
document = self._upload_service.upload(filename, media_type, source)
|
||||||
|
parsed = self._parsing_service.process(document.id)
|
||||||
|
document.status = DocumentStatus.PROCESSING
|
||||||
|
document.parsed_storage_key = f"parsed/{document.id}/document.json"
|
||||||
|
return document, parsed
|
||||||
184
packages/core/src/ekp_core/document_upload.py
Normal file
184
packages/core/src/ekp_core/document_upload.py
Normal file
@ -0,0 +1,184 @@
|
|||||||
|
"""Validated, transactional source-document upload application service."""
|
||||||
|
|
||||||
|
from pathlib import Path
|
||||||
|
from uuid import UUID, uuid4
|
||||||
|
|
||||||
|
from sqlalchemy import delete
|
||||||
|
from sqlalchemy.orm import Session, sessionmaker
|
||||||
|
|
||||||
|
from ekp_domain import Document
|
||||||
|
from ekp_object_storage import BinarySource, ObjectStorage
|
||||||
|
|
||||||
|
GENERIC_MEDIA_TYPE = "application/octet-stream"
|
||||||
|
SUPPORTED_MEDIA_TYPES = {
|
||||||
|
".pdf": ("application/pdf", frozenset({"application/pdf", GENERIC_MEDIA_TYPE})),
|
||||||
|
".docx": (
|
||||||
|
"application/vnd.openxmlformats-officedocument.wordprocessingml.document",
|
||||||
|
frozenset(
|
||||||
|
{
|
||||||
|
"application/vnd.openxmlformats-officedocument.wordprocessingml.document",
|
||||||
|
GENERIC_MEDIA_TYPE,
|
||||||
|
}
|
||||||
|
),
|
||||||
|
),
|
||||||
|
".txt": ("text/plain", frozenset({"text/plain", GENERIC_MEDIA_TYPE})),
|
||||||
|
".md": (
|
||||||
|
"text/markdown",
|
||||||
|
frozenset({"text/markdown", "text/plain", GENERIC_MEDIA_TYPE}),
|
||||||
|
),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
class DocumentUploadError(ValueError):
|
||||||
|
"""Base error for rejected source-document uploads."""
|
||||||
|
|
||||||
|
|
||||||
|
class InvalidDocumentFilenameError(DocumentUploadError):
|
||||||
|
"""Raised when a filename is blank, unsafe, or too long."""
|
||||||
|
|
||||||
|
|
||||||
|
class UnsupportedDocumentTypeError(DocumentUploadError):
|
||||||
|
"""Raised when a filename extension is outside the Demo scope."""
|
||||||
|
|
||||||
|
|
||||||
|
class MediaTypeMismatchError(DocumentUploadError):
|
||||||
|
"""Raised when the declared media type conflicts with the extension."""
|
||||||
|
|
||||||
|
|
||||||
|
class EmptyDocumentError(DocumentUploadError):
|
||||||
|
"""Raised when an uploaded source contains no bytes."""
|
||||||
|
|
||||||
|
|
||||||
|
class DocumentTooLargeError(DocumentUploadError):
|
||||||
|
"""Raised when a source exceeds the configured streaming limit."""
|
||||||
|
|
||||||
|
|
||||||
|
class UploadRollbackError(RuntimeError):
|
||||||
|
"""Raised when database failure compensation cannot remove source bytes."""
|
||||||
|
|
||||||
|
|
||||||
|
class _SizeLimitedSource:
|
||||||
|
"""Count streamed bytes and stop before an oversized object is committed."""
|
||||||
|
|
||||||
|
def __init__(self, source: BinarySource, limit_bytes: int) -> None:
|
||||||
|
self._source = source
|
||||||
|
self._limit_bytes = limit_bytes
|
||||||
|
self._consumed_bytes = 0
|
||||||
|
|
||||||
|
def read(self, size: int = -1, /) -> bytes:
|
||||||
|
remaining_with_probe = self._limit_bytes - self._consumed_bytes + 1
|
||||||
|
requested = remaining_with_probe if size < 0 else min(size, remaining_with_probe)
|
||||||
|
chunk = self._source.read(requested)
|
||||||
|
self._consumed_bytes += len(chunk)
|
||||||
|
if self._consumed_bytes > self._limit_bytes:
|
||||||
|
raise DocumentTooLargeError(
|
||||||
|
f"document exceeds the {self._limit_bytes}-byte upload limit"
|
||||||
|
)
|
||||||
|
return chunk
|
||||||
|
|
||||||
|
|
||||||
|
class DocumentUploadService:
|
||||||
|
"""Persist immutable source bytes and their canonical Document metadata."""
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
session_factory: sessionmaker[Session],
|
||||||
|
storage: ObjectStorage,
|
||||||
|
max_upload_bytes: int,
|
||||||
|
) -> None:
|
||||||
|
if max_upload_bytes <= 0:
|
||||||
|
raise ValueError("max_upload_bytes must be positive")
|
||||||
|
self._session_factory = session_factory
|
||||||
|
self._storage = storage
|
||||||
|
self._max_upload_bytes = max_upload_bytes
|
||||||
|
|
||||||
|
def upload(
|
||||||
|
self,
|
||||||
|
filename: str,
|
||||||
|
media_type: str | None,
|
||||||
|
source: BinarySource,
|
||||||
|
) -> Document:
|
||||||
|
"""Validate and persist one source, compensating if metadata commit fails."""
|
||||||
|
safe_filename = _validate_filename(filename)
|
||||||
|
canonical_media_type = _resolve_media_type(safe_filename, media_type)
|
||||||
|
document_id = uuid4()
|
||||||
|
storage_key = f"documents/{document_id}/{safe_filename}"
|
||||||
|
limited_source = _SizeLimitedSource(source, self._max_upload_bytes)
|
||||||
|
|
||||||
|
with self._session_factory() as session:
|
||||||
|
stored = self._storage.put(storage_key, limited_source)
|
||||||
|
if stored.size_bytes == 0:
|
||||||
|
self._storage.delete(stored.key)
|
||||||
|
raise EmptyDocumentError("document must contain at least one byte")
|
||||||
|
|
||||||
|
try:
|
||||||
|
document = Document(
|
||||||
|
id=document_id,
|
||||||
|
filename=safe_filename,
|
||||||
|
media_type=canonical_media_type,
|
||||||
|
size_bytes=stored.size_bytes,
|
||||||
|
storage_key=stored.key,
|
||||||
|
)
|
||||||
|
session.add(document)
|
||||||
|
session.commit()
|
||||||
|
except Exception:
|
||||||
|
session.rollback()
|
||||||
|
try:
|
||||||
|
self._storage.delete(stored.key)
|
||||||
|
except Exception as cleanup_error:
|
||||||
|
raise UploadRollbackError(
|
||||||
|
f"metadata failed and source cleanup failed for {stored.key}"
|
||||||
|
) from cleanup_error
|
||||||
|
raise
|
||||||
|
|
||||||
|
return document
|
||||||
|
|
||||||
|
def rollback_acceptance(self, document_id: UUID) -> None:
|
||||||
|
"""Compensate a just-uploaded source when durable Job creation fails.
|
||||||
|
|
||||||
|
This narrow method is only valid before parsing starts; normal document
|
||||||
|
deletion remains coordinated by ``DocumentManagementService``.
|
||||||
|
"""
|
||||||
|
with self._session_factory() as session:
|
||||||
|
document = session.get(Document, document_id)
|
||||||
|
if document is None:
|
||||||
|
return
|
||||||
|
storage_key = document.storage_key
|
||||||
|
session.execute(delete(Document).where(Document.id == document.id))
|
||||||
|
session.commit()
|
||||||
|
try:
|
||||||
|
self._storage.delete(storage_key)
|
||||||
|
except Exception as cleanup_error:
|
||||||
|
raise UploadRollbackError(
|
||||||
|
f"job creation failed and source cleanup failed for {storage_key}"
|
||||||
|
) from cleanup_error
|
||||||
|
|
||||||
|
|
||||||
|
def _validate_filename(filename: str) -> str:
|
||||||
|
"""Accept a plain basename that can be traced without path ambiguity."""
|
||||||
|
if (
|
||||||
|
not filename.strip()
|
||||||
|
or len(filename) > 255
|
||||||
|
or "\x00" in filename
|
||||||
|
or "/" in filename
|
||||||
|
or "\\" in filename
|
||||||
|
):
|
||||||
|
raise InvalidDocumentFilenameError(filename)
|
||||||
|
return filename
|
||||||
|
|
||||||
|
|
||||||
|
def _resolve_media_type(filename: str, media_type: str | None) -> str:
|
||||||
|
"""Return the canonical media type after extension and declaration checks."""
|
||||||
|
# File signatures and actual parseability are validated by the DEMO-004 parser.
|
||||||
|
extension = Path(filename).suffix.lower()
|
||||||
|
type_config = SUPPORTED_MEDIA_TYPES.get(extension)
|
||||||
|
if type_config is None:
|
||||||
|
raise UnsupportedDocumentTypeError(extension or "<missing>")
|
||||||
|
|
||||||
|
canonical_media_type, accepted_media_types = type_config
|
||||||
|
declared_media_type = (media_type or GENERIC_MEDIA_TYPE).partition(";")[0].strip().lower()
|
||||||
|
if declared_media_type not in accepted_media_types:
|
||||||
|
raise MediaTypeMismatchError(
|
||||||
|
f"media type {declared_media_type!r} does not match extension {extension!r}"
|
||||||
|
)
|
||||||
|
return canonical_media_type
|
||||||
279
packages/core/src/ekp_core/index_rebuild.py
Normal file
279
packages/core/src/ekp_core/index_rebuild.py
Normal file
@ -0,0 +1,279 @@
|
|||||||
|
"""Retry-safe shadow index rebuild, verification, and activation services."""
|
||||||
|
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from datetime import UTC, datetime
|
||||||
|
from uuid import UUID
|
||||||
|
|
||||||
|
from sqlalchemy import delete, select
|
||||||
|
from sqlalchemy.orm import Session, sessionmaker
|
||||||
|
|
||||||
|
from ekp_core.document_indexing import DocumentIndexingService
|
||||||
|
from ekp_domain import (
|
||||||
|
Chunk,
|
||||||
|
ChunkIndexState,
|
||||||
|
IndexProfile,
|
||||||
|
IndexProfileStatus,
|
||||||
|
StoredIndexProfile,
|
||||||
|
)
|
||||||
|
from ekp_embedding import EmbeddingProvider
|
||||||
|
from ekp_vector_store import VectorStore
|
||||||
|
|
||||||
|
|
||||||
|
class IndexProfileLifecycleError(RuntimeError):
|
||||||
|
"""Raised when a shadow profile cannot safely move through its lifecycle."""
|
||||||
|
|
||||||
|
|
||||||
|
class IndexProfileMetadataConflictError(IndexProfileLifecycleError):
|
||||||
|
"""Raised when a profile ID is reused with different immutable metadata."""
|
||||||
|
|
||||||
|
|
||||||
|
class IndexProfileNotFoundError(IndexProfileLifecycleError):
|
||||||
|
"""Raised when verification or activation targets an unknown profile."""
|
||||||
|
|
||||||
|
|
||||||
|
class IndexProfileIncompleteError(IndexProfileLifecycleError):
|
||||||
|
"""Raised when activation is attempted before every canonical chunk is indexed."""
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class IndexBuildSummary:
|
||||||
|
"""Safe progress counters for rebuild commands and deployment checks."""
|
||||||
|
|
||||||
|
profile_id: str
|
||||||
|
expected_documents: int
|
||||||
|
built_documents: int
|
||||||
|
skipped_documents: int
|
||||||
|
expected_chunks: int
|
||||||
|
indexed_chunks: int
|
||||||
|
complete: bool
|
||||||
|
|
||||||
|
|
||||||
|
class IndexRebuildService:
|
||||||
|
"""Build a new profile from canonical chunks without mutating an active index."""
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
session_factory: sessionmaker[Session],
|
||||||
|
embedding_provider: EmbeddingProvider,
|
||||||
|
vector_store: VectorStore,
|
||||||
|
target_profile: IndexProfile,
|
||||||
|
) -> None:
|
||||||
|
self._session_factory = session_factory
|
||||||
|
self._vector_store = vector_store
|
||||||
|
self._target_profile = target_profile
|
||||||
|
self._indexing = DocumentIndexingService(
|
||||||
|
session_factory,
|
||||||
|
embedding_provider,
|
||||||
|
vector_store,
|
||||||
|
target_profile,
|
||||||
|
)
|
||||||
|
|
||||||
|
def rebuild(self) -> IndexBuildSummary:
|
||||||
|
"""Register and populate a shadow profile, skipping verified documents on retry."""
|
||||||
|
self._register_shadow()
|
||||||
|
document_ids = self._target_document_ids()
|
||||||
|
built_documents = 0
|
||||||
|
skipped_documents = 0
|
||||||
|
for document_id in document_ids:
|
||||||
|
if self._document_is_complete(document_id):
|
||||||
|
skipped_documents += 1
|
||||||
|
continue
|
||||||
|
self._reset_partial_document(document_id)
|
||||||
|
self._indexing.process(document_id)
|
||||||
|
built_documents += 1
|
||||||
|
|
||||||
|
verified = self.verify()
|
||||||
|
return IndexBuildSummary(
|
||||||
|
profile_id=verified.profile_id,
|
||||||
|
expected_documents=verified.expected_documents,
|
||||||
|
built_documents=built_documents,
|
||||||
|
skipped_documents=skipped_documents,
|
||||||
|
expected_chunks=verified.expected_chunks,
|
||||||
|
indexed_chunks=verified.indexed_chunks,
|
||||||
|
complete=verified.complete,
|
||||||
|
)
|
||||||
|
|
||||||
|
def verify(self) -> IndexBuildSummary:
|
||||||
|
"""Compare canonical chunks and successful index states for the target profile."""
|
||||||
|
with self._session_factory() as session:
|
||||||
|
self._require_registered_profile(session)
|
||||||
|
return self._verification_summary(session)
|
||||||
|
|
||||||
|
def activate(self) -> IndexBuildSummary:
|
||||||
|
"""Atomically retire the previous profile after target completeness is proven."""
|
||||||
|
with self._session_factory() as session:
|
||||||
|
target = self._require_registered_profile(session, for_update=True)
|
||||||
|
self._lock_canonical_chunks(session)
|
||||||
|
summary = self._verification_summary(session)
|
||||||
|
if not summary.complete:
|
||||||
|
raise IndexProfileIncompleteError(
|
||||||
|
"shadow index is incomplete and cannot be activated"
|
||||||
|
)
|
||||||
|
if target.status is IndexProfileStatus.ACTIVE:
|
||||||
|
return summary
|
||||||
|
if target.status is not IndexProfileStatus.SHADOW:
|
||||||
|
raise IndexProfileLifecycleError("only a shadow profile can be activated")
|
||||||
|
|
||||||
|
active_profiles = session.scalars(
|
||||||
|
select(StoredIndexProfile)
|
||||||
|
.where(StoredIndexProfile.status == IndexProfileStatus.ACTIVE)
|
||||||
|
.with_for_update()
|
||||||
|
).all()
|
||||||
|
for active in active_profiles:
|
||||||
|
active.status = IndexProfileStatus.RETIRED
|
||||||
|
target.status = IndexProfileStatus.ACTIVE
|
||||||
|
target.activated_at = datetime.now(UTC)
|
||||||
|
session.commit()
|
||||||
|
return summary
|
||||||
|
|
||||||
|
def _register_shadow(self) -> None:
|
||||||
|
"""Create an idempotent shadow registration without reusing active IDs."""
|
||||||
|
with self._session_factory() as session:
|
||||||
|
stored = session.get(StoredIndexProfile, self._target_profile.profile_id)
|
||||||
|
if stored is None:
|
||||||
|
session.add(
|
||||||
|
StoredIndexProfile(
|
||||||
|
profile_id=self._target_profile.profile_id,
|
||||||
|
chunk_strategy_version=self._target_profile.chunk_strategy_version,
|
||||||
|
embedding_model=self._target_profile.embedding_model,
|
||||||
|
vector_dimension=self._target_profile.vector_dimension,
|
||||||
|
status=IndexProfileStatus.SHADOW,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
session.commit()
|
||||||
|
return
|
||||||
|
self._validate_stored_metadata(stored)
|
||||||
|
if stored.status is not IndexProfileStatus.SHADOW:
|
||||||
|
raise IndexProfileLifecycleError(
|
||||||
|
"rebuild target must be a shadow profile with a new profile ID"
|
||||||
|
)
|
||||||
|
|
||||||
|
def _target_document_ids(self) -> tuple[UUID, ...]:
|
||||||
|
"""List documents that have canonical chunks for the target strategy."""
|
||||||
|
with self._session_factory() as session:
|
||||||
|
return tuple(
|
||||||
|
session.scalars(
|
||||||
|
select(Chunk.document_id)
|
||||||
|
.where(Chunk.strategy_version == self._target_profile.chunk_strategy_version)
|
||||||
|
.distinct()
|
||||||
|
.order_by(Chunk.document_id)
|
||||||
|
).all()
|
||||||
|
)
|
||||||
|
|
||||||
|
def _document_is_complete(self, document_id: UUID) -> bool:
|
||||||
|
"""Return true only when every selected chunk has matching profile evidence."""
|
||||||
|
with self._session_factory() as session:
|
||||||
|
chunks = session.scalars(
|
||||||
|
select(Chunk.chunk_id).where(
|
||||||
|
Chunk.document_id == document_id,
|
||||||
|
Chunk.strategy_version == self._target_profile.chunk_strategy_version,
|
||||||
|
)
|
||||||
|
).all()
|
||||||
|
states = session.scalars(
|
||||||
|
select(ChunkIndexState).where(
|
||||||
|
ChunkIndexState.chunk_id.in_(chunks),
|
||||||
|
ChunkIndexState.index_profile_id == self._target_profile.profile_id,
|
||||||
|
)
|
||||||
|
).all()
|
||||||
|
return len(states) == len(chunks) and all(
|
||||||
|
self._state_matches_profile(state) for state in states
|
||||||
|
)
|
||||||
|
|
||||||
|
def _reset_partial_document(self, document_id: UUID) -> None:
|
||||||
|
"""Remove only target-profile remnants before retrying one document."""
|
||||||
|
self._vector_store.delete_document(document_id, self._target_profile.profile_id)
|
||||||
|
with self._session_factory() as session:
|
||||||
|
chunk_ids = select(Chunk.chunk_id).where(Chunk.document_id == document_id)
|
||||||
|
session.execute(
|
||||||
|
delete(ChunkIndexState).where(
|
||||||
|
ChunkIndexState.chunk_id.in_(chunk_ids),
|
||||||
|
ChunkIndexState.index_profile_id == self._target_profile.profile_id,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
session.commit()
|
||||||
|
|
||||||
|
def _require_registered_profile(
|
||||||
|
self,
|
||||||
|
session: Session,
|
||||||
|
*,
|
||||||
|
for_update: bool = False,
|
||||||
|
) -> StoredIndexProfile:
|
||||||
|
statement = select(StoredIndexProfile).where(
|
||||||
|
StoredIndexProfile.profile_id == self._target_profile.profile_id
|
||||||
|
)
|
||||||
|
if for_update:
|
||||||
|
statement = statement.with_for_update()
|
||||||
|
stored = session.scalar(statement)
|
||||||
|
if stored is None:
|
||||||
|
raise IndexProfileNotFoundError(self._target_profile.profile_id)
|
||||||
|
self._validate_stored_metadata(stored)
|
||||||
|
return stored
|
||||||
|
|
||||||
|
def _validate_stored_metadata(self, stored: StoredIndexProfile) -> None:
|
||||||
|
expected = self._target_profile
|
||||||
|
if (
|
||||||
|
stored.chunk_strategy_version != expected.chunk_strategy_version
|
||||||
|
or stored.embedding_model != expected.embedding_model
|
||||||
|
or stored.vector_dimension != expected.vector_dimension
|
||||||
|
):
|
||||||
|
raise IndexProfileMetadataConflictError(
|
||||||
|
"index profile ID is already registered with different metadata"
|
||||||
|
)
|
||||||
|
|
||||||
|
def _verification_summary(self, session: Session) -> IndexBuildSummary:
|
||||||
|
expected_rows = session.execute(
|
||||||
|
select(Chunk.chunk_id, Chunk.document_id).where(
|
||||||
|
Chunk.strategy_version == self._target_profile.chunk_strategy_version
|
||||||
|
)
|
||||||
|
).all()
|
||||||
|
expected_chunk_ids = {row.chunk_id for row in expected_rows}
|
||||||
|
expected_document_ids = {row.document_id for row in expected_rows}
|
||||||
|
states = session.scalars(
|
||||||
|
select(ChunkIndexState).where(
|
||||||
|
ChunkIndexState.index_profile_id == self._target_profile.profile_id
|
||||||
|
)
|
||||||
|
).all()
|
||||||
|
indexed_chunk_ids = {
|
||||||
|
state.chunk_id
|
||||||
|
for state in states
|
||||||
|
if state.chunk_id in expected_chunk_ids and self._state_matches_profile(state)
|
||||||
|
}
|
||||||
|
complete = (
|
||||||
|
bool(expected_chunk_ids)
|
||||||
|
and len(states) == len(expected_chunk_ids)
|
||||||
|
and indexed_chunk_ids == expected_chunk_ids
|
||||||
|
)
|
||||||
|
return IndexBuildSummary(
|
||||||
|
profile_id=self._target_profile.profile_id,
|
||||||
|
expected_documents=len(expected_document_ids),
|
||||||
|
built_documents=0,
|
||||||
|
skipped_documents=0,
|
||||||
|
expected_chunks=len(expected_chunk_ids),
|
||||||
|
indexed_chunks=len(indexed_chunk_ids),
|
||||||
|
complete=complete,
|
||||||
|
)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _lock_canonical_chunks(session: Session) -> None:
|
||||||
|
"""Prevent canonical chunk writes between verification and activation."""
|
||||||
|
bind = session.get_bind()
|
||||||
|
if bind.dialect.name == "mysql":
|
||||||
|
session.execute(select(Chunk.chunk_id).with_for_update()).all()
|
||||||
|
|
||||||
|
def _state_matches_profile(self, state: ChunkIndexState) -> bool:
|
||||||
|
return (
|
||||||
|
state.embedding_model == self._target_profile.embedding_model
|
||||||
|
and state.vector_dimension == self._target_profile.vector_dimension
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def resolve_active_index_profile(
|
||||||
|
session_factory: sessionmaker[Session],
|
||||||
|
fallback: IndexProfile,
|
||||||
|
) -> IndexProfile:
|
||||||
|
"""Resolve the canonical active profile, with a fresh-deployment fallback."""
|
||||||
|
with session_factory() as session:
|
||||||
|
stored = session.scalar(
|
||||||
|
select(StoredIndexProfile).where(StoredIndexProfile.status == IndexProfileStatus.ACTIVE)
|
||||||
|
)
|
||||||
|
return fallback if stored is None else stored.to_profile()
|
||||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
x
Reference in New Issue
Block a user