Compare commits
No commits in common. "main" and "codex/demo-001" have entirely different histories.
main
...
codex/demo
@ -1,21 +0,0 @@
|
||||
# Keep credentials, local state, caches, and developer artifacts out of every image context.
|
||||
.env
|
||||
.env.*
|
||||
!.env.example
|
||||
.git
|
||||
.github
|
||||
.idea
|
||||
.vscode
|
||||
.DS_Store
|
||||
.mypy_cache
|
||||
.pytest_cache
|
||||
.ruff_cache
|
||||
.venv
|
||||
__pycache__
|
||||
*.py[cod]
|
||||
*.sqlite*
|
||||
dist
|
||||
storage
|
||||
tests
|
||||
docs
|
||||
docker-compose.dev.yml
|
||||
88
.env.example
88
.env.example
@ -4,86 +4,32 @@ LOG_LEVEL=INFO
|
||||
API_HOST=0.0.0.0
|
||||
API_PORT=8000
|
||||
API_PREFIX=/api/v1
|
||||
# Optional outside source checkouts; production image fixes this to /app/apps/admin-web.
|
||||
# ADMIN_WEB_ROOT=/absolute/path/to/apps/admin-web
|
||||
|
||||
DATABASE_URL=mysql+pymysql://ekp:change-me@localhost:2881/ekp
|
||||
DATABASE_POOL_SIZE=10
|
||||
DATABASE_MAX_OVERFLOW=10
|
||||
DATABASE_POOL_RECYCLE_SECONDS=600
|
||||
DATABASE_POOL_TIMEOUT_SECONDS=30
|
||||
DATABASE_CONNECT_TIMEOUT_SECONDS=30
|
||||
# DEMO-015 queue. Inject the real credential at runtime; never commit its value.
|
||||
RABBITMQ_URL=
|
||||
INGESTION_MAX_ATTEMPTS=3
|
||||
INGESTION_RETRY_BACKOFF_SECONDS=2
|
||||
INGESTION_RETRY_BACKOFF_MAX_SECONDS=60
|
||||
INGESTION_WORKER_LEASE_SECONDS=300
|
||||
INGESTION_RECOVERY_INTERVAL_SECONDS=30
|
||||
BATCH_MAX_FILES=20
|
||||
BATCH_MAX_TOTAL_BYTES=209715200
|
||||
DATABASE_URL=postgresql+psycopg://ekp:ekp@localhost:5432/ekp
|
||||
REDIS_URL=redis://localhost:6379/0
|
||||
RABBITMQ_URL=amqp://guest:guest@localhost:5672//
|
||||
|
||||
OBJECT_STORAGE_PROVIDER=bos
|
||||
OBJECT_STORAGE_PATH=storage
|
||||
BOS_ENDPOINT=
|
||||
BOS_ACCESS_KEY_ID=
|
||||
BOS_SECRET_ACCESS_KEY=
|
||||
BOS_BUCKET=
|
||||
BOS_KEY_PREFIX=knowledge-platform
|
||||
BOS_TIMEOUT_SECONDS=30
|
||||
MAX_UPLOAD_BYTES=52428800
|
||||
CHUNK_MAX_TOKENS=450
|
||||
CHUNK_OVERLAP_TOKENS=75
|
||||
CHUNK_MAX_CHARACTERS=2000
|
||||
OBJECT_STORAGE_PROVIDER=s3
|
||||
OBJECT_STORAGE_ENDPOINT=http://localhost:9000
|
||||
OBJECT_STORAGE_ACCESS_KEY=
|
||||
OBJECT_STORAGE_SECRET_KEY=
|
||||
OBJECT_STORAGE_BUCKET=ekp
|
||||
|
||||
VECTOR_STORE_PROVIDER=oceanbase
|
||||
OCEANBASE_VECTOR_URI=
|
||||
OCEANBASE_VECTOR_USER=
|
||||
OCEANBASE_VECTOR_PASSWORD=
|
||||
OCEANBASE_VECTOR_DATABASE=
|
||||
OCEANBASE_VECTOR_TABLE=knowledge_chunks_v1
|
||||
OCEANBASE_VECTOR_INDEX=vidx_knowledge_chunks_embedding
|
||||
OCEANBASE_VECTOR_BATCH_SIZE=100
|
||||
OCEANBASE_VECTOR_TIMEOUT_SECONDS=30
|
||||
VECTOR_STORE_PROVIDER=mochow
|
||||
MOCHOW_ENDPOINT=
|
||||
MOCHOW_API_KEY=
|
||||
MOCHOW_DATABASE=
|
||||
MOCHOW_TABLE=
|
||||
|
||||
EMBEDDING_PROVIDER=qianfan
|
||||
EMBEDDING_MODEL=bge-large-zh
|
||||
EMBEDDING_ENDPOINT=https://qianfan.baidubce.com/v2/embeddings
|
||||
BAIDU_QIANFAN_API_KEY=
|
||||
EMBEDDING_TIMEOUT_SECONDS=30
|
||||
INDEX_PROFILE_ID=demo-bge-large-zh-structure-v1-oceanbase-v1
|
||||
DENSE_TOP_N=20
|
||||
EMBEDDING_ENDPOINT=
|
||||
EMBEDDING_API_KEY=
|
||||
|
||||
RERANKER_PROVIDER=qianfan
|
||||
RERANKER_MODEL=bce-reranker-base
|
||||
RERANKER_ENDPOINT=https://qianfan.baidubce.com/v2/rerank
|
||||
RERANKER_MODEL=
|
||||
RERANKER_ENDPOINT=
|
||||
RERANKER_API_KEY=
|
||||
RERANKER_TIMEOUT_SECONDS=30
|
||||
SEARCH_TOP_K=5
|
||||
|
||||
# DEMO-012 MCP gateway. Inject MCP_AUTH_TOKEN at runtime; never commit its value.
|
||||
MCP_HOST=127.0.0.1
|
||||
MCP_PORT=8766
|
||||
MCP_PATH=/mcp
|
||||
MCP_PUBLIC_URL=http://127.0.0.1:8766/mcp
|
||||
MCP_ISSUER_URL=http://127.0.0.1:8766
|
||||
MCP_AUTH_TOKEN=
|
||||
MCP_CLIENT_ID=workbuddy-enterprise
|
||||
MCP_SERVICE_IDENTITY=workbuddy-demo
|
||||
MCP_REQUIRED_SCOPE=knowledge:search
|
||||
MCP_ALLOWED_HOSTS='["127.0.0.1:*","localhost:*","[::1]:*","testserver"]'
|
||||
MCP_ALLOWED_ORIGINS='["http://127.0.0.1:*","http://localhost:*","http://[::1]:*"]'
|
||||
MCP_MAX_REQUEST_BODY_BYTES=1048576
|
||||
|
||||
# Controlled ARCH-006 acceptance; keep disabled outside an authorized test run.
|
||||
EKP_RUN_LIVE_TESTS=0
|
||||
EKP_CREDENTIALS_AUTHORIZED=0
|
||||
EKP_LIVE_RESOURCE_ACK=0
|
||||
EKP_LIVE_MIN_RECALL_AT_1=0.90
|
||||
EKP_LIVE_MIN_RECALL_AT_5=1.0
|
||||
EKP_LIVE_MIN_MRR=0.90
|
||||
EKP_LIVE_MIN_NDCG_AT_5=0.90
|
||||
EKP_LIVE_MAX_P95_MS=10000
|
||||
|
||||
LLM_PROVIDER=qianfan
|
||||
LLM_MODEL=
|
||||
|
||||
@ -19,8 +19,8 @@ Do not redesign accepted architecture unless the task explicitly asks for an arc
|
||||
|
||||
1. API First.
|
||||
2. V1 uses a modular monolith plus independent async/GPU workers and an MCP gateway.
|
||||
3. OceanBase relational tables + object storage hold canonical data.
|
||||
4. OceanBase vector tables are rebuildable indexes, not sources of truth, even when co-located with canonical relational tables.
|
||||
3. PostgreSQL + object storage hold canonical data.
|
||||
4. Vector databases are rebuildable indexes, not sources of truth.
|
||||
5. Business services depend on `VectorStore` abstractions, never directly on Mochow.
|
||||
6. Embedding, reranking, parsing, OCR, object storage, and LLMs use provider interfaces.
|
||||
7. Original source documents are immutable.
|
||||
|
||||
39
Makefile
39
Makefile
@ -1,20 +1,12 @@
|
||||
.PHONY: help install run run-mcp run-worker images images-print images-push deploy deploy-lint deploy-render format format-check lint typecheck test test-unit test-integration migration-check eval live-acceptance up down
|
||||
.PHONY: help install run format format-check lint typecheck test test-unit test-integration eval up down
|
||||
|
||||
UV_CACHE_DIR ?= /tmp/ekp-uv-cache
|
||||
UV_CACHE_DIR ?= $(CURDIR)/tmp/uv-cache
|
||||
export UV_CACHE_DIR
|
||||
|
||||
help:
|
||||
@echo "Enterprise Knowledge Platform"
|
||||
@echo " install Install development dependencies"
|
||||
@echo " run Start the development API"
|
||||
@echo " run-mcp Start the authenticated MCP gateway"
|
||||
@echo " run-worker Start the ingestion worker and recovery scheduler"
|
||||
@echo " images Build production images for the current builder"
|
||||
@echo " images-print Render the multi-architecture image build plan"
|
||||
@echo " images-push Build and push amd64/arm64 production images"
|
||||
@echo " deploy Install or upgrade Kubernetes with VALUES=<file>"
|
||||
@echo " deploy-lint Validate Helm values and templates"
|
||||
@echo " deploy-render Render non-secret Kubernetes manifests"
|
||||
@echo " format Format code"
|
||||
@echo " format-check Check formatting without modifying files"
|
||||
@echo " lint Run linters"
|
||||
@ -22,9 +14,7 @@ help:
|
||||
@echo " test Run default tests"
|
||||
@echo " test-unit Run unit tests"
|
||||
@echo " test-integration Run integration tests"
|
||||
@echo " migration-check Validate the forward migration path"
|
||||
@echo " eval Run RAG evaluation"
|
||||
@echo " live-acceptance Run controlled target-provider acceptance"
|
||||
@echo " up Start local dependencies"
|
||||
@echo " down Stop local dependencies"
|
||||
|
||||
@ -32,24 +22,6 @@ install:
|
||||
uv sync --all-groups
|
||||
run:
|
||||
uv run uvicorn ekp_api.main:app --host "$${API_HOST:-0.0.0.0}" --port "$${API_PORT:-8000}"
|
||||
run-mcp:
|
||||
uv run ekp-mcp
|
||||
run-worker:
|
||||
uv run ekp-worker
|
||||
images:
|
||||
docker buildx bake -f deploy/docker/docker-bake.hcl
|
||||
images-print:
|
||||
docker buildx bake -f deploy/docker/docker-bake.hcl --print
|
||||
images-push:
|
||||
docker buildx bake -f deploy/docker/docker-bake.hcl --push
|
||||
deploy:
|
||||
./scripts/deploy.sh "$(VALUES)"
|
||||
deploy-lint:
|
||||
@test -n "$(VALUES)" || (echo "VALUES must point to an environment values file" >&2; exit 2)
|
||||
helm lint deploy/helm/knowledge-platform --values "$(VALUES)"
|
||||
deploy-render:
|
||||
@test -n "$(VALUES)" || (echo "VALUES must point to an environment values file" >&2; exit 2)
|
||||
helm template "$${RELEASE:-knowledge-platform}" deploy/helm/knowledge-platform --namespace "$${NAMESPACE:-knowledge-platform}" --values "$(VALUES)"
|
||||
format:
|
||||
uv run ruff format .
|
||||
format-check:
|
||||
@ -57,20 +29,15 @@ format-check:
|
||||
lint:
|
||||
uv run ruff check .
|
||||
typecheck:
|
||||
uv run mypy apps/api/src apps/mcp-server/src packages tests
|
||||
uv run mypy apps/api/src packages/core/src tests
|
||||
test:
|
||||
uv run pytest tests/unit tests/integration
|
||||
test-unit:
|
||||
uv run pytest tests/unit
|
||||
test-integration:
|
||||
uv run pytest tests/integration
|
||||
migration-check:
|
||||
uv run alembic heads
|
||||
uv run alembic upgrade head --sql >/dev/null
|
||||
eval:
|
||||
uv run pytest tests/evaluation
|
||||
live-acceptance:
|
||||
uv run ekp-live-acceptance
|
||||
up:
|
||||
docker compose -f docker-compose.dev.yml up -d
|
||||
down:
|
||||
|
||||
92
README.md
92
README.md
@ -19,9 +19,8 @@ Enterprise internal knowledge infrastructure.
|
||||
|
||||
- API First + Admin Console
|
||||
- Modular Monolith + async workers + MCP gateway
|
||||
- OceanBase MySQL relational tables + object storage as canonical source of truth
|
||||
- Co-located OceanBase Vector tables as rebuildable retrieval indexes behind an adapter
|
||||
- Baidu BOS as target object storage; local storage remains for tests/development
|
||||
- PostgreSQL + object storage as canonical source of truth
|
||||
- Mochow as the initial retrieval index, behind a replaceable adapter
|
||||
- Embedding / reranker / parser / LLM providers are replaceable
|
||||
- Raw source is immutable
|
||||
- Compiled Knowledge is derived and traceable
|
||||
@ -46,105 +45,22 @@ The current implementation target is the retrieval demo defined in
|
||||
Prerequisites:
|
||||
|
||||
- `uv`
|
||||
- Docker (for optional local integration dependencies)
|
||||
- Docker (for PostgreSQL, MinIO, and later integration dependencies)
|
||||
|
||||
Install dependencies and run the API:
|
||||
|
||||
```bash
|
||||
cp .env.example .env
|
||||
make install
|
||||
make run-worker
|
||||
make run
|
||||
```
|
||||
|
||||
Run the independent authenticated MCP Gateway after injecting a dedicated
|
||||
runtime token of at least 32 characters:
|
||||
|
||||
```bash
|
||||
export MCP_AUTH_TOKEN="<runtime-secret-with-at-least-32-characters>"
|
||||
make run-mcp
|
||||
```
|
||||
|
||||
The Streamable HTTP endpoint defaults to `http://127.0.0.1:8766/mcp` and exposes
|
||||
only `search_knowledge`. See `docs/development/MCP_SETUP.md` for WorkBuddy configuration,
|
||||
identity boundaries, remote allowlists, and protocol verification.
|
||||
|
||||
The health endpoint is available at `http://localhost:8000/healthz`, and the
|
||||
versioned OpenAPI document is available at
|
||||
`http://localhost:8000/api/v1/openapi.json`.
|
||||
The Demo Admin Console is available at `http://localhost:8000/admin` and contains
|
||||
only document upload, document management, and Search Lab pages. See
|
||||
`docs/development/ADMIN_CONSOLE.md` for the operation and scope boundaries.
|
||||
|
||||
DEPLOY-001 provides locked, non-root `api`, `mcp`, and `worker` production image targets with an
|
||||
amd64/arm64 Buildx Bake plan. DEPLOY-002 adds the Kubernetes/Helm Chart, runtime Secret references,
|
||||
serialized Migration, Ingress/TLS and the `make deploy VALUES=<file>` entrypoint. See
|
||||
`docs/development/CONTAINER_IMAGES.md` and `docs/development/KUBERNETES_DEPLOYMENT.md`.
|
||||
|
||||
Run the development checks with:
|
||||
|
||||
```bash
|
||||
make format-check lint typecheck test migration-check eval
|
||||
make format-check lint typecheck test eval
|
||||
```
|
||||
|
||||
Controlled target-provider acceptance uses 32 synthetic queries and is disabled by
|
||||
default. It requires rotated secrets, dedicated test resources, and all safety
|
||||
acknowledgements described in `docs/development/LIVE_ACCEPTANCE.md`; only then run
|
||||
`make live-acceptance`.
|
||||
|
||||
Document metadata is canonical in OceanBase MySQL relational tables. Demo source files use the local
|
||||
`storage/` directory through the provider-neutral `ObjectStorage` interface;
|
||||
existing objects cannot be overwritten. The upload service accepts PDF, DOCX,
|
||||
TXT, and MD sources, streams at most `MAX_UPLOAD_BYTES`, and compensates stored
|
||||
bytes if metadata persistence fails. The ingestion service then routes PDF to
|
||||
PyMuPDF, DOCX to python-docx, and UTF-8 TXT/Markdown to lightweight parsers. It
|
||||
normalizes whitespace, retains available page/section context, and stores a
|
||||
versioned parsed JSON artifact under `parsed/{document_id}/document.json`.
|
||||
The `structure_v1` chunk strategy keeps heading/section boundaries, combines
|
||||
paragraphs, and creates configurable 450-token windows with a 75-token overlap.
|
||||
Chunk metadata, including deterministic IDs, filename, page range, section,
|
||||
index, and strategy version, is stored atomically in canonical relational tables. The lightweight
|
||||
token counter is replaceable so DEMO-006 can inject model-specific behavior.
|
||||
Documents remain `processing` until embedding/indexing completes; pipeline
|
||||
failures become `failed` without deleting the immutable original. DEMO-015 now
|
||||
accepts a source plus durable Job with HTTP `202`, while a Celery/RabbitMQ worker
|
||||
runs the pipeline. Bounded transient retries clean every derived artifact before
|
||||
starting the next Attempt; broker redelivery and expired worker leases converge on
|
||||
OceanBase Job state. See `docs/development/ASYNC_INGESTION_SETUP.md`.
|
||||
|
||||
DEMO-006 added a provider-neutral `EmbeddingProvider` and `VectorStore`, plus a
|
||||
validated Qianfan `bge-large-zh` adapter. ARCH-005 now wires that boundary to OceanBase.
|
||||
Chunk vectors are upserted with source metadata under a versioned index profile;
|
||||
OceanBase relational tables record canonical index state while vector rows remain rebuildable. A
|
||||
document becomes `ready` only after every chunk is indexed. See
|
||||
`docs/development/OCEANBASE_VECTOR_SETUP.md` for the required environment variables
|
||||
and pre-provisioned table schema.
|
||||
|
||||
The accepted provider migration is tracked by `ARCH-001` through `ARCH-006`:
|
||||
Baidu BOS, OceanBase Vector, and Qianfan `bce-reranker-base` replace the initial
|
||||
demo providers, and ARCH-006 aligns canonical ownership with the verified OceanBase target. BOS and OceanBase
|
||||
are now the runtime defaults; LocalObjectStorage and FakeVectorStore remain available
|
||||
for explicit local/offline tests. ARCH-005 also adds the controlled shadow rebuild
|
||||
workflow documented in `docs/development/INDEX_REBUILD.md`.
|
||||
|
||||
DEMO-007 adds query embedding and profile-scoped Dense Top-N retrieval. The
|
||||
default is Top 20; the in-memory fake uses
|
||||
exact cosine search for deterministic offline regression. Returned objects are
|
||||
raw dense candidates only.
|
||||
|
||||
DEMO-008 adds a provider-neutral reranker, and ARCH-004 aligns its validated Qianfan
|
||||
adapter with the `bce-reranker-base` baseline. The final Top20-to-Top5 pipeline and
|
||||
source results are rehydrated from canonical relational chunks and retain both
|
||||
Dense and reranker scores; stale vector content fails closed. See
|
||||
`docs/development/RERANKER_SETUP.md` for configuration and validation limits.
|
||||
DEMO-009 exposes document and search REST endpoints under `/api/v1`; DEMO-015
|
||||
changes upload to asynchronous acceptance and adds Job status. DEMO-016 adds bounded multi-file
|
||||
submission, batch/per-file status, and automatic Admin Console polling. Upload, list, detail, direct delete,
|
||||
and reranked source search are documented in
|
||||
`docs/development/REST_API_SETUP.md`; provider and storage internals remain
|
||||
outside the public response contract.
|
||||
|
||||
DEMO-012 adds the independent authenticated MCP Gateway. It uses the official
|
||||
MCP Python SDK, maps a runtime-only Bearer credential to a least-privilege service
|
||||
identity, and delegates `search_knowledge` to the same `SearchService` used by REST.
|
||||
No provider, storage, database, model, or index-profile details are exposed.
|
||||
|
||||
38
alembic.ini
38
alembic.ini
@ -1,38 +0,0 @@
|
||||
[alembic]
|
||||
script_location = %(here)s/migrations
|
||||
prepend_sys_path = .
|
||||
path_separator = os
|
||||
|
||||
[loggers]
|
||||
keys = root,sqlalchemy,alembic
|
||||
|
||||
[handlers]
|
||||
keys = console
|
||||
|
||||
[formatters]
|
||||
keys = generic
|
||||
|
||||
[logger_root]
|
||||
level = WARN
|
||||
handlers = console
|
||||
qualname =
|
||||
|
||||
[logger_sqlalchemy]
|
||||
level = WARN
|
||||
handlers =
|
||||
qualname = sqlalchemy.engine
|
||||
|
||||
[logger_alembic]
|
||||
level = INFO
|
||||
handlers =
|
||||
qualname = alembic
|
||||
|
||||
[handler_console]
|
||||
class = StreamHandler
|
||||
args = (sys.stderr,)
|
||||
level = NOTSET
|
||||
formatter = generic
|
||||
|
||||
[formatter_generic]
|
||||
format = %(levelname)-5.5s [%(name)s] %(message)s
|
||||
datefmt = %H:%M:%S
|
||||
@ -1,411 +0,0 @@
|
||||
// The Admin Console owns presentation state only. Batch acceptance, Job state,
|
||||
// and retrieval remain behind the versioned public REST contract.
|
||||
const API_BASE = "/api/v1";
|
||||
const BATCH_POLL_INTERVAL_MS = 1500;
|
||||
const terminalBatchStatuses = new Set(["completed", "partial_failure", "failed"]);
|
||||
let batchPollTimer;
|
||||
let activeBatchId;
|
||||
|
||||
const statusLabels = {
|
||||
uploaded: "Uploaded",
|
||||
processing: "Processing",
|
||||
ready: "Ready",
|
||||
failed: "Failed",
|
||||
pending: "Pending",
|
||||
accepted: "Accepted",
|
||||
rejected: "Rejected",
|
||||
queued: "Queued",
|
||||
running: "Running",
|
||||
retrying: "Retrying",
|
||||
succeeded: "Succeeded",
|
||||
completed: "Completed",
|
||||
partial_failure: "Partial failure",
|
||||
};
|
||||
|
||||
const stageLabels = {
|
||||
parsing: "解析",
|
||||
cleaning: "清理",
|
||||
chunking: "切分",
|
||||
embedding: "向量化",
|
||||
indexing: "索引",
|
||||
};
|
||||
|
||||
const pageTitles = {
|
||||
upload: "上传文档",
|
||||
documents: "文档列表",
|
||||
search: "Search Lab",
|
||||
};
|
||||
|
||||
function currentPage() {
|
||||
const route = window.location.pathname.split("/").filter(Boolean).at(-1);
|
||||
return Object.hasOwn(pageTitles, route) ? route : "upload";
|
||||
}
|
||||
|
||||
function showPage(page) {
|
||||
document.querySelectorAll("[data-page]").forEach((section) => {
|
||||
section.hidden = section.dataset.page !== page;
|
||||
});
|
||||
document.querySelectorAll("[data-nav]").forEach((link) => {
|
||||
const active = link.dataset.nav === page;
|
||||
link.classList.toggle("active", active);
|
||||
if (active) link.setAttribute("aria-current", "page");
|
||||
else link.removeAttribute("aria-current");
|
||||
});
|
||||
document.querySelector("#page-title").textContent = pageTitles[page];
|
||||
}
|
||||
|
||||
function showNotice(message, kind = "success") {
|
||||
const notice = document.querySelector("#notice");
|
||||
notice.textContent = message;
|
||||
notice.className = `notice ${kind}`;
|
||||
notice.hidden = false;
|
||||
window.clearTimeout(showNotice.timeout);
|
||||
showNotice.timeout = window.setTimeout(() => {
|
||||
notice.hidden = true;
|
||||
}, 6000);
|
||||
}
|
||||
|
||||
async function requestJson(path, options = {}) {
|
||||
const response = await fetch(`${API_BASE}${path}`, options);
|
||||
if (!response.ok) {
|
||||
let message = `请求失败(HTTP ${response.status})`;
|
||||
try {
|
||||
const body = await response.json();
|
||||
if (body.detail) message = body.detail;
|
||||
} catch {
|
||||
// A non-JSON upstream error is represented by the stable HTTP message above.
|
||||
}
|
||||
throw new Error(message);
|
||||
}
|
||||
return response.status === 204 ? null : response.json();
|
||||
}
|
||||
|
||||
function formatDate(value) {
|
||||
if (!value) return "—";
|
||||
return new Intl.DateTimeFormat("zh-CN", {
|
||||
dateStyle: "medium",
|
||||
timeStyle: "short",
|
||||
}).format(new Date(value));
|
||||
}
|
||||
|
||||
function formatSize(bytes) {
|
||||
if (bytes < 1024) return `${bytes} B`;
|
||||
if (bytes < 1024 ** 2) return `${(bytes / 1024).toFixed(1)} KB`;
|
||||
return `${(bytes / 1024 ** 2).toFixed(1)} MB`;
|
||||
}
|
||||
|
||||
function statusPill(status) {
|
||||
const pill = document.createElement("span");
|
||||
pill.className = `status-pill status-${status}`;
|
||||
pill.textContent = statusLabels[status] ?? status;
|
||||
return pill;
|
||||
}
|
||||
|
||||
function setButtonBusy(button, busy, busyText) {
|
||||
if (!button.dataset.label) button.dataset.label = button.textContent;
|
||||
button.disabled = busy;
|
||||
button.textContent = busy ? busyText : button.dataset.label;
|
||||
}
|
||||
|
||||
function initializeUpload() {
|
||||
const form = document.querySelector("#upload-form");
|
||||
const input = document.querySelector("#file-input");
|
||||
const selected = document.querySelector("#selected-file");
|
||||
const button = document.querySelector("#upload-button");
|
||||
|
||||
const updateFilename = () => {
|
||||
const files = Array.from(input.files);
|
||||
if (!files.length) {
|
||||
selected.textContent = "尚未选择文件";
|
||||
} else if (files.length === 1) {
|
||||
selected.textContent = files[0].name;
|
||||
} else {
|
||||
selected.textContent = `已选择 ${files.length} 个文件`;
|
||||
}
|
||||
};
|
||||
input.addEventListener("change", updateFilename);
|
||||
|
||||
form.addEventListener("submit", async (event) => {
|
||||
event.preventDefault();
|
||||
if (!input.files.length) return;
|
||||
window.clearTimeout(batchPollTimer);
|
||||
activeBatchId = undefined;
|
||||
setButtonBusy(button, true, "提交中…");
|
||||
try {
|
||||
const formData = new FormData();
|
||||
Array.from(input.files).forEach((file) => formData.append("files", file));
|
||||
const batch = await requestJson("/ingestion-batches", {
|
||||
method: "POST",
|
||||
body: formData,
|
||||
});
|
||||
activeBatchId = batch.batch_id;
|
||||
renderBatchResult(batch);
|
||||
scheduleBatchPoll(batch);
|
||||
showNotice(
|
||||
`批次已接收:${batch.accepted_files} 个进入后台队列,${batch.rejected_files} 个被拒绝。`,
|
||||
batch.rejected_files ? "error" : "success",
|
||||
);
|
||||
form.reset();
|
||||
updateFilename();
|
||||
} catch (error) {
|
||||
showNotice(`上传失败:${error.message}`, "error");
|
||||
} finally {
|
||||
setButtonBusy(button, false, "");
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
function scheduleBatchPoll(batch) {
|
||||
window.clearTimeout(batchPollTimer);
|
||||
if (batch.batch_id !== activeBatchId) return;
|
||||
if (terminalBatchStatuses.has(batch.status)) return;
|
||||
batchPollTimer = window.setTimeout(async () => {
|
||||
try {
|
||||
const latest = await requestJson(`/ingestion-batches/${batch.batch_id}`);
|
||||
renderBatchResult(latest);
|
||||
scheduleBatchPoll(latest);
|
||||
} catch (error) {
|
||||
showNotice(`批次状态刷新失败:${error.message}`, "error");
|
||||
scheduleBatchPoll(batch);
|
||||
}
|
||||
}, BATCH_POLL_INTERVAL_MS);
|
||||
}
|
||||
|
||||
function renderBatchResult(batch) {
|
||||
const card = document.querySelector("#upload-result");
|
||||
document.querySelector("#upload-batch-title").textContent =
|
||||
`批次 ${batch.batch_id.slice(0, 8)} · ${formatDate(batch.created_at)}`;
|
||||
document.querySelector("#upload-total").textContent = `${batch.total_files} 个`;
|
||||
document.querySelector("#upload-completed").textContent =
|
||||
`${batch.completed_files} / ${batch.total_files}`;
|
||||
document.querySelector("#upload-accepted").textContent =
|
||||
`${batch.accepted_files} / ${batch.rejected_files}`;
|
||||
document.querySelector("#upload-status").replaceWith(statusPill(batch.status));
|
||||
const status = card.querySelector(".status-pill");
|
||||
status.id = "upload-status";
|
||||
const items = document.querySelector("#batch-items");
|
||||
items.replaceChildren(...batch.items.map(batchItem));
|
||||
card.hidden = false;
|
||||
}
|
||||
|
||||
function batchItem(item) {
|
||||
const row = document.createElement("article");
|
||||
row.className = "batch-item";
|
||||
|
||||
const order = document.createElement("span");
|
||||
order.className = "batch-item-order";
|
||||
order.textContent = String(item.item_index + 1).padStart(2, "0");
|
||||
|
||||
const body = document.createElement("div");
|
||||
body.className = "batch-item-body";
|
||||
const name = document.createElement("strong");
|
||||
name.textContent = item.filename;
|
||||
const detail = document.createElement("span");
|
||||
const stage = item.stage ? ` · ${stageLabels[item.stage] ?? item.stage}` : "";
|
||||
const attempts = item.max_attempts == null ? "" : ` · 尝试 ${item.attempt}/${item.max_attempts}`;
|
||||
detail.textContent = item.error_message ?? `后台任务${stage}${attempts}`;
|
||||
body.append(name, detail);
|
||||
|
||||
const state = item.job_status ?? item.acceptance_status;
|
||||
row.append(order, body, statusPill(state));
|
||||
return row;
|
||||
}
|
||||
|
||||
async function loadDocuments() {
|
||||
const refresh = document.querySelector("#refresh-documents");
|
||||
setButtonBusy(refresh, true, "加载中…");
|
||||
try {
|
||||
const documents = await requestJson("/documents");
|
||||
renderDocuments(documents);
|
||||
} catch (error) {
|
||||
showNotice(`文档列表加载失败:${error.message}`, "error");
|
||||
} finally {
|
||||
setButtonBusy(refresh, false, "");
|
||||
}
|
||||
}
|
||||
|
||||
function renderDocuments(documents) {
|
||||
const rows = document.querySelector("#document-rows");
|
||||
const empty = document.querySelector("#document-empty");
|
||||
rows.replaceChildren();
|
||||
empty.hidden = documents.length !== 0;
|
||||
document.querySelector("#document-count").textContent = `${documents.length} 个文档`;
|
||||
document.querySelector("#document-refresh-time").textContent =
|
||||
`更新于 ${new Intl.DateTimeFormat("zh-CN", { timeStyle: "short" }).format(new Date())}`;
|
||||
|
||||
documents.forEach((documentData) => {
|
||||
const row = document.createElement("tr");
|
||||
row.append(
|
||||
tableCell(documentData.filename, "filename-cell"),
|
||||
tableCell(documentData.media_type),
|
||||
tableCell(formatDate(documentData.created_at)),
|
||||
);
|
||||
const statusCell = document.createElement("td");
|
||||
statusCell.append(statusPill(documentData.status));
|
||||
row.append(statusCell);
|
||||
|
||||
const actions = document.createElement("td");
|
||||
actions.className = "row-actions";
|
||||
const view = actionButton("查看", () => showDocumentDetail(documentData));
|
||||
const remove = actionButton("删除", () => deleteDocument(documentData), "danger-link");
|
||||
actions.append(view, remove);
|
||||
row.append(actions);
|
||||
rows.append(row);
|
||||
});
|
||||
}
|
||||
|
||||
function tableCell(value, className = "") {
|
||||
const cell = document.createElement("td");
|
||||
cell.textContent = value;
|
||||
cell.className = className;
|
||||
return cell;
|
||||
}
|
||||
|
||||
function actionButton(label, handler, className = "") {
|
||||
const button = document.createElement("button");
|
||||
button.type = "button";
|
||||
button.className = `table-action ${className}`;
|
||||
button.textContent = label;
|
||||
button.addEventListener("click", handler);
|
||||
return button;
|
||||
}
|
||||
|
||||
function showDocumentDetail(documentData) {
|
||||
const dialog = document.querySelector("#document-dialog");
|
||||
document.querySelector("#dialog-filename").textContent = documentData.filename;
|
||||
const metadata = document.querySelector("#dialog-metadata");
|
||||
metadata.replaceChildren();
|
||||
[
|
||||
["文档 ID", documentData.id],
|
||||
["文件类型", documentData.media_type],
|
||||
["文件大小", formatSize(documentData.size_bytes)],
|
||||
["当前状态", statusLabels[documentData.status] ?? documentData.status],
|
||||
["上传时间", formatDate(documentData.created_at)],
|
||||
["更新时间", formatDate(documentData.updated_at)],
|
||||
["错误信息", documentData.error_message ?? "无"],
|
||||
].forEach(([term, value]) => {
|
||||
const wrapper = document.createElement("div");
|
||||
const dt = document.createElement("dt");
|
||||
const dd = document.createElement("dd");
|
||||
dt.textContent = term;
|
||||
dd.textContent = value;
|
||||
wrapper.append(dt, dd);
|
||||
metadata.append(wrapper);
|
||||
});
|
||||
dialog.showModal();
|
||||
}
|
||||
|
||||
async function deleteDocument(documentData) {
|
||||
if (!window.confirm(`确认删除“${documentData.filename}”及其索引数据?`)) return;
|
||||
try {
|
||||
await requestJson(`/documents/${documentData.id}`, { method: "DELETE" });
|
||||
showNotice(`已删除 ${documentData.filename}`, "success");
|
||||
await loadDocuments();
|
||||
} catch (error) {
|
||||
showNotice(`删除失败:${error.message}`, "error");
|
||||
}
|
||||
}
|
||||
|
||||
function initializeSearch() {
|
||||
const form = document.querySelector("#search-form");
|
||||
const button = document.querySelector("#search-button");
|
||||
form.addEventListener("submit", async (event) => {
|
||||
event.preventDefault();
|
||||
const query = document.querySelector("#search-query").value.trim();
|
||||
const topK = Number(document.querySelector("#search-top-k").value);
|
||||
if (!query) return;
|
||||
setButtonBusy(button, true, "检索中…");
|
||||
try {
|
||||
const startedAt = performance.now();
|
||||
const response = await requestJson("/search", {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ query, top_k: topK }),
|
||||
});
|
||||
renderSearchResults(response.results, performance.now() - startedAt);
|
||||
} catch (error) {
|
||||
showNotice(`检索失败:${error.message}`, "error");
|
||||
} finally {
|
||||
setButtonBusy(button, false, "");
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
function renderSearchResults(results, elapsedMs) {
|
||||
const container = document.querySelector("#search-results");
|
||||
container.replaceChildren();
|
||||
document.querySelector("#search-summary").textContent =
|
||||
`${results.length} 条结果 · ${(elapsedMs / 1000).toFixed(2)} 秒`;
|
||||
if (!results.length) {
|
||||
const empty = document.createElement("div");
|
||||
empty.className = "empty-state search-empty";
|
||||
const strong = document.createElement("strong");
|
||||
strong.textContent = "没有找到匹配内容";
|
||||
const detail = document.createElement("span");
|
||||
detail.textContent = "可以换一种问法,或先确认文档状态为 Ready。";
|
||||
empty.append(strong, detail);
|
||||
container.append(empty);
|
||||
return;
|
||||
}
|
||||
|
||||
results.forEach((result, index) => {
|
||||
const card = document.createElement("article");
|
||||
card.className = "search-result-card";
|
||||
const rank = document.createElement("span");
|
||||
rank.className = "rank";
|
||||
rank.textContent = String(index + 1).padStart(2, "0");
|
||||
const body = document.createElement("div");
|
||||
body.className = "result-body";
|
||||
const heading = document.createElement("div");
|
||||
heading.className = "result-title-row";
|
||||
const title = document.createElement("h4");
|
||||
title.textContent = result.filename;
|
||||
const sourceMeta = document.createElement("div");
|
||||
sourceMeta.className = "source-meta";
|
||||
sourceMeta.append(
|
||||
metadataItem("Page", result.page == null ? "—" : String(result.page)),
|
||||
metadataItem("Section", result.section ?? "—"),
|
||||
);
|
||||
heading.append(title, sourceMeta);
|
||||
const content = document.createElement("p");
|
||||
content.className = "chunk-content";
|
||||
content.textContent = result.content;
|
||||
const scores = document.createElement("div");
|
||||
scores.className = "score-row";
|
||||
scores.append(
|
||||
scoreItem("Dense Score", result.score),
|
||||
scoreItem("Rerank Score", result.rerank_score),
|
||||
);
|
||||
body.append(heading, content, scores);
|
||||
card.append(rank, body);
|
||||
container.append(card);
|
||||
});
|
||||
}
|
||||
|
||||
function scoreItem(label, value) {
|
||||
const item = document.createElement("span");
|
||||
const labelNode = document.createElement("small");
|
||||
const valueNode = document.createElement("strong");
|
||||
labelNode.textContent = label;
|
||||
valueNode.textContent = Number(value).toFixed(4);
|
||||
item.append(labelNode, valueNode);
|
||||
return item;
|
||||
}
|
||||
|
||||
function metadataItem(label, value) {
|
||||
const item = document.createElement("span");
|
||||
const labelNode = document.createElement("small");
|
||||
const valueNode = document.createElement("strong");
|
||||
labelNode.textContent = label;
|
||||
valueNode.textContent = value;
|
||||
item.append(labelNode, valueNode);
|
||||
return item;
|
||||
}
|
||||
|
||||
const page = currentPage();
|
||||
showPage(page);
|
||||
initializeUpload();
|
||||
initializeSearch();
|
||||
document.querySelector("#refresh-documents").addEventListener("click", loadDocuments);
|
||||
if (page === "documents") loadDocuments();
|
||||
window.addEventListener("pagehide", () => window.clearTimeout(batchPollTimer));
|
||||
File diff suppressed because it is too large
Load Diff
@ -1,201 +0,0 @@
|
||||
<!doctype html>
|
||||
<html lang="zh-CN">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
<meta
|
||||
name="description"
|
||||
content="Enterprise Knowledge Platform Demo 管理控制台"
|
||||
/>
|
||||
<title>Enterprise Knowledge Platform</title>
|
||||
<link rel="stylesheet" href="/admin/assets/styles.css" />
|
||||
<script type="module" src="/admin/assets/app.js"></script>
|
||||
</head>
|
||||
<body>
|
||||
<div class="app-shell">
|
||||
<aside class="sidebar" aria-label="管理控制台导航">
|
||||
<a class="brand" href="/admin/upload" aria-label="Enterprise Knowledge Platform 首页">
|
||||
<svg class="brand-mark" viewBox="0 0 48 36" aria-hidden="true">
|
||||
<path d="M15 3h17a15 15 0 0 1 0 30H15v-9h17a6 6 0 0 0 0-12H15z" />
|
||||
<circle cx="11" cy="18" r="8" />
|
||||
</svg>
|
||||
<span>
|
||||
<strong>地瓜机器人</strong>
|
||||
<small>Knowledge Platform</small>
|
||||
</span>
|
||||
</a>
|
||||
<div class="workspace-label">
|
||||
<span>CONTROL DESK</span>
|
||||
<strong>企业知识运营台</strong>
|
||||
</div>
|
||||
<nav class="primary-nav">
|
||||
<a data-nav="upload" href="/admin/upload">
|
||||
<span aria-hidden="true">01</span>上传文档
|
||||
</a>
|
||||
<a data-nav="documents" href="/admin/documents">
|
||||
<span aria-hidden="true">02</span>文档列表
|
||||
</a>
|
||||
<a data-nav="search" href="/admin/search">
|
||||
<span aria-hidden="true">03</span>Search Lab
|
||||
</a>
|
||||
</nav>
|
||||
<div class="sidebar-foot">
|
||||
<span class="environment-dot" aria-hidden="true"></span>
|
||||
DEMO ENVIRONMENT
|
||||
</div>
|
||||
</aside>
|
||||
|
||||
<main class="workspace">
|
||||
<header class="topbar">
|
||||
<div>
|
||||
<p class="eyebrow">KNOWLEDGE OPERATIONS</p>
|
||||
<h1 id="page-title">上传文档</h1>
|
||||
</div>
|
||||
<div class="topbar-actions">
|
||||
<span class="system-state"><i aria-hidden="true"></i>API FIRST · V1</span>
|
||||
<a class="api-link" href="/api/v1/docs" target="_blank" rel="noreferrer">
|
||||
API 文档 <span aria-hidden="true">↗</span>
|
||||
</a>
|
||||
</div>
|
||||
</header>
|
||||
<div class="signal-line" aria-hidden="true"><span></span><span></span><span></span></div>
|
||||
|
||||
<div id="notice" class="notice" role="status" aria-live="polite" hidden></div>
|
||||
|
||||
<section id="upload-page" class="page" data-page="upload" aria-labelledby="page-title">
|
||||
<div class="page-heading">
|
||||
<div>
|
||||
<p class="section-label">01 / INGESTION</p>
|
||||
<h2>让知识进入检索链路</h2>
|
||||
<p>批量提交 PDF、DOCX、TXT 或 Markdown 文件,系统将逐个接收并在后台独立完成解析、切分与索引。</p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<form id="upload-form" class="upload-panel">
|
||||
<label id="drop-zone" class="drop-zone" for="file-input">
|
||||
<input
|
||||
id="file-input"
|
||||
name="files"
|
||||
type="file"
|
||||
multiple
|
||||
accept=".pdf,.docx,.txt,.md,application/pdf,application/vnd.openxmlformats-officedocument.wordprocessingml.document,text/plain,text/markdown"
|
||||
required
|
||||
/>
|
||||
<span class="upload-icon" aria-hidden="true">↑</span>
|
||||
<strong>选择一批知识文件</strong>
|
||||
<span>支持 PDF、DOCX、TXT、Markdown · 每个文件独立处理</span>
|
||||
<span id="selected-file" class="selected-file">尚未选择文件</span>
|
||||
</label>
|
||||
<button id="upload-button" class="primary-button" type="submit">上传并处理</button>
|
||||
</form>
|
||||
|
||||
<article id="upload-result" class="result-card" hidden>
|
||||
<div>
|
||||
<p class="section-label">LATEST BATCH</p>
|
||||
<h3 id="upload-batch-title"></h3>
|
||||
</div>
|
||||
<span id="upload-status" class="status-pill"></span>
|
||||
<dl class="metadata-grid">
|
||||
<div><dt>总文件</dt><dd id="upload-total"></dd></div>
|
||||
<div><dt>已完成</dt><dd id="upload-completed"></dd></div>
|
||||
<div><dt>接收 / 拒绝</dt><dd id="upload-accepted"></dd></div>
|
||||
</dl>
|
||||
<div id="batch-items" class="batch-items" aria-live="polite"></div>
|
||||
<a class="text-link" href="/admin/search">前往 Search Lab 检索 <span>→</span></a>
|
||||
</article>
|
||||
</section>
|
||||
|
||||
<section id="documents-page" class="page" data-page="documents" aria-labelledby="page-title" hidden>
|
||||
<div class="page-heading heading-row">
|
||||
<div>
|
||||
<p class="section-label">02 / DOCUMENTS</p>
|
||||
<h2>文档列表</h2>
|
||||
<p>查看知识文档的状态、元数据并管理 Demo 数据。</p>
|
||||
</div>
|
||||
<button id="refresh-documents" class="secondary-button" type="button">刷新列表</button>
|
||||
</div>
|
||||
|
||||
<div class="table-card">
|
||||
<div class="table-meta">
|
||||
<span id="document-count">0 个文档</span>
|
||||
<span id="document-refresh-time">尚未加载</span>
|
||||
</div>
|
||||
<div class="table-scroll">
|
||||
<table>
|
||||
<thead>
|
||||
<tr>
|
||||
<th>文件名</th>
|
||||
<th>类型</th>
|
||||
<th>上传时间</th>
|
||||
<th>状态</th>
|
||||
<th aria-label="操作"></th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody id="document-rows"></tbody>
|
||||
</table>
|
||||
</div>
|
||||
<div id="document-empty" class="empty-state" hidden>
|
||||
<strong>还没有文档</strong>
|
||||
<span>上传第一个文件后,它会出现在这里。</span>
|
||||
<a class="primary-button compact" href="/admin/upload">上传文档</a>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section id="search-page" class="page" data-page="search" aria-labelledby="page-title" hidden>
|
||||
<div class="page-heading">
|
||||
<div>
|
||||
<p class="section-label">03 / RETRIEVAL</p>
|
||||
<h2>Search Lab</h2>
|
||||
<p>输入问题,检查 Dense 召回与 Reranker 精排后的 Top-K 来源。</p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<form id="search-form" class="search-panel">
|
||||
<label for="search-query">检索问题</label>
|
||||
<div class="search-controls">
|
||||
<textarea
|
||||
id="search-query"
|
||||
name="query"
|
||||
rows="3"
|
||||
maxlength="1600"
|
||||
placeholder="例如:产品 A 的退款周期是多少?"
|
||||
required
|
||||
></textarea>
|
||||
<label class="top-k-control" for="search-top-k">
|
||||
Top-K
|
||||
<select id="search-top-k" name="top_k">
|
||||
<option value="3">3</option>
|
||||
<option value="5" selected>5</option>
|
||||
<option value="10">10</option>
|
||||
</select>
|
||||
</label>
|
||||
<button id="search-button" class="primary-button" type="submit">开始检索</button>
|
||||
</div>
|
||||
</form>
|
||||
|
||||
<div class="results-heading">
|
||||
<div>
|
||||
<p class="section-label">RESULTS</p>
|
||||
<h3>来源结果</h3>
|
||||
</div>
|
||||
<span id="search-summary">等待检索</span>
|
||||
</div>
|
||||
<div id="search-results" class="search-results" aria-live="polite">
|
||||
<div class="empty-state search-empty">
|
||||
<strong>检索结果会显示在这里</strong>
|
||||
<span>结果包含排名、文件、页码、章节、Chunk 内容和两阶段分数。</span>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
</main>
|
||||
</div>
|
||||
|
||||
<dialog id="document-dialog" class="document-dialog">
|
||||
<form method="dialog"><button class="icon-button" aria-label="关闭详情">×</button></form>
|
||||
<p class="section-label">DOCUMENT DETAIL</p>
|
||||
<h2 id="dialog-filename"></h2>
|
||||
<dl id="dialog-metadata" class="dialog-metadata"></dl>
|
||||
</dialog>
|
||||
</body>
|
||||
</html>
|
||||
@ -1,55 +0,0 @@
|
||||
"""Celery/RabbitMQ adapter behind the queue-neutral ingestion dispatcher."""
|
||||
|
||||
from uuid import UUID
|
||||
|
||||
from celery import Celery # type: ignore[import-untyped]
|
||||
|
||||
from ekp_core.config import Settings
|
||||
|
||||
INGESTION_TASK_NAME = "ekp.ingestion.run"
|
||||
RECOVERY_TASK_NAME = "ekp.ingestion.recover"
|
||||
|
||||
|
||||
def create_celery_app(settings: Settings) -> Celery:
|
||||
"""Create a secret-safe broker client without opening a connection eagerly."""
|
||||
if settings.rabbitmq_url is None or not settings.rabbitmq_url.get_secret_value().strip():
|
||||
raise ValueError("RABBITMQ_URL is required for asynchronous ingestion")
|
||||
app = Celery(
|
||||
"enterprise-knowledge-platform",
|
||||
broker=settings.rabbitmq_url.get_secret_value(),
|
||||
)
|
||||
app.conf.update(
|
||||
task_ignore_result=True,
|
||||
task_acks_late=True,
|
||||
task_reject_on_worker_lost=True,
|
||||
worker_prefetch_multiplier=1,
|
||||
broker_connection_retry_on_startup=True,
|
||||
timezone="UTC",
|
||||
enable_utc=True,
|
||||
task_routes={
|
||||
INGESTION_TASK_NAME: {"queue": "ingestion"},
|
||||
RECOVERY_TASK_NAME: {"queue": "ingestion"},
|
||||
},
|
||||
beat_schedule={
|
||||
"recover-durable-ingestion-jobs": {
|
||||
"task": RECOVERY_TASK_NAME,
|
||||
"schedule": float(settings.ingestion_recovery_interval_seconds),
|
||||
}
|
||||
},
|
||||
)
|
||||
return app
|
||||
|
||||
|
||||
class CeleryIngestionDispatcher:
|
||||
"""Publish only a durable Job id; source bytes and state stay canonical."""
|
||||
|
||||
def __init__(self, app: Celery) -> None:
|
||||
self._app = app
|
||||
|
||||
def enqueue(self, job_id: UUID) -> None:
|
||||
"""Deliver one idempotent execution hint to the ingestion queue."""
|
||||
self._app.send_task(
|
||||
INGESTION_TASK_NAME,
|
||||
args=[str(job_id)],
|
||||
queue="ingestion",
|
||||
)
|
||||
@ -1,67 +0,0 @@
|
||||
"""Controlled CLI for the ARCH-005 shadow index cutover workflow."""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
from collections.abc import Sequence
|
||||
from dataclasses import asdict
|
||||
|
||||
from ekp_api.providers import (
|
||||
create_embedding_provider,
|
||||
create_index_profile,
|
||||
create_vector_store,
|
||||
)
|
||||
from ekp_core.config import Settings
|
||||
from ekp_core.database import create_database_engine, create_session_factory
|
||||
from ekp_core.index_rebuild import IndexBuildSummary, IndexRebuildService
|
||||
|
||||
|
||||
def _parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Build, verify, and activate a versioned shadow vector index."
|
||||
)
|
||||
parser.add_argument(
|
||||
"command",
|
||||
choices=("build", "verify", "activate"),
|
||||
help="Lifecycle step to execute for INDEX_PROFILE_ID.",
|
||||
)
|
||||
return parser
|
||||
|
||||
|
||||
def _create_service(settings: Settings) -> IndexRebuildService:
|
||||
"""Compose the command from the same provider settings as the API runtime."""
|
||||
engine = create_database_engine(settings)
|
||||
session_factory = create_session_factory(engine)
|
||||
return IndexRebuildService(
|
||||
session_factory=session_factory,
|
||||
embedding_provider=create_embedding_provider(settings),
|
||||
vector_store=create_vector_store(settings),
|
||||
target_profile=create_index_profile(settings),
|
||||
)
|
||||
|
||||
|
||||
def _execute(service: IndexRebuildService, command: str) -> IndexBuildSummary:
|
||||
"""Run exactly one explicit lifecycle step."""
|
||||
if command == "build":
|
||||
return service.rebuild()
|
||||
if command == "verify":
|
||||
return service.verify()
|
||||
if command == "activate":
|
||||
return service.activate()
|
||||
raise ValueError("unsupported index rebuild command")
|
||||
|
||||
|
||||
def main(argv: Sequence[str] | None = None) -> int:
|
||||
"""Return a shell-safe status without printing endpoints, SQL, or credentials."""
|
||||
args = _parser().parse_args(argv)
|
||||
try:
|
||||
summary = _execute(_create_service(Settings()), args.command)
|
||||
except Exception:
|
||||
print("index profile command failed; inspect sanitized service logs", file=sys.stderr)
|
||||
return 1
|
||||
print(json.dumps(asdict(summary), sort_keys=True, separators=(",", ":")))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@ -1,279 +0,0 @@
|
||||
"""Controlled end-to-end acceptance for the target provider chain."""
|
||||
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import sys
|
||||
from collections.abc import Sequence
|
||||
from dataclasses import asdict, dataclass
|
||||
from io import BytesIO
|
||||
from pathlib import Path
|
||||
from time import perf_counter
|
||||
from typing import cast
|
||||
from uuid import uuid4
|
||||
|
||||
from ekp_api.providers import ApiServices, create_api_services
|
||||
from ekp_core.config import Settings
|
||||
from ekp_core.search import SourceResult
|
||||
|
||||
MIN_GOLDEN_QUERIES = 30
|
||||
MAX_GOLDEN_QUERIES = 50
|
||||
DEFAULT_MIN_RECALL_AT_1 = 0.90
|
||||
DEFAULT_MIN_RECALL_AT_5 = 1.0
|
||||
DEFAULT_MIN_MRR = 0.90
|
||||
DEFAULT_MIN_NDCG_AT_5 = 0.90
|
||||
DEFAULT_MAX_P95_MS = 10_000.0
|
||||
|
||||
|
||||
class LiveAcceptanceError(RuntimeError):
|
||||
"""Raised for a safe precondition or acceptance failure."""
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class GoldenQuery:
|
||||
"""One anonymized expected source and section."""
|
||||
|
||||
query_id: str
|
||||
query: str
|
||||
expected_document: str
|
||||
expected_sections: tuple[str, ...]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class AcceptanceReport:
|
||||
"""Provider-neutral quality and latency metrics without retrieved content."""
|
||||
|
||||
query_count: int
|
||||
recall_at_1: float
|
||||
recall_at_3: float
|
||||
recall_at_5: float
|
||||
mrr: float
|
||||
ndcg_at_5: float
|
||||
latency_p50_ms: float
|
||||
latency_p95_ms: float
|
||||
passed: bool
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class AcceptanceThresholds:
|
||||
"""Explicit gates for controlled live acceptance."""
|
||||
|
||||
min_recall_at_1: float = DEFAULT_MIN_RECALL_AT_1
|
||||
min_recall_at_5: float = DEFAULT_MIN_RECALL_AT_5
|
||||
min_mrr: float = DEFAULT_MIN_MRR
|
||||
min_ndcg_at_5: float = DEFAULT_MIN_NDCG_AT_5
|
||||
max_p95_ms: float = DEFAULT_MAX_P95_MS
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
quality_values = (
|
||||
self.min_recall_at_1,
|
||||
self.min_recall_at_5,
|
||||
self.min_mrr,
|
||||
self.min_ndcg_at_5,
|
||||
)
|
||||
if any(not 0 <= value <= 1 for value in quality_values):
|
||||
raise ValueError("live quality thresholds must be between zero and one")
|
||||
if self.max_p95_ms <= 0:
|
||||
raise ValueError("live latency threshold must be positive")
|
||||
|
||||
|
||||
def _project_root() -> Path:
|
||||
return Path(__file__).resolve().parents[4]
|
||||
|
||||
|
||||
def _load_golden_queries(path: Path) -> tuple[GoldenQuery, ...]:
|
||||
"""Load only the minimal fields needed by the live evaluator."""
|
||||
records: list[GoldenQuery] = []
|
||||
for line in path.read_text(encoding="utf-8").splitlines():
|
||||
if not line:
|
||||
continue
|
||||
raw: object = json.loads(line)
|
||||
if not isinstance(raw, dict):
|
||||
raise LiveAcceptanceError("Golden Query record must be an object")
|
||||
documents = raw.get("expected_documents")
|
||||
sections = raw.get("expected_sections")
|
||||
if (
|
||||
not isinstance(raw.get("id"), str)
|
||||
or not isinstance(raw.get("query"), str)
|
||||
or not isinstance(documents, list)
|
||||
or len(documents) != 1
|
||||
or not isinstance(documents[0], str)
|
||||
or not isinstance(sections, list)
|
||||
or not sections
|
||||
or any(not isinstance(section, str) for section in sections)
|
||||
):
|
||||
raise LiveAcceptanceError("Golden Query record violates the live contract")
|
||||
records.append(
|
||||
GoldenQuery(
|
||||
query_id=cast(str, raw["id"]),
|
||||
query=cast(str, raw["query"]),
|
||||
expected_document=documents[0],
|
||||
expected_sections=tuple(cast(list[str], sections)),
|
||||
)
|
||||
)
|
||||
if not MIN_GOLDEN_QUERIES <= len(records) <= MAX_GOLDEN_QUERIES:
|
||||
raise LiveAcceptanceError("live Golden Dataset must contain 30 to 50 queries")
|
||||
if len({record.query_id for record in records}) != len(records):
|
||||
raise LiveAcceptanceError("live Golden Query IDs must be unique")
|
||||
return tuple(records)
|
||||
|
||||
|
||||
def _matching_rank(
|
||||
results: Sequence[SourceResult],
|
||||
*,
|
||||
expected_filename: str,
|
||||
expected_sections: tuple[str, ...],
|
||||
) -> int | None:
|
||||
"""Return the one-based rank of the first matching canonical source."""
|
||||
for rank, result in enumerate(results, start=1):
|
||||
if result.filename != expected_filename or result.section is None:
|
||||
continue
|
||||
if any(section in result.section for section in expected_sections):
|
||||
return rank
|
||||
return None
|
||||
|
||||
|
||||
def _percentile(values: Sequence[float], percentile: float) -> float:
|
||||
"""Use nearest-rank percentiles for a small, deterministic acceptance set."""
|
||||
if not values:
|
||||
raise ValueError("latency sample must not be empty")
|
||||
ordered = sorted(values)
|
||||
index = max(0, math.ceil(percentile * len(ordered)) - 1)
|
||||
return ordered[index]
|
||||
|
||||
|
||||
def _build_report(
|
||||
ranks: Sequence[int | None],
|
||||
latencies_ms: Sequence[float],
|
||||
thresholds: AcceptanceThresholds,
|
||||
) -> AcceptanceReport:
|
||||
"""Calculate binary-relevance metrics and apply explicit acceptance gates."""
|
||||
if not ranks or len(ranks) != len(latencies_ms):
|
||||
raise ValueError("live ranks and latencies must have equal non-zero length")
|
||||
query_count = len(ranks)
|
||||
|
||||
def recall_at(limit: int) -> float:
|
||||
return sum(rank is not None and rank <= limit for rank in ranks) / query_count
|
||||
|
||||
reciprocal_ranks = [0.0 if rank is None else 1.0 / rank for rank in ranks]
|
||||
discounted_gains = [
|
||||
0.0 if rank is None or rank > 5 else 1.0 / math.log2(rank + 1) for rank in ranks
|
||||
]
|
||||
recall_at_1 = recall_at(1)
|
||||
recall_at_3 = recall_at(3)
|
||||
recall_at_5 = recall_at(5)
|
||||
mrr = sum(reciprocal_ranks) / query_count
|
||||
ndcg_at_5 = sum(discounted_gains) / query_count
|
||||
p50 = _percentile(latencies_ms, 0.50)
|
||||
p95 = _percentile(latencies_ms, 0.95)
|
||||
passed = (
|
||||
recall_at_1 >= thresholds.min_recall_at_1
|
||||
and recall_at_5 >= thresholds.min_recall_at_5
|
||||
and mrr >= thresholds.min_mrr
|
||||
and ndcg_at_5 >= thresholds.min_ndcg_at_5
|
||||
and p95 <= thresholds.max_p95_ms
|
||||
)
|
||||
return AcceptanceReport(
|
||||
query_count=query_count,
|
||||
recall_at_1=recall_at_1,
|
||||
recall_at_3=recall_at_3,
|
||||
recall_at_5=recall_at_5,
|
||||
mrr=mrr,
|
||||
ndcg_at_5=ndcg_at_5,
|
||||
latency_p50_ms=p50,
|
||||
latency_p95_ms=p95,
|
||||
passed=passed,
|
||||
)
|
||||
|
||||
|
||||
def _thresholds_from_environment() -> AcceptanceThresholds:
|
||||
return AcceptanceThresholds(
|
||||
min_recall_at_1=float(os.getenv("EKP_LIVE_MIN_RECALL_AT_1", str(DEFAULT_MIN_RECALL_AT_1))),
|
||||
min_recall_at_5=float(os.getenv("EKP_LIVE_MIN_RECALL_AT_5", str(DEFAULT_MIN_RECALL_AT_5))),
|
||||
min_mrr=float(os.getenv("EKP_LIVE_MIN_MRR", str(DEFAULT_MIN_MRR))),
|
||||
min_ndcg_at_5=float(os.getenv("EKP_LIVE_MIN_NDCG_AT_5", str(DEFAULT_MIN_NDCG_AT_5))),
|
||||
max_p95_ms=float(os.getenv("EKP_LIVE_MAX_P95_MS", str(DEFAULT_MAX_P95_MS))),
|
||||
)
|
||||
|
||||
|
||||
def _require_safe_live_settings(settings: Settings) -> None:
|
||||
"""Refuse external writes unless rotation and resource authorization are explicit."""
|
||||
required_flags = (
|
||||
"EKP_RUN_LIVE_TESTS",
|
||||
"EKP_CREDENTIALS_AUTHORIZED",
|
||||
"EKP_LIVE_RESOURCE_ACK",
|
||||
)
|
||||
if any(os.getenv(name) != "1" for name in required_flags):
|
||||
raise LiveAcceptanceError("live acceptance safety flags are not enabled")
|
||||
if settings.app_env != "test":
|
||||
raise LiveAcceptanceError("live acceptance requires APP_ENV=test")
|
||||
if settings.object_storage_provider != "bos" or settings.vector_store_provider != "oceanbase":
|
||||
raise LiveAcceptanceError("live acceptance requires the target provider baseline")
|
||||
if not settings.bos_key_prefix.startswith("live-acceptance/"):
|
||||
raise LiveAcceptanceError("BOS_KEY_PREFIX must use a dedicated live-acceptance prefix")
|
||||
|
||||
|
||||
def _cleanup_run_documents(services: ApiServices, filename_prefix: str) -> None:
|
||||
"""Delete only documents created under this random acceptance-run prefix."""
|
||||
documents = tuple(
|
||||
document
|
||||
for document in services.documents.list_documents()
|
||||
if document.filename.startswith(filename_prefix)
|
||||
)
|
||||
for document in documents:
|
||||
services.documents.delete_document(document.id)
|
||||
|
||||
|
||||
def _run_live(settings: Settings) -> AcceptanceReport:
|
||||
"""Upload synthetic sources, execute every query, and clean run-owned data."""
|
||||
_require_safe_live_settings(settings)
|
||||
project_root = _project_root()
|
||||
golden = _load_golden_queries(project_root / "fixtures/evaluation/golden.jsonl")
|
||||
services = create_api_services(settings)
|
||||
filename_prefix = f"live-{uuid4().hex[:12]}-"
|
||||
uploaded_names: dict[str, str] = {}
|
||||
cleanup_error: Exception | None = None
|
||||
try:
|
||||
source_names = sorted({record.expected_document for record in golden})
|
||||
for source_name in source_names:
|
||||
live_name = f"{filename_prefix}{source_name}"
|
||||
payload = (project_root / "fixtures/documents" / source_name).read_bytes()
|
||||
services.ingestion.ingest(live_name, "text/markdown", BytesIO(payload))
|
||||
uploaded_names[source_name] = live_name
|
||||
|
||||
ranks: list[int | None] = []
|
||||
latencies_ms: list[float] = []
|
||||
for record in golden:
|
||||
started = perf_counter()
|
||||
result = services.search.search(record.query, top_k=5)
|
||||
latencies_ms.append((perf_counter() - started) * 1000)
|
||||
ranks.append(
|
||||
_matching_rank(
|
||||
result.results,
|
||||
expected_filename=uploaded_names[record.expected_document],
|
||||
expected_sections=record.expected_sections,
|
||||
)
|
||||
)
|
||||
return _build_report(ranks, latencies_ms, _thresholds_from_environment())
|
||||
finally:
|
||||
try:
|
||||
_cleanup_run_documents(services, filename_prefix)
|
||||
except Exception as error:
|
||||
cleanup_error = error
|
||||
if cleanup_error is not None:
|
||||
raise LiveAcceptanceError("live acceptance cleanup failed") from cleanup_error
|
||||
|
||||
|
||||
def main() -> int:
|
||||
"""Print only aggregate metrics and return non-zero for unsafe or failed runs."""
|
||||
try:
|
||||
report = _run_live(Settings())
|
||||
except Exception:
|
||||
print("live acceptance failed; no provider details were emitted", file=sys.stderr)
|
||||
return 1
|
||||
print(json.dumps(asdict(report), sort_keys=True, separators=(",", ":")))
|
||||
return 0 if report.passed else 2
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@ -1,18 +1,10 @@
|
||||
"""FastAPI application entry point."""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from fastapi import FastAPI
|
||||
from fastapi.responses import FileResponse
|
||||
from fastapi.staticfiles import StaticFiles
|
||||
from pydantic import BaseModel
|
||||
|
||||
from ekp_api.providers import ApiServices
|
||||
from ekp_api.routes import router
|
||||
from ekp_core.config import Settings, get_settings
|
||||
|
||||
ADMIN_WEB_ROOT = Path(__file__).resolve().parents[3] / "admin-web"
|
||||
|
||||
|
||||
class HealthResponse(BaseModel):
|
||||
"""Health endpoint response."""
|
||||
@ -22,10 +14,7 @@ class HealthResponse(BaseModel):
|
||||
environment: str
|
||||
|
||||
|
||||
def create_app(
|
||||
settings: Settings | None = None,
|
||||
api_services: ApiServices | None = None,
|
||||
) -> FastAPI:
|
||||
def create_app(settings: Settings | None = None) -> FastAPI:
|
||||
"""Create the API application with explicit, testable settings."""
|
||||
resolved_settings = settings or get_settings()
|
||||
application = FastAPI(
|
||||
@ -34,26 +23,6 @@ def create_app(
|
||||
docs_url=f"{resolved_settings.api_prefix}/docs",
|
||||
openapi_url=f"{resolved_settings.api_prefix}/openapi.json",
|
||||
)
|
||||
application.state.settings = resolved_settings
|
||||
application.state.api_services = api_services
|
||||
application.include_router(router, prefix=resolved_settings.api_prefix)
|
||||
admin_web_root = (resolved_settings.admin_web_root or ADMIN_WEB_ROOT).resolve()
|
||||
|
||||
# The Demo console is a deliberately thin, same-origin client. It contains
|
||||
# no business logic and calls only the versioned public REST API.
|
||||
application.mount(
|
||||
"/admin/assets",
|
||||
StaticFiles(directory=admin_web_root / "assets"),
|
||||
name="admin-assets",
|
||||
)
|
||||
|
||||
@application.get("/admin", include_in_schema=False)
|
||||
@application.get("/admin/upload", include_in_schema=False)
|
||||
@application.get("/admin/documents", include_in_schema=False)
|
||||
@application.get("/admin/search", include_in_schema=False)
|
||||
async def admin_console() -> FileResponse:
|
||||
"""Serve the three-page Demo Admin Console shell."""
|
||||
return FileResponse(admin_web_root / "index.html")
|
||||
|
||||
@application.get("/healthz", response_model=HealthResponse, tags=["system"])
|
||||
async def health() -> HealthResponse:
|
||||
|
||||
@ -1,347 +0,0 @@
|
||||
"""Runtime provider and application-service composition from safe settings."""
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
from sqlalchemy.orm import Session, sessionmaker
|
||||
|
||||
from ekp_api.celery_queue import CeleryIngestionDispatcher, create_celery_app
|
||||
from ekp_core.chunking import StructureAwareChunker
|
||||
from ekp_core.config import Settings
|
||||
from ekp_core.database import create_database_engine, create_session_factory
|
||||
from ekp_core.dense_retrieval import DenseRetrievalService
|
||||
from ekp_core.document_chunking import DocumentChunkingService, DocumentPipelineService
|
||||
from ekp_core.document_indexing import (
|
||||
DocumentIndexingPipelineService,
|
||||
DocumentIndexingService,
|
||||
)
|
||||
from ekp_core.document_management import DocumentManagementService
|
||||
from ekp_core.document_parsing import DocumentParsingService
|
||||
from ekp_core.document_upload import DocumentUploadService
|
||||
from ekp_core.index_rebuild import resolve_active_index_profile
|
||||
from ekp_core.ingestion_batches import IngestionBatchService
|
||||
from ekp_core.ingestion_jobs import (
|
||||
AsyncIngestionSubmissionService,
|
||||
DerivedDocumentCleanupService,
|
||||
IngestionJobRunner,
|
||||
IngestionJobService,
|
||||
)
|
||||
from ekp_core.search import SearchService
|
||||
from ekp_domain import IndexProfile
|
||||
from ekp_embedding import EmbeddingProvider, QianfanEmbeddingProvider
|
||||
from ekp_object_storage import ObjectStorage, create_bos_object_storage
|
||||
from ekp_object_storage.local import LocalObjectStorage
|
||||
from ekp_parser import ParserRouter
|
||||
from ekp_reranker import QianfanRerankerProvider, RerankerProvider
|
||||
from ekp_vector_store import (
|
||||
VectorStore,
|
||||
create_oceanbase_vector_store,
|
||||
)
|
||||
|
||||
BGE_LARGE_ZH_DIMENSIONS = 1024
|
||||
DEMO_CHUNK_STRATEGY_VERSION = "structure_v1"
|
||||
|
||||
|
||||
class ProviderConfigurationError(ValueError):
|
||||
"""Raised when a runtime provider lacks required injected configuration."""
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ApiServices:
|
||||
"""Business services injected into REST routes without provider leakage."""
|
||||
|
||||
ingestion: DocumentIndexingPipelineService
|
||||
submissions: AsyncIngestionSubmissionService
|
||||
batches: IngestionBatchService
|
||||
jobs: IngestionJobService
|
||||
documents: DocumentManagementService
|
||||
search: SearchService
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class WorkerServices:
|
||||
"""Provider graph required by Celery without REST search-only services."""
|
||||
|
||||
runner: IngestionJobRunner
|
||||
jobs: IngestionJobService
|
||||
dispatcher: CeleryIngestionDispatcher
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _IngestionComponents:
|
||||
"""Shared composition result used by API and independent worker processes."""
|
||||
|
||||
session_factory: sessionmaker[Session]
|
||||
storage: ObjectStorage
|
||||
vector_store: VectorStore
|
||||
embedding_provider: EmbeddingProvider
|
||||
index_profile: IndexProfile
|
||||
upload: DocumentUploadService
|
||||
parsing: DocumentParsingService
|
||||
chunking: DocumentChunkingService
|
||||
indexing: DocumentIndexingService
|
||||
jobs: IngestionJobService
|
||||
dispatcher: CeleryIngestionDispatcher
|
||||
|
||||
|
||||
def create_index_profile(
|
||||
settings: Settings,
|
||||
session_factory: sessionmaker[Session] | None = None,
|
||||
) -> IndexProfile:
|
||||
"""Resolve the canonical active profile or the fresh-deployment baseline."""
|
||||
configured = IndexProfile(
|
||||
profile_id=settings.index_profile_id,
|
||||
chunk_strategy_version=DEMO_CHUNK_STRATEGY_VERSION,
|
||||
embedding_model=settings.embedding_model,
|
||||
vector_dimension=BGE_LARGE_ZH_DIMENSIONS,
|
||||
)
|
||||
if session_factory is None:
|
||||
return configured
|
||||
return resolve_active_index_profile(session_factory, configured)
|
||||
|
||||
|
||||
def create_embedding_provider(settings: Settings) -> QianfanEmbeddingProvider:
|
||||
"""Create Qianfan only when its API key is supplied by the runtime."""
|
||||
if settings.baidu_qianfan_api_key is None:
|
||||
raise ProviderConfigurationError("BAIDU_QIANFAN_API_KEY is required")
|
||||
return QianfanEmbeddingProvider(
|
||||
api_key=settings.baidu_qianfan_api_key.get_secret_value(),
|
||||
endpoint=settings.embedding_endpoint,
|
||||
timeout_seconds=settings.embedding_timeout_seconds,
|
||||
)
|
||||
|
||||
|
||||
def create_object_storage(settings: Settings) -> ObjectStorage:
|
||||
"""Create local or BOS storage while keeping credentials out of core services."""
|
||||
if settings.object_storage_provider == "local":
|
||||
return LocalObjectStorage(settings.object_storage_path)
|
||||
if settings.bos_endpoint is None:
|
||||
raise ProviderConfigurationError("BOS_ENDPOINT is required")
|
||||
if settings.bos_access_key_id is None:
|
||||
raise ProviderConfigurationError("BOS_ACCESS_KEY_ID is required")
|
||||
if settings.bos_secret_access_key is None:
|
||||
raise ProviderConfigurationError("BOS_SECRET_ACCESS_KEY is required")
|
||||
if settings.bos_bucket is None:
|
||||
raise ProviderConfigurationError("BOS_BUCKET is required")
|
||||
return create_bos_object_storage(
|
||||
endpoint=settings.bos_endpoint,
|
||||
access_key_id=settings.bos_access_key_id.get_secret_value(),
|
||||
secret_access_key=settings.bos_secret_access_key.get_secret_value(),
|
||||
bucket_name=settings.bos_bucket,
|
||||
key_prefix=settings.bos_key_prefix,
|
||||
timeout_seconds=settings.bos_timeout_seconds,
|
||||
)
|
||||
|
||||
|
||||
def create_vector_store(settings: Settings) -> VectorStore:
|
||||
"""Create the OceanBase adapter from explicit, secret-safe settings."""
|
||||
if settings.oceanbase_vector_uri is None:
|
||||
raise ProviderConfigurationError("OCEANBASE_VECTOR_URI is required")
|
||||
if settings.oceanbase_vector_user is None:
|
||||
raise ProviderConfigurationError("OCEANBASE_VECTOR_USER is required")
|
||||
if settings.oceanbase_vector_password is None:
|
||||
raise ProviderConfigurationError("OCEANBASE_VECTOR_PASSWORD is required")
|
||||
if settings.oceanbase_vector_database is None:
|
||||
raise ProviderConfigurationError("OCEANBASE_VECTOR_DATABASE is required")
|
||||
return create_oceanbase_vector_store(
|
||||
uri=settings.oceanbase_vector_uri,
|
||||
user=settings.oceanbase_vector_user,
|
||||
password=settings.oceanbase_vector_password.get_secret_value(),
|
||||
database_name=settings.oceanbase_vector_database,
|
||||
table_name=settings.oceanbase_vector_table,
|
||||
index_name=settings.oceanbase_vector_index,
|
||||
vector_dimension=BGE_LARGE_ZH_DIMENSIONS,
|
||||
batch_size=settings.oceanbase_vector_batch_size,
|
||||
timeout_seconds=settings.oceanbase_vector_timeout_seconds,
|
||||
)
|
||||
|
||||
|
||||
def create_dense_retrieval_service(
|
||||
settings: Settings,
|
||||
embedding_provider: EmbeddingProvider,
|
||||
vector_store: VectorStore,
|
||||
index_profile: IndexProfile,
|
||||
) -> DenseRetrievalService:
|
||||
"""Compose the profile-scoped dense stage reused by ingestion search APIs."""
|
||||
return DenseRetrievalService(
|
||||
embedding_provider=embedding_provider,
|
||||
vector_store=vector_store,
|
||||
index_profile=index_profile,
|
||||
default_top_n=settings.dense_top_n,
|
||||
)
|
||||
|
||||
|
||||
def create_reranker_provider(settings: Settings) -> QianfanRerankerProvider:
|
||||
"""Create the Demo BCE reranker using an explicit or shared Qianfan key."""
|
||||
secret = settings.reranker_api_key or settings.baidu_qianfan_api_key
|
||||
if secret is None:
|
||||
raise ProviderConfigurationError("RERANKER_API_KEY or BAIDU_QIANFAN_API_KEY is required")
|
||||
return QianfanRerankerProvider(
|
||||
api_key=secret.get_secret_value(),
|
||||
endpoint=settings.reranker_endpoint,
|
||||
model_name=settings.reranker_model,
|
||||
timeout_seconds=settings.reranker_timeout_seconds,
|
||||
)
|
||||
|
||||
|
||||
def create_search_service(
|
||||
settings: Settings,
|
||||
session_factory: sessionmaker[Session],
|
||||
dense_retriever: DenseRetrievalService,
|
||||
reranker_provider: RerankerProvider,
|
||||
) -> SearchService:
|
||||
"""Compose reranked search independently from REST response schemas."""
|
||||
return SearchService(
|
||||
session_factory=session_factory,
|
||||
dense_retriever=dense_retriever,
|
||||
reranker_provider=reranker_provider,
|
||||
default_top_k=settings.search_top_k,
|
||||
)
|
||||
|
||||
|
||||
def create_runtime_search_service(settings: Settings) -> SearchService:
|
||||
"""Compose only read-path providers for the independent MCP Gateway."""
|
||||
engine = create_database_engine(settings)
|
||||
session_factory = create_session_factory(engine)
|
||||
index_profile = create_index_profile(settings, session_factory)
|
||||
embedding_provider = create_embedding_provider(settings)
|
||||
vector_store = create_vector_store(settings)
|
||||
dense_retrieval = create_dense_retrieval_service(
|
||||
settings,
|
||||
embedding_provider,
|
||||
vector_store,
|
||||
index_profile,
|
||||
)
|
||||
return create_search_service(
|
||||
settings,
|
||||
session_factory,
|
||||
dense_retrieval,
|
||||
create_reranker_provider(settings),
|
||||
)
|
||||
|
||||
|
||||
def _create_ingestion_components(settings: Settings) -> _IngestionComponents:
|
||||
"""Compose provider-neutral ingestion stages and the Celery delivery adapter."""
|
||||
if settings.rabbitmq_url is None or not settings.rabbitmq_url.get_secret_value().strip():
|
||||
raise ProviderConfigurationError("RABBITMQ_URL is required")
|
||||
engine = create_database_engine(settings)
|
||||
session_factory = create_session_factory(engine)
|
||||
storage = create_object_storage(settings)
|
||||
index_profile = create_index_profile(settings, session_factory)
|
||||
embedding_provider = create_embedding_provider(settings)
|
||||
vector_store = create_vector_store(settings)
|
||||
|
||||
upload_service = DocumentUploadService(
|
||||
session_factory,
|
||||
storage,
|
||||
max_upload_bytes=settings.max_upload_bytes,
|
||||
)
|
||||
parsing_service = DocumentParsingService(session_factory, storage, ParserRouter())
|
||||
chunking_service = DocumentChunkingService(
|
||||
session_factory,
|
||||
storage,
|
||||
StructureAwareChunker(
|
||||
max_tokens=settings.chunk_max_tokens,
|
||||
overlap_tokens=settings.chunk_overlap_tokens,
|
||||
max_characters=settings.chunk_max_characters,
|
||||
),
|
||||
)
|
||||
indexing_service = DocumentIndexingService(
|
||||
session_factory,
|
||||
embedding_provider,
|
||||
vector_store,
|
||||
index_profile,
|
||||
)
|
||||
jobs = IngestionJobService(session_factory, settings.ingestion_max_attempts)
|
||||
dispatcher = CeleryIngestionDispatcher(create_celery_app(settings))
|
||||
return _IngestionComponents(
|
||||
session_factory=session_factory,
|
||||
storage=storage,
|
||||
vector_store=vector_store,
|
||||
embedding_provider=embedding_provider,
|
||||
index_profile=index_profile,
|
||||
upload=upload_service,
|
||||
parsing=parsing_service,
|
||||
chunking=chunking_service,
|
||||
indexing=indexing_service,
|
||||
jobs=jobs,
|
||||
dispatcher=dispatcher,
|
||||
)
|
||||
|
||||
|
||||
def _create_runner(settings: Settings, components: _IngestionComponents) -> IngestionJobRunner:
|
||||
"""Compose retry cleanup and execution policy from non-sensitive settings."""
|
||||
cleanup = DerivedDocumentCleanupService(
|
||||
components.session_factory,
|
||||
components.storage,
|
||||
components.vector_store,
|
||||
components.index_profile.profile_id,
|
||||
)
|
||||
return IngestionJobRunner(
|
||||
components.session_factory,
|
||||
components.parsing,
|
||||
components.chunking,
|
||||
components.indexing,
|
||||
cleanup,
|
||||
retry_backoff_seconds=settings.ingestion_retry_backoff_seconds,
|
||||
retry_backoff_max_seconds=settings.ingestion_retry_backoff_max_seconds,
|
||||
lease_seconds=settings.ingestion_worker_lease_seconds,
|
||||
)
|
||||
|
||||
|
||||
def create_worker_services(settings: Settings) -> WorkerServices:
|
||||
"""Build the independent worker runtime from the same application boundaries."""
|
||||
components = _create_ingestion_components(settings)
|
||||
return WorkerServices(
|
||||
runner=_create_runner(settings, components),
|
||||
jobs=components.jobs,
|
||||
dispatcher=components.dispatcher,
|
||||
)
|
||||
|
||||
|
||||
def create_api_services(settings: Settings) -> ApiServices:
|
||||
"""Compose REST services while ingestion execution remains in workers."""
|
||||
components = _create_ingestion_components(settings)
|
||||
document_pipeline = DocumentPipelineService(
|
||||
components.upload,
|
||||
components.parsing,
|
||||
components.chunking,
|
||||
)
|
||||
ingestion = DocumentIndexingPipelineService(document_pipeline, components.indexing)
|
||||
submissions = AsyncIngestionSubmissionService(
|
||||
components.upload,
|
||||
components.jobs,
|
||||
components.dispatcher,
|
||||
)
|
||||
batches = IngestionBatchService(
|
||||
components.session_factory,
|
||||
submissions,
|
||||
max_files=settings.batch_max_files,
|
||||
max_total_bytes=settings.batch_max_total_bytes,
|
||||
)
|
||||
dense_retrieval = create_dense_retrieval_service(
|
||||
settings,
|
||||
components.embedding_provider,
|
||||
components.vector_store,
|
||||
components.index_profile,
|
||||
)
|
||||
reranker_provider = create_reranker_provider(settings)
|
||||
search = create_search_service(
|
||||
settings,
|
||||
components.session_factory,
|
||||
dense_retrieval,
|
||||
reranker_provider,
|
||||
)
|
||||
documents = DocumentManagementService(
|
||||
components.session_factory,
|
||||
components.storage,
|
||||
components.vector_store,
|
||||
active_index_profile_id=components.index_profile.profile_id,
|
||||
)
|
||||
return ApiServices(
|
||||
ingestion=ingestion,
|
||||
submissions=submissions,
|
||||
batches=batches,
|
||||
jobs=components.jobs,
|
||||
documents=documents,
|
||||
search=search,
|
||||
)
|
||||
@ -1,224 +0,0 @@
|
||||
"""Versioned document and search routes for the retrieval Demo."""
|
||||
|
||||
from typing import Annotated
|
||||
from uuid import UUID
|
||||
|
||||
from fastapi import APIRouter, Depends, File, HTTPException, Request, Response, UploadFile, status
|
||||
|
||||
from ekp_api.providers import ApiServices, ProviderConfigurationError, create_api_services
|
||||
from ekp_api.schemas import (
|
||||
DocumentResponse,
|
||||
IngestionAcceptedResponse,
|
||||
IngestionBatchResponse,
|
||||
IngestionJobResponse,
|
||||
SearchRequest,
|
||||
SearchResponse,
|
||||
)
|
||||
from ekp_core.dense_retrieval import InvalidRetrievalQueryError
|
||||
from ekp_core.document_management import (
|
||||
DocumentArtifactCleanupError,
|
||||
DocumentIngestionActiveError,
|
||||
DocumentNotFoundError,
|
||||
)
|
||||
from ekp_core.document_upload import (
|
||||
DocumentTooLargeError,
|
||||
DocumentUploadError,
|
||||
MediaTypeMismatchError,
|
||||
UnsupportedDocumentTypeError,
|
||||
)
|
||||
from ekp_core.ingestion_batches import BatchFileCountLimitError, BatchNotFoundError, BatchSource
|
||||
from ekp_core.ingestion_jobs import IngestionJobNotFoundError
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
def get_api_services(request: Request) -> ApiServices:
|
||||
"""Resolve provider composition lazily so health checks need no cloud credentials."""
|
||||
configured = getattr(request.app.state, "api_services", None)
|
||||
if isinstance(configured, ApiServices):
|
||||
return configured
|
||||
try:
|
||||
services = create_api_services(request.app.state.settings)
|
||||
except ProviderConfigurationError as error:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
|
||||
detail="API providers are not fully configured",
|
||||
) from error
|
||||
request.app.state.api_services = services
|
||||
return services
|
||||
|
||||
|
||||
Services = Annotated[ApiServices, Depends(get_api_services)]
|
||||
|
||||
|
||||
@router.post(
|
||||
"/documents",
|
||||
response_model=IngestionAcceptedResponse,
|
||||
status_code=status.HTTP_202_ACCEPTED,
|
||||
tags=["documents"],
|
||||
)
|
||||
def upload_document(
|
||||
services: Services,
|
||||
file: Annotated[UploadFile, File(description="One PDF, DOCX, TXT, or Markdown file")],
|
||||
) -> IngestionAcceptedResponse:
|
||||
"""Persist one immutable source and durable Job, then return immediately."""
|
||||
try:
|
||||
submission = services.submissions.submit(file.filename or "", file.content_type, file.file)
|
||||
except DocumentTooLargeError as error:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_413_CONTENT_TOO_LARGE,
|
||||
detail="document exceeds the configured upload limit",
|
||||
) from error
|
||||
except (UnsupportedDocumentTypeError, MediaTypeMismatchError) as error:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_415_UNSUPPORTED_MEDIA_TYPE,
|
||||
detail="unsupported document type or media type",
|
||||
) from error
|
||||
except DocumentUploadError as error:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_422_UNPROCESSABLE_CONTENT,
|
||||
detail="document could not be accepted",
|
||||
) from error
|
||||
except Exception as error:
|
||||
# Provider and persistence exceptions are intentionally not reflected to clients.
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_502_BAD_GATEWAY,
|
||||
detail="document acceptance failed",
|
||||
) from error
|
||||
return IngestionAcceptedResponse.from_submission(submission)
|
||||
|
||||
|
||||
@router.get("/jobs/{job_id}", response_model=IngestionJobResponse, tags=["jobs"])
|
||||
def get_ingestion_job(job_id: UUID, services: Services) -> IngestionJobResponse:
|
||||
"""Return durable stage, attempt and safe failure state for one Job."""
|
||||
try:
|
||||
job = services.jobs.get(job_id)
|
||||
except IngestionJobNotFoundError as error:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND, detail="job not found"
|
||||
) from error
|
||||
return IngestionJobResponse.from_job(job)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/ingestion-batches",
|
||||
response_model=IngestionBatchResponse,
|
||||
status_code=status.HTTP_202_ACCEPTED,
|
||||
tags=["jobs"],
|
||||
)
|
||||
def upload_ingestion_batch(
|
||||
services: Services,
|
||||
files: Annotated[
|
||||
list[UploadFile],
|
||||
File(description="One or more PDF, DOCX, TXT, or Markdown files"),
|
||||
],
|
||||
) -> IngestionBatchResponse:
|
||||
"""Accept a bounded batch while isolating every file's result."""
|
||||
try:
|
||||
view = services.batches.submit(
|
||||
tuple(
|
||||
BatchSource(
|
||||
filename=file.filename or "",
|
||||
media_type=file.content_type,
|
||||
source=file.file,
|
||||
)
|
||||
for file in files
|
||||
)
|
||||
)
|
||||
except BatchFileCountLimitError as error:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_413_CONTENT_TOO_LARGE,
|
||||
detail="batch file count exceeds the configured limit",
|
||||
) from error
|
||||
except Exception as error:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_502_BAD_GATEWAY,
|
||||
detail="batch could not be accepted",
|
||||
) from error
|
||||
return IngestionBatchResponse.from_view(view)
|
||||
|
||||
|
||||
@router.get(
|
||||
"/ingestion-batches/{batch_id}",
|
||||
response_model=IngestionBatchResponse,
|
||||
tags=["jobs"],
|
||||
)
|
||||
def get_ingestion_batch(batch_id: UUID, services: Services) -> IngestionBatchResponse:
|
||||
"""Return aggregate and per-file status for Admin Console polling."""
|
||||
try:
|
||||
view = services.batches.get(batch_id)
|
||||
except BatchNotFoundError as error:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND,
|
||||
detail="ingestion batch not found",
|
||||
) from error
|
||||
return IngestionBatchResponse.from_view(view)
|
||||
|
||||
|
||||
@router.get("/documents", response_model=list[DocumentResponse], tags=["documents"])
|
||||
def list_documents(services: Services) -> list[DocumentResponse]:
|
||||
"""List canonical document metadata for the Demo admin page."""
|
||||
return [
|
||||
DocumentResponse.from_document(document) for document in services.documents.list_documents()
|
||||
]
|
||||
|
||||
|
||||
@router.get("/documents/{document_id}", response_model=DocumentResponse, tags=["documents"])
|
||||
def get_document(document_id: UUID, services: Services) -> DocumentResponse:
|
||||
"""Return one document's status and safe processing diagnostic."""
|
||||
try:
|
||||
document = services.documents.get_document(document_id)
|
||||
except DocumentNotFoundError as error:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND, detail="document not found"
|
||||
) from error
|
||||
return DocumentResponse.from_document(document)
|
||||
|
||||
|
||||
@router.delete(
|
||||
"/documents/{document_id}",
|
||||
status_code=status.HTTP_204_NO_CONTENT,
|
||||
tags=["documents"],
|
||||
)
|
||||
def delete_document(document_id: UUID, services: Services) -> Response:
|
||||
"""Directly remove Demo metadata, source objects, and vector rows."""
|
||||
try:
|
||||
services.documents.delete_document(document_id)
|
||||
except DocumentNotFoundError as error:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND, detail="document not found"
|
||||
) from error
|
||||
except DocumentIngestionActiveError as error:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_409_CONFLICT,
|
||||
detail="document ingestion is still active",
|
||||
) from error
|
||||
except DocumentArtifactCleanupError as error:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail="document metadata was deleted but artifact cleanup failed",
|
||||
) from error
|
||||
except Exception as error:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_502_BAD_GATEWAY,
|
||||
detail="document deletion failed",
|
||||
) from error
|
||||
return Response(status_code=status.HTTP_204_NO_CONTENT)
|
||||
|
||||
|
||||
@router.post("/search", response_model=SearchResponse, tags=["search"])
|
||||
def search_knowledge(payload: SearchRequest, services: Services) -> SearchResponse:
|
||||
"""Return reranked Top-K sources without leaking provider or index details."""
|
||||
try:
|
||||
result = services.search.search(payload.query, payload.top_k)
|
||||
except InvalidRetrievalQueryError as error:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_422_UNPROCESSABLE_CONTENT,
|
||||
detail="search query must contain non-whitespace text",
|
||||
) from error
|
||||
except Exception as error:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_502_BAD_GATEWAY,
|
||||
detail="search providers failed",
|
||||
) from error
|
||||
return SearchResponse.from_result(result)
|
||||
@ -1,215 +0,0 @@
|
||||
"""Provider-neutral REST request and response schemas for the Demo API."""
|
||||
|
||||
from datetime import datetime
|
||||
from uuid import UUID
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
|
||||
from ekp_core.ingestion_batches import BatchAggregateStatus, BatchItemView, BatchView
|
||||
from ekp_core.ingestion_jobs import IngestionSubmission
|
||||
from ekp_core.search import SearchResult
|
||||
from ekp_domain import (
|
||||
BatchItemAcceptanceStatus,
|
||||
Document,
|
||||
DocumentStatus,
|
||||
IngestionJob,
|
||||
IngestionJobStatus,
|
||||
IngestionStage,
|
||||
)
|
||||
|
||||
|
||||
class DocumentResponse(BaseModel):
|
||||
"""Public document metadata without internal storage or provider fields."""
|
||||
|
||||
model_config = ConfigDict(from_attributes=True)
|
||||
|
||||
id: UUID
|
||||
filename: str
|
||||
media_type: str
|
||||
size_bytes: int
|
||||
status: DocumentStatus
|
||||
error_message: str | None
|
||||
created_at: datetime
|
||||
updated_at: datetime
|
||||
|
||||
@classmethod
|
||||
def from_document(cls, document: Document) -> "DocumentResponse":
|
||||
"""Serialize a detached canonical model through the explicit API shape."""
|
||||
return cls.model_validate(document)
|
||||
|
||||
|
||||
class IngestionAcceptedResponse(DocumentResponse):
|
||||
"""Single-file acceptance with the durable Job needed for status polling."""
|
||||
|
||||
job_id: UUID
|
||||
job_status: IngestionJobStatus
|
||||
max_attempts: int
|
||||
|
||||
@classmethod
|
||||
def from_submission(cls, submission: IngestionSubmission) -> "IngestionAcceptedResponse":
|
||||
"""Flatten identifiers while retaining the existing document fields."""
|
||||
document = submission.document
|
||||
job = submission.job
|
||||
return cls(
|
||||
id=document.id,
|
||||
filename=document.filename,
|
||||
media_type=document.media_type,
|
||||
size_bytes=document.size_bytes,
|
||||
status=document.status,
|
||||
error_message=document.error_message,
|
||||
created_at=document.created_at,
|
||||
updated_at=document.updated_at,
|
||||
job_id=job.id,
|
||||
job_status=job.status,
|
||||
max_attempts=job.max_attempts,
|
||||
)
|
||||
|
||||
|
||||
class IngestionJobResponse(BaseModel):
|
||||
"""Public queue-independent Job status without provider or broker details."""
|
||||
|
||||
id: UUID
|
||||
document_id: UUID
|
||||
status: IngestionJobStatus
|
||||
stage: IngestionStage | None
|
||||
attempt: int
|
||||
max_attempts: int
|
||||
error_message: str | None
|
||||
created_at: datetime
|
||||
updated_at: datetime
|
||||
completed_at: datetime | None
|
||||
|
||||
@classmethod
|
||||
def from_job(cls, job: IngestionJob) -> "IngestionJobResponse":
|
||||
"""Serialize only client-actionable state and safe diagnostics."""
|
||||
return cls(
|
||||
id=job.id,
|
||||
document_id=job.document_id,
|
||||
status=job.status,
|
||||
stage=job.stage,
|
||||
attempt=job.attempt_count,
|
||||
max_attempts=job.max_attempts,
|
||||
error_message=job.error_message,
|
||||
created_at=job.created_at,
|
||||
updated_at=job.updated_at,
|
||||
completed_at=job.completed_at,
|
||||
)
|
||||
|
||||
|
||||
class IngestionBatchItemResponse(BaseModel):
|
||||
"""Per-file acceptance and latest asynchronous processing state."""
|
||||
|
||||
item_index: int
|
||||
filename: str
|
||||
acceptance_status: BatchItemAcceptanceStatus
|
||||
document_id: UUID | None
|
||||
job_id: UUID | None
|
||||
job_status: IngestionJobStatus | None
|
||||
stage: IngestionStage | None
|
||||
attempt: int
|
||||
max_attempts: int | None
|
||||
error_message: str | None
|
||||
|
||||
@classmethod
|
||||
def from_view(cls, view: BatchItemView) -> "IngestionBatchItemResponse":
|
||||
job = view.job
|
||||
return cls(
|
||||
item_index=view.item.item_index,
|
||||
filename=view.item.filename,
|
||||
acceptance_status=view.item.acceptance_status,
|
||||
document_id=view.item.document_id,
|
||||
job_id=view.item.job_id,
|
||||
job_status=job.status if job is not None else None,
|
||||
stage=job.stage if job is not None else None,
|
||||
attempt=job.attempt_count if job is not None else 0,
|
||||
max_attempts=job.max_attempts if job is not None else None,
|
||||
error_message=(job.error_message if job is not None else view.item.error_message),
|
||||
)
|
||||
|
||||
|
||||
class IngestionBatchResponse(BaseModel):
|
||||
"""Batch aggregate with ordered independently processed files."""
|
||||
|
||||
batch_id: UUID
|
||||
status: BatchAggregateStatus
|
||||
total_files: int
|
||||
accepted_files: int
|
||||
rejected_files: int
|
||||
completed_files: int
|
||||
created_at: datetime
|
||||
items: list[IngestionBatchItemResponse]
|
||||
|
||||
@classmethod
|
||||
def from_view(cls, view: BatchView) -> "IngestionBatchResponse":
|
||||
items = [IngestionBatchItemResponse.from_view(item) for item in view.items]
|
||||
accepted = sum(
|
||||
item.acceptance_status == BatchItemAcceptanceStatus.ACCEPTED for item in items
|
||||
)
|
||||
rejected = sum(
|
||||
item.acceptance_status == BatchItemAcceptanceStatus.REJECTED for item in items
|
||||
)
|
||||
completed = sum(
|
||||
item.acceptance_status == BatchItemAcceptanceStatus.REJECTED
|
||||
or item.job_status
|
||||
in {
|
||||
IngestionJobStatus.SUCCEEDED,
|
||||
IngestionJobStatus.FAILED,
|
||||
IngestionJobStatus.CANCELLED,
|
||||
}
|
||||
for item in items
|
||||
)
|
||||
return cls(
|
||||
batch_id=view.batch.id,
|
||||
status=view.status,
|
||||
total_files=view.batch.total_files,
|
||||
accepted_files=accepted,
|
||||
rejected_files=rejected,
|
||||
completed_files=completed,
|
||||
created_at=view.batch.created_at,
|
||||
items=items,
|
||||
)
|
||||
|
||||
|
||||
class SearchRequest(BaseModel):
|
||||
"""Validated public search input; normalization remains in the core service."""
|
||||
|
||||
query: str = Field(min_length=1, max_length=1600)
|
||||
top_k: int = Field(default=5, ge=1, le=64, strict=True)
|
||||
|
||||
|
||||
class SourceResponse(BaseModel):
|
||||
"""Traceable source returned after dense retrieval and reranking."""
|
||||
|
||||
document_id: UUID
|
||||
filename: str
|
||||
page: int | None
|
||||
section: str | None
|
||||
chunk_id: UUID
|
||||
content: str
|
||||
score: float
|
||||
rerank_score: float
|
||||
|
||||
|
||||
class SearchResponse(BaseModel):
|
||||
"""Minimal response contract consumed by the DEMO-010 Search Lab."""
|
||||
|
||||
results: list[SourceResponse]
|
||||
|
||||
@classmethod
|
||||
def from_result(cls, result: SearchResult) -> "SearchResponse":
|
||||
"""Hide index-profile internals while preserving every required source field."""
|
||||
return cls(
|
||||
results=[
|
||||
SourceResponse(
|
||||
document_id=source.document_id,
|
||||
filename=source.filename,
|
||||
page=source.page,
|
||||
section=source.section,
|
||||
chunk_id=source.chunk_id,
|
||||
content=source.content,
|
||||
score=source.score,
|
||||
rerank_score=source.rerank_score,
|
||||
)
|
||||
for source in result.results
|
||||
]
|
||||
)
|
||||
@ -1,51 +0,0 @@
|
||||
"""Celery worker entry point for durable ingestion Jobs."""
|
||||
|
||||
from functools import lru_cache
|
||||
from typing import Any
|
||||
from uuid import UUID
|
||||
|
||||
from ekp_api.celery_queue import (
|
||||
INGESTION_TASK_NAME,
|
||||
RECOVERY_TASK_NAME,
|
||||
create_celery_app,
|
||||
)
|
||||
from ekp_api.providers import WorkerServices, create_worker_services
|
||||
from ekp_core.config import get_settings
|
||||
from ekp_core.ingestion_jobs import JobRunAction, dispatch_pending_jobs
|
||||
|
||||
_settings = get_settings()
|
||||
celery_app = create_celery_app(_settings)
|
||||
|
||||
|
||||
@lru_cache
|
||||
def _worker_services() -> WorkerServices:
|
||||
"""Build one provider/service graph per worker process."""
|
||||
return create_worker_services(_settings)
|
||||
|
||||
|
||||
@celery_app.task( # type: ignore[untyped-decorator]
|
||||
bind=True,
|
||||
name=INGESTION_TASK_NAME,
|
||||
max_retries=None,
|
||||
)
|
||||
def run_ingestion(self: Any, job_id: str) -> None:
|
||||
"""Execute one delivery and ask Celery to redeliver only when required.
|
||||
|
||||
Celery delivery retries are unbounded because valid-lease deferrals are not
|
||||
business Attempts; the durable runner independently enforces max_attempts.
|
||||
"""
|
||||
result = _worker_services().runner.run(UUID(job_id))
|
||||
if result.action == JobRunAction.RETRY:
|
||||
raise self.retry(countdown=result.retry_delay_seconds or 1, max_retries=None)
|
||||
|
||||
|
||||
@celery_app.task(name=RECOVERY_TASK_NAME) # type: ignore[untyped-decorator]
|
||||
def recover_ingestion_jobs() -> int:
|
||||
"""Redeliver due durable Jobs after API/broker or worker interruptions."""
|
||||
services = _worker_services()
|
||||
return dispatch_pending_jobs(services.jobs, services.dispatcher)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""Run one worker with the embedded Demo recovery scheduler."""
|
||||
celery_app.worker_main(["worker", "--loglevel=INFO", "--queues=ingestion", "--beat"])
|
||||
@ -1,5 +0,0 @@
|
||||
"""Enterprise Knowledge MCP gateway."""
|
||||
|
||||
from ekp_mcp.server import create_mcp_http_app, create_mcp_server
|
||||
|
||||
__all__ = ["create_mcp_http_app", "create_mcp_server"]
|
||||
@ -1,76 +0,0 @@
|
||||
"""Request-scoped MCP service identity and bearer-token verification."""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from secrets import compare_digest
|
||||
|
||||
from mcp.server.auth.middleware.auth_context import get_access_token
|
||||
from mcp.server.auth.provider import AccessToken
|
||||
from mcp.server.mcpserver.exceptions import ToolError
|
||||
from pydantic import SecretStr
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class McpIdentity:
|
||||
"""Authenticated client/service identity passed across the application boundary."""
|
||||
|
||||
client_id: str
|
||||
subject: str
|
||||
scopes: frozenset[str]
|
||||
|
||||
|
||||
class StaticServiceTokenVerifier:
|
||||
"""Validate one runtime-injected, least-privilege Demo service credential.
|
||||
|
||||
The verifier is intentionally small for DEMO-012. It establishes the MCP
|
||||
identity boundary without pretending to be the V1 OAuth or ACL solution.
|
||||
"""
|
||||
|
||||
__slots__ = (
|
||||
"_client_id",
|
||||
"_expected_token",
|
||||
"_issuer",
|
||||
"_scope",
|
||||
"_subject",
|
||||
)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
expected_token: SecretStr,
|
||||
*,
|
||||
client_id: str,
|
||||
subject: str,
|
||||
scope: str,
|
||||
issuer: str,
|
||||
) -> None:
|
||||
self._expected_token = expected_token
|
||||
self._client_id = client_id
|
||||
self._subject = subject
|
||||
self._scope = scope
|
||||
self._issuer = issuer
|
||||
|
||||
async def verify_token(self, token: str) -> AccessToken | None:
|
||||
"""Return scoped identity metadata only for a constant-time token match."""
|
||||
if not compare_digest(token, self._expected_token.get_secret_value()):
|
||||
return None
|
||||
return AccessToken(
|
||||
token=token,
|
||||
client_id=self._client_id,
|
||||
subject=self._subject,
|
||||
scopes=[self._scope],
|
||||
claims={"iss": self._issuer},
|
||||
)
|
||||
|
||||
|
||||
def current_mcp_identity() -> McpIdentity:
|
||||
"""Resolve the SDK-authenticated principal for the active tool request."""
|
||||
access_token = get_access_token()
|
||||
if access_token is None:
|
||||
# HTTP middleware normally rejects this earlier; retaining the guard
|
||||
# prevents accidental unauthenticated in-process tool invocation.
|
||||
raise ToolError("authentication required")
|
||||
subject = access_token.subject or access_token.client_id
|
||||
return McpIdentity(
|
||||
client_id=access_token.client_id,
|
||||
subject=subject,
|
||||
scopes=frozenset(access_token.scopes),
|
||||
)
|
||||
@ -1,30 +0,0 @@
|
||||
"""Command-line entry point for the independent MCP gateway process."""
|
||||
|
||||
from mcp.server.transport_security import TransportSecuritySettings
|
||||
|
||||
from ekp_core.config import get_settings
|
||||
from ekp_mcp.server import create_mcp_server
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""Run the authenticated Streamable HTTP server from environment settings."""
|
||||
settings = get_settings()
|
||||
server = create_mcp_server(settings)
|
||||
server.run(
|
||||
transport="streamable-http",
|
||||
host=settings.mcp_host,
|
||||
port=settings.mcp_port,
|
||||
streamable_http_path=settings.mcp_path,
|
||||
json_response=True,
|
||||
stateless_http=True,
|
||||
max_request_body_size=settings.mcp_max_request_body_bytes,
|
||||
transport_security=TransportSecuritySettings(
|
||||
enable_dns_rebinding_protection=True,
|
||||
allowed_hosts=settings.mcp_allowed_hosts,
|
||||
allowed_origins=settings.mcp_allowed_origins,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__": # pragma: no cover - exercised through the console script.
|
||||
main()
|
||||
@ -1,77 +0,0 @@
|
||||
"""MCP-specific application boundary over the provider-neutral SearchService."""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Protocol
|
||||
from uuid import UUID
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
from ekp_core.search import SearchResult
|
||||
from ekp_mcp.auth import McpIdentity
|
||||
|
||||
|
||||
class KnowledgeSearcher(Protocol):
|
||||
"""Subset of SearchService required by the MCP gateway."""
|
||||
|
||||
def search(self, query: str, top_k: int | None = None) -> SearchResult:
|
||||
"""Return reranked canonical sources for a normalized query."""
|
||||
|
||||
|
||||
class McpSourceResponse(BaseModel):
|
||||
"""Provider-neutral source shape exposed by `search_knowledge`."""
|
||||
|
||||
document_id: UUID
|
||||
filename: str
|
||||
page: int | None
|
||||
section: str | None
|
||||
chunk_id: UUID
|
||||
content: str
|
||||
score: float
|
||||
rerank_score: float
|
||||
|
||||
|
||||
class McpSearchResponse(BaseModel):
|
||||
"""Structured MCP result aligned with the REST search source fields."""
|
||||
|
||||
results: list[McpSourceResponse]
|
||||
|
||||
|
||||
class McpKnowledgeSearch(Protocol):
|
||||
"""Identity-aware search boundary consumed by the protocol server."""
|
||||
|
||||
def search(self, identity: McpIdentity, query: str, top_k: int) -> McpSearchResponse:
|
||||
"""Return authorized, structured knowledge sources."""
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class McpSearchApplication:
|
||||
"""Require an identity before delegating to the shared SearchService.
|
||||
|
||||
Document ACL is outside the current Demo scope. Keeping identity explicit
|
||||
here prevents the gateway from becoming an anonymous bypass when the
|
||||
permission resolver is added in the V1 tasks.
|
||||
"""
|
||||
|
||||
searcher: KnowledgeSearcher
|
||||
required_scope: str
|
||||
|
||||
def search(self, identity: McpIdentity, query: str, top_k: int) -> McpSearchResponse:
|
||||
"""Authorize the Demo scope and serialize canonical search results."""
|
||||
if self.required_scope not in identity.scopes:
|
||||
raise PermissionError("MCP identity lacks the knowledge search scope")
|
||||
result = self.searcher.search(query, top_k)
|
||||
return McpSearchResponse(
|
||||
results=[
|
||||
McpSourceResponse(
|
||||
document_id=source.document_id,
|
||||
filename=source.filename,
|
||||
page=source.page,
|
||||
section=source.section,
|
||||
chunk_id=source.chunk_id,
|
||||
content=source.content,
|
||||
score=source.score,
|
||||
rerank_score=source.rerank_score,
|
||||
)
|
||||
for source in result.results
|
||||
]
|
||||
)
|
||||
@ -1,133 +0,0 @@
|
||||
"""Authenticated Streamable HTTP MCP server exposing Demo business tools."""
|
||||
|
||||
import logging
|
||||
from functools import partial
|
||||
from typing import Annotated
|
||||
|
||||
import anyio
|
||||
from mcp.server import MCPServer
|
||||
from mcp.server.auth.settings import AuthSettings
|
||||
from mcp.server.mcpserver.exceptions import ToolError
|
||||
from mcp.server.transport_security import TransportSecuritySettings
|
||||
from pydantic import Field
|
||||
from starlette.applications import Starlette
|
||||
|
||||
from ekp_api.providers import ApiServices, create_runtime_search_service
|
||||
from ekp_core.config import Settings, get_settings
|
||||
from ekp_core.dense_retrieval import InvalidRetrievalQueryError
|
||||
from ekp_mcp.auth import StaticServiceTokenVerifier, current_mcp_identity
|
||||
from ekp_mcp.search import McpKnowledgeSearch, McpSearchApplication, McpSearchResponse
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class McpConfigurationError(ValueError):
|
||||
"""Raised when the gateway lacks its dedicated runtime credential."""
|
||||
|
||||
|
||||
def _create_search_application(
|
||||
settings: Settings,
|
||||
api_services: ApiServices | None,
|
||||
) -> McpSearchApplication:
|
||||
searcher = (
|
||||
api_services.search if api_services is not None else create_runtime_search_service(settings)
|
||||
)
|
||||
return McpSearchApplication(searcher, settings.mcp_required_scope)
|
||||
|
||||
|
||||
def create_mcp_server(
|
||||
settings: Settings | None = None,
|
||||
*,
|
||||
api_services: ApiServices | None = None,
|
||||
search_application: McpKnowledgeSearch | None = None,
|
||||
) -> MCPServer[dict[str, object]]:
|
||||
"""Create the authenticated MCP gateway with only `search_knowledge`."""
|
||||
resolved_settings = settings or get_settings()
|
||||
if (
|
||||
resolved_settings.mcp_auth_token is None
|
||||
or len(resolved_settings.mcp_auth_token.get_secret_value()) < 32
|
||||
):
|
||||
raise McpConfigurationError("MCP_AUTH_TOKEN must contain at least 32 characters")
|
||||
|
||||
application = search_application or _create_search_application(
|
||||
resolved_settings,
|
||||
api_services,
|
||||
)
|
||||
token_verifier = StaticServiceTokenVerifier(
|
||||
resolved_settings.mcp_auth_token,
|
||||
client_id=resolved_settings.mcp_client_id,
|
||||
subject=resolved_settings.mcp_service_identity,
|
||||
scope=resolved_settings.mcp_required_scope,
|
||||
issuer=str(resolved_settings.mcp_issuer_url),
|
||||
)
|
||||
server: MCPServer[dict[str, object]] = MCPServer(
|
||||
name="enterprise-knowledge",
|
||||
title="Enterprise Knowledge MCP Server",
|
||||
description="Authenticated enterprise knowledge search backed by canonical sources.",
|
||||
version="0.1.0",
|
||||
token_verifier=token_verifier,
|
||||
auth=AuthSettings(
|
||||
issuer_url=resolved_settings.mcp_issuer_url,
|
||||
resource_server_url=resolved_settings.mcp_public_url,
|
||||
required_scopes=[resolved_settings.mcp_required_scope],
|
||||
),
|
||||
log_level=resolved_settings.log_level,
|
||||
)
|
||||
|
||||
@server.tool(
|
||||
name="search_knowledge",
|
||||
description=(
|
||||
"Search authorized enterprise knowledge and return traceable source passages. "
|
||||
"Use for factual questions that should be grounded in indexed company documents."
|
||||
),
|
||||
structured_output=True,
|
||||
)
|
||||
async def search_knowledge(
|
||||
query: Annotated[str, Field(min_length=1, max_length=1600)],
|
||||
top_k: Annotated[int, Field(ge=1, le=64, strict=True)] = 5,
|
||||
) -> McpSearchResponse:
|
||||
"""Run the shared synchronous search pipeline outside the ASGI event loop."""
|
||||
identity = current_mcp_identity()
|
||||
try:
|
||||
return await anyio.to_thread.run_sync(
|
||||
partial(application.search, identity, query, top_k)
|
||||
)
|
||||
except InvalidRetrievalQueryError as error:
|
||||
raise ToolError("search query must contain non-whitespace text") from error
|
||||
except PermissionError as error:
|
||||
raise ToolError("knowledge search is not authorized") from error
|
||||
except Exception:
|
||||
# Do not reflect provider, database, endpoint, or credential details
|
||||
# into MCP content or logs. Operations can correlate this generic
|
||||
# event with upstream sanitized service telemetry.
|
||||
logger.error("MCP knowledge search failed")
|
||||
raise ToolError("knowledge search failed") from None
|
||||
|
||||
return server
|
||||
|
||||
|
||||
def create_mcp_http_app(
|
||||
settings: Settings | None = None,
|
||||
*,
|
||||
api_services: ApiServices | None = None,
|
||||
search_application: McpKnowledgeSearch | None = None,
|
||||
) -> Starlette:
|
||||
"""Build the stateless Streamable HTTP app used by WorkBuddy and tests."""
|
||||
resolved_settings = settings or get_settings()
|
||||
server = create_mcp_server(
|
||||
resolved_settings,
|
||||
api_services=api_services,
|
||||
search_application=search_application,
|
||||
)
|
||||
return server.streamable_http_app(
|
||||
streamable_http_path=resolved_settings.mcp_path,
|
||||
json_response=True,
|
||||
stateless_http=True,
|
||||
max_request_body_size=resolved_settings.mcp_max_request_body_bytes,
|
||||
host=resolved_settings.mcp_host,
|
||||
transport_security=TransportSecuritySettings(
|
||||
enable_dns_rebinding_protection=True,
|
||||
allowed_hosts=resolved_settings.mcp_allowed_hosts,
|
||||
allowed_origins=resolved_settings.mcp_allowed_origins,
|
||||
),
|
||||
)
|
||||
@ -1,61 +0,0 @@
|
||||
# syntax=docker/dockerfile:1.7
|
||||
# One locked runtime produces three independently deployable process images.
|
||||
ARG PYTHON_IMAGE=python:3.12.14-slim-bookworm@sha256:782412e85d0f0984994c290652577d4018aff08145c85b262bb63dc0c7522254
|
||||
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.12.5@sha256:e85be844203885286c60ffad8a858d48afb6c5a5c237ca0e67f12e74b8f174b1
|
||||
|
||||
FROM ${UV_IMAGE} AS uv-bin
|
||||
|
||||
FROM ${PYTHON_IMAGE} AS runtime
|
||||
|
||||
ENV PYTHONUNBUFFERED=1 \
|
||||
PYTHONDONTWRITEBYTECODE=1 \
|
||||
UV_COMPILE_BYTECODE=1 \
|
||||
UV_LINK_MODE=copy \
|
||||
VIRTUAL_ENV=/app/.venv \
|
||||
PATH=/app/.venv/bin:${PATH} \
|
||||
HOME=/tmp/ekp \
|
||||
XDG_CACHE_HOME=/tmp/ekp/cache \
|
||||
APP_ENV=production \
|
||||
API_HOST=0.0.0.0 \
|
||||
MCP_HOST=0.0.0.0 \
|
||||
ADMIN_WEB_ROOT=/app/apps/admin-web
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY --from=uv-bin /uv /uvx /usr/local/bin/
|
||||
COPY pyproject.toml uv.lock README.md alembic.ini ./
|
||||
COPY apps ./apps
|
||||
COPY packages ./packages
|
||||
COPY migrations ./migrations
|
||||
|
||||
# --frozen prevents lock changes; --no-dev and --no-editable keep the runtime
|
||||
# independent from source-tree import hooks while retaining migration sources.
|
||||
RUN uv sync --frozen --no-dev --no-editable \
|
||||
&& test -f /app/apps/admin-web/index.html \
|
||||
&& groupadd --gid 10001 ekp \
|
||||
&& useradd --uid 10001 --gid 10001 --no-create-home --shell /usr/sbin/nologin ekp \
|
||||
&& mkdir -p /tmp/ekp/cache \
|
||||
&& chown -R 10001:10001 /tmp/ekp \
|
||||
&& rm -rf /root/.cache/uv
|
||||
|
||||
USER 10001:10001
|
||||
|
||||
FROM runtime AS api
|
||||
EXPOSE 8000
|
||||
# This endpoint is intentionally credential-free and suitable for container liveness.
|
||||
HEALTHCHECK --interval=30s --timeout=3s --start-period=10s --retries=3 \
|
||||
CMD ["python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/healthz', timeout=2)"]
|
||||
CMD ["uvicorn", "ekp_api.main:app", "--host", "0.0.0.0", "--port", "8000", "--no-access-log"]
|
||||
|
||||
FROM runtime AS mcp
|
||||
EXPOSE 8766
|
||||
# MCP requests require authentication, so the image-level check only verifies PID 1.
|
||||
HEALTHCHECK --interval=30s --timeout=3s --start-period=10s --retries=3 \
|
||||
CMD ["python", "-c", "import os; os.kill(1, 0)"]
|
||||
CMD ["ekp-mcp"]
|
||||
|
||||
FROM runtime AS worker
|
||||
# Worker health is process-level here; durable Job recovery is validated by the service tests.
|
||||
HEALTHCHECK --interval=30s --timeout=3s --start-period=10s --retries=3 \
|
||||
CMD ["python", "-c", "import os; os.kill(1, 0)"]
|
||||
CMD ["ekp-worker"]
|
||||
@ -1,36 +0,0 @@
|
||||
# Reproducible target names consumed by local and CI buildx invocations.
|
||||
variable "IMAGE_REGISTRY" {
|
||||
default = "knowledge-platform"
|
||||
}
|
||||
|
||||
variable "IMAGE_TAG" {
|
||||
default = "dev"
|
||||
}
|
||||
|
||||
group "default" {
|
||||
targets = ["api", "mcp", "worker"]
|
||||
}
|
||||
|
||||
target "common" {
|
||||
context = "."
|
||||
dockerfile = "deploy/docker/Dockerfile"
|
||||
platforms = ["linux/amd64", "linux/arm64"]
|
||||
}
|
||||
|
||||
target "api" {
|
||||
inherits = ["common"]
|
||||
target = "api"
|
||||
tags = ["${IMAGE_REGISTRY}/enterprise-knowledge-api:${IMAGE_TAG}"]
|
||||
}
|
||||
|
||||
target "mcp" {
|
||||
inherits = ["common"]
|
||||
target = "mcp"
|
||||
tags = ["${IMAGE_REGISTRY}/enterprise-knowledge-mcp:${IMAGE_TAG}"]
|
||||
}
|
||||
|
||||
target "worker" {
|
||||
inherits = ["common"]
|
||||
target = "worker"
|
||||
tags = ["${IMAGE_REGISTRY}/enterprise-knowledge-worker:${IMAGE_TAG}"]
|
||||
}
|
||||
@ -1,50 +0,0 @@
|
||||
# Copy this file outside the repository for one environment. It contains only
|
||||
# non-sensitive settings and Kubernetes Secret references; never add values for
|
||||
# DATABASE_URL, RABBITMQ_URL, provider keys, passwords or MCP_AUTH_TOKEN here.
|
||||
existingSecret: knowledge-platform-runtime
|
||||
|
||||
images:
|
||||
api:
|
||||
repository: registry.example.internal/knowledge/enterprise-knowledge-api
|
||||
tag: "0.1.0"
|
||||
digest: ""
|
||||
mcp:
|
||||
repository: registry.example.internal/knowledge/enterprise-knowledge-mcp
|
||||
tag: "0.1.0"
|
||||
digest: ""
|
||||
worker:
|
||||
repository: registry.example.internal/knowledge/enterprise-knowledge-worker
|
||||
tag: "0.1.0"
|
||||
digest: ""
|
||||
|
||||
ingress:
|
||||
enabled: true
|
||||
className: nginx
|
||||
api:
|
||||
host: knowledge.example.internal
|
||||
tlsSecretName: knowledge-platform-api-tls
|
||||
mcp:
|
||||
host: mcp.example.internal
|
||||
tlsSecretName: knowledge-platform-mcp-tls
|
||||
|
||||
config:
|
||||
objectStorage:
|
||||
endpoint: https://s3.example.internal
|
||||
bucket: knowledge-platform
|
||||
vectorStore:
|
||||
uri: oceanbase.example.internal:2881
|
||||
user: knowledge-platform
|
||||
database: knowledge_platform
|
||||
mcp:
|
||||
publicUrl: https://mcp.example.internal/mcp
|
||||
issuerUrl: https://mcp.example.internal
|
||||
allowedHosts:
|
||||
- mcp.example.internal
|
||||
allowedOrigins:
|
||||
- https://mcp.example.internal
|
||||
|
||||
# External RabbitMQ is the production default. If this local dependency is
|
||||
# explicitly enabled for a demo cluster, create its Secret before deployment.
|
||||
rabbitmq:
|
||||
enabled: false
|
||||
existingSecret: knowledge-platform-rabbitmq
|
||||
@ -1,15 +0,0 @@
|
||||
apiVersion: v2
|
||||
name: knowledge-platform
|
||||
description: Enterprise Knowledge Platform public-knowledge deployment
|
||||
type: application
|
||||
version: 0.1.0
|
||||
appVersion: "0.1.0"
|
||||
|
||||
# This local dependency is disabled by default. Production deployments should
|
||||
# use an externally managed RabbitMQ service with an independently owned data
|
||||
# lifecycle; the subchart exists only for explicit development/demo use.
|
||||
dependencies:
|
||||
- name: rabbitmq
|
||||
version: 0.1.0
|
||||
repository: file://charts/rabbitmq
|
||||
condition: rabbitmq.enabled
|
||||
@ -1,6 +0,0 @@
|
||||
apiVersion: v2
|
||||
name: rabbitmq
|
||||
description: Minimal single-node RabbitMQ dependency for local demo clusters only
|
||||
type: application
|
||||
version: 0.1.0
|
||||
appVersion: "4.1.4"
|
||||
@ -1,11 +0,0 @@
|
||||
{{- define "rabbitmq.fullname" -}}
|
||||
{{- printf "%s-rabbitmq" .Release.Name | trunc 63 | trimSuffix "-" -}}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "rabbitmq.image" -}}
|
||||
{{- if .Values.image.digest -}}
|
||||
{{- printf "%s@%s" .Values.image.repository .Values.image.digest -}}
|
||||
{{- else -}}
|
||||
{{- printf "%s:%s" .Values.image.repository .Values.image.tag -}}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
@ -1,16 +0,0 @@
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: {{ include "rabbitmq.fullname" . }}
|
||||
labels:
|
||||
app.kubernetes.io/name: rabbitmq
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
spec:
|
||||
clusterIP: None
|
||||
ports:
|
||||
- name: amqp
|
||||
port: {{ .Values.service.port }}
|
||||
targetPort: amqp
|
||||
selector:
|
||||
app.kubernetes.io/name: rabbitmq
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
@ -1,8 +0,0 @@
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: {{ include "rabbitmq.fullname" . }}
|
||||
labels:
|
||||
app.kubernetes.io/name: rabbitmq
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
automountServiceAccountToken: false
|
||||
@ -1,109 +0,0 @@
|
||||
apiVersion: apps/v1
|
||||
kind: StatefulSet
|
||||
metadata:
|
||||
name: {{ include "rabbitmq.fullname" . }}
|
||||
labels:
|
||||
app.kubernetes.io/name: rabbitmq
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
spec:
|
||||
serviceName: {{ include "rabbitmq.fullname" . }}
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: rabbitmq
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/name: rabbitmq
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
spec:
|
||||
serviceAccountName: {{ include "rabbitmq.fullname" . }}
|
||||
automountServiceAccountToken: false
|
||||
securityContext:
|
||||
runAsNonRoot: true
|
||||
runAsUser: 999
|
||||
runAsGroup: 999
|
||||
fsGroup: 999
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
containers:
|
||||
- name: rabbitmq
|
||||
image: {{ include "rabbitmq.image" . | quote }}
|
||||
imagePullPolicy: {{ .Values.image.pullPolicy }}
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
readOnlyRootFilesystem: true
|
||||
capabilities:
|
||||
drop: ["ALL"]
|
||||
ports:
|
||||
- name: amqp
|
||||
containerPort: 5672
|
||||
env:
|
||||
- name: RABBITMQ_DEFAULT_USER
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: {{ .Values.existingSecret }}
|
||||
key: {{ .Values.secretKeys.username }}
|
||||
- name: RABBITMQ_DEFAULT_PASS
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: {{ .Values.existingSecret }}
|
||||
key: {{ .Values.secretKeys.password }}
|
||||
- name: RABBITMQ_ERLANG_COOKIE
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: {{ .Values.existingSecret }}
|
||||
key: {{ .Values.secretKeys.erlangCookie }}
|
||||
startupProbe:
|
||||
exec:
|
||||
command: ["rabbitmq-diagnostics", "-q", "ping"]
|
||||
periodSeconds: 5
|
||||
failureThreshold: 24
|
||||
readinessProbe:
|
||||
exec:
|
||||
command: ["rabbitmq-diagnostics", "-q", "check_running"]
|
||||
periodSeconds: 10
|
||||
timeoutSeconds: 5
|
||||
failureThreshold: 3
|
||||
livenessProbe:
|
||||
exec:
|
||||
command: ["rabbitmq-diagnostics", "-q", "ping"]
|
||||
periodSeconds: 20
|
||||
timeoutSeconds: 5
|
||||
failureThreshold: 3
|
||||
resources:
|
||||
{{- toYaml .Values.resources | nindent 12 }}
|
||||
volumeMounts:
|
||||
- name: data
|
||||
mountPath: /var/lib/rabbitmq
|
||||
- name: config
|
||||
mountPath: /etc/rabbitmq/conf.d
|
||||
- name: log
|
||||
mountPath: /var/log/rabbitmq
|
||||
- name: tmp
|
||||
mountPath: /tmp
|
||||
volumes:
|
||||
- name: config
|
||||
emptyDir: {}
|
||||
- name: log
|
||||
emptyDir: {}
|
||||
- name: tmp
|
||||
emptyDir: {}
|
||||
{{- if not .Values.persistence.enabled }}
|
||||
- name: data
|
||||
emptyDir: {}
|
||||
{{- end }}
|
||||
{{- if .Values.persistence.enabled }}
|
||||
volumeClaimTemplates:
|
||||
- metadata:
|
||||
name: data
|
||||
spec:
|
||||
accessModes: [{{ .Values.persistence.accessMode | quote }}]
|
||||
{{- if .Values.persistence.storageClass }}
|
||||
storageClassName: {{ .Values.persistence.storageClass | quote }}
|
||||
{{- end }}
|
||||
resources:
|
||||
requests:
|
||||
storage: {{ .Values.persistence.size }}
|
||||
{{- end }}
|
||||
@ -1,69 +0,0 @@
|
||||
{
|
||||
"$schema": "https://json-schema.org/draft-07/schema#",
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["enabled", "existingSecret", "secretKeys", "image", "service", "persistence", "resources"],
|
||||
"properties": {
|
||||
"global": {"type": "object"},
|
||||
"enabled": {"type": "boolean"},
|
||||
"existingSecret": {"type": "string", "minLength": 1},
|
||||
"secretKeys": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["username", "password", "erlangCookie"],
|
||||
"properties": {
|
||||
"username": {"type": "string", "minLength": 1},
|
||||
"password": {"type": "string", "minLength": 1},
|
||||
"erlangCookie": {"type": "string", "minLength": 1}
|
||||
}
|
||||
},
|
||||
"image": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["repository", "tag", "digest", "pullPolicy"],
|
||||
"properties": {
|
||||
"repository": {"type": "string", "minLength": 1},
|
||||
"tag": {"type": "string", "minLength": 1, "not": {"enum": ["latest"]}},
|
||||
"digest": {"type": "string", "pattern": "^$|^sha256:[a-f0-9]{64}$"},
|
||||
"pullPolicy": {"enum": ["Always", "IfNotPresent", "Never"]}
|
||||
}
|
||||
},
|
||||
"service": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["port"],
|
||||
"properties": {"port": {"type": "integer", "minimum": 1, "maximum": 65535}}
|
||||
},
|
||||
"persistence": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["enabled", "storageClass", "accessMode", "size"],
|
||||
"properties": {
|
||||
"enabled": {"type": "boolean"},
|
||||
"storageClass": {"type": "string"},
|
||||
"accessMode": {"enum": ["ReadWriteOnce", "ReadWriteMany"]},
|
||||
"size": {"type": "string", "minLength": 2}
|
||||
}
|
||||
},
|
||||
"resources": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["requests", "limits"],
|
||||
"properties": {
|
||||
"requests": {"$ref": "#/definitions/resourcePair"},
|
||||
"limits": {"$ref": "#/definitions/resourcePair"}
|
||||
}
|
||||
}
|
||||
},
|
||||
"definitions": {
|
||||
"resourcePair": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["cpu", "memory"],
|
||||
"properties": {
|
||||
"cpu": {"type": ["string", "number"]},
|
||||
"memory": {"type": "string", "minLength": 2}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,25 +0,0 @@
|
||||
enabled: false
|
||||
existingSecret: knowledge-platform-rabbitmq
|
||||
secretKeys:
|
||||
username: username
|
||||
password: password
|
||||
erlangCookie: erlang-cookie
|
||||
image:
|
||||
repository: rabbitmq
|
||||
tag: 4.1.4-alpine
|
||||
digest: sha256:b736d649308e1b3e1a116c3f36986b605ee3d03e88f10166be2900083d2e63f2
|
||||
pullPolicy: IfNotPresent
|
||||
service:
|
||||
port: 5672
|
||||
persistence:
|
||||
enabled: true
|
||||
storageClass: ""
|
||||
accessMode: ReadWriteOnce
|
||||
size: 8Gi
|
||||
resources:
|
||||
requests:
|
||||
cpu: 250m
|
||||
memory: 512Mi
|
||||
limits:
|
||||
cpu: "1"
|
||||
memory: 1Gi
|
||||
@ -1,12 +0,0 @@
|
||||
Enterprise Knowledge Platform {{ .Chart.AppVersion }} is managed by Helm release {{ .Release.Name }}.
|
||||
|
||||
Runtime credentials are read only from Secret {{ .Values.existingSecret }}.
|
||||
{{- if .Values.ingress.enabled }}
|
||||
Admin/API: https://{{ .Values.ingress.api.host }}{{ .Values.ingress.api.path }}
|
||||
MCP: https://{{ .Values.ingress.mcp.host }}{{ .Values.ingress.mcp.path }}
|
||||
{{- else }}
|
||||
Ingress is disabled. Enable it in the environment values file before remote use.
|
||||
{{- end }}
|
||||
|
||||
Inspect rollout state without printing environment variables:
|
||||
kubectl -n {{ .Release.Namespace }} get deploy,job,pod
|
||||
@ -1,106 +0,0 @@
|
||||
{{/* Stable resource name helpers keep every workload under one release identity. */}}
|
||||
{{- define "knowledge-platform.name" -}}
|
||||
{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "knowledge-platform.fullname" -}}
|
||||
{{- if .Values.fullnameOverride -}}
|
||||
{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
|
||||
{{- else -}}
|
||||
{{- $name := include "knowledge-platform.name" . -}}
|
||||
{{- if contains $name .Release.Name -}}
|
||||
{{- .Release.Name | trunc 63 | trimSuffix "-" -}}
|
||||
{{- else -}}
|
||||
{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "knowledge-platform.labels" -}}
|
||||
helm.sh/chart: {{ printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | quote }}
|
||||
app.kubernetes.io/name: {{ include "knowledge-platform.name" . }}
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
|
||||
app.kubernetes.io/managed-by: {{ .Release.Service }}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "knowledge-platform.selectorLabels" -}}
|
||||
app.kubernetes.io/name: {{ include "knowledge-platform.name" . }}
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "knowledge-platform.serviceAccountName" -}}
|
||||
{{- if .Values.serviceAccount.create -}}
|
||||
{{- default (include "knowledge-platform.fullname" .) .Values.serviceAccount.name -}}
|
||||
{{- else -}}
|
||||
{{- required "serviceAccount.name is required when serviceAccount.create=false" .Values.serviceAccount.name -}}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/* A supplied digest takes precedence over the immutable release tag. */}}
|
||||
{{- define "knowledge-platform.image" -}}
|
||||
{{- if .digest -}}
|
||||
{{- printf "%s@%s" .repository .digest -}}
|
||||
{{- else -}}
|
||||
{{- printf "%s:%s" .repository .tag -}}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "knowledge-platform.configMapName" -}}
|
||||
{{- printf "%s-runtime" (include "knowledge-platform.fullname" .) -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/* Cross-field rules complement values.schema.json's structural checks. */}}
|
||||
{{- define "knowledge-platform.validateValues" -}}
|
||||
{{- if lt (float64 .Values.config.ingestion.retryBackoffMaxSeconds) (float64 .Values.config.ingestion.retryBackoffSeconds) -}}
|
||||
{{- fail "config.ingestion.retryBackoffMaxSeconds must be >= retryBackoffSeconds" -}}
|
||||
{{- end -}}
|
||||
{{- if lt (float64 .Values.config.ingestion.batchMaxTotalBytes) (float64 .Values.config.ingestion.maxUploadBytes) -}}
|
||||
{{- fail "config.ingestion.batchMaxTotalBytes must be >= maxUploadBytes" -}}
|
||||
{{- end -}}
|
||||
{{- if ge (float64 .Values.config.chunking.overlapTokens) (float64 .Values.config.chunking.maxTokens) -}}
|
||||
{{- fail "config.chunking.overlapTokens must be < maxTokens" -}}
|
||||
{{- end -}}
|
||||
{{- if gt (float64 .Values.config.retrieval.searchTopK) (float64 .Values.config.retrieval.denseTopN) -}}
|
||||
{{- fail "config.retrieval.searchTopK must be <= denseTopN" -}}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "knowledge-platform.podSecurityContext" -}}
|
||||
runAsNonRoot: true
|
||||
runAsUser: 10001
|
||||
runAsGroup: 10001
|
||||
fsGroup: 10001
|
||||
fsGroupChangePolicy: OnRootMismatch
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
{{- end -}}
|
||||
|
||||
{{- define "knowledge-platform.containerSecurityContext" -}}
|
||||
allowPrivilegeEscalation: false
|
||||
readOnlyRootFilesystem: true
|
||||
runAsNonRoot: true
|
||||
runAsUser: 10001
|
||||
runAsGroup: 10001
|
||||
capabilities:
|
||||
drop:
|
||||
- ALL
|
||||
{{- end -}}
|
||||
|
||||
{{- define "knowledge-platform.envFrom" -}}
|
||||
- configMapRef:
|
||||
name: {{ include "knowledge-platform.configMapName" . }}
|
||||
- secretRef:
|
||||
name: {{ .Values.existingSecret }}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "knowledge-platform.tmpVolume" -}}
|
||||
- name: tmp
|
||||
emptyDir:
|
||||
sizeLimit: 1Gi
|
||||
{{- end -}}
|
||||
|
||||
{{- define "knowledge-platform.tmpMount" -}}
|
||||
- name: tmp
|
||||
mountPath: /tmp
|
||||
{{- end -}}
|
||||
@ -1,75 +0,0 @@
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: {{ include "knowledge-platform.fullname" . }}-api
|
||||
labels:
|
||||
{{- include "knowledge-platform.labels" . | nindent 4 }}
|
||||
app.kubernetes.io/component: api
|
||||
spec:
|
||||
replicas: {{ .Values.api.replicaCount }}
|
||||
minReadySeconds: 5
|
||||
strategy:
|
||||
type: RollingUpdate
|
||||
rollingUpdate:
|
||||
maxUnavailable: 0
|
||||
maxSurge: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
{{- include "knowledge-platform.selectorLabels" . | nindent 6 }}
|
||||
app.kubernetes.io/component: api
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
{{- include "knowledge-platform.selectorLabels" . | nindent 8 }}
|
||||
app.kubernetes.io/component: api
|
||||
{{- with .Values.podLabels }}
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
annotations:
|
||||
checksum/runtime-config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }}
|
||||
checksum/runtime-secret-reference: {{ .Values.existingSecret | sha256sum }}
|
||||
{{- with .Values.podAnnotations }}
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
serviceAccountName: {{ include "knowledge-platform.serviceAccountName" . }}
|
||||
automountServiceAccountToken: false
|
||||
securityContext:
|
||||
{{- include "knowledge-platform.podSecurityContext" . | nindent 8 }}
|
||||
{{- with .Values.imagePullSecrets }}
|
||||
imagePullSecrets:
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
containers:
|
||||
- name: api
|
||||
image: {{ include "knowledge-platform.image" .Values.images.api | quote }}
|
||||
imagePullPolicy: {{ .Values.images.api.pullPolicy }}
|
||||
securityContext:
|
||||
{{- include "knowledge-platform.containerSecurityContext" . | nindent 12 }}
|
||||
ports:
|
||||
- name: http
|
||||
containerPort: 8000
|
||||
protocol: TCP
|
||||
envFrom:
|
||||
{{- include "knowledge-platform.envFrom" . | nindent 12 }}
|
||||
startupProbe:
|
||||
httpGet:
|
||||
path: /healthz
|
||||
port: http
|
||||
{{- toYaml .Values.api.probes.startup | nindent 12 }}
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /healthz
|
||||
port: http
|
||||
{{- toYaml .Values.api.probes.readiness | nindent 12 }}
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /healthz
|
||||
port: http
|
||||
{{- toYaml .Values.api.probes.liveness | nindent 12 }}
|
||||
resources:
|
||||
{{- toYaml .Values.api.resources | nindent 12 }}
|
||||
volumeMounts:
|
||||
{{- include "knowledge-platform.tmpMount" . | nindent 12 }}
|
||||
volumes:
|
||||
{{- include "knowledge-platform.tmpVolume" . | nindent 8 }}
|
||||
@ -1,17 +0,0 @@
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: {{ include "knowledge-platform.fullname" . }}-api
|
||||
labels:
|
||||
{{- include "knowledge-platform.labels" . | nindent 4 }}
|
||||
app.kubernetes.io/component: api
|
||||
spec:
|
||||
type: {{ .Values.api.service.type }}
|
||||
ports:
|
||||
- name: http
|
||||
port: {{ .Values.api.service.port }}
|
||||
targetPort: http
|
||||
protocol: TCP
|
||||
selector:
|
||||
{{- include "knowledge-platform.selectorLabels" . | nindent 4 }}
|
||||
app.kubernetes.io/component: api
|
||||
@ -1,66 +0,0 @@
|
||||
{{- include "knowledge-platform.validateValues" . }}
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: {{ include "knowledge-platform.configMapName" . }}
|
||||
labels:
|
||||
{{- include "knowledge-platform.labels" . | nindent 4 }}
|
||||
data:
|
||||
# Only the explicitly enumerated non-sensitive settings are rendered here.
|
||||
APP_ENV: "production"
|
||||
APP_NAME: {{ .Values.config.appName | quote }}
|
||||
LOG_LEVEL: {{ .Values.config.logLevel | quote }}
|
||||
API_HOST: "0.0.0.0"
|
||||
API_PORT: "8000"
|
||||
API_PREFIX: "/api/v1"
|
||||
DATABASE_POOL_SIZE: {{ .Values.config.database.poolSize | quote }}
|
||||
DATABASE_MAX_OVERFLOW: {{ .Values.config.database.maxOverflow | quote }}
|
||||
DATABASE_POOL_RECYCLE_SECONDS: {{ .Values.config.database.poolRecycleSeconds | quote }}
|
||||
DATABASE_POOL_TIMEOUT_SECONDS: {{ .Values.config.database.poolTimeoutSeconds | quote }}
|
||||
DATABASE_CONNECT_TIMEOUT_SECONDS: {{ .Values.config.database.connectTimeoutSeconds | quote }}
|
||||
INGESTION_MAX_ATTEMPTS: {{ .Values.config.ingestion.maxAttempts | quote }}
|
||||
INGESTION_RETRY_BACKOFF_SECONDS: {{ .Values.config.ingestion.retryBackoffSeconds | quote }}
|
||||
INGESTION_RETRY_BACKOFF_MAX_SECONDS: {{ .Values.config.ingestion.retryBackoffMaxSeconds | quote }}
|
||||
INGESTION_WORKER_LEASE_SECONDS: {{ .Values.config.ingestion.workerLeaseSeconds | quote }}
|
||||
INGESTION_RECOVERY_INTERVAL_SECONDS: {{ .Values.config.ingestion.recoveryIntervalSeconds | quote }}
|
||||
BATCH_MAX_FILES: {{ .Values.config.ingestion.batchMaxFiles | quote }}
|
||||
BATCH_MAX_TOTAL_BYTES: {{ printf "%.0f" .Values.config.ingestion.batchMaxTotalBytes | quote }}
|
||||
MAX_UPLOAD_BYTES: {{ printf "%.0f" .Values.config.ingestion.maxUploadBytes | quote }}
|
||||
OBJECT_STORAGE_PROVIDER: {{ .Values.config.objectStorage.provider | quote }}
|
||||
BOS_ENDPOINT: {{ .Values.config.objectStorage.endpoint | quote }}
|
||||
BOS_BUCKET: {{ .Values.config.objectStorage.bucket | quote }}
|
||||
BOS_KEY_PREFIX: {{ .Values.config.objectStorage.keyPrefix | quote }}
|
||||
BOS_TIMEOUT_SECONDS: {{ .Values.config.objectStorage.timeoutSeconds | quote }}
|
||||
CHUNK_MAX_TOKENS: {{ .Values.config.chunking.maxTokens | quote }}
|
||||
CHUNK_OVERLAP_TOKENS: {{ .Values.config.chunking.overlapTokens | quote }}
|
||||
CHUNK_MAX_CHARACTERS: {{ .Values.config.chunking.maxCharacters | quote }}
|
||||
EMBEDDING_PROVIDER: {{ .Values.config.embedding.provider | quote }}
|
||||
EMBEDDING_MODEL: {{ .Values.config.embedding.model | quote }}
|
||||
EMBEDDING_ENDPOINT: {{ .Values.config.embedding.endpoint | quote }}
|
||||
EMBEDDING_TIMEOUT_SECONDS: {{ .Values.config.embedding.timeoutSeconds | quote }}
|
||||
VECTOR_STORE_PROVIDER: {{ .Values.config.vectorStore.provider | quote }}
|
||||
OCEANBASE_VECTOR_URI: {{ .Values.config.vectorStore.uri | quote }}
|
||||
OCEANBASE_VECTOR_USER: {{ .Values.config.vectorStore.user | quote }}
|
||||
OCEANBASE_VECTOR_DATABASE: {{ .Values.config.vectorStore.database | quote }}
|
||||
OCEANBASE_VECTOR_TABLE: {{ .Values.config.vectorStore.table | quote }}
|
||||
OCEANBASE_VECTOR_INDEX: {{ .Values.config.vectorStore.index | quote }}
|
||||
OCEANBASE_VECTOR_BATCH_SIZE: {{ .Values.config.vectorStore.batchSize | quote }}
|
||||
OCEANBASE_VECTOR_TIMEOUT_SECONDS: {{ .Values.config.vectorStore.timeoutSeconds | quote }}
|
||||
INDEX_PROFILE_ID: {{ .Values.config.retrieval.indexProfileId | quote }}
|
||||
DENSE_TOP_N: {{ .Values.config.retrieval.denseTopN | quote }}
|
||||
SEARCH_TOP_K: {{ .Values.config.retrieval.searchTopK | quote }}
|
||||
RERANKER_PROVIDER: {{ .Values.config.reranker.provider | quote }}
|
||||
RERANKER_MODEL: {{ .Values.config.reranker.model | quote }}
|
||||
RERANKER_ENDPOINT: {{ .Values.config.reranker.endpoint | quote }}
|
||||
RERANKER_TIMEOUT_SECONDS: {{ .Values.config.reranker.timeoutSeconds | quote }}
|
||||
MCP_HOST: "0.0.0.0"
|
||||
MCP_PORT: "8766"
|
||||
MCP_PATH: "/mcp"
|
||||
MCP_PUBLIC_URL: {{ .Values.config.mcp.publicUrl | quote }}
|
||||
MCP_ISSUER_URL: {{ .Values.config.mcp.issuerUrl | quote }}
|
||||
MCP_CLIENT_ID: {{ .Values.config.mcp.clientId | quote }}
|
||||
MCP_SERVICE_IDENTITY: {{ .Values.config.mcp.serviceIdentity | quote }}
|
||||
MCP_REQUIRED_SCOPE: {{ .Values.config.mcp.requiredScope | quote }}
|
||||
MCP_ALLOWED_HOSTS: {{ .Values.config.mcp.allowedHosts | toJson | quote }}
|
||||
MCP_ALLOWED_ORIGINS: {{ .Values.config.mcp.allowedOrigins | toJson | quote }}
|
||||
MCP_MAX_REQUEST_BODY_BYTES: {{ printf "%.0f" .Values.config.mcp.maxRequestBodyBytes | quote }}
|
||||
@ -1,42 +0,0 @@
|
||||
{{- if .Values.ingress.enabled }}
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: Ingress
|
||||
metadata:
|
||||
name: {{ include "knowledge-platform.fullname" . }}
|
||||
labels:
|
||||
{{- include "knowledge-platform.labels" . | nindent 4 }}
|
||||
{{- with .Values.ingress.annotations }}
|
||||
annotations:
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
ingressClassName: {{ .Values.ingress.className }}
|
||||
tls:
|
||||
- hosts:
|
||||
- {{ .Values.ingress.api.host | quote }}
|
||||
secretName: {{ .Values.ingress.api.tlsSecretName }}
|
||||
- hosts:
|
||||
- {{ .Values.ingress.mcp.host | quote }}
|
||||
secretName: {{ .Values.ingress.mcp.tlsSecretName }}
|
||||
rules:
|
||||
- host: {{ .Values.ingress.api.host | quote }}
|
||||
http:
|
||||
paths:
|
||||
- path: {{ .Values.ingress.api.path }}
|
||||
pathType: Prefix
|
||||
backend:
|
||||
service:
|
||||
name: {{ include "knowledge-platform.fullname" . }}-api
|
||||
port:
|
||||
number: {{ .Values.api.service.port }}
|
||||
- host: {{ .Values.ingress.mcp.host | quote }}
|
||||
http:
|
||||
paths:
|
||||
- path: {{ .Values.ingress.mcp.path }}
|
||||
pathType: Prefix
|
||||
backend:
|
||||
service:
|
||||
name: {{ include "knowledge-platform.fullname" . }}-mcp
|
||||
port:
|
||||
number: {{ .Values.mcp.service.port }}
|
||||
{{- end }}
|
||||
@ -1,72 +0,0 @@
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: {{ include "knowledge-platform.fullname" . }}-mcp
|
||||
labels:
|
||||
{{- include "knowledge-platform.labels" . | nindent 4 }}
|
||||
app.kubernetes.io/component: mcp
|
||||
spec:
|
||||
replicas: {{ .Values.mcp.replicaCount }}
|
||||
minReadySeconds: 5
|
||||
strategy:
|
||||
type: RollingUpdate
|
||||
rollingUpdate:
|
||||
maxUnavailable: 0
|
||||
maxSurge: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
{{- include "knowledge-platform.selectorLabels" . | nindent 6 }}
|
||||
app.kubernetes.io/component: mcp
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
{{- include "knowledge-platform.selectorLabels" . | nindent 8 }}
|
||||
app.kubernetes.io/component: mcp
|
||||
{{- with .Values.podLabels }}
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
annotations:
|
||||
checksum/runtime-config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }}
|
||||
checksum/runtime-secret-reference: {{ .Values.existingSecret | sha256sum }}
|
||||
{{- with .Values.podAnnotations }}
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
serviceAccountName: {{ include "knowledge-platform.serviceAccountName" . }}
|
||||
automountServiceAccountToken: false
|
||||
securityContext:
|
||||
{{- include "knowledge-platform.podSecurityContext" . | nindent 8 }}
|
||||
{{- with .Values.imagePullSecrets }}
|
||||
imagePullSecrets:
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
containers:
|
||||
- name: mcp
|
||||
image: {{ include "knowledge-platform.image" .Values.images.mcp | quote }}
|
||||
imagePullPolicy: {{ .Values.images.mcp.pullPolicy }}
|
||||
securityContext:
|
||||
{{- include "knowledge-platform.containerSecurityContext" . | nindent 12 }}
|
||||
ports:
|
||||
- name: mcp
|
||||
containerPort: 8766
|
||||
protocol: TCP
|
||||
envFrom:
|
||||
{{- include "knowledge-platform.envFrom" . | nindent 12 }}
|
||||
startupProbe:
|
||||
tcpSocket:
|
||||
port: mcp
|
||||
{{- toYaml .Values.mcp.probes.startup | nindent 12 }}
|
||||
readinessProbe:
|
||||
tcpSocket:
|
||||
port: mcp
|
||||
{{- toYaml .Values.mcp.probes.readiness | nindent 12 }}
|
||||
livenessProbe:
|
||||
tcpSocket:
|
||||
port: mcp
|
||||
{{- toYaml .Values.mcp.probes.liveness | nindent 12 }}
|
||||
resources:
|
||||
{{- toYaml .Values.mcp.resources | nindent 12 }}
|
||||
volumeMounts:
|
||||
{{- include "knowledge-platform.tmpMount" . | nindent 12 }}
|
||||
volumes:
|
||||
{{- include "knowledge-platform.tmpVolume" . | nindent 8 }}
|
||||
@ -1,17 +0,0 @@
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: {{ include "knowledge-platform.fullname" . }}-mcp
|
||||
labels:
|
||||
{{- include "knowledge-platform.labels" . | nindent 4 }}
|
||||
app.kubernetes.io/component: mcp
|
||||
spec:
|
||||
type: {{ .Values.mcp.service.type }}
|
||||
ports:
|
||||
- name: mcp
|
||||
port: {{ .Values.mcp.service.port }}
|
||||
targetPort: mcp
|
||||
protocol: TCP
|
||||
selector:
|
||||
{{- include "knowledge-platform.selectorLabels" . | nindent 4 }}
|
||||
app.kubernetes.io/component: mcp
|
||||
@ -1,59 +0,0 @@
|
||||
{{- if .Values.migration.enabled }}
|
||||
apiVersion: batch/v1
|
||||
kind: Job
|
||||
metadata:
|
||||
name: {{ include "knowledge-platform.fullname" . }}-migration
|
||||
labels:
|
||||
{{- include "knowledge-platform.labels" . | nindent 4 }}
|
||||
app.kubernetes.io/component: migration
|
||||
annotations:
|
||||
# Helm waits for pre-install/pre-upgrade hooks. A failed migration blocks
|
||||
# the application rollout, and the previous Job remains until the next run.
|
||||
helm.sh/hook: pre-install,pre-upgrade
|
||||
helm.sh/hook-weight: "-10"
|
||||
helm.sh/hook-delete-policy: before-hook-creation
|
||||
spec:
|
||||
backoffLimit: {{ .Values.migration.backoffLimit }}
|
||||
activeDeadlineSeconds: {{ .Values.migration.activeDeadlineSeconds }}
|
||||
ttlSecondsAfterFinished: {{ .Values.migration.ttlSecondsAfterFinished }}
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
{{- include "knowledge-platform.selectorLabels" . | nindent 8 }}
|
||||
app.kubernetes.io/component: migration
|
||||
spec:
|
||||
restartPolicy: Never
|
||||
# Pre-install hooks run before the release ServiceAccount exists. The
|
||||
# namespace default account is used with token mounting disabled; the
|
||||
# migration needs no Kubernetes API permissions.
|
||||
serviceAccountName: default
|
||||
automountServiceAccountToken: false
|
||||
securityContext:
|
||||
{{- include "knowledge-platform.podSecurityContext" . | nindent 8 }}
|
||||
{{- with .Values.imagePullSecrets }}
|
||||
imagePullSecrets:
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
containers:
|
||||
- name: migration
|
||||
image: {{ include "knowledge-platform.image" .Values.images.api | quote }}
|
||||
imagePullPolicy: {{ .Values.images.api.pullPolicy }}
|
||||
command: ["alembic"]
|
||||
args: ["upgrade", "head"]
|
||||
securityContext:
|
||||
{{- include "knowledge-platform.containerSecurityContext" . | nindent 12 }}
|
||||
env:
|
||||
- name: APP_ENV
|
||||
value: production
|
||||
- name: DATABASE_URL
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: {{ .Values.existingSecret }}
|
||||
key: DATABASE_URL
|
||||
resources:
|
||||
{{- toYaml .Values.migration.resources | nindent 12 }}
|
||||
volumeMounts:
|
||||
{{- include "knowledge-platform.tmpMount" . | nindent 12 }}
|
||||
volumes:
|
||||
{{- include "knowledge-platform.tmpVolume" . | nindent 8 }}
|
||||
{{- end }}
|
||||
@ -1,71 +0,0 @@
|
||||
{{- if .Values.scheduler.enabled }}
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: {{ include "knowledge-platform.fullname" . }}-scheduler
|
||||
labels:
|
||||
{{- include "knowledge-platform.labels" . | nindent 4 }}
|
||||
app.kubernetes.io/component: scheduler
|
||||
spec:
|
||||
# A single Celery Beat owns periodic durable-Job recovery. Recreate prevents
|
||||
# two schedulers from overlapping during upgrades.
|
||||
replicas: 1
|
||||
strategy:
|
||||
type: Recreate
|
||||
selector:
|
||||
matchLabels:
|
||||
{{- include "knowledge-platform.selectorLabels" . | nindent 6 }}
|
||||
app.kubernetes.io/component: scheduler
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
{{- include "knowledge-platform.selectorLabels" . | nindent 8 }}
|
||||
app.kubernetes.io/component: scheduler
|
||||
{{- with .Values.podLabels }}
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
annotations:
|
||||
checksum/runtime-config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }}
|
||||
checksum/runtime-secret-reference: {{ .Values.existingSecret | sha256sum }}
|
||||
{{- with .Values.podAnnotations }}
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
serviceAccountName: {{ include "knowledge-platform.serviceAccountName" . }}
|
||||
automountServiceAccountToken: false
|
||||
securityContext:
|
||||
{{- include "knowledge-platform.podSecurityContext" . | nindent 8 }}
|
||||
{{- with .Values.imagePullSecrets }}
|
||||
imagePullSecrets:
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
containers:
|
||||
- name: scheduler
|
||||
image: {{ include "knowledge-platform.image" .Values.images.worker | quote }}
|
||||
imagePullPolicy: {{ .Values.images.worker.pullPolicy }}
|
||||
workingDir: /tmp
|
||||
command: ["celery"]
|
||||
args:
|
||||
- --app=ekp_api.worker:celery_app
|
||||
- beat
|
||||
- --loglevel=INFO
|
||||
- --schedule=/tmp/celerybeat-schedule
|
||||
securityContext:
|
||||
{{- include "knowledge-platform.containerSecurityContext" . | nindent 12 }}
|
||||
envFrom:
|
||||
{{- include "knowledge-platform.envFrom" . | nindent 12 }}
|
||||
startupProbe:
|
||||
exec:
|
||||
command: ["python", "-c", "import os; os.kill(1, 0)"]
|
||||
{{- toYaml .Values.scheduler.probes.startup | nindent 12 }}
|
||||
livenessProbe:
|
||||
exec:
|
||||
command: ["python", "-c", "import os; os.kill(1, 0)"]
|
||||
{{- toYaml .Values.scheduler.probes.liveness | nindent 12 }}
|
||||
resources:
|
||||
{{- toYaml .Values.scheduler.resources | nindent 12 }}
|
||||
volumeMounts:
|
||||
{{- include "knowledge-platform.tmpMount" . | nindent 12 }}
|
||||
volumes:
|
||||
{{- include "knowledge-platform.tmpVolume" . | nindent 8 }}
|
||||
{{- end }}
|
||||
@ -1,13 +0,0 @@
|
||||
{{- if .Values.serviceAccount.create }}
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: {{ include "knowledge-platform.serviceAccountName" . }}
|
||||
labels:
|
||||
{{- include "knowledge-platform.labels" . | nindent 4 }}
|
||||
{{- with .Values.serviceAccount.annotations }}
|
||||
annotations:
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
automountServiceAccountToken: false
|
||||
{{- end }}
|
||||
@ -1,17 +0,0 @@
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: {{ include "knowledge-platform.fullname" . }}-preflight-references
|
||||
labels:
|
||||
{{- include "knowledge-platform.labels" . | nindent 4 }}
|
||||
annotations:
|
||||
# The Make preflight renders this test hook to discover only resource
|
||||
# names. It never renders or reads Secret data.
|
||||
helm.sh/hook: test
|
||||
data:
|
||||
runtimeSecret: {{ .Values.existingSecret | quote }}
|
||||
ingressEnabled: {{ .Values.ingress.enabled | quote }}
|
||||
apiTlsSecret: {{ .Values.ingress.api.tlsSecretName | quote }}
|
||||
mcpTlsSecret: {{ .Values.ingress.mcp.tlsSecretName | quote }}
|
||||
rabbitmqEnabled: {{ .Values.rabbitmq.enabled | quote }}
|
||||
rabbitmqSecret: {{ .Values.rabbitmq.existingSecret | quote }}
|
||||
@ -1,71 +0,0 @@
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: {{ include "knowledge-platform.fullname" . }}-worker
|
||||
labels:
|
||||
{{- include "knowledge-platform.labels" . | nindent 4 }}
|
||||
app.kubernetes.io/component: worker
|
||||
spec:
|
||||
replicas: {{ .Values.worker.replicaCount }}
|
||||
strategy:
|
||||
type: RollingUpdate
|
||||
rollingUpdate:
|
||||
maxUnavailable: 1
|
||||
maxSurge: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
{{- include "knowledge-platform.selectorLabels" . | nindent 6 }}
|
||||
app.kubernetes.io/component: worker
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
{{- include "knowledge-platform.selectorLabels" . | nindent 8 }}
|
||||
app.kubernetes.io/component: worker
|
||||
{{- with .Values.podLabels }}
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
annotations:
|
||||
checksum/runtime-config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }}
|
||||
checksum/runtime-secret-reference: {{ .Values.existingSecret | sha256sum }}
|
||||
{{- with .Values.podAnnotations }}
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
serviceAccountName: {{ include "knowledge-platform.serviceAccountName" . }}
|
||||
automountServiceAccountToken: false
|
||||
terminationGracePeriodSeconds: {{ .Values.worker.terminationGracePeriodSeconds }}
|
||||
securityContext:
|
||||
{{- include "knowledge-platform.podSecurityContext" . | nindent 8 }}
|
||||
{{- with .Values.imagePullSecrets }}
|
||||
imagePullSecrets:
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
containers:
|
||||
- name: worker
|
||||
image: {{ include "knowledge-platform.image" .Values.images.worker | quote }}
|
||||
imagePullPolicy: {{ .Values.images.worker.pullPolicy }}
|
||||
workingDir: /tmp
|
||||
command: ["celery"]
|
||||
args:
|
||||
- --app=ekp_api.worker:celery_app
|
||||
- worker
|
||||
- --loglevel=INFO
|
||||
- --queues=ingestion
|
||||
securityContext:
|
||||
{{- include "knowledge-platform.containerSecurityContext" . | nindent 12 }}
|
||||
envFrom:
|
||||
{{- include "knowledge-platform.envFrom" . | nindent 12 }}
|
||||
startupProbe:
|
||||
exec:
|
||||
command: ["python", "-c", "import os; os.kill(1, 0)"]
|
||||
{{- toYaml .Values.worker.probes.startup | nindent 12 }}
|
||||
livenessProbe:
|
||||
exec:
|
||||
command: ["python", "-c", "import os; os.kill(1, 0)"]
|
||||
{{- toYaml .Values.worker.probes.liveness | nindent 12 }}
|
||||
resources:
|
||||
{{- toYaml .Values.worker.resources | nindent 12 }}
|
||||
volumeMounts:
|
||||
{{- include "knowledge-platform.tmpMount" . | nindent 12 }}
|
||||
volumes:
|
||||
{{- include "knowledge-platform.tmpVolume" . | nindent 8 }}
|
||||
@ -1,338 +0,0 @@
|
||||
{
|
||||
"$schema": "https://json-schema.org/draft-07/schema#",
|
||||
"title": "Enterprise Knowledge Platform Helm values",
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": [
|
||||
"existingSecret",
|
||||
"images",
|
||||
"serviceAccount",
|
||||
"api",
|
||||
"mcp",
|
||||
"worker",
|
||||
"scheduler",
|
||||
"migration",
|
||||
"ingress",
|
||||
"config",
|
||||
"rabbitmq"
|
||||
],
|
||||
"properties": {
|
||||
"nameOverride": {"type": "string"},
|
||||
"fullnameOverride": {"type": "string"},
|
||||
"existingSecret": {"$ref": "#/definitions/kubernetesName"},
|
||||
"imagePullSecrets": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["name"],
|
||||
"properties": {"name": {"$ref": "#/definitions/kubernetesName"}}
|
||||
}
|
||||
},
|
||||
"images": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["api", "mcp", "worker"],
|
||||
"properties": {
|
||||
"api": {"$ref": "#/definitions/image"},
|
||||
"mcp": {"$ref": "#/definitions/image"},
|
||||
"worker": {"$ref": "#/definitions/image"}
|
||||
}
|
||||
},
|
||||
"serviceAccount": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["create", "name", "annotations"],
|
||||
"properties": {
|
||||
"create": {"type": "boolean"},
|
||||
"name": {"type": "string"},
|
||||
"annotations": {"$ref": "#/definitions/stringMap"}
|
||||
}
|
||||
},
|
||||
"podAnnotations": {"$ref": "#/definitions/stringMap"},
|
||||
"podLabels": {"$ref": "#/definitions/stringMap"},
|
||||
"api": {"$ref": "#/definitions/httpWorkload"},
|
||||
"mcp": {"$ref": "#/definitions/httpWorkload"},
|
||||
"worker": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["replicaCount", "terminationGracePeriodSeconds", "resources", "probes"],
|
||||
"properties": {
|
||||
"replicaCount": {"type": "integer", "minimum": 1, "maximum": 100},
|
||||
"terminationGracePeriodSeconds": {"type": "integer", "minimum": 30, "maximum": 3600},
|
||||
"resources": {"$ref": "#/definitions/resources"},
|
||||
"probes": {"$ref": "#/definitions/processProbes"}
|
||||
}
|
||||
},
|
||||
"scheduler": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["enabled", "resources", "probes"],
|
||||
"properties": {
|
||||
"enabled": {"type": "boolean"},
|
||||
"resources": {"$ref": "#/definitions/resources"},
|
||||
"probes": {"$ref": "#/definitions/processProbes"}
|
||||
}
|
||||
},
|
||||
"migration": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["enabled", "activeDeadlineSeconds", "backoffLimit", "ttlSecondsAfterFinished", "resources"],
|
||||
"properties": {
|
||||
"enabled": {"type": "boolean"},
|
||||
"activeDeadlineSeconds": {"type": "integer", "minimum": 60, "maximum": 3600},
|
||||
"backoffLimit": {"type": "integer", "minimum": 0, "maximum": 3},
|
||||
"ttlSecondsAfterFinished": {"type": "integer", "minimum": 0},
|
||||
"resources": {"$ref": "#/definitions/resources"}
|
||||
}
|
||||
},
|
||||
"ingress": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["enabled", "className", "annotations", "api", "mcp"],
|
||||
"properties": {
|
||||
"enabled": {"type": "boolean"},
|
||||
"className": {"type": "string", "minLength": 1},
|
||||
"annotations": {"$ref": "#/definitions/stringMap"},
|
||||
"api": {"$ref": "#/definitions/ingressRoute"},
|
||||
"mcp": {"$ref": "#/definitions/ingressRoute"}
|
||||
}
|
||||
},
|
||||
"config": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["appName", "logLevel", "database", "ingestion", "objectStorage", "chunking", "embedding", "vectorStore", "retrieval", "reranker", "mcp"],
|
||||
"properties": {
|
||||
"appName": {"type": "string", "minLength": 1},
|
||||
"logLevel": {"enum": ["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"]},
|
||||
"database": {"$ref": "#/definitions/databaseConfig"},
|
||||
"ingestion": {"$ref": "#/definitions/ingestionConfig"},
|
||||
"objectStorage": {"$ref": "#/definitions/objectStorageConfig"},
|
||||
"chunking": {"$ref": "#/definitions/chunkingConfig"},
|
||||
"embedding": {"$ref": "#/definitions/embeddingConfig"},
|
||||
"vectorStore": {"$ref": "#/definitions/vectorStoreConfig"},
|
||||
"retrieval": {"$ref": "#/definitions/retrievalConfig"},
|
||||
"reranker": {"$ref": "#/definitions/rerankerConfig"},
|
||||
"mcp": {"$ref": "#/definitions/mcpConfig"}
|
||||
}
|
||||
},
|
||||
"rabbitmq": {
|
||||
"type": "object",
|
||||
"required": ["enabled", "existingSecret"],
|
||||
"properties": {
|
||||
"enabled": {"type": "boolean"},
|
||||
"existingSecret": {"$ref": "#/definitions/kubernetesName"}
|
||||
}
|
||||
}
|
||||
},
|
||||
"definitions": {
|
||||
"kubernetesName": {
|
||||
"type": "string",
|
||||
"minLength": 1,
|
||||
"maxLength": 253,
|
||||
"pattern": "^[a-z0-9]([-a-z0-9.]*[a-z0-9])?$"
|
||||
},
|
||||
"stringMap": {
|
||||
"type": "object",
|
||||
"additionalProperties": {"type": "string"}
|
||||
},
|
||||
"image": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["repository", "tag", "digest", "pullPolicy"],
|
||||
"properties": {
|
||||
"repository": {"type": "string", "minLength": 1},
|
||||
"tag": {"type": "string", "minLength": 1, "not": {"enum": ["latest"]}},
|
||||
"digest": {"type": "string", "pattern": "^$|^sha256:[a-f0-9]{64}$"},
|
||||
"pullPolicy": {"enum": ["Always", "IfNotPresent", "Never"]}
|
||||
}
|
||||
},
|
||||
"probeTiming": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["failureThreshold", "periodSeconds"],
|
||||
"properties": {
|
||||
"failureThreshold": {"type": "integer", "minimum": 1},
|
||||
"periodSeconds": {"type": "integer", "minimum": 1},
|
||||
"timeoutSeconds": {"type": "integer", "minimum": 1}
|
||||
}
|
||||
},
|
||||
"httpProbes": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["startup", "readiness", "liveness"],
|
||||
"properties": {
|
||||
"startup": {"$ref": "#/definitions/probeTiming"},
|
||||
"readiness": {"$ref": "#/definitions/probeTiming"},
|
||||
"liveness": {"$ref": "#/definitions/probeTiming"}
|
||||
}
|
||||
},
|
||||
"processProbes": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["startup", "liveness"],
|
||||
"properties": {
|
||||
"startup": {"$ref": "#/definitions/probeTiming"},
|
||||
"liveness": {"$ref": "#/definitions/probeTiming"}
|
||||
}
|
||||
},
|
||||
"resources": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["requests", "limits"],
|
||||
"properties": {
|
||||
"requests": {"$ref": "#/definitions/resourcePair"},
|
||||
"limits": {"$ref": "#/definitions/resourcePair"}
|
||||
}
|
||||
},
|
||||
"resourcePair": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["cpu", "memory"],
|
||||
"properties": {
|
||||
"cpu": {"type": ["string", "number"]},
|
||||
"memory": {"type": "string", "minLength": 2}
|
||||
}
|
||||
},
|
||||
"httpWorkload": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["replicaCount", "service", "resources", "probes"],
|
||||
"properties": {
|
||||
"replicaCount": {"type": "integer", "minimum": 2, "maximum": 100},
|
||||
"service": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["type", "port"],
|
||||
"properties": {
|
||||
"type": {"enum": ["ClusterIP"]},
|
||||
"port": {"type": "integer", "minimum": 1, "maximum": 65535}
|
||||
}
|
||||
},
|
||||
"resources": {"$ref": "#/definitions/resources"},
|
||||
"probes": {"$ref": "#/definitions/httpProbes"}
|
||||
}
|
||||
},
|
||||
"ingressRoute": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["host", "path", "tlsSecretName"],
|
||||
"properties": {
|
||||
"host": {"type": "string", "minLength": 1},
|
||||
"path": {"type": "string", "pattern": "^/"},
|
||||
"tlsSecretName": {"$ref": "#/definitions/kubernetesName"}
|
||||
}
|
||||
},
|
||||
"databaseConfig": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["poolSize", "maxOverflow", "poolRecycleSeconds", "poolTimeoutSeconds", "connectTimeoutSeconds"],
|
||||
"properties": {
|
||||
"poolSize": {"type": "integer", "minimum": 1, "maximum": 100},
|
||||
"maxOverflow": {"type": "integer", "minimum": 0, "maximum": 100},
|
||||
"poolRecycleSeconds": {"type": "integer", "minimum": 1},
|
||||
"poolTimeoutSeconds": {"type": "number", "exclusiveMinimum": 0},
|
||||
"connectTimeoutSeconds": {"type": "integer", "minimum": 1}
|
||||
}
|
||||
},
|
||||
"ingestionConfig": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["maxAttempts", "retryBackoffSeconds", "retryBackoffMaxSeconds", "workerLeaseSeconds", "recoveryIntervalSeconds", "batchMaxFiles", "batchMaxTotalBytes", "maxUploadBytes"],
|
||||
"properties": {
|
||||
"maxAttempts": {"type": "integer", "minimum": 1, "maximum": 10},
|
||||
"retryBackoffSeconds": {"type": "integer", "minimum": 1},
|
||||
"retryBackoffMaxSeconds": {"type": "integer", "minimum": 1},
|
||||
"workerLeaseSeconds": {"type": "integer", "minimum": 30},
|
||||
"recoveryIntervalSeconds": {"type": "integer", "minimum": 5},
|
||||
"batchMaxFiles": {"type": "integer", "minimum": 1, "maximum": 100},
|
||||
"batchMaxTotalBytes": {"type": "integer", "minimum": 1},
|
||||
"maxUploadBytes": {"type": "integer", "minimum": 1}
|
||||
}
|
||||
},
|
||||
"objectStorageConfig": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["provider", "endpoint", "bucket", "keyPrefix", "timeoutSeconds"],
|
||||
"properties": {
|
||||
"provider": {"const": "bos"},
|
||||
"endpoint": {"type": "string", "minLength": 1},
|
||||
"bucket": {"type": "string", "minLength": 1},
|
||||
"keyPrefix": {"type": "string", "minLength": 1},
|
||||
"timeoutSeconds": {"type": "number", "exclusiveMinimum": 0}
|
||||
}
|
||||
},
|
||||
"chunkingConfig": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["maxTokens", "overlapTokens", "maxCharacters"],
|
||||
"properties": {
|
||||
"maxTokens": {"type": "integer", "minimum": 1},
|
||||
"overlapTokens": {"type": "integer", "minimum": 0},
|
||||
"maxCharacters": {"type": "integer", "minimum": 4}
|
||||
}
|
||||
},
|
||||
"embeddingConfig": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["provider", "model", "endpoint", "timeoutSeconds"],
|
||||
"properties": {
|
||||
"provider": {"const": "qianfan"},
|
||||
"model": {"const": "bge-large-zh"},
|
||||
"endpoint": {"type": "string", "minLength": 1},
|
||||
"timeoutSeconds": {"type": "number", "exclusiveMinimum": 0}
|
||||
}
|
||||
},
|
||||
"vectorStoreConfig": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["provider", "uri", "user", "database", "table", "index", "batchSize", "timeoutSeconds"],
|
||||
"properties": {
|
||||
"provider": {"const": "oceanbase"},
|
||||
"uri": {"type": "string", "minLength": 1},
|
||||
"user": {"type": "string", "minLength": 1},
|
||||
"database": {"type": "string", "minLength": 1},
|
||||
"table": {"type": "string", "minLength": 1},
|
||||
"index": {"type": "string", "minLength": 1},
|
||||
"batchSize": {"type": "integer", "minimum": 1, "maximum": 1000},
|
||||
"timeoutSeconds": {"type": "number", "exclusiveMinimum": 0}
|
||||
}
|
||||
},
|
||||
"retrievalConfig": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["indexProfileId", "denseTopN", "searchTopK"],
|
||||
"properties": {
|
||||
"indexProfileId": {"type": "string", "minLength": 1},
|
||||
"denseTopN": {"type": "integer", "minimum": 1, "maximum": 64},
|
||||
"searchTopK": {"type": "integer", "minimum": 1, "maximum": 64}
|
||||
}
|
||||
},
|
||||
"rerankerConfig": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["provider", "model", "endpoint", "timeoutSeconds"],
|
||||
"properties": {
|
||||
"provider": {"const": "qianfan"},
|
||||
"model": {"const": "bce-reranker-base"},
|
||||
"endpoint": {"type": "string", "minLength": 1},
|
||||
"timeoutSeconds": {"type": "number", "exclusiveMinimum": 0}
|
||||
}
|
||||
},
|
||||
"mcpConfig": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["publicUrl", "issuerUrl", "clientId", "serviceIdentity", "requiredScope", "allowedHosts", "allowedOrigins", "maxRequestBodyBytes"],
|
||||
"properties": {
|
||||
"publicUrl": {"type": "string", "pattern": "^https://"},
|
||||
"issuerUrl": {"type": "string", "pattern": "^https://"},
|
||||
"clientId": {"type": "string", "minLength": 1},
|
||||
"serviceIdentity": {"type": "string", "minLength": 1},
|
||||
"requiredScope": {"const": "knowledge:search"},
|
||||
"allowedHosts": {"type": "array", "minItems": 1, "items": {"type": "string", "minLength": 1}},
|
||||
"allowedOrigins": {"type": "array", "minItems": 1, "items": {"type": "string", "pattern": "^https://"}},
|
||||
"maxRequestBodyBytes": {"type": "integer", "minimum": 1024, "maximum": 4194304}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,218 +0,0 @@
|
||||
# DEPLOY-002 safe defaults. Real endpoints, immutable image coordinates and
|
||||
# Secret names are supplied in an environment-specific values file.
|
||||
nameOverride: ""
|
||||
fullnameOverride: ""
|
||||
|
||||
existingSecret: knowledge-platform-runtime
|
||||
imagePullSecrets: []
|
||||
|
||||
images:
|
||||
api:
|
||||
repository: knowledge-platform/enterprise-knowledge-api
|
||||
tag: "0.1.0"
|
||||
digest: ""
|
||||
pullPolicy: IfNotPresent
|
||||
mcp:
|
||||
repository: knowledge-platform/enterprise-knowledge-mcp
|
||||
tag: "0.1.0"
|
||||
digest: ""
|
||||
pullPolicy: IfNotPresent
|
||||
worker:
|
||||
repository: knowledge-platform/enterprise-knowledge-worker
|
||||
tag: "0.1.0"
|
||||
digest: ""
|
||||
pullPolicy: IfNotPresent
|
||||
|
||||
serviceAccount:
|
||||
create: true
|
||||
name: ""
|
||||
annotations: {}
|
||||
|
||||
podAnnotations: {}
|
||||
podLabels: {}
|
||||
|
||||
api:
|
||||
replicaCount: 2
|
||||
service:
|
||||
type: ClusterIP
|
||||
port: 80
|
||||
resources:
|
||||
requests:
|
||||
cpu: 250m
|
||||
memory: 512Mi
|
||||
limits:
|
||||
cpu: "1"
|
||||
memory: 1Gi
|
||||
probes:
|
||||
startup:
|
||||
failureThreshold: 30
|
||||
periodSeconds: 2
|
||||
readiness:
|
||||
periodSeconds: 10
|
||||
timeoutSeconds: 2
|
||||
failureThreshold: 3
|
||||
liveness:
|
||||
periodSeconds: 20
|
||||
timeoutSeconds: 2
|
||||
failureThreshold: 3
|
||||
|
||||
mcp:
|
||||
replicaCount: 2
|
||||
service:
|
||||
type: ClusterIP
|
||||
port: 80
|
||||
resources:
|
||||
requests:
|
||||
cpu: 250m
|
||||
memory: 512Mi
|
||||
limits:
|
||||
cpu: "1"
|
||||
memory: 1Gi
|
||||
probes:
|
||||
startup:
|
||||
failureThreshold: 30
|
||||
periodSeconds: 2
|
||||
readiness:
|
||||
periodSeconds: 10
|
||||
timeoutSeconds: 2
|
||||
failureThreshold: 3
|
||||
liveness:
|
||||
periodSeconds: 20
|
||||
timeoutSeconds: 2
|
||||
failureThreshold: 3
|
||||
|
||||
worker:
|
||||
replicaCount: 2
|
||||
terminationGracePeriodSeconds: 60
|
||||
resources:
|
||||
requests:
|
||||
cpu: 500m
|
||||
memory: 1Gi
|
||||
limits:
|
||||
cpu: "2"
|
||||
memory: 2Gi
|
||||
probes:
|
||||
startup:
|
||||
failureThreshold: 30
|
||||
periodSeconds: 2
|
||||
liveness:
|
||||
periodSeconds: 20
|
||||
timeoutSeconds: 2
|
||||
failureThreshold: 3
|
||||
|
||||
# Celery Beat is separated from scalable workers so only one recovery
|
||||
# scheduler publishes due durable Jobs at a time.
|
||||
scheduler:
|
||||
enabled: true
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 256Mi
|
||||
limits:
|
||||
cpu: 500m
|
||||
memory: 512Mi
|
||||
probes:
|
||||
startup:
|
||||
failureThreshold: 30
|
||||
periodSeconds: 2
|
||||
liveness:
|
||||
periodSeconds: 20
|
||||
timeoutSeconds: 2
|
||||
failureThreshold: 3
|
||||
|
||||
migration:
|
||||
enabled: true
|
||||
activeDeadlineSeconds: 600
|
||||
backoffLimit: 0
|
||||
ttlSecondsAfterFinished: 86400
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 256Mi
|
||||
limits:
|
||||
cpu: 500m
|
||||
memory: 512Mi
|
||||
|
||||
ingress:
|
||||
enabled: false
|
||||
className: nginx
|
||||
annotations: {}
|
||||
api:
|
||||
host: knowledge.example.invalid
|
||||
path: /
|
||||
tlsSecretName: knowledge-platform-api-tls
|
||||
mcp:
|
||||
host: mcp.example.invalid
|
||||
path: /mcp
|
||||
tlsSecretName: knowledge-platform-mcp-tls
|
||||
|
||||
# Only non-sensitive runtime settings belong here. The templates enumerate
|
||||
# keys explicitly so credentials cannot be added to this ConfigMap by mistake.
|
||||
config:
|
||||
appName: enterprise-knowledge-platform
|
||||
logLevel: INFO
|
||||
database:
|
||||
poolSize: 10
|
||||
maxOverflow: 10
|
||||
poolRecycleSeconds: 600
|
||||
poolTimeoutSeconds: 30
|
||||
connectTimeoutSeconds: 30
|
||||
ingestion:
|
||||
maxAttempts: 3
|
||||
retryBackoffSeconds: 2
|
||||
retryBackoffMaxSeconds: 60
|
||||
workerLeaseSeconds: 300
|
||||
recoveryIntervalSeconds: 30
|
||||
batchMaxFiles: 20
|
||||
batchMaxTotalBytes: 209715200
|
||||
maxUploadBytes: 52428800
|
||||
objectStorage:
|
||||
provider: bos
|
||||
endpoint: https://s3.example.invalid
|
||||
bucket: knowledge-platform
|
||||
keyPrefix: knowledge-platform
|
||||
timeoutSeconds: 30
|
||||
chunking:
|
||||
maxTokens: 450
|
||||
overlapTokens: 75
|
||||
maxCharacters: 2000
|
||||
embedding:
|
||||
provider: qianfan
|
||||
model: bge-large-zh
|
||||
endpoint: https://qianfan.baidubce.com/v2/embeddings
|
||||
timeoutSeconds: 30
|
||||
vectorStore:
|
||||
provider: oceanbase
|
||||
uri: oceanbase.example.invalid:2881
|
||||
user: knowledge-platform
|
||||
database: knowledge_platform
|
||||
table: knowledge_chunks_v1
|
||||
index: vidx_knowledge_chunks_embedding
|
||||
batchSize: 100
|
||||
timeoutSeconds: 30
|
||||
retrieval:
|
||||
indexProfileId: demo-bge-large-zh-structure-v1-oceanbase-v1
|
||||
denseTopN: 20
|
||||
searchTopK: 5
|
||||
reranker:
|
||||
provider: qianfan
|
||||
model: bce-reranker-base
|
||||
endpoint: https://qianfan.baidubce.com/v2/rerank
|
||||
timeoutSeconds: 30
|
||||
mcp:
|
||||
publicUrl: https://mcp.example.invalid/mcp
|
||||
issuerUrl: https://mcp.example.invalid
|
||||
clientId: workbuddy-enterprise
|
||||
serviceIdentity: workbuddy-public-knowledge
|
||||
requiredScope: knowledge:search
|
||||
allowedHosts:
|
||||
- mcp.example.invalid
|
||||
allowedOrigins:
|
||||
- https://mcp.example.invalid
|
||||
maxRequestBodyBytes: 1048576
|
||||
|
||||
# Optional local dependency. Keep disabled for production and provide
|
||||
# RABBITMQ_URL through existingSecret for an externally managed broker.
|
||||
rabbitmq:
|
||||
enabled: false
|
||||
existingSecret: knowledge-platform-rabbitmq
|
||||
@ -1,31 +0,0 @@
|
||||
-- ARCH-003 rebuildable OceanBase Vector index for bge-large-zh (1024 dimensions).
|
||||
-- Canonical metadata is co-located in OceanBase relational tables; this vector
|
||||
-- table remains a derived index and may be rebuilt from canonical chunks.
|
||||
-- Requires OceanBase 4.3.5.4+; ARCH-006 verifies PRE-FILTER in the target test cluster.
|
||||
|
||||
CREATE TABLE IF NOT EXISTS knowledge_chunks_v1 (
|
||||
index_record_id VARCHAR(36) NOT NULL,
|
||||
chunk_id VARCHAR(36) NOT NULL,
|
||||
document_id VARCHAR(36) NOT NULL,
|
||||
filename VARCHAR(255) NOT NULL,
|
||||
page INT NULL,
|
||||
page_end INT NULL,
|
||||
section VARCHAR(1024) NULL,
|
||||
chunk_index INT NOT NULL,
|
||||
content MEDIUMTEXT NOT NULL,
|
||||
embedding VECTOR(1024) NOT NULL,
|
||||
embedding_model VARCHAR(128) NOT NULL,
|
||||
index_profile_id VARCHAR(128) NOT NULL,
|
||||
PRIMARY KEY (index_record_id),
|
||||
KEY idx_knowledge_chunks_document_profile (document_id, index_profile_id),
|
||||
KEY idx_knowledge_chunks_profile (index_profile_id),
|
||||
VECTOR KEY vidx_knowledge_chunks_embedding (embedding)
|
||||
WITH (
|
||||
DISTANCE = COSINE,
|
||||
TYPE = HNSW,
|
||||
LIB = VSAG,
|
||||
M = 16,
|
||||
EF_CONSTRUCTION = 256,
|
||||
EF_SEARCH = 200
|
||||
) BLOCK_SIZE 16384
|
||||
) ORGANIZATION = HEAP;
|
||||
@ -1,12 +0,0 @@
|
||||
{
|
||||
"mcpServers": {
|
||||
"enterprise-knowledge": {
|
||||
"type": "http",
|
||||
"url": "${EKP_MCP_URL:-http://127.0.0.1:8766/mcp}",
|
||||
"headers": {
|
||||
"Authorization": "Bearer ${EKP_MCP_TOKEN}"
|
||||
},
|
||||
"description": "企业知识检索"
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -5,139 +5,3 @@ Version all public APIs under `/api/v1`.
|
||||
Initial domains: auth, users/groups, knowledge-bases, documents, compiled-knowledge, search, evaluations, jobs, audit-logs, providers and MCP administration.
|
||||
|
||||
Use OpenAPI as executable source of truth once implementation starts. Provider-specific concepts must not leak into public contracts.
|
||||
|
||||
## Demo v0.1 P0
|
||||
|
||||
DEMO-009 公开以下最小接口。`docs/demo/DEMO_SCOPE.md` 中的 `/api/*` 逻辑路径按本契约统一
|
||||
版本化为 `/api/v1/*`。
|
||||
|
||||
`DEMO-015` 已将单文件上传切换为持久化异步 Job;`DEMO-016` 已增加批量提交和聚合查询。
|
||||
|
||||
| Method | Path | 行为 |
|
||||
|---|---|---|
|
||||
| `POST` | `/api/v1/documents` | multipart 单文件接收;持久化 Source、Document、Job 后返回 `202` |
|
||||
| `POST` | `/api/v1/ingestion-batches` | multipart `files` 多文件接收;返回批次与逐文件状态 |
|
||||
| `GET` | `/api/v1/ingestion-batches/{batch_id}` | 返回批次汇总和逐文件最新 Job 状态 |
|
||||
| `GET` | `/api/v1/documents` | 按创建时间倒序返回文档列表 |
|
||||
| `GET` | `/api/v1/documents/{document_id}` | 返回单个文档状态与安全错误信息 |
|
||||
| `DELETE` | `/api/v1/documents/{document_id}` | 直接删除向量、Metadata、解析产物和原始文件 |
|
||||
| `GET` | `/api/v1/jobs/{job_id}` | 返回阶段、状态、attempt、最大尝试次数和安全错误信息 |
|
||||
| `POST` | `/api/v1/search` | Dense Top20、Reranker、TopK、来源补全 |
|
||||
|
||||
上传字段名为 `file`,支持 PDF、DOCX、TXT 和 Markdown。成功返回 `202`、Document 字段及
|
||||
`job_id`、`job_status=queued`、`max_attempts`;响应不等待 Parse、Clean、Chunk、Embedding 或
|
||||
Index。Document 响应不暴露 `storage_key`、Provider 配置或索引实现。
|
||||
|
||||
Search 请求:
|
||||
|
||||
```json
|
||||
{
|
||||
"query": "产品A退款周期是多少?",
|
||||
"top_k": 5
|
||||
}
|
||||
```
|
||||
|
||||
Search 响应:
|
||||
|
||||
```json
|
||||
{
|
||||
"results": [
|
||||
{
|
||||
"document_id": "6e5ec370-5a45-4eb7-95d7-e84267ad8752",
|
||||
"filename": "产品A说明书.pdf",
|
||||
"page": 12,
|
||||
"section": "退款政策",
|
||||
"chunk_id": "76ba1c30-bff7-44e5-9989-fb8372f54f46",
|
||||
"content": "……",
|
||||
"score": 0.82,
|
||||
"rerank_score": 0.94
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
公开错误语义:请求校验失败返回 `422`,不支持的上传类型返回 `415`,文件过大返回 `413`,
|
||||
文档、Job 或批次不存在返回 `404`,活动 Job 的文档删除返回 `409`,运行时 Provider 未配置返回
|
||||
`503`。上游 Provider 或持久化异常只返回经过收敛的 `5xx` 信息,不回显凭证、请求正文或
|
||||
厂商响应。
|
||||
|
||||
## Demo Admin 客户端路由
|
||||
|
||||
DEMO-010 提供 `/admin/upload`、`/admin/documents` 和 `/admin/search` 三个客户端路由,
|
||||
`/admin` 默认展示上传页。它们是同源静态页面,不是新增业务 API,也不改变上述公开 REST
|
||||
契约;所有业务操作仍调用 `/api/v1`。
|
||||
|
||||
## 当前公共知识库部署边界(PLAN-002)
|
||||
|
||||
当前阶段所有 Document 属于同一个公共知识可见域,不新增用户、知识库或 ACL 字段,也不把
|
||||
这些完整未实现能力伪装为默认放行。Admin 写接口只允许部署在企业受控网络或统一接入网关后;
|
||||
Ingress 是否进一步认证属于部署配置,但前端隐藏按钮或隐藏路由不构成安全控制。
|
||||
|
||||
集群重启、扩缩容和滚动部署不得改变本契约或要求浏览器重新注入 Provider 配置。数据库、BOS、
|
||||
模型与队列凭据由服务端 Secret 提供,任何 API 响应都不得返回这些值。
|
||||
|
||||
## 异步入库契约(DEMO-015~016 已实现)
|
||||
|
||||
| Method | Path | 行为 |
|
||||
|---|---|---|
|
||||
| `POST` | `/api/v1/documents` | 已实现:单文件接收并返回 Document/Job 标识 |
|
||||
| `POST` | `/api/v1/ingestion-batches` | multipart 多文件接收,返回批次及逐文件 Document/Job 标识 |
|
||||
| `GET` | `/api/v1/ingestion-batches/{batch_id}` | 返回批次汇总和逐文件最新状态 |
|
||||
| `GET` | `/api/v1/jobs/{job_id}` | 已实现:返回持久化 Job 状态 |
|
||||
|
||||
单文件响应示意:
|
||||
|
||||
```json
|
||||
{
|
||||
"id": "6e5ec370-5a45-4eb7-95d7-e84267ad8752",
|
||||
"filename": "退款政策.md",
|
||||
"media_type": "text/markdown",
|
||||
"size_bytes": 1024,
|
||||
"status": "uploaded",
|
||||
"error_message": null,
|
||||
"created_at": "2026-09-02T12:00:00Z",
|
||||
"updated_at": "2026-09-02T12:00:00Z",
|
||||
"job_id": "7adc4278-1157-4c3d-95a4-6926ad77a04e",
|
||||
"job_status": "queued",
|
||||
"max_attempts": 3
|
||||
}
|
||||
```
|
||||
|
||||
批量字段名为 `files`,成功返回 `202`。响应示意:
|
||||
|
||||
```json
|
||||
{
|
||||
"batch_id": "d21f8012-4c4e-4bb4-89af-e11c0f9f470e",
|
||||
"status": "processing",
|
||||
"total_files": 2,
|
||||
"accepted_files": 1,
|
||||
"rejected_files": 1,
|
||||
"completed_files": 1,
|
||||
"created_at": "2026-09-02T12:00:00Z",
|
||||
"items": [
|
||||
{
|
||||
"item_index": 0,
|
||||
"filename": "退款政策.md",
|
||||
"acceptance_status": "accepted",
|
||||
"document_id": "6e5ec370-5a45-4eb7-95d7-e84267ad8752",
|
||||
"job_id": "7adc4278-1157-4c3d-95a4-6926ad77a04e",
|
||||
"job_status": "queued",
|
||||
"stage": null,
|
||||
"attempt": 0,
|
||||
"max_attempts": 3,
|
||||
"error_message": null
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
Job 公开状态为 `queued | running | retrying | succeeded | failed | cancelled`,并通过 `stage`
|
||||
表示 `parsing | cleaning | chunking | embedding | indexing`。错误字段只返回有界、安全摘要。
|
||||
批次状态为 `processing | completed | partial_failure | failed`;逐文件接收状态为
|
||||
`pending | accepted | rejected`。批次允许逐文件部分接收/失败,不以一个文件的处理结果回滚
|
||||
其他文件。文件数量、单文件大小和请求总大小分别由 `BATCH_MAX_FILES`、`MAX_UPLOAD_BYTES`
|
||||
和 `BATCH_MAX_TOTAL_BYTES` 限制。数量超限返回 `413`;类型、大小或内容不合规作为该文件的
|
||||
`rejected` 安全结果返回,其他合规文件继续接收。
|
||||
|
||||
自动重试是服务端 Job 行为,不要求客户端再次上传,也不新增公开手动重试接口。每次重试前
|
||||
清理派生数据但保留原始文件和稳定的 Document ID;达到次数上限或清理失败后进入 `failed`。
|
||||
|
||||
@ -8,44 +8,4 @@ P0 business tools:
|
||||
- `get_source`
|
||||
- `list_knowledge_bases`
|
||||
|
||||
Every request maps to a user/service identity and uses the same authorization/application services as REST. Do not expose OceanBase Vector, BOS, model names, parsers or other infrastructure choices.
|
||||
|
||||
## Demo priority slice(DEMO-012,已实现)
|
||||
|
||||
`DEMO-012` 已提升为当前最高实现优先级。首个可运行 Gateway 只注册
|
||||
`search_knowledge`,其输入与 `/api/v1/search` 对齐:
|
||||
|
||||
```json
|
||||
{
|
||||
"query": "产品A退款周期是多少?",
|
||||
"top_k": 5
|
||||
}
|
||||
```
|
||||
|
||||
输出保留业务来源字段:`document_id`、`filename`、`page`、`section`、`chunk_id`、`content`
|
||||
和排序分数。Gateway 必须复用现有 Search Service 或正式 REST API,不得复制检索、来源补全
|
||||
或排序逻辑。
|
||||
|
||||
Demo 验证也必须建立 user/service identity 映射边界,禁止使用共享超级管理员 Token。由于
|
||||
Document ACL 尚未进入 Demo Scope,`DEMO-012` 只能证明身份传递、认证拒绝和业务服务复用,
|
||||
不能声称完成 V1 ACL 授权闭环。错误响应和日志不得包含 Provider Endpoint、凭据、数据库表名
|
||||
或模型名。
|
||||
|
||||
验收至少覆盖:MCP 工具发现、参数校验、成功检索、无结果、认证失败、安全错误收敛,以及
|
||||
WorkBuddy Enterprise 的一次端到端调用。其余 P0 tools 继续留在完整 V1 `EKP-071`。
|
||||
|
||||
实现使用独立 Streamable HTTP Gateway,默认端点为 `http://127.0.0.1:8766/mcp`。请求必须
|
||||
携带 `Authorization: Bearer <token>`;Token 只从运行环境读取,并映射为显式 service identity
|
||||
和 `knowledge:search` scope。当前静态 service credential 是 Demo 兼容层,不替代 V1 OAuth、
|
||||
ACL 或用户委托授权。
|
||||
|
||||
公开错误边界:缺失/无效 Token 为 HTTP `401`,scope 不足为 `403`;Tool 参数不合法返回 MCP
|
||||
参数错误;检索运行失败只返回 `knowledge search failed`。Host/Origin allowlist 默认只允许
|
||||
本机,远程部署必须显式注入公开域名。
|
||||
|
||||
## 当前公共知识库部署边界(PLAN-002)
|
||||
|
||||
集群阶段仍只暴露只读 `search_knowledge`,并为 WorkBuddy 配置稳定的 HTTPS MCP 地址。
|
||||
Credential 由 Kubernetes Secret 在服务端运行时注入,客户端只保存管理员分发的最小
|
||||
`knowledge:search` Token;Pod 重启或滚动升级不得要求重新生成配置。公共知识域不等于匿名
|
||||
访问或超级管理员身份,未来引入非公共知识前必须升级为用户委托身份与 ACL-aware Search。
|
||||
Every request maps to a user/service identity and uses the same authorization/application services as REST. Do not expose Mochow, Qwen, BGE, MinerU or other infrastructure choices.
|
||||
|
||||
@ -2,40 +2,9 @@
|
||||
|
||||
- ADR-001: API First + Admin Console — Accepted
|
||||
- ADR-002: Modular monolith + workers + MCP gateway for V1 — Accepted
|
||||
- ADR-003: PostgreSQL + object storage are canonical — Superseded by ADR-012
|
||||
- ADR-004: Mochow is initial retrieval backend but replaceable — Superseded by ADR-010
|
||||
- ADR-003: PostgreSQL + object storage are canonical — Accepted
|
||||
- ADR-004: Mochow is initial retrieval backend but replaceable — Accepted
|
||||
- ADR-005: Provider abstraction for embedding/reranker/parser/LLM — Accepted
|
||||
- ADR-006: RAG + Knowledge Compilation — Accepted
|
||||
- ADR-007: ACL before/during retrieval — Accepted
|
||||
- ADR-008: MCP hides infrastructure details — Accepted
|
||||
- ADR-009: Baidu BOS is the primary object-storage provider; local storage remains for tests and local development — Accepted
|
||||
- ADR-010: OceanBase Vector is the primary rebuildable retrieval index behind `VectorStore`; PostgreSQL remains canonical — Superseded by ADR-012
|
||||
- ADR-011: Qianfan `bge-large-zh` plus `bce-reranker-base` is the initial Demo model baseline — Accepted
|
||||
- ADR-012: OceanBase MySQL relational tables + BOS are canonical; OceanBase vector tables remain rebuildable indexes — Accepted
|
||||
|
||||
## Architecture migration boundary
|
||||
|
||||
ARCH-006 的只读环境核验确认,目标平台没有独立 PostgreSQL,OceanBase MySQL 模式同时承载
|
||||
关系数据和向量数据。因此 ADR-012 取代 ADR-003/010 中的 PostgreSQL 部署假设:OceanBase
|
||||
关系表保存 canonical metadata,BOS 保存不可变源文件,OceanBase 专用向量表仍只保存可由
|
||||
canonical Chunk 重建的检索行。共用数据库实例不改变两类数据的所有权边界,也不允许业务层
|
||||
绕过 `VectorStore` 直接依赖 `pyobvector`。
|
||||
|
||||
目标 test 集群为 OceanBase 4.3.5.4 MySQL 模式。ARCH-006 已用只读 `EXPLAIN` 验证
|
||||
`VECTOR_INDEX(... pre_filter)` 产生 `VECTOR INDEX ADAPTIVE SCAN (PRE-FILTER)`;版本下限与
|
||||
运行检查据此调整为 4.3.5.4。生产升级和重新验收仍由运维流程控制。
|
||||
|
||||
## Public knowledge delivery boundary
|
||||
|
||||
PLAN-002 将当前交付阶段限定为企业受控网络内的单一公共知识可见域,并加入 Kubernetes/Helm
|
||||
集群交付任务。这是部署范围和实施顺序调整,不是新的架构选型:继续采用 ADR-002 的模块化
|
||||
单体 API、独立 Worker 和 MCP Gateway,以及架构既定的 Celery/RabbitMQ;OceanBase/BOS
|
||||
边界继续遵循 ADR-012。
|
||||
|
||||
ADR-007 仍为 Accepted。当前阶段只是延后用户/组/角色和 Document ACL;开始接收任何非公共
|
||||
知识前,必须先实现 Permission Resolver 和检索前/检索中 ACL 下推。公共知识库也不等于匿名
|
||||
公网开放,Admin 写操作只允许经受控网络或统一接入网关访问,MCP 使用最小只读服务身份。
|
||||
|
||||
The migration retains the Demo's Dense Top20 → Reranker Top5 behavior and current lightweight
|
||||
Parser scope. Top50 retrieval, distance thresholds, dual-query RRF, Marker/Pandoc and LLM-derived
|
||||
index content require separate product and evaluation decisions.
|
||||
|
||||
@ -63,11 +63,10 @@ Wiki / FAQ / SOP / Summary 等派生知识
|
||||
┌────────────────────────┼────────────────────────┐
|
||||
│ │ │
|
||||
▼ ▼ ▼
|
||||
┌────────────────────────┐ ┌──────────────┐
|
||||
│ OceanBase MySQL │ │ Object Store │
|
||||
│ Canonical + Vector │ │ BOS │
|
||||
│ (logical separation) │ └──────────────┘
|
||||
└────────────────────────┘
|
||||
┌──────────────┐ ┌──────────────┐ ┌──────────────┐
|
||||
│ PostgreSQL │ │ Object Store │ │ Mochow │
|
||||
│ Canonical DB │ │ S3/MinIO │ │ Search Index │
|
||||
└──────────────┘ └──────────────┘ └──────────────┘
|
||||
▲ ▲
|
||||
│ │
|
||||
┌───────┴────────────────────────┘
|
||||
@ -89,7 +88,7 @@ Wiki / FAQ / SOP / Summary 等派生知识
|
||||
|
||||
1. **API First**:后台、MCP、其他业务系统均调用统一 API/Service。
|
||||
2. **模块化单体优先**:V1 不拆大量微服务,API 服务 + 异步 Worker + MCP Gateway 足够。
|
||||
3. **Canonical Data 逻辑独立**:OceanBase 关系表 + BOS 保存事实数据,同库向量表仅保存可重建索引。
|
||||
3. **Canonical Data 独立**:PostgreSQL + 对象存储保存事实数据,Vector DB 仅保存可重建索引。
|
||||
4. **Provider 可替换**:Embedding、Reranker、Vector DB、Parser、LLM 均通过 Adapter 抽象。
|
||||
5. **MCP 不暴露底层实现**:客户端只看到 `search_knowledge` 等业务工具。
|
||||
|
||||
@ -100,14 +99,14 @@ Wiki / FAQ / SOP / Summary 等派生知识
|
||||
| 模块 | V1 当前选型 | 说明 |
|
||||
|---|---|---|
|
||||
| Backend | Python + FastAPI | AI / 数据处理生态成熟,适合 MCP、异步任务和模型调用 |
|
||||
| Metadata DB | OceanBase MySQL | 保存用户、权限、文档、版本、任务、Compiled Knowledge 等 Canonical Data |
|
||||
| Object Storage | Baidu BOS | 保存不可变原始文件、解析产物和中间文件;Local Adapter 仅用于测试/本地开发 |
|
||||
| Metadata DB | PostgreSQL | 保存用户、权限、文档、版本、任务、Compiled Knowledge 等 Canonical Data |
|
||||
| Object Storage | S3 Compatible / MinIO | 保存原始文件、解析产物、中间文件 |
|
||||
| Task Queue | Celery + RabbitMQ | 文件解析、Embedding、索引、Compilation 等异步任务 |
|
||||
| PDF / OCR Parser | MinerU 优先 | 中文复杂 PDF、扫描件、表格、版面解析能力较强 |
|
||||
| Office Parser | Docling 补充 | DOCX/PPTX/XLSX/HTML 等格式统一解析 |
|
||||
| Vector/Search DB | OceanBase Vector | 与关系表同库部署但逻辑隔离,必须通过 `VectorStore` 并可从 canonical Chunk 重建 |
|
||||
| Embedding | Qianfan bge-large-zh | 当前 1024 维基线,后续模型变更使用版本化 Shadow Index |
|
||||
| Reranker | Qianfan bce-reranker-base | 当前 Dense Top20 → Reranker Top5 基线 |
|
||||
| Vector/Search DB | Mochow | 延续公司现有经验,支持向量、BM25、Hybrid、Filter |
|
||||
| Embedding | 多模型 Benchmark 决定 | 基线 bge-large-zh,对比 Qwen3 Embedding 等 |
|
||||
| Reranker | Qwen3 Reranker 等候选 | V1 正式检索链必选能力 |
|
||||
| LLM | Provider 抽象,云端优先 | 用于 Knowledge Compilation、分类、摘要等 |
|
||||
| MCP | 独立 MCP Gateway | 面向 WorkBuddy 等客户端,隐藏内部技术栈 |
|
||||
| Frontend | Admin Console | 知识库、文档、权限、检索调试、Benchmark、任务管理 |
|
||||
@ -174,46 +173,6 @@ Queued / Parsing / Cleaning / Chunking / Embedding / Indexing / Completed / Fail
|
||||
- 软删除;
|
||||
- 来源页码和章节追踪。
|
||||
|
||||
### 4.2.1 批量异步提交与失败恢复
|
||||
|
||||
API 只负责校验并持久化不可变源文件、Document 和 Ingestion Job,然后返回 `202`。批次是
|
||||
提交与聚合查询边界,不是跨文件数据库事务;每个文件独立排队、执行和失败。
|
||||
|
||||
DEMO-016 以 canonical `ingestion_batches` / `ingestion_batch_items` 保存输入顺序和接收引用;
|
||||
处理阶段、Attempt 与最终结果继续以 `ingestion_jobs` / `ingestion_attempts` 为事实源,批次表
|
||||
不复制 Worker 状态。
|
||||
|
||||
```text
|
||||
Batch Request
|
||||
├─ Source A + Document A + Job A ─→ Worker
|
||||
├─ Source B + Document B + Job B ─→ Worker
|
||||
└─ Source C + Document C + Job C ─→ Worker
|
||||
```
|
||||
|
||||
Worker 对可重试错误执行有界退避。重新执行前必须通过独立的派生数据清理服务按以下顺序
|
||||
恢复干净状态:
|
||||
|
||||
```text
|
||||
Vector/Sparse rows
|
||||
→ ChunkIndexState
|
||||
→ Chunks
|
||||
→ Parsed artifact + parsed_storage_key
|
||||
→ reset processing state
|
||||
→ next attempt from immutable source
|
||||
```
|
||||
|
||||
清理服务与 Document 删除语义分离:重试清理保留原始 BOS 对象、Document 和 Job/Attempt
|
||||
历史。清理失败时 Job 终止,不能在未知状态继续执行。Job 通过数据库状态、唯一有效 Attempt
|
||||
和幂等键避免重复投递产生并发执行;具体队列实现保持可替换,不进入业务服务接口。
|
||||
|
||||
公共知识库集群交付阶段由 Celery/RabbitMQ 作为首个生产执行器。API 只调用队列执行器抽象,
|
||||
Celery task 只负责恢复应用上下文并调用 Ingestion Application Service;重试分类、Attempt 状态、
|
||||
清理顺序和业务幂等仍由应用层持有。Broker 消息不是 Job 的 source of truth,Worker 重启或消息
|
||||
重复投递必须以 OceanBase 中的持久化 Job/Attempt 状态收敛。
|
||||
|
||||
DEPLOY-001 将 API/Admin、MCP Gateway 与 Async Worker 构建为三个共享锁定 Python 运行时的
|
||||
非 root 镜像 target;进程边界与上述模块边界一致,不新增服务或把 Provider 配置烘焙进镜像。
|
||||
|
||||
---
|
||||
|
||||
### 4.3 文档结构与 Chunk
|
||||
@ -367,28 +326,21 @@ profile_v2
|
||||
|
||||
避免直接原地覆盖现有生产索引。
|
||||
|
||||
ARCH-005/006 将 Profile 生命周期以 `shadow / active / retired` 记录在 OceanBase 关系表;任一时刻至多
|
||||
一个 Active Profile。重建从 canonical Chunk 生成确定性 `chunk_id + profile_id` 索引行,
|
||||
重复执行跳过已完整文档,激活前重新核对全部 ChunkIndexState,并在同一事务内锁定 Chunk
|
||||
写入、退役旧 Profile、激活新 Profile。OceanBase 表本身仍不是 canonical source。
|
||||
|
||||
---
|
||||
|
||||
## 8. Vector Store 设计
|
||||
|
||||
### 当前默认:OceanBase Vector
|
||||
### 当前默认:Mochow
|
||||
|
||||
选择理由:
|
||||
|
||||
- 公司现有知识平台主链已有使用经验;
|
||||
- 可复用现有 OceanBase 运维体系;
|
||||
- 公司已有使用经验;
|
||||
- 已有百度云体系;
|
||||
- 支持 Dense Vector;
|
||||
- 支持 BM25 / Full Text;
|
||||
- 支持 Hybrid Search;
|
||||
- 支持 Metadata / Scalar Filter;
|
||||
- 满足当前 Demo 的 Profile 过滤和后续 ACL 下推边界;
|
||||
- `pyobvector` 仅允许出现在 Provider Adapter 内。
|
||||
|
||||
BM25/Hybrid 仍是 V1 产品能力,但不在本轮 Provider 迁移中提前扩展;相关实现必须经
|
||||
独立 Task、契约测试和 RAG 回归后启用。
|
||||
- 满足 V1 ACL 下推需求。
|
||||
|
||||
### 保留替换能力
|
||||
|
||||
@ -396,18 +348,15 @@ BM25/Hybrid 仍是 V1 产品能力,但不在本轮 Provider 迁移中提前扩
|
||||
|
||||
```text
|
||||
VectorStoreProvider
|
||||
├─ OceanBase Vector (Primary)
|
||||
├─ Mochow
|
||||
├─ Qdrant
|
||||
├─ Milvus
|
||||
└─ pgvector
|
||||
```
|
||||
|
||||
业务 Domain Model 不依赖 OceanBase/`pyobvector` SDK 数据结构。
|
||||
业务 Domain Model 不依赖 Mochow SDK 数据结构。
|
||||
|
||||
OceanBase Vector 表中的数据全部视为 **可重建索引**。同一数据库的关系表保存文档、Chunk、
|
||||
Profile 和索引状态等 canonical 数据,BOS 保存不可变原始文件和派生产物。物理共库不允许
|
||||
向量表反向成为业务事实来源。
|
||||
Mochow 中的数据全部视为 **可重建索引**,真实 Canonical Data 保存在 PostgreSQL / Object Storage。
|
||||
|
||||
---
|
||||
|
||||
@ -492,7 +441,7 @@ Manage
|
||||
User Identity
|
||||
→ Permission Resolver
|
||||
→ Allowed Scope / ACL Filter
|
||||
→ Provider-neutral Retrieval(OceanBase Vector 当前实现)
|
||||
→ Mochow Hybrid Retrieval
|
||||
```
|
||||
|
||||
MCP / REST / Admin Console 必须复用同一权限服务。
|
||||
@ -532,8 +481,7 @@ V1 建议工具:
|
||||
MCP Response 不出现:
|
||||
|
||||
```text
|
||||
OceanBase Vector
|
||||
BOS
|
||||
Mochow
|
||||
Qdrant
|
||||
bge-large-zh
|
||||
Qwen3
|
||||
@ -544,17 +492,6 @@ MinerU
|
||||
|
||||
首要验证客户端:**WorkBuddy Enterprise**。
|
||||
|
||||
### 11.1 DEMO-012 实现边界
|
||||
|
||||
当前已落地独立 Streamable HTTP Gateway,默认只绑定 `127.0.0.1:8766/mcp`,只注册
|
||||
`search_knowledge`。Gateway 与 REST 共享 `SearchService`,同步检索通过工作线程执行,避免
|
||||
阻塞 MCP ASGI 事件循环。
|
||||
|
||||
Demo 使用运行时专用 Bearer Token 映射到 client/service identity 和最小
|
||||
`knowledge:search` scope,并启用 Host/Origin allowlist。该边界用于 WorkBuddy 接入验证,
|
||||
后续 `EKP-070` 仍需用正式 OAuth/企业身份系统和 Permission Resolver 替换静态 Token;
|
||||
`DEMO-012` 不改变 ADR-007 的 ACL 要求。
|
||||
|
||||
---
|
||||
|
||||
## 12. REST API
|
||||
@ -655,11 +592,6 @@ ReindexJob
|
||||
EvaluationJob
|
||||
```
|
||||
|
||||
当前增量实现先提供单个 `IngestionJob` 编排既有 Parse/Clean/Chunk/Embedding/Indexing 阶段,
|
||||
保留阶段级状态和 Attempt 历史;以后拆分为独立阶段 Job 时不得改变公开批次/Job 查询契约。
|
||||
Demo 默认最多 3 次总尝试,只有瞬时 Provider、网络、存储或数据库错误进入自动重试;输入
|
||||
校验和确定性内容错误直接终止。
|
||||
|
||||
GPU 节点优先承担:
|
||||
|
||||
- MinerU / OCR;
|
||||
@ -672,7 +604,7 @@ LLM V1 可继续使用云端 Provider,不要求部署本地大模型。
|
||||
|
||||
## 15. 数据存储原则
|
||||
|
||||
### OceanBase MySQL 关系表
|
||||
### PostgreSQL
|
||||
|
||||
保存:
|
||||
|
||||
@ -697,14 +629,14 @@ LLM V1 可继续使用云端 Provider,不要求部署本地大模型。
|
||||
- Markdown / JSON 中间产物;
|
||||
- 需要持久化的大对象。
|
||||
|
||||
### OceanBase Vector
|
||||
### Mochow
|
||||
|
||||
仅保存:
|
||||
|
||||
- Dense Vector;
|
||||
- Sparse/BM25 Index;
|
||||
- Retrieval metadata;
|
||||
- Profile/ACL 所需过滤字段;
|
||||
- 经独立能力验证后启用的 Sparse/BM25 索引。
|
||||
- ACL 所需过滤字段。
|
||||
|
||||
数据库损坏或更换时,可由 Canonical Data 全量重建。
|
||||
|
||||
@ -721,13 +653,15 @@ Reverse Proxy / Ingress
|
||||
├─ Knowledge API
|
||||
└─ MCP Gateway
|
||||
|
||||
OceanBase MySQL(Canonical + Vector,逻辑隔离)
|
||||
PostgreSQL
|
||||
RabbitMQ
|
||||
Baidu BOS
|
||||
Object Storage
|
||||
|
||||
CPU Worker
|
||||
GPU Worker
|
||||
|
||||
Mochow
|
||||
|
||||
Observability
|
||||
```
|
||||
|
||||
@ -744,30 +678,6 @@ Observability
|
||||
|
||||
未来根据负载再拆服务。
|
||||
|
||||
### 16.1 当前公共知识库集群交付基线
|
||||
|
||||
PLAN-002 将当前交付范围收敛为企业受控网络内的单一公共知识可见域。该范围不重开 ADR-007:
|
||||
所有现有文档共享同一可见性,因而当前检索不执行事后 ACL 过滤;接收任何非公共知识前,必须
|
||||
先实现 Permission Resolver 和数据库/向量检索前置过滤。Admin 写接口不得直接暴露到匿名公网,
|
||||
MCP 使用运行时 Secret 注入的最小 `knowledge:search` 服务身份。
|
||||
|
||||
集群中 API/Admin、MCP Gateway 和 Async Worker 分别部署并独立扩缩容,业务核心继续保持模块化
|
||||
单体。当前 Parser 及云端 Embedding/Reranker 不需要 GPU Worker,因此它不进入最小交付,但角色
|
||||
边界保留。OceanBase、BOS 和 AI Provider 使用外部服务;RabbitMQ 默认使用外部地址,可为开发/
|
||||
演示显式启用 Chart 依赖。
|
||||
|
||||
Kubernetes/Helm 必须提供:生产镜像、`existingSecret`、串行 Migration Job、Ingress/TLS、
|
||||
readiness/liveness、资源限制、非 root 与只读根文件系统、多副本和滚动更新。仓库的一键入口只
|
||||
编排已有集群和外部服务,不创建云账号或把凭据写入 Values、镜像、命令行及日志。现有
|
||||
`docker-compose.dev.yml` 包含已被 ADR-012 取代的 PostgreSQL/MinIO 假设,只能视为待清理的
|
||||
本地开发资产,不能作为集群模板。详细依赖和验收见
|
||||
[`PUBLIC_KNOWLEDGE_CLUSTER_PLAN.md`](../plans/PUBLIC_KNOWLEDGE_CLUSTER_PLAN.md)。
|
||||
|
||||
DEPLOY-002 的运行拓扑将 Celery Worker 与 Celery Beat 恢复调度器拆成独立 Pod:Worker 可横向
|
||||
扩缩,Beat 固定单副本并使用 `Recreate`,避免每个 Worker 都发布周期恢复消息。二者仍调用同一
|
||||
应用服务与持久化 Job/Attempt 状态,不形成新业务服务,也不改变 ADR-002。Migration 使用 Helm
|
||||
`pre-install,pre-upgrade` Hook;失败会在业务 Deployment 更新前终止原子升级。
|
||||
|
||||
---
|
||||
|
||||
## 17. V1 验收重点
|
||||
@ -822,13 +732,13 @@ WorkBuddy 可以通过 MCP 完成知识搜索和 Source 获取。
|
||||
|
||||
```text
|
||||
Backend Python + FastAPI
|
||||
Metadata OceanBase MySQL relational tables
|
||||
Object Storage Baidu BOS(Local Adapter 仅测试/本地开发)
|
||||
Metadata PostgreSQL
|
||||
Object Storage S3 Compatible / MinIO
|
||||
Queue RabbitMQ + Celery
|
||||
Parser MinerU 主力 + Docling 补充
|
||||
Vector Store OceanBase Vector Primary,Adapter 可替换,索引可重建
|
||||
Embedding Qianfan bge-large-zh(1024 维)
|
||||
Reranker Qianfan bce-reranker-base,Dense Top20 → Top5
|
||||
Vector Store Mochow Primary,Adapter 可替换
|
||||
Embedding bge-large-zh 基线,多模型 Benchmark 后决定生产模型
|
||||
Reranker Qwen3 Reranker 等候选,V1 必选
|
||||
LLM Provider 抽象,允许云端
|
||||
Frontend Admin Console
|
||||
Protocol REST API + MCP
|
||||
|
||||
@ -6,7 +6,7 @@
|
||||
|
||||
该 Demo 的目标不是完成完整企业级知识平台,而是优先验证最核心的技术闭环:
|
||||
|
||||
> 文件上传 → 文档解析/清洗 → Chunk → Embedding → OceanBase Vector 入库 → 检索召回 → Reranker → 来源返回 → 简单管理页面
|
||||
> 文件上传 → 文档解析/清洗 → Chunk → Embedding → Mochow 入库 → 检索召回 → Reranker → 来源返回 → 简单管理页面
|
||||
|
||||
当前 Demo Scope 的优先级高于完整 V1 PRD。
|
||||
|
||||
@ -20,7 +20,7 @@
|
||||
|
||||
1. 中文企业文档是否能够稳定上传和解析;
|
||||
2. 清洗后的内容是否可以形成质量可接受的 Chunk;
|
||||
3. 使用现有 Embedding + OceanBase Vector 能否完成稳定召回;
|
||||
3. 使用现有 Embedding + Mochow 能否完成稳定召回;
|
||||
4. Reranker 是否能提升最终 TopK 结果质量;
|
||||
5. 检索结果是否能够准确返回文件名、页码、章节和原文;
|
||||
6. 非研发人员是否能够通过简单页面完成上传和检索验证。
|
||||
@ -49,13 +49,13 @@ FastAPI
|
||||
│ ↓
|
||||
│ Embedding
|
||||
│ ↓
|
||||
│ OceanBase Vector
|
||||
│ Mochow
|
||||
│
|
||||
└── Search
|
||||
↓
|
||||
Query Embedding
|
||||
↓
|
||||
OceanBase Vector TopN
|
||||
Mochow TopN
|
||||
↓
|
||||
Reranker
|
||||
↓
|
||||
@ -67,10 +67,10 @@ FastAPI
|
||||
辅助存储:
|
||||
|
||||
```text
|
||||
OceanBase MySQL
|
||||
└── canonical document / chunk metadata
|
||||
SQLite / PostgreSQL
|
||||
└── document / chunk metadata
|
||||
|
||||
Local Storage / BOS
|
||||
Local Storage / MinIO
|
||||
└── original files
|
||||
```
|
||||
|
||||
@ -83,16 +83,16 @@ Demo 阶段优先简单实现,不引入不必要的基础设施。
|
||||
| 模块 | Demo 推荐选型 | 说明 |
|
||||
|---|---|---|
|
||||
| Backend | FastAPI | 简单、开发效率高 |
|
||||
| Metadata DB | OceanBase MySQL | 目标环境使用 OceanBase;SQLite 仅用于离线测试 |
|
||||
| File Storage | 百度 BOS | Local Adapter 仅用于测试和本地开发,目标环境使用 BOS |
|
||||
| Metadata DB | SQLite 或 PostgreSQL | 单机 Demo 可 SQLite;已有环境可直接 PostgreSQL |
|
||||
| File Storage | 本地目录 | Demo 阶段优先简单;后续可替换 MinIO/S3 |
|
||||
| Parser | PyMuPDF + python-docx 或 MinerU | 优先快速跑通;复杂 PDF 后续增强 |
|
||||
| Cleaner | 自定义基础清洗 | 处理空行、空白、异常换行等 |
|
||||
| Chunker | 标题/段落优先 + Token 限制 | 暂不设计复杂 Chunk 策略 |
|
||||
| Embedding | bge-large-zh | 延续公司现有百度千帆方案 |
|
||||
| Vector DB | OceanBase Vector | 当前默认,通过 `VectorStore` 隔离 SDK,不做多数据库运行时切换 |
|
||||
| Reranker | `bce-reranker-base` | 延续当前可调用的千帆 Reranker,Demo 中保留 |
|
||||
| Vector DB | Mochow | 当前默认,不做多数据库实现 |
|
||||
| Reranker | Qwen3 Reranker 或现有可调用 Reranker | Demo 中保留 |
|
||||
| Frontend | 简单 React/Vue 或 FastAPI HTML | 以最小可用为目标 |
|
||||
| Deployment | Kubernetes/Helm | PLAN-002 起面向企业受控网络交付;本地模式仅用于开发测试 |
|
||||
| Deployment | 本地 / 单机 Docker | 暂不考虑 K8s |
|
||||
|
||||
---
|
||||
|
||||
@ -171,9 +171,9 @@ content
|
||||
|
||||
Demo 阶段不实现多 Embedding Benchmark 平台和动态模型切换,但代码层建议保留简单接口,避免业务逻辑直接写死调用细节。
|
||||
|
||||
### 5.6 OceanBase Vector 入库
|
||||
### 5.6 Mochow 入库
|
||||
|
||||
将 Chunk 写入 OceanBase Vector 专用索引表,至少保存:
|
||||
将 Chunk 写入 Mochow,至少保存:
|
||||
|
||||
```text
|
||||
chunk_id
|
||||
@ -187,9 +187,7 @@ embedding
|
||||
|
||||
支持:Upsert、按 document_id 删除相关 Chunk、Dense Search。
|
||||
|
||||
Demo 阶段不实现 Qdrant/Milvus/pgvector,也不实现多 Vector Store 动态切换,但避免
|
||||
OceanBase/pyobvector 调用散落在整个项目中。OceanBase Vector 行仍是可重建索引,不能替代
|
||||
同库关系表中的 canonical Chunk。
|
||||
Demo 阶段不实现 Qdrant/Milvus/pgvector,也不实现完整 VectorStore Adapter 体系,但避免 Mochow SDK 调用散落在整个项目中。
|
||||
|
||||
### 5.7 Dense Retrieval
|
||||
|
||||
@ -198,7 +196,7 @@ Query
|
||||
↓
|
||||
bge-large-zh Embedding
|
||||
↓
|
||||
OceanBase Vector Dense Search
|
||||
Mochow Dense Search
|
||||
↓
|
||||
Top 20 Candidates
|
||||
```
|
||||
@ -248,8 +246,7 @@ DELETE /api/documents/{id}
|
||||
POST /api/search
|
||||
```
|
||||
|
||||
Demo 阶段删除可以直接执行:删除 BOS/本地原始文件 + 删除 Metadata + 删除 OceanBase
|
||||
Vector 对应 Chunk,暂不要求 Soft Delete。
|
||||
Demo 阶段删除可以直接执行:删除原始文件 + 删除 Metadata + 删除 Mochow 对应 Chunk,暂不要求 Soft Delete。
|
||||
|
||||
Search Request 示例:
|
||||
|
||||
@ -305,7 +302,7 @@ Search Response 示例:
|
||||
|
||||
---
|
||||
|
||||
## 8. Demo 后续增强
|
||||
## 8. P1 可选增强
|
||||
|
||||
### DEMO-011 Hybrid Search
|
||||
|
||||
@ -323,26 +320,12 @@ Reranker
|
||||
|
||||
### DEMO-012 MCP
|
||||
|
||||
**已完成。** 只实现一个最小 MCP Tool:`search_knowledge`,复用现有
|
||||
Search Service 或 `/api/v1/search`,用于验证标准 MCP、identity 边界和 WorkBuddy 接入。
|
||||
不在本 Task 扩展其他 P0 tools 或声称完成尚未实施的 Document ACL。
|
||||
只实现一个最小 MCP Tool:`search_knowledge`,内部调用 `/api/search`,用于验证 WorkBuddy 接入。
|
||||
|
||||
### DEMO-013 OCR / MinerU
|
||||
|
||||
增加扫描 PDF、图片、OCR 和复杂中文 PDF 处理能力。
|
||||
|
||||
### DEMO-015 异步入库 Job 与自动重试清理
|
||||
|
||||
**已完成。** 将现有同步入库流水线放入持久化 Ingestion Job。可重试失败自动执行有界重试;每次重试前
|
||||
删除旧解析产物、Chunk、索引状态和向量数据,但保留不可变原始文件、Document 和 Attempt
|
||||
历史。清理失败时终止,不在未知脏状态继续执行。通过执行器抽象接入 Celery/RabbitMQ,使
|
||||
API 与 Worker 可独立部署,并以持久化状态处理 Worker 重启和重复消息。
|
||||
|
||||
### DEMO-016 批量异步上传
|
||||
|
||||
**已完成。** 在 DEMO-015 基础上支持一次提交多个文件、逐文件 Document/Job 状态、批次聚合查询和
|
||||
Admin Console 状态展示。批次内文件相互隔离,不包含目录上传、压缩包解包或断点续传。
|
||||
|
||||
---
|
||||
|
||||
## 9. 明确后置能力
|
||||
@ -362,67 +345,37 @@ Admin Console 状态展示。批次内文件相互隔离,不包含目录上传
|
||||
- 多 Embedding Benchmark Platform;
|
||||
- 多 Vector DB;
|
||||
- 动态 Provider;
|
||||
- RabbitMQ / Celery;
|
||||
- Kubernetes;
|
||||
- 高可用;
|
||||
- 灾备;
|
||||
- 完整 Observability。
|
||||
|
||||
用户/权限隔离属于完整 V1 后置能力,不因“公共知识库”而取消;本阶段所有已接收知识共享一个
|
||||
可见域,且仅部署在企业受控网络或统一接入网关后。RabbitMQ/Celery 与最小 Kubernetes 可用性
|
||||
已由 PLAN-002 纳入当前交付,跨区域容灾和完整平台级高可用仍不在范围内。
|
||||
|
||||
---
|
||||
|
||||
## 10. Demo Task 清单
|
||||
|
||||
```text
|
||||
DEMO-001 项目骨架 + FastAPI + 配置
|
||||
DEMO-002 Document 数据模型 + 本地文件存储
|
||||
DEMO-002 Document 数据模型 + 文件存储
|
||||
DEMO-003 PDF/DOCX/TXT/MD 上传
|
||||
DEMO-004 Parser + Cleaner
|
||||
DEMO-005 Chunk Pipeline
|
||||
DEMO-006 bge-large-zh Embedding + 初始 Mochow 入库(ARCH-003~005 已迁移为 OceanBase)
|
||||
DEMO-006 bge-large-zh Embedding + Mochow 入库
|
||||
DEMO-007 Dense Retrieval
|
||||
DEMO-008 Reranker + Source Result
|
||||
DEMO-009 Document/Search REST API
|
||||
DEMO-010 简单 Admin Console
|
||||
DEMO-012 MCP search_knowledge(已提升优先级)
|
||||
DEMO-014 地瓜机器人品牌化前端优化(用户新增)
|
||||
DEMO-015 异步入库 Job + 自动重试前清理(用户新增)
|
||||
DEMO-016 批量异步上传 + 状态展示(用户新增)
|
||||
ARCH-001~006 Provider 架构迁移与真实验收
|
||||
PLAN-002 公共知识库集群交付规划(已完成)
|
||||
DEPLOY-001 API/Admin、MCP Gateway、Async Worker 生产镜像
|
||||
DEPLOY-002 Kubernetes/Helm 一键部署
|
||||
DEPLOY-003 可用性、滚动升级和远程 WorkBuddy 验收
|
||||
```
|
||||
|
||||
实施状态(2026-09-02):`DEMO-001~DEMO-010`、用户新增的 `DEMO-014` 与
|
||||
`ARCH-001~ARCH-006` 均已完成并通过当前 Scope 的闭环验收。`PLAN-001` 已完成现状审计和
|
||||
任务拆分;`DEMO-012`、`DEMO-015~016` 已完成。`PLAN-002` 已将公共知识库集群交付纳入范围,
|
||||
后续严格按 `DEPLOY-001~003` 顺序实施。`DEMO-011` 和 `DEMO-013` 仍为
|
||||
可选项,不因本状态记录自动进入范围。
|
||||
|
||||
可选:
|
||||
|
||||
```text
|
||||
DEMO-011 Hybrid Search
|
||||
DEMO-012 MCP search_knowledge
|
||||
DEMO-013 OCR / MinerU
|
||||
```
|
||||
|
||||
已批准顺序:
|
||||
|
||||
```text
|
||||
PLAN-002 Public Knowledge Cluster Plan
|
||||
→ DEMO-015 Celery/RabbitMQ Async Job + Retry Cleanup
|
||||
→ DEMO-016 Batch Async Upload + UI
|
||||
→ DEPLOY-001 Production Images
|
||||
→ DEPLOY-002 Kubernetes/Helm One-click Deploy
|
||||
→ DEPLOY-003 Availability + Remote Acceptance
|
||||
```
|
||||
|
||||
详细 Task 映射、依赖与验收边界见
|
||||
[`docs/plans/ASYNC_INGESTION_MCP_PLAN.md`](../plans/ASYNC_INGESTION_MCP_PLAN.md) 和
|
||||
[`docs/plans/PUBLIC_KNOWLEDGE_CLUSTER_PLAN.md`](../plans/PUBLIC_KNOWLEDGE_CLUSTER_PLAN.md)。
|
||||
|
||||
---
|
||||
|
||||
## 11. Demo 测试数据
|
||||
@ -469,7 +422,7 @@ Upload
|
||||
→ Clean
|
||||
→ Chunk
|
||||
→ Embedding
|
||||
→ OceanBase Vector
|
||||
→ Mochow
|
||||
```
|
||||
|
||||
完整跑通。
|
||||
@ -479,7 +432,7 @@ Upload
|
||||
```text
|
||||
Query
|
||||
→ Embedding
|
||||
→ OceanBase Vector Dense Top20
|
||||
→ Mochow Dense Top20
|
||||
→ Reranker
|
||||
→ Top5
|
||||
```
|
||||
@ -511,13 +464,9 @@ Demo 阶段不设严格生产指标。
|
||||
|
||||
## 14. Codex 开发约束
|
||||
|
||||
Codex 当前只能实现 `DEMO-001 ~ DEMO-010`、`DEMO-012`、`DEMO-014 ~ DEMO-016`、
|
||||
`ARCH-001 ~ ARCH-006`、`PLAN-001 ~ PLAN-002` 和 `DEPLOY-001 ~ DEPLOY-003`。
|
||||
PLAN-002 只授权实现 Celery/RabbitMQ 异步执行器与最小 Kubernetes/Helm 集群交付,不授权
|
||||
拆分额外微服务或扩大业务能力。
|
||||
Codex 当前只能实现 `DEMO-001 ~ DEMO-010`。
|
||||
|
||||
除非明确要求,否则禁止提前实现权限、审计、Knowledge Compilation、多数据库、多模型平台、
|
||||
额外微服务、跨区域容灾和完整 Observability 平台。
|
||||
除非明确要求,否则禁止提前实现权限、审计、Knowledge Compilation、多数据库、多模型平台、微服务、Kubernetes 和企业级 Job Queue。
|
||||
|
||||
当本文件与完整 PRD 的实现优先级冲突时:
|
||||
|
||||
@ -546,29 +495,20 @@ docs/demo/DEMO_SCOPE.md
|
||||
>
|
||||
完整 V1 PRD
|
||||
|
||||
只允许实现本文件明确列出的 Demo、Architecture、Plan 与 Deploy Task,并且一次只执行一个
|
||||
Task。当前严格顺序为 DEMO-016 → DEPLOY-001 → DEPLOY-002 → DEPLOY-003。
|
||||
只允许实现 DEMO-001 ~ DEMO-010。
|
||||
|
||||
当前公共知识库阶段的业务目标保持为:
|
||||
当前 Demo 的目标只有:
|
||||
|
||||
Upload
|
||||
→ Parse/Clean
|
||||
→ Chunk
|
||||
→ bge-large-zh
|
||||
→ OceanBase Vector
|
||||
→ Mochow
|
||||
→ Dense Retrieval
|
||||
→ Reranker
|
||||
→ Source Result
|
||||
→ Search UI
|
||||
|
||||
并补齐以下交付能力:
|
||||
|
||||
Async Job / Retry Cleanup
|
||||
→ Batch Async Upload
|
||||
→ Production Images
|
||||
→ Kubernetes/Helm One-click Deploy
|
||||
→ Availability + Remote WorkBuddy Acceptance
|
||||
|
||||
不要提前实现:
|
||||
|
||||
- ACL
|
||||
@ -578,9 +518,9 @@ Async Job / Retry Cleanup
|
||||
- Knowledge Compilation
|
||||
- Multi Vector Store
|
||||
- Embedding Benchmark Platform
|
||||
- 额外微服务
|
||||
- 跨区域容灾
|
||||
- 完整 Observability 平台
|
||||
- Celery/RabbitMQ
|
||||
- Kubernetes
|
||||
- 企业级高可用
|
||||
|
||||
先不要修改代码。
|
||||
|
||||
@ -589,7 +529,7 @@ Async Job / Retry Cleanup
|
||||
1. 当前仓库状态;
|
||||
2. 为完成 Demo 需要保留哪些现有结构;
|
||||
3. 哪些完整 V1 设计可以暂时不实现;
|
||||
4. 当前已批准 Task 的执行顺序;
|
||||
4. DEMO-001 ~ DEMO-010 的建议执行顺序;
|
||||
5. 下一步应该实施的第一个 Task。
|
||||
```
|
||||
|
||||
@ -600,20 +540,10 @@ Async Job / Retry Cleanup
|
||||
```text
|
||||
Demo
|
||||
↓
|
||||
MCP
|
||||
↓
|
||||
Async Job + Retry Cleanup
|
||||
↓
|
||||
Batch Async Upload
|
||||
↓
|
||||
Production Images
|
||||
↓
|
||||
Kubernetes/Helm One-click Deploy
|
||||
↓
|
||||
Availability + Remote WorkBuddy Acceptance
|
||||
↓
|
||||
Hybrid Search
|
||||
↓
|
||||
MCP
|
||||
↓
|
||||
OCR / Complex Parsing
|
||||
↓
|
||||
Document Version
|
||||
|
||||
@ -1,59 +0,0 @@
|
||||
# Demo Admin Console 使用说明
|
||||
|
||||
DEMO-010 在 FastAPI 服务中提供一个轻量、同源的管理客户端。页面只负责展示和交互,上传、
|
||||
文档管理与检索仍全部通过 `/api/v1` 公开业务接口完成,不复制服务端业务逻辑。
|
||||
|
||||
## 启动与访问
|
||||
|
||||
完成运行时配置后启动 Worker 与 API:
|
||||
|
||||
```bash
|
||||
make run-worker
|
||||
make run
|
||||
```
|
||||
|
||||
页面地址:
|
||||
|
||||
| 页面 | 地址 | 对应业务接口 |
|
||||
|---|---|---|
|
||||
| 文档上传 | `/admin/upload` | `POST/GET /api/v1/ingestion-batches*` |
|
||||
| 文档列表与详情 | `/admin/documents` | `GET/DELETE /api/v1/documents*` |
|
||||
| Search Lab | `/admin/search` | `POST /api/v1/search` |
|
||||
|
||||
`/admin` 默认展示上传页。OpenAPI 文档仍位于 `/api/v1/docs`。
|
||||
|
||||
## 品牌视觉基线
|
||||
|
||||
DEMO-014 参考地瓜机器人公开官网的设计语言,使用品牌橙 `#FF3C00`、黑白高对比、中文
|
||||
无衬线标题、紧凑数据字体、低圆角容器和宽松留白。贯穿页面标题的“计算信号轨”是唯一的
|
||||
强调性图形,序号对应上传、文档、检索三步真实操作顺序。
|
||||
|
||||
品牌图形使用仓库内的代码原生 SVG/CSS 绘制,不复制或热链官网图片、字体和脚本;页面
|
||||
运行时不依赖品牌官网。前端设计方法参考 GitHub `anthropics/skills` 的 Apache-2.0
|
||||
`frontend-design` Skill,并以现有 Demo 信息架构、可访问性和业务正确性为优先约束。
|
||||
|
||||
## Demo 操作闭环
|
||||
|
||||
1. 在上传页选择一个或多个 PDF、DOCX、TXT 或 Markdown 文件并提交;
|
||||
2. 接口返回 `202` 后,页面自动轮询批次,逐文件显示接收状态、处理阶段、尝试次数和安全错误;
|
||||
3. 批次进入 `Completed`、`Partial failure` 或 `Failed` 后停止轮询,再在文档列表查看元数据;
|
||||
4. 在 Search Lab 输入问题并查看 Top-K 的文件名、页码、章节、Chunk 内容、Dense Score 和
|
||||
Rerank Score;
|
||||
5. 如需清理 Demo 数据,在文档列表确认后直接删除。
|
||||
|
||||
## 范围与安全边界
|
||||
|
||||
- 页面静态文件位于 `apps/admin-web`,由同一个 FastAPI 进程提供,避免额外前端运行时;
|
||||
- API 返回内容仅通过 DOM `textContent` 渲染,不作为 HTML 注入;
|
||||
- 页面不保存或展示 Provider 凭据、内部 Endpoint、storage key 或 Index Profile;
|
||||
- Console 只轮询当前批次的聚合接口,不读取 Provider/队列细节,也不提供人工重试按钮;
|
||||
- 批量上传不包含目录上传、压缩包解包、断点续传或跨批次去重;
|
||||
- 本任务不包含 ACL/RBAC、审计、Document Version、Compiled Knowledge、MCP、Hybrid/BM25
|
||||
或完整 V1 Admin Console 导航;
|
||||
- 删除是 Demo 的直接删除语义,确认后会清理 canonical Metadata、派生产物、源文件与向量行。
|
||||
|
||||
## 验证
|
||||
|
||||
静态路由和范围契约由 `tests/integration/test_admin_console.py` 覆盖;现有 REST 全链路测试覆盖
|
||||
上传、列表、详情、搜索与删除。浏览器验收使用确定性本地 Provider,不读取真实企业文档,
|
||||
并覆盖桌面与移动布局。
|
||||
@ -1,97 +0,0 @@
|
||||
# 异步入库与失败恢复说明
|
||||
|
||||
## 范围
|
||||
|
||||
DEMO-015 将单文件上传从 API 请求线程迁移到独立 Celery Worker,DEMO-016 在其上增加逐文件
|
||||
隔离的批量接收和状态聚合。OceanBase 中的
|
||||
`ingestion_jobs` 和 `ingestion_attempts` 是执行状态事实源;RabbitMQ 消息只携带 Job ID,
|
||||
不携带文件内容、Provider 凭据或业务状态。
|
||||
|
||||
批次只聚合接收与 Job 状态,不改变每个文件的独立执行和重试边界。
|
||||
|
||||
## 运行时配置
|
||||
|
||||
以下值必须通过环境变量或部署 Secret 注入,不得提交真实值:
|
||||
|
||||
```text
|
||||
RABBITMQ_URL=amqp://<user>:<password>@<host>:5672/<vhost>
|
||||
INGESTION_MAX_ATTEMPTS=3
|
||||
INGESTION_RETRY_BACKOFF_SECONDS=2
|
||||
INGESTION_RETRY_BACKOFF_MAX_SECONDS=60
|
||||
INGESTION_WORKER_LEASE_SECONDS=300
|
||||
INGESTION_RECOVERY_INTERVAL_SECONDS=30
|
||||
BATCH_MAX_FILES=20
|
||||
BATCH_MAX_TOTAL_BYTES=209715200
|
||||
```
|
||||
|
||||
Worker 与 API 还需要相同的 OceanBase、BOS、Embedding、VectorStore 和 Index Profile 配置。
|
||||
RabbitMQ 用户应只具备指定 vhost/queue 的最小权限。
|
||||
|
||||
## 启动顺序
|
||||
|
||||
```bash
|
||||
uv run alembic upgrade head
|
||||
make run-worker
|
||||
make run
|
||||
```
|
||||
|
||||
`make run-worker` 在当前 Demo 中同时启动 Worker 和恢复调度器。恢复调度器周期扫描:
|
||||
|
||||
- 尚未成功投递的 `queued` Job;
|
||||
- 已到重试时间的 `retrying` Job;
|
||||
- Worker 丢失后租约已过期的 `running` Job。
|
||||
|
||||
任务投递使用 late acknowledgement、worker-lost reject 和单任务 prefetch。重复消息会先锁定
|
||||
Job 行;有效租约内不会创建第二个 Attempt。租约过期后,旧 Attempt 以安全错误关闭,再按重试
|
||||
规则恢复。
|
||||
|
||||
## 重试与清理
|
||||
|
||||
确定性的内容错误(无法解析、清洗后为空、无 Chunk、派生数据身份不一致)直接进入 `failed`。
|
||||
其余运行时/Provider 错误按指数退避重试,默认最多三次总 Attempt。
|
||||
|
||||
第二次及后续 Attempt 创建前严格执行:
|
||||
|
||||
```text
|
||||
OceanBase Vector rows
|
||||
→ ChunkIndexState
|
||||
→ Chunk
|
||||
→ parsed JSON object
|
||||
→ reset Document processing state
|
||||
```
|
||||
|
||||
不可变源文件、Document、Job 和已结束 Attempt 始终保留。任一清理步骤失败时不创建新 Attempt,
|
||||
Job 直接以 `retry cleanup failed` 终止,禁止在未知脏状态继续执行。
|
||||
|
||||
## API 状态
|
||||
|
||||
`POST /api/v1/documents` 成功返回 `202` 和 `job_id`。查询:
|
||||
|
||||
```text
|
||||
GET /api/v1/jobs/{job_id}
|
||||
```
|
||||
|
||||
Job 状态为 `queued | running | retrying | succeeded | failed | cancelled`;阶段为
|
||||
`parsing | cleaning | chunking | embedding | indexing`。公开错误只返回固定安全摘要,不保存或
|
||||
返回 Broker/Provider 异常正文。活动 Job 的 Document 暂不允许删除,API 返回 `409`。
|
||||
|
||||
批量提交与查询:
|
||||
|
||||
```text
|
||||
POST /api/v1/ingestion-batches # multipart 字段 files,可重复
|
||||
GET /api/v1/ingestion-batches/{batch_id}
|
||||
```
|
||||
|
||||
批次创建后逐文件返回 `accepted` 或 `rejected`;已接收文件继续通过上述 Job 状态异步推进。
|
||||
批次聚合状态为 `processing | completed | partial_failure | failed`。请求文件数超过
|
||||
`BATCH_MAX_FILES` 时整体返回 `413`;单文件或共享总字节预算超限只拒绝当前文件,不删除此前
|
||||
已接收文件。默认总预算 200 MiB,且配置不得小于单文件 `MAX_UPLOAD_BYTES`。
|
||||
|
||||
## 故障检查
|
||||
|
||||
1. Job 长期为 `queued`:检查 RabbitMQ 连通性、恢复调度器以及 `ingestion` queue;
|
||||
2. Job 长期为 `running`:检查 Worker,租约到期后恢复调度器会重新投递;
|
||||
3. Job 为 `retrying`:等待 `next_run_at`,客户端不应重新上传同一文件;
|
||||
4. Job 为 `failed` 且错误为 `retry cleanup failed`:停止人工重试,先核对向量、关系数据与 BOS
|
||||
派生对象的一致性;
|
||||
5. 不得通过日志打印 `RABBITMQ_URL`、数据库 URL、BOS/模型凭据或原始 Provider 响应。
|
||||
@ -1,63 +0,0 @@
|
||||
# ARCH-002 BOS 配置与对象契约
|
||||
|
||||
ARCH-002 在既有 `ObjectStorage` 契约后新增百度 BOS Adapter。业务服务仍只使用逻辑 key,
|
||||
不知道 Bucket、Prefix、SDK 类型或凭据。ARCH-005 已将 BOS 设为正式运行默认值;Local
|
||||
Adapter 继续用于单元测试和显式配置的本地开发。
|
||||
|
||||
## 运行配置
|
||||
|
||||
配置只能由部署环境或 Secret Manager 注入,禁止把真实值写入 `.env`、镜像、日志、测试
|
||||
快照或提交记录。
|
||||
|
||||
```text
|
||||
OBJECT_STORAGE_PROVIDER=bos
|
||||
BOS_ENDPOINT=<已授权的 BOS Host 或 HTTP(S) Origin>
|
||||
BOS_ACCESS_KEY_ID=<运行时注入>
|
||||
BOS_SECRET_ACCESS_KEY=<运行时注入>
|
||||
BOS_BUCKET=<预创建的 Bucket>
|
||||
BOS_KEY_PREFIX=knowledge-platform
|
||||
BOS_TIMEOUT_SECONDS=30
|
||||
```
|
||||
|
||||
- Endpoint 必须是无用户名、密码、Path、Query 和 Fragment 的 Host 或 HTTP(S) Origin;旧配置
|
||||
的纯 Host 会在 Provider 边界强制规范化为 HTTPS,禁止沿用 SDK 的默认 HTTP;
|
||||
- Bucket 名必须符合 BOS 的小写字母、数字和连字符规则;
|
||||
- Prefix 可为空;非空时使用安全的 POSIX 相对 key,不允许空段、`.`、`..` 或反斜杠;
|
||||
- 应用不会创建、删除或修改 Bucket 配置。
|
||||
|
||||
## 对象与不可变约束
|
||||
|
||||
canonical `Document.storage_key` 保存与 Provider 无关的逻辑 key,例如:
|
||||
|
||||
```text
|
||||
documents/<document_id>/<filename>
|
||||
```
|
||||
|
||||
BOS Adapter 发起请求时才映射为:
|
||||
|
||||
```text
|
||||
<BOS_KEY_PREFIX>/documents/<document_id>/<filename>
|
||||
```
|
||||
|
||||
上传先写入可回卷的临时流,以满足官方 SDK 对长度和 CRC 计算的要求;随后使用
|
||||
`If-None-Match: *` 执行单对象 Put,服务端返回 412 时统一映射为
|
||||
`ObjectAlreadyExistsError`。这样不会用“先 HEAD、再 PUT”的竞态检查破坏不可变来源约束。
|
||||
|
||||
下载返回 SDK 的流式 Response Body,由调用方负责关闭;存在性检查只发 HEAD,不下载正文。
|
||||
404、412 和其他 SDK 异常映射为稳定的 Provider-neutral 错误,外层错误不包含 Endpoint、
|
||||
SDK Response Body 或凭据。
|
||||
|
||||
## 权限与验收边界
|
||||
|
||||
运行身份只授予目标 Bucket/Prefix 所需的 Put、Get/Head 和 Delete 权限,不授予 Bucket 管理或
|
||||
全局 List 权限。若生产策略要求来源不可物理删除,应在后续 Document Version/Retention
|
||||
设计中单独处理;当前 Demo 的直接删除行为保持不变。
|
||||
|
||||
默认测试使用 Fake Client,不连接 BOS。真实连接、权限和临时对象清理已在 ARCH-006 的
|
||||
受控 test 资源中验证;凭据只允许运行时注入,不得写入仓库、日志或测试快照。
|
||||
|
||||
官方参考:
|
||||
|
||||
- [BOS Python SDK 上传文件](https://cloud.baidu.com/doc/BOS/s/yl5jfscvi)
|
||||
- [BOS 获取 Object](https://cloud.baidu.com/doc/BOS/s/Mk4xtzv8v)
|
||||
- [BOS 访问控制](https://cloud.baidu.com/doc/BOS/s/Tjwvysda9)
|
||||
@ -1,91 +0,0 @@
|
||||
# 生产容器镜像说明
|
||||
|
||||
## 范围
|
||||
|
||||
DEPLOY-001 使用同一个锁定运行时构建三个独立镜像目标:
|
||||
|
||||
| Target | 默认镜像名 | 入口 | 端口 | 镜像健康检查 |
|
||||
|---|---|---|---|---|
|
||||
| `api` | `enterprise-knowledge-api` | Uvicorn + API/Admin | `8000` | `GET /healthz` |
|
||||
| `mcp` | `enterprise-knowledge-mcp` | `ekp-mcp` | `8766` | PID 1 存活 |
|
||||
| `worker` | `enterprise-knowledge-worker` | `ekp-worker` | 无 | PID 1 存活 |
|
||||
|
||||
镜像没有内置数据库迁移、Helm、Ingress 或 Secret 对象;DEPLOY-002 已在独立 Chart 中提供这些
|
||||
编排能力。API 镜像包含 Alembic 和 migrations,由一次性 Migration Job 覆盖命令后复用。详见
|
||||
[`KUBERNETES_DEPLOYMENT.md`](KUBERNETES_DEPLOYMENT.md)。
|
||||
|
||||
## 可重复构建基线
|
||||
|
||||
- Python 基础镜像固定为 `3.12.14-slim-bookworm`,uv 固定为 `0.12.5`,两者同时锁定 manifest
|
||||
digest;
|
||||
- Python 依赖只使用提交的 `uv.lock`,以 `uv sync --frozen --no-dev --no-editable` 安装;
|
||||
- 三个目标共享相同依赖层,不复制不同版本的业务运行时;
|
||||
- 进程均以固定 UID/GID `10001:10001` 运行,不具备 root 权限;
|
||||
- `.dockerignore` 排除 `.env`、Git、虚拟环境、缓存、本地存储、测试和开发产物;
|
||||
- Bake 计划同时声明 `linux/amd64` 与 `linux/arm64`,不下载或打包平台专属模型。
|
||||
|
||||
基础镜像 tag/digest 与应用依赖均已固定;镜像发布系统仍应保存本项目镜像的构建 digest,并让
|
||||
Helm 按 digest 部署。本项目镜像 digest 由实际 Registry 发布产生,不把某个环境的 Registry
|
||||
地址写入仓库。
|
||||
|
||||
## 构建命令
|
||||
|
||||
先确认 Docker Buildx builder 支持两个目标平台:
|
||||
|
||||
```bash
|
||||
make images-print
|
||||
```
|
||||
|
||||
构建全部目标:
|
||||
|
||||
```bash
|
||||
IMAGE_REGISTRY=registry.example.internal/knowledge \
|
||||
IMAGE_TAG=<immutable-release-tag> \
|
||||
make images
|
||||
```
|
||||
|
||||
只有发布流水线才执行推送:
|
||||
|
||||
```bash
|
||||
IMAGE_REGISTRY=registry.example.internal/knowledge \
|
||||
IMAGE_TAG=<immutable-release-tag> \
|
||||
make images-push
|
||||
```
|
||||
|
||||
不要把 Registry 密码放入变量或命令行;构建节点应提前通过其凭据助手登录。禁止使用
|
||||
`latest` 作为集群部署版本。
|
||||
|
||||
## 运行时配置
|
||||
|
||||
镜像只内置以下非敏感进程默认值:`APP_ENV=production`、容器监听地址、静态页面绝对路径和
|
||||
临时缓存目录。OceanBase、BOS、RabbitMQ、Embedding、Reranker 与 MCP Token 必须在容器启动
|
||||
时由 Secret 注入。镜像构建参数、Layer、标签、健康检查和启动命令都不得包含这些值。
|
||||
|
||||
API 的 `/healthz` 只证明进程存活,不读取云端凭据或探测依赖。Kubernetes readiness、资源限制、
|
||||
只读根文件系统所需临时卷及依赖就绪策略由 DEPLOY-002 Chart 配置。MCP endpoint 需要认证,Worker
|
||||
没有 HTTP server,因此这两个镜像的 Docker 健康检查只验证主进程;业务可恢复性仍以持久化
|
||||
Job/Attempt 和 DEPLOY-003 故障验收为准。
|
||||
|
||||
## 本地单目标检查
|
||||
|
||||
需要加载到本地 Docker 时,可直接构建当前平台的一个 target:
|
||||
|
||||
```bash
|
||||
docker build --target api -f deploy/docker/Dockerfile -t ekp-api:test .
|
||||
docker build --target mcp -f deploy/docker/Dockerfile -t ekp-mcp:test .
|
||||
docker build --target worker -f deploy/docker/Dockerfile -t ekp-worker:test .
|
||||
```
|
||||
|
||||
启动时使用 `--env-file` 或编排系统的 Secret 引用,不在命令中展开敏感值。API/Admin smoke test
|
||||
至少检查 `/healthz`、`/admin/upload` 和 `/api/v1/openapi.json`;MCP 与 Worker 的真实依赖启动
|
||||
验收在具备 RabbitMQ 和目标 Provider 的受控环境执行。
|
||||
|
||||
## DEPLOY-001 构建验收
|
||||
|
||||
2026-09-02 已使用 Buildx 分别构建并加载三个 `linux/arm64` target 和三个 `linux/amd64`
|
||||
target。API/Admin 以非 root 用户完成健康、静态页面和 OpenAPI smoke test;MCP/Worker 完成非
|
||||
root 安装入口检查。构建历史与容器文件系统检查未发现 `.env`、Git、测试文件或敏感配置。
|
||||
|
||||
这次验收只生成本地 `deploy001-*` 测试 tag,没有登录或推送 Registry,因此不产生可供集群
|
||||
引用的项目镜像 digest。DEPLOY-002 必须使用操作者指定的不可变 Registry tag/digest,不得把
|
||||
本地测试 tag 当作发布版本。
|
||||
@ -1,19 +1,3 @@
|
||||
# Definition of Done
|
||||
|
||||
A task is done only when scope is implemented, architecture invariants hold, format/lint/typecheck pass, relevant tests pass, migrations/contracts/docs are updated as needed, no secrets are committed, retrieval changes include RAG regression results, and known limitations are documented.
|
||||
|
||||
For asynchronous ingestion or retry tasks, relevant tests must also prove durable Job/Attempt state,
|
||||
bounded retry classification, idempotent duplicate delivery, cleanup-before-retry, preservation of the
|
||||
immutable source, and terminal behavior when cleanup fails. Batch ingestion must cover per-file partial
|
||||
failure and configured limits.
|
||||
|
||||
For MCP tasks, contract tests must cover tool discovery, identity propagation, authorization/error
|
||||
boundaries, REST/application-service parity, infrastructure redaction, and the explicitly targeted MCP
|
||||
client compatibility check.
|
||||
|
||||
For cluster delivery tasks, completion also requires reproducible production images, Secret-only
|
||||
sensitive configuration, serialized migration, readiness/liveness probes, resource and pod-security
|
||||
defaults, idempotent Helm install/upgrade, rolling-restart verification, durable worker recovery, stable
|
||||
HTTPS Admin/MCP endpoints, and remote WorkBuddy acceptance. Manifests, rendered output, logs and images
|
||||
must be checked for credential leakage. Public-knowledge delivery does not complete or waive V1
|
||||
user/ACL tasks.
|
||||
|
||||
@ -1,53 +0,0 @@
|
||||
# ARCH-005 Shadow Index 重建与切换
|
||||
|
||||
ARCH-005 将正式运行默认值切换为 BOS、OceanBase Vector 和千帆 BCE Reranker;ARCH-006
|
||||
进一步确认 canonical 关系数据与向量索引同处 OceanBase MySQL 数据库。关系表保存 Document、
|
||||
Chunk、ChunkIndexState 和 Index Profile;向量表只保存可以重建的检索行。旧 Mochow 运行配置、依赖和代码已移除,但本流程不会自动删除
|
||||
旧外部索引,物理退役必须由受控运维流程另行确认。
|
||||
|
||||
## Profile 与迁移前提
|
||||
|
||||
默认新 Profile 为:
|
||||
|
||||
```text
|
||||
demo-bge-large-zh-structure-v1-oceanbase-v1
|
||||
```
|
||||
|
||||
它与旧 Demo Profile 使用不同 ID,避免已有 Mochow `ChunkIndexState` 被误认为 OceanBase
|
||||
写入证据。自定义 ID 仍由 `INDEX_PROFILE_ID` 注入,并且必须是全新的、稳定的版本标识。
|
||||
|
||||
执行前必须:
|
||||
|
||||
1. 运行 `uv run alembic upgrade head`,创建 canonical `index_profiles` 表;
|
||||
2. 使用 `deploy/oceanbase/001_knowledge_chunks_v1.sql` 预创建目标表和 HNSW 索引;
|
||||
3. 仅在运行时注入已授权的 OceanBase 与千帆凭据,不写入仓库或日志;
|
||||
4. 确认目标 OceanBase 为 4.3.5.4+,且应用账号只有目标数据库/表所需权限;
|
||||
5. 在重建期间避免修改 Chunk 策略;激活事务会阻止 canonical Chunk 并发写入穿过最终核对。
|
||||
|
||||
## 顺序执行
|
||||
|
||||
每一步只输出 Profile ID 和计数,不输出 Endpoint、SQL、Chunk 内容或凭据。
|
||||
|
||||
```bash
|
||||
uv run ekp-index-rebuild build
|
||||
uv run ekp-index-rebuild verify
|
||||
uv run ekp-index-rebuild activate
|
||||
```
|
||||
|
||||
- `build` 注册 Shadow Profile,从 OceanBase canonical Chunk 分文档生成 Embedding 并写入
|
||||
OceanBase。命令中断后可以原样重试;已完整文档会跳过,部分目标 Profile 数据会先定向
|
||||
清理再重建,不触碰 Active Profile;
|
||||
- `verify` 要求目标策略下至少存在一个 canonical Chunk,并核对全部 Chunk 与匹配模型、
|
||||
维度的 ChunkIndexState。输出中的 `complete` 必须为 `true`;
|
||||
- `activate` 在 OceanBase 事务内再次验证,锁定 Chunk 写入,将原 Active Profile 标为
|
||||
`retired`,再将目标 Shadow Profile 标为 `active`。重复执行已成功的激活是幂等的。
|
||||
|
||||
激活后 API 从 OceanBase canonical 关系表解析 Active Profile,不再仅依赖进程环境中的 ID。新上传与查询
|
||||
统一使用 OceanBase Active Profile。部署仍应保留前一版本制品和旧索引回滚窗口;本 Task
|
||||
不自动销毁任何旧外部数据。
|
||||
|
||||
## 验证与已知边界
|
||||
|
||||
离线测试覆盖多文档构建、重试跳过、完整性失败关闭、单 Active 约束、原子激活、迁移和
|
||||
CLI 输出脱敏。真实 BOS/OceanBase/Qianfan 连通性、DDL 权限、30~50 条脱敏 Golden Query、
|
||||
召回质量和延迟属于 ARCH-006,且只能使用运行时授权凭据与受控测试资源。
|
||||
@ -1,143 +0,0 @@
|
||||
# Kubernetes / Helm 部署说明
|
||||
|
||||
## 1. 交付范围
|
||||
|
||||
DEPLOY-002 提供单一 Helm Chart,将以下运行角色部署到已有 Kubernetes 集群:
|
||||
|
||||
- API/Admin:默认两个副本,提供 `/healthz`、`/admin/*` 和 `/api/v1/*`;
|
||||
- MCP Gateway:默认两个副本,提供经 Bearer Token 保护的 `/mcp`;
|
||||
- Async Worker:默认两个副本,只消费 `ingestion` 队列;
|
||||
- Recovery Scheduler:固定单副本,以 `Recreate` 策略运行 Celery Beat;
|
||||
- Migration Job:每次安装/升级前串行执行 `alembic upgrade head`;
|
||||
- 可选 RabbitMQ:只供开发/演示集群显式启用,生产默认使用外部 Broker。
|
||||
|
||||
OceanBase MySQL、BOS、Embedding/Reranker 服务继续使用集群外已有资源。Chart 不创建数据库、
|
||||
BOS Bucket、域名、证书或云账号,也没有扩大公共知识库阶段的业务范围。
|
||||
|
||||
## 2. 前置条件
|
||||
|
||||
操作者需要提前准备:
|
||||
|
||||
1. 可用的 Kubernetes Namespace 和对应 `kubectl` 上下文;
|
||||
2. Helm 3/4 与 Kubernetes 客户端;
|
||||
3. 已推送到目标 Registry 的 API、MCP、Worker 不可变 tag 或 digest;
|
||||
4. Namespace 内的运行时 Secret、TLS Secret,以及私有 Registry 所需的 ImagePullSecret;
|
||||
5. 集群到 OceanBase、BOS、RabbitMQ 和模型服务的网络连通性。
|
||||
|
||||
Chart 默认引用 `knowledge-platform-runtime`,但不会创建它。该 Secret 至少包含以下键:
|
||||
|
||||
```text
|
||||
DATABASE_URL
|
||||
RABBITMQ_URL
|
||||
BOS_ACCESS_KEY_ID
|
||||
BOS_SECRET_ACCESS_KEY
|
||||
BAIDU_QIANFAN_API_KEY
|
||||
OCEANBASE_VECTOR_PASSWORD
|
||||
MCP_AUTH_TOKEN
|
||||
```
|
||||
|
||||
`RERANKER_API_KEY` 可单独提供;未提供时应用按既有契约复用
|
||||
`BAIDU_QIANFAN_API_KEY`。`MCP_AUTH_TOKEN` 至少 32 个字符,只具备
|
||||
`knowledge:search` 服务范围。不要使用 `kubectl --from-literal`、Shell 参数或 Helm
|
||||
`--set` 传递以上值;应由受控 Secret 管理系统或运维流程预先写入 Namespace。
|
||||
|
||||
## 3. 环境 Values
|
||||
|
||||
复制 [`values.example.yaml`](../../deploy/helm/environments/values.example.yaml) 到仓库外的受控
|
||||
路径,填写以下非敏感信息:
|
||||
|
||||
- 三个镜像的 Registry 路径与不可变 tag/digest;
|
||||
- `existingSecret` 和 `imagePullSecrets` 的资源名称;
|
||||
- OceanBase/BOS 的非敏感 Endpoint、数据库标识和 Bucket;
|
||||
- Admin/API 与 MCP 的 Ingress Host、TLS Secret 名称;
|
||||
- 与外部地址一致的 MCP Public URL、Issuer、Allowed Host/Origin;
|
||||
- 副本、资源、超时和批量上传限制。
|
||||
|
||||
`values.schema.json` 拒绝未知顶层字段、`latest` tag、无效 digest、单副本 API/MCP、错误类型和
|
||||
空的关键引用;模板还会校验 Chunk、检索和入库限制之间的大小关系。Values 只允许非敏感配置,
|
||||
模板逐项生成 ConfigMap,不提供任意 `extraEnv` 入口,避免把凭据误写进渲染产物。
|
||||
|
||||
## 4. 安装与幂等升级
|
||||
|
||||
先做离线校验和清单检查:
|
||||
|
||||
```bash
|
||||
make deploy-lint VALUES=/secure/path/environment-values.yaml
|
||||
make deploy-render VALUES=/secure/path/environment-values.yaml
|
||||
```
|
||||
|
||||
安装或升级:
|
||||
|
||||
```bash
|
||||
NAMESPACE=knowledge-platform \
|
||||
RELEASE=knowledge-platform \
|
||||
make deploy VALUES=/secure/path/environment-values.yaml
|
||||
```
|
||||
|
||||
`make deploy` 只接收 Values 文件路径。脚本会先验证 Chart、Namespace、运行时 Secret 及必需
|
||||
键名、可选 RabbitMQ Secret 键名,以及启用 Ingress 时的两个 TLS Secret 引用,然后执行带失败
|
||||
回滚和等待语义的 `helm upgrade --install`;脚本对 Helm 3 使用 `--atomic`,对 Helm 4 使用等价
|
||||
的 `--rollback-on-failure`。它只验证 Secret 资源和键是否存在,不读取或输出 Secret 值。
|
||||
|
||||
Migration 是 `pre-install,pre-upgrade` Hook,固定名称配合
|
||||
`before-hook-creation` 保证同一 Release 内不会并发执行。Helm 等待 Migration 成功后才更新业务
|
||||
工作负载;失败时 Helm 保留上一版本。完成的 Job 最长保留一天供日志追查,并在下次升级前删除。
|
||||
安装命令可以重复执行;数据库迁移必须保持 forward-only 与幂等。
|
||||
|
||||
## 5. Ingress、探针与 Pod 安全
|
||||
|
||||
生产环境启用 Ingress 后,Admin/API 与 MCP 使用独立稳定 HTTPS Host,也可由同一 Ingress
|
||||
Controller 托管。证书只通过预建 TLS Secret 引用,不进入 Values。
|
||||
|
||||
- API 的 startup/readiness/liveness 调用无凭据 `/healthz`;
|
||||
- MCP 的三类探针使用 TCP 端口,不发送或记录 Token;
|
||||
- Worker/Scheduler 使用 PID 1 存活探针,业务恢复正确性由持久化 Job/Attempt 保证;
|
||||
- 全部应用容器固定 UID/GID `10001`、禁止提权、丢弃 Linux capabilities、使用
|
||||
`RuntimeDefault` seccomp 和只读根文件系统;
|
||||
- `/tmp` 使用有大小上限的 `emptyDir`,ServiceAccount 不挂载 API Token;
|
||||
- API/MCP 滚动更新使用 `maxUnavailable: 0`,未 Ready Pod 不进入 Service Endpoint。
|
||||
|
||||
`/healthz` 当前只表示 API 进程存活,不探测所有外部 Provider;真实依赖故障、滚动重启和
|
||||
Worker 恢复行为由 DEPLOY-003 在目标集群验收。
|
||||
|
||||
## 6. 可选 Demo RabbitMQ
|
||||
|
||||
生产默认 `rabbitmq.enabled=false`,`RABBITMQ_URL` 指向外部受管 Broker。仅开发/演示时可开启
|
||||
本地子 Chart;它使用单节点 StatefulSet 和 PVC,并要求预建 Secret 包含:
|
||||
|
||||
```text
|
||||
username
|
||||
password
|
||||
erlang-cookie
|
||||
```
|
||||
|
||||
同时必须让运行时 Secret 中的 `RABBITMQ_URL` 指向该 Service。子 Chart 使用固定
|
||||
`rabbitmq:4.1.4-alpine` tag 与 multi-architecture manifest digest;它不具备生产 RabbitMQ 的
|
||||
高可用、备份或升级保证,禁止作为生产默认值。卸载 Release 时 StatefulSet PVC 默认保留,
|
||||
删除前必须由数据责任人确认。
|
||||
|
||||
## 7. 回滚、卸载与 Secret 轮换
|
||||
|
||||
查看历史并回滚时不要输出 Pod 环境变量:
|
||||
|
||||
```bash
|
||||
helm -n knowledge-platform history knowledge-platform
|
||||
helm -n knowledge-platform rollback knowledge-platform <revision> --wait
|
||||
```
|
||||
|
||||
更新外部 Secret 后,Secret 名称不变;用户和 WorkBuddy 配置无需重新注入。当前应通过受控
|
||||
滚动重启让 Pod 读取新值,目标集群的无中断验证属于 DEPLOY-003。
|
||||
|
||||
卸载只删除本 Release 管理的 Kubernetes 对象:
|
||||
|
||||
```bash
|
||||
helm -n knowledge-platform uninstall knowledge-platform
|
||||
```
|
||||
|
||||
外部 OceanBase、BOS、RabbitMQ、TLS/运行时 Secret 和可选 RabbitMQ PVC 不由卸载命令删除。
|
||||
|
||||
## 8. DEPLOY-002 验收边界
|
||||
|
||||
本任务完成 Chart lint、双路径渲染、严格 Kubernetes Schema、Values 失败关闭、Secret 泄漏扫描
|
||||
以及本地集群两次安装/升级验证。目标企业集群的稳定 HTTPS、滚动重启、Worker 故障恢复、真实
|
||||
Provider 和远程 WorkBuddy 验收统一属于后续 `DEPLOY-003`,不得提前宣称完成。
|
||||
@ -1,67 +0,0 @@
|
||||
# ARCH-006 受控真实链路验收
|
||||
|
||||
本验收覆盖 OceanBase canonical relational metadata → BOS immutable source → 千帆
|
||||
`bge-large-zh` → OceanBase Vector → 千帆 `bce-reranker-base` → canonical Source Result。数据集包含
|
||||
32 条完全合成、脱敏 Query,不含真实客户、产品、政策或内部知识。
|
||||
|
||||
## 强制安全门禁
|
||||
|
||||
只有同时满足以下条件才允许执行:
|
||||
|
||||
1. 数据库、BOS 和千帆凭据已经所有者明确授权;配置清单中的现有值只允许本次 test
|
||||
验收临时注入,禁止保存为 `.env`、写入日志或用于 fat/prod,完成后仍必须轮换;
|
||||
2. OceanBase Database/Table 和 BOS Bucket/Prefix 已明确授权用于测试;
|
||||
3. `APP_ENV=test`;
|
||||
4. `BOS_KEY_PREFIX` 以 `live-acceptance/` 开头,避免写入共享业务前缀;
|
||||
5. 显式设置 `EKP_RUN_LIVE_TESTS=1`、`EKP_CREDENTIALS_AUTHORIZED=1` 和
|
||||
`EKP_LIVE_RESOURCE_ACK=1`。
|
||||
|
||||
任一条件不满足时,命令在创建 Provider 或外部连接前失败关闭,并且不输出 Endpoint、用户
|
||||
名、SQL、响应正文、Chunk 内容或 Secret。仓库 `.env.example` 中三个开关固定为 `0`。
|
||||
|
||||
## 执行顺序
|
||||
|
||||
先完成 ARCH-005 切换:
|
||||
|
||||
```bash
|
||||
uv run alembic upgrade head
|
||||
uv run ekp-index-rebuild build
|
||||
uv run ekp-index-rebuild verify
|
||||
uv run ekp-index-rebuild activate
|
||||
```
|
||||
|
||||
确认 `verify` 和 `activate` 输出的 `complete` 均为 `true` 后,再运行:
|
||||
|
||||
```bash
|
||||
make live-acceptance
|
||||
```
|
||||
|
||||
命令会使用随机运行前缀上传两份合成 Markdown,执行全部 32 条 Query,并在 `finally` 清理
|
||||
本次运行的 OceanBase Vector 行、canonical Document/Chunk/State 以及 BOS 对象。清理失败会使验收
|
||||
失败,且不会尝试删除不带本次随机文件名前缀的数据。
|
||||
|
||||
## 验收指标
|
||||
|
||||
默认门槛:
|
||||
|
||||
```text
|
||||
Recall@1 >= 0.90
|
||||
Recall@5 = 1.00
|
||||
MRR >= 0.90
|
||||
NDCG@5 >= 0.90
|
||||
Search P95 <= 10000 ms
|
||||
```
|
||||
|
||||
输出只包含 Query 数、Recall@1/3/5、MRR、NDCG@5、P50/P95 和 `passed`。如受控环境需要调整
|
||||
门槛,只能通过 `EKP_LIVE_MIN_*` / `EKP_LIVE_MAX_P95_MS` 显式注入,并把实际门槛与结果记录
|
||||
到受保护的验收日志;不要把 Endpoint、账号、凭据或原始响应提交到仓库。
|
||||
|
||||
## 验收结果(2026-09-02)
|
||||
|
||||
目标 test OceanBase 已完成 migration、专用向量表/HNSW、PRE-FILTER、BOS、Embedding、
|
||||
Reranker 和 canonical Source Result 验收。32 条查询结果:Recall@1/3/5、MRR、NDCG@5 均为
|
||||
`1.0`,P50 `3787.61 ms`,P95 `4759.21 ms`,`passed=true`。正式命令退出码为 `0`,数据库
|
||||
复核 `live-` Document/Chunk/Vector 残留为 `0`。
|
||||
|
||||
首次正式运行出现过一次无细节瞬时失败;分阶段 Provider/单文档/32 查询诊断以及随后正式
|
||||
运行均通过。后续稳定性任务应补充有界重试、限流指标与告警;这不阻塞本次受控验收。
|
||||
@ -1,102 +0,0 @@
|
||||
# DEMO-012 MCP Gateway 使用说明
|
||||
|
||||
## 范围
|
||||
|
||||
DEMO-012 提供独立的 Enterprise Knowledge MCP Gateway,通过标准 Streamable HTTP 暴露一个
|
||||
只读工具:`search_knowledge`。Gateway 直接复用 API 进程使用的 `SearchService` 组合,不复制
|
||||
Dense Retrieval、Reranker 或 Source Enrichment 逻辑。
|
||||
|
||||
当前 Demo 只建立认证和身份传递边界:一个运行时注入的专用 Bearer Token 映射到一个
|
||||
user/service identity,并且只有 `knowledge:search` scope。它不是共享超级管理员 Token,也不
|
||||
代表 V1 OAuth、Document ACL 或完整 MCP P0 工具已经完成。
|
||||
|
||||
## 配置
|
||||
|
||||
必须通过运行环境提供至少 32 字符的随机 Token:
|
||||
|
||||
```bash
|
||||
export MCP_AUTH_TOKEN="<runtime-secret-with-at-least-32-characters>"
|
||||
make run-mcp
|
||||
```
|
||||
|
||||
默认地址为 `http://127.0.0.1:8766/mcp`。除现有数据库、BOS、Embedding、Vector 和 Reranker
|
||||
配置外,Gateway 支持:
|
||||
|
||||
| 变量 | 默认值 | 说明 |
|
||||
|---|---|---|
|
||||
| `MCP_HOST` | `127.0.0.1` | 监听地址;本地默认不暴露到其他网卡 |
|
||||
| `MCP_PORT` | `8766` | Gateway 端口 |
|
||||
| `MCP_PATH` | `/mcp` | Streamable HTTP 单一端点 |
|
||||
| `MCP_PUBLIC_URL` | `http://127.0.0.1:8766/mcp` | MCP Resource Server URL |
|
||||
| `MCP_ISSUER_URL` | `http://127.0.0.1:8766` | Demo Token 的身份颁发方标识 |
|
||||
| `MCP_CLIENT_ID` | `workbuddy-enterprise` | MCP Client 类型标识 |
|
||||
| `MCP_SERVICE_IDENTITY` | `workbuddy-demo` | 传入业务边界的服务身份 |
|
||||
| `MCP_REQUIRED_SCOPE` | `knowledge:search` | Tool 所需最小 scope |
|
||||
| `MCP_ALLOWED_HOSTS` | 仅本机/test | DNS rebinding Host allowlist,JSON 数组 |
|
||||
| `MCP_ALLOWED_ORIGINS` | 仅本机 | Origin allowlist,JSON 数组 |
|
||||
|
||||
远程部署必须使用 TLS 反向代理,并把公开域名加入 `MCP_ALLOWED_HOSTS`;有 `Origin` 的客户端还
|
||||
必须显式配置 `MCP_ALLOWED_ORIGINS`。Token、真实域名和内部 Endpoint 不得提交 Git。
|
||||
|
||||
## WorkBuddy / CodeBuddy 配置
|
||||
|
||||
WorkBuddy Enterprise 官方客户端支持 `type: "http"`、远程 URL、自定义 Header 和环境变量
|
||||
展开。可复制 `deploy/workbuddy-mcp.example.json` 作为项目配置,内容如下:
|
||||
|
||||
```jsonc
|
||||
{
|
||||
"mcpServers": {
|
||||
"enterprise-knowledge": {
|
||||
"type": "http",
|
||||
"url": "${EKP_MCP_URL:-http://127.0.0.1:8766/mcp}",
|
||||
"headers": {
|
||||
"Authorization": "Bearer ${EKP_MCP_TOKEN}"
|
||||
},
|
||||
"description": "企业知识检索"
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
只提交包含环境变量占位符的配置;`EKP_MCP_TOKEN` 必须由每个运行环境安全注入。连接后应只
|
||||
发现 `mcp__enterprise_knowledge__search_knowledge` 一个工具。
|
||||
|
||||
## 安全和错误语义
|
||||
|
||||
- 缺失或错误 Token 返回 `401`,缺少 scope 返回 `403`;
|
||||
- 未批准 Host 返回 `421`,未批准 Origin 返回 `403`;
|
||||
- Query 长度和 `top_k` 在 MCP schema 层校验;
|
||||
- 预期业务错误返回安全 Tool Error;未知 Provider/数据库错误收敛为
|
||||
`knowledge search failed`;
|
||||
- MCP 输出和 Gateway 日志不得包含 Token、Provider Endpoint、数据库表名、模型名或
|
||||
Index Profile。
|
||||
|
||||
## 验证
|
||||
|
||||
```bash
|
||||
uv run pytest tests/unit/test_mcp_auth.py tests/unit/test_mcp_search.py
|
||||
uv run pytest tests/integration/test_mcp_server.py
|
||||
```
|
||||
|
||||
集成测试使用官方 MCP Python Client 通过 Streamable HTTP 完成 initialize、tools/list 和
|
||||
tools/call,并使用与 WorkBuddy 相同的 Bearer Header 配置验证身份传递、无结果、参数错误、
|
||||
认证失败、Origin 防护和错误脱敏。
|
||||
|
||||
### WorkBuddy 现场验收记录
|
||||
|
||||
2026-09-02 使用 WorkBuddy Desktop `5.3.14` 自带的 CodeBuddy CLI `2.115.0` 完成兼容性
|
||||
验收。客户端直接加载 `deploy/workbuddy-mcp.example.json`,健康检查为 `Connected`;工具
|
||||
发现结果只有 `mcp__enterprise-knowledge__search_knowledge`,随后以字符串 `query` 和
|
||||
`top_k=1` 完成 Tool 调用并读取结构化 `filename`、`section`、`content` 等来源字段。
|
||||
|
||||
现场验收使用 DEMO-012 的真实 Gateway、Bearer 验证、service identity 传递、输入 Schema 和
|
||||
Tool 注册代码。为避免外部数据库与模型波动掩盖客户端兼容性,检索应用使用确定性验收返回;
|
||||
真实 Provider 检索链路由 `ARCH-006` 的受控验收负责。非交互 CLI 需要批准 MCP Tool 调用,
|
||||
本次只在单次会话中允许 `DeferExecuteTool`,没有永久启用跳过权限。测试结束后已删除临时 MCP
|
||||
配置并停止本地 Gateway,未写入真实凭据。
|
||||
|
||||
参考:
|
||||
|
||||
- [MCP Python SDK](https://github.com/modelcontextprotocol/python-sdk)
|
||||
- [MCP Streamable HTTP 规范](https://modelcontextprotocol.io/specification/2025-11-25/basic/transports)
|
||||
- [WorkBuddy Enterprise MCP 使用文档](https://cloud.tencent.cn/document/product/1831/137039)
|
||||
@ -1,72 +0,0 @@
|
||||
# ARCH-003 OceanBase Vector 配置与索引契约
|
||||
|
||||
ARCH-003 在既有 `VectorStore` 契约后新增 OceanBase Vector Adapter。ARCH-006 确认 OceanBase
|
||||
MySQL 关系表保存 canonical Document、Chunk、Index Profile 和 Chunk Index State;同库向量表
|
||||
只保存可由这些数据重建的检索行。ARCH-005 已完成运行时默认切换,并移除 Mochow 运行配置与依赖。
|
||||
|
||||
## 数据库前提
|
||||
|
||||
- OceanBase MySQL 租户版本至少为 4.3.5.4:ARCH-006 已在目标 test 集群用只读
|
||||
`EXPLAIN` 验证 `VECTOR_INDEX(... pre_filter)` 产生 PRE-FILTER 执行计划;
|
||||
- 目标数据库和最小权限账号由受控部署流程预创建;
|
||||
- Vector 内存、资源规格和 HNSW 参数由 DBA 在受控环境确认;
|
||||
- 应用启动和请求路径不会创建、修改或删除表结构。
|
||||
|
||||
使用 [001_knowledge_chunks_v1.sql](../../deploy/oceanbase/001_knowledge_chunks_v1.sql)
|
||||
在专用数据库中创建索引表。脚本固定:
|
||||
|
||||
```text
|
||||
VECTOR(1024)
|
||||
DISTANCE = COSINE
|
||||
TYPE = HNSW
|
||||
EF_CONSTRUCTION = 256
|
||||
EF_SEARCH = 200
|
||||
```
|
||||
|
||||
表的 `index_record_id` 是确定性的 Chunk/Profile 主键;`document_id + index_profile_id` 和
|
||||
`index_profile_id` 是清理与 Profile 过滤的标量索引。表名或向量维度变化必须建立新表或
|
||||
Shadow Profile,验证后再激活,禁止在生产索引上原地修改。
|
||||
|
||||
## 运行配置
|
||||
|
||||
配置只能由部署环境或 Secret Manager 注入,禁止把真实地址、用户名和密码写入 `.env`、
|
||||
镜像、日志、测试快照或提交记录。
|
||||
|
||||
```text
|
||||
VECTOR_STORE_PROVIDER=oceanbase
|
||||
OCEANBASE_VECTOR_URI=<host:port>
|
||||
OCEANBASE_VECTOR_USER=<运行时注入>
|
||||
OCEANBASE_VECTOR_PASSWORD=<运行时注入>
|
||||
OCEANBASE_VECTOR_DATABASE=<预创建数据库>
|
||||
OCEANBASE_VECTOR_TABLE=knowledge_chunks_v1
|
||||
OCEANBASE_VECTOR_INDEX=vidx_knowledge_chunks_embedding
|
||||
OCEANBASE_VECTOR_BATCH_SIZE=100
|
||||
OCEANBASE_VECTOR_TIMEOUT_SECONDS=30
|
||||
```
|
||||
|
||||
- URI 只接受 `host:port`,不得包含 Scheme、用户信息、Path、Query 或 Fragment;
|
||||
- Database/Table/Index 只接受安全标识符;
|
||||
- Password 使用 `SecretStr`,Provider 异常只返回收敛后的错误,不回显 SQL、连接信息、
|
||||
Chunk 内容或 SDK Response;
|
||||
- `pyobvector` 只存在于 Adapter/Factory 内,业务服务不依赖其 Row、Engine 或距离函数类型。
|
||||
|
||||
## 写入、删除和检索语义
|
||||
|
||||
- Upsert 使用 `index_record_id` 主键,按可配置批次写入;
|
||||
- 删除使用参数化的 `document_id + index_profile_id` 条件,不拼接用户输入;
|
||||
- Dense Search 在同一 ANN 查询中使用 `VECTOR_INDEX(... pre_filter)`,保证
|
||||
`index_profile_id` 在候选近邻搜索前过滤,并返回 Dense Top20;禁止依赖普通 `WHERE`
|
||||
的自适应过滤语义,也禁止全局 TopK 后再过滤;
|
||||
- OceanBase `cosine_distance` 是低值优先距离,Adapter 统一转换为
|
||||
`score = 1 - cosine_distance`,保持 `VectorStore` 的高分优先契约;
|
||||
- 返回 UUID、Profile、字段数、距离范围和排序不满足契约时失败关闭,不生成 Source Result。
|
||||
|
||||
默认测试使用 Fake Client,不连接 OceanBase。真实版本、DDL、权限、Cosine HNSW 召回、过滤
|
||||
行为与 30~50 条脱敏 Golden Query 统一在 ARCH-006 的受控资源中验证。凭据只能在运行时
|
||||
临时注入,禁止进入仓库、日志或测试快照。
|
||||
|
||||
官方参考:
|
||||
|
||||
- [pyobvector 官方仓库](https://github.com/oceanbase/pyobvector)
|
||||
- [OceanBase 创建向量索引](https://en.oceanbase.com/docs/common-oceanbase-database-10000000003075357)
|
||||
- [OceanBase 向量索引查询 Hint](https://en.oceanbase.com/docs/common-oceanbase-database-10000000004311163)
|
||||
@ -1,76 +0,0 @@
|
||||
# ARCH-004 千帆 BCE Reranker 与 Source Result 配置
|
||||
|
||||
本文记录 Knowledge Platform Demo 的重排序连接方式、请求边界和 canonical Source Result
|
||||
契约。真实 API Key 只允许由运行环境注入,不得写入代码、示例配置、日志或提交记录。
|
||||
|
||||
## 运行配置
|
||||
|
||||
```dotenv
|
||||
RERANKER_PROVIDER=qianfan
|
||||
RERANKER_MODEL=bce-reranker-base
|
||||
RERANKER_ENDPOINT=https://qianfan.baidubce.com/v2/rerank
|
||||
RERANKER_API_KEY=<运行环境注入,可空>
|
||||
RERANKER_TIMEOUT_SECONDS=30
|
||||
DENSE_TOP_N=20
|
||||
SEARCH_TOP_K=5
|
||||
```
|
||||
|
||||
`RERANKER_API_KEY` 未单独配置时,Demo 会复用 `BAIDU_QIANFAN_API_KEY`。两者都不存在时
|
||||
Provider 创建失败。凭证使用 `SecretStr`,HTTP/JSON 异常不会包含请求头、请求正文或
|
||||
响应正文。
|
||||
|
||||
## Reranker 请求契约
|
||||
|
||||
ARCH-004 固定使用千帆 `POST /v2/rerank`,模型基线为
|
||||
`bce-reranker-base`:
|
||||
|
||||
- Query 去除首尾空白后不能为空,最多 1600 字符;
|
||||
- documents 不能为空,最多 64 条,每条最多 4096 字符;
|
||||
- 当前标准链路发送 Dense Top20,并请求 Reranker Top5;
|
||||
- 响应必须恰好返回请求的 Top-N,`index` 唯一且指向原输入位置;
|
||||
- `relevance_score` 必须是有限数值,并保持降序;
|
||||
- 响应若回显 document,必须与对应输入文本完全一致。
|
||||
|
||||
上述边界与千帆重排序接口当前约束一致。BCE 和原 Demo Qwen3 模型共享请求/响应结构,
|
||||
因此本次变更不修改 `RerankerProvider`、Search REST API 或最终 Top5 行为。Qwen3-Reranker
|
||||
保留为 PRD 中的后续 Benchmark 候选,不作为当前运行默认值。
|
||||
|
||||
Adapter 只向业务层返回原候选索引和 Reranker 分数,不暴露千帆响应结构。默认测试使用
|
||||
`FakeRerankerProvider` 的确定性字符覆盖率排序,不发起网络请求。
|
||||
|
||||
## Source Result 契约
|
||||
|
||||
Reranker 对 provider-neutral Dense Candidate 的 `content` 评分;选出 Top5 后,
|
||||
`SearchService` 使用
|
||||
`chunk_id` 一次性回查 OceanBase canonical `chunks` 关系表,并由 canonical row 生成:
|
||||
|
||||
```text
|
||||
document_id
|
||||
filename
|
||||
page
|
||||
section
|
||||
chunk_id
|
||||
content
|
||||
score
|
||||
rerank_score
|
||||
```
|
||||
|
||||
其中 `score` 保留 Dense 分数,`rerank_score` 保存精排分数。若 Chunk 缺失、document_id
|
||||
不一致或向量索引 content 已与 canonical content 不一致,搜索会失败关闭,避免返回来源
|
||||
与评分文本不一致的结果。
|
||||
|
||||
DEMO-009 已通过 `POST /api/v1/search` 提供序列化和错误映射,参见
|
||||
`docs/development/REST_API_SETUP.md`。Demo 仍不实现 RBAC/Document ACL,正式 V1 权限
|
||||
范围必须在 Dense 召回阶段下推,不能在 Reranker 后过滤。
|
||||
|
||||
## 验证边界
|
||||
|
||||
默认测试离线覆盖 Dense Top20 → Reranker → Top5 → canonical Source Result,并使用合成
|
||||
Golden Dataset 验证预期文档和章节升至首位。真实千帆联调前还需在受控运行环境确认:
|
||||
|
||||
- API Key 已开通目标 Reranker 模型权限;
|
||||
- Endpoint 网络可达;
|
||||
- 模型 API 名称与控制台服务一致;
|
||||
- 真实中文 Query 的延迟、配额和 Top5 质量满足 Demo 演示需求。
|
||||
|
||||
官方接口参考:[千帆重排序 API](https://cloud.baidu.com/doc/qianfan-api/s/2m7u4zt74)。
|
||||
@ -1,76 +0,0 @@
|
||||
# Demo REST API 使用说明
|
||||
|
||||
## 范围
|
||||
|
||||
DEMO-009 将入库和检索服务公开为版本化 REST API,DEMO-015 将单文件入库切换为持久化异步
|
||||
Job,DEMO-016 增加批量接收与聚合查询。API 层只负责输入校验、
|
||||
错误映射和响应序列化,不重复实现 Parser、Embedding、VectorStore 或 Reranker 业务逻辑。
|
||||
|
||||
当前不包含 ACL/RBAC、软删除、目录/压缩包上传、断点续传或 Hybrid Search。
|
||||
|
||||
## 启动前提
|
||||
|
||||
1. 按 `docs/development/BOS_SETUP.md` 配置 BOS;
|
||||
2. 按 `docs/development/OCEANBASE_VECTOR_SETUP.md` 配置 OceanBase Vector 和千帆 Embedding;
|
||||
3. 按 `docs/development/RERANKER_SETUP.md` 配置 Reranker;
|
||||
4. 按 `docs/development/ASYNC_INGESTION_SETUP.md` 配置 RabbitMQ 并启动 Worker;
|
||||
5. 执行 `uv run alembic upgrade head`;
|
||||
6. 新旧索引迁移按 `docs/development/INDEX_REBUILD.md` 构建、验证并激活 Profile;
|
||||
7. 执行 `make run`。
|
||||
|
||||
`/healthz` 和 OpenAPI 可以在没有云端凭证时启动和访问。业务依赖在首次访问 P0 业务接口时
|
||||
延迟创建;配置不完整时返回 `503`,不会将凭证名以外的配置值暴露给客户端。
|
||||
|
||||
OpenAPI 地址:`/api/v1/openapi.json`
|
||||
|
||||
Swagger UI:`/api/v1/docs`
|
||||
|
||||
## 示例
|
||||
|
||||
上传并异步排队:
|
||||
|
||||
```bash
|
||||
curl -F 'file=@./policy.md;type=text/markdown' \
|
||||
http://127.0.0.1:8000/api/v1/documents
|
||||
```
|
||||
|
||||
使用响应中的 `job_id` 查询进度:
|
||||
|
||||
```bash
|
||||
curl http://127.0.0.1:8000/api/v1/jobs/<job-id>
|
||||
```
|
||||
|
||||
批量上传并查询聚合进度:
|
||||
|
||||
```bash
|
||||
curl -F 'files=@./policy.md;type=text/markdown' \
|
||||
-F 'files=@./manual.pdf;type=application/pdf' \
|
||||
http://127.0.0.1:8000/api/v1/ingestion-batches
|
||||
|
||||
curl http://127.0.0.1:8000/api/v1/ingestion-batches/<batch-id>
|
||||
```
|
||||
|
||||
检索:
|
||||
|
||||
```bash
|
||||
curl -H 'Content-Type: application/json' \
|
||||
-d '{"query":"产品A退款周期是多少?","top_k":5}' \
|
||||
http://127.0.0.1:8000/api/v1/search
|
||||
```
|
||||
|
||||
上传只等待不可变源文件、Document 与 Job 持久化,成功返回 `202`。Worker 后台执行 Parse、
|
||||
Clean、Chunk、Embedding 和 OceanBase Upsert;客户端通过 Job 状态判断何时可检索。
|
||||
|
||||
## 删除一致性
|
||||
|
||||
删除按以下顺序执行:
|
||||
|
||||
1. 删除所有已记录 Index Profile 以及当前 Profile 的 OceanBase 行;
|
||||
2. 删除 OceanBase canonical 关系表中的 ChunkIndexState、Chunk 和 Document;
|
||||
3. 删除解析 JSON 与不可变原始文件。
|
||||
|
||||
`queued`、`running` 或 `retrying` Job 对应的 Document 返回 `409`,避免删除与 Worker 并发。
|
||||
|
||||
向量清理失败时保留 canonical Metadata 和原始文件。Metadata 提交后若文件清理失败,接口
|
||||
返回明确的 `500`,可能留下不可访问的孤立文件;这是 Demo 直接删除的已知边界,正式 V1
|
||||
应通过可重试清理任务和审计记录强化。
|
||||
@ -1,6 +1,6 @@
|
||||
# Test Strategy
|
||||
|
||||
Unit tests must run without OceanBase, BOS, Qianfan, internet, GPU or production credentials.
|
||||
Unit tests must run without Mochow, Qianfan, internet, GPU or production credentials.
|
||||
|
||||
Provide deterministic fakes:
|
||||
- FakeEmbeddingProvider
|
||||
@ -11,11 +11,6 @@ Provide deterministic fakes:
|
||||
|
||||
External integrations use separate integration suites/markers.
|
||||
|
||||
Async ingestion tests run with an in-process recording dispatcher and deterministic providers. They
|
||||
must cover persisted Job/Attempt history, safe error summaries, bounded retry, cleanup-before-retry,
|
||||
active-lease duplicate delivery, expired-lease recovery, immutable-source preservation and terminal
|
||||
cleanup failure without requiring a live RabbitMQ broker.
|
||||
|
||||
E2E covers upload -> parse -> chunk -> embed -> index -> retrieve -> source trace, and source -> compilation draft -> review -> publish -> retrieval.
|
||||
|
||||
Any parsing/chunking/embedding/fusion/retrieval/reranking/query-routing change must run the Golden Dataset evaluation.
|
||||
|
||||
@ -6,20 +6,4 @@ Benchmark the full pipeline: Embedding × Retrieval Strategy × Fusion × Rerank
|
||||
|
||||
Initial embedding candidates: bge-large-zh, Qwen3-Embedding-0.6B, Qwen3-Embedding-4B, optional 8B.
|
||||
|
||||
Use synthetic/public/anonymized fixtures only. ARCH-006 expands the checked-in dataset to 32
|
||||
fully synthetic queries across refund, technical support, lodging, and reimbursement-material
|
||||
sources. Offline Fake Provider regression must remain deterministic; target-provider acceptance
|
||||
uses the same records through `make live-acceptance` and reports aggregate metrics only.
|
||||
|
||||
Current live gates:
|
||||
|
||||
```text
|
||||
Recall@1 >= 0.90
|
||||
Recall@5 = 1.00
|
||||
MRR >= 0.90
|
||||
NDCG@5 >= 0.90
|
||||
Latency P95 <= 10000 ms
|
||||
```
|
||||
|
||||
Threshold overrides are allowed only as explicit test-run environment inputs and must be recorded
|
||||
with the run result. See `docs/development/LIVE_ACCEPTANCE.md` for safety and cleanup requirements.
|
||||
Use synthetic/public/anonymized fixtures only.
|
||||
|
||||
@ -1,85 +0,0 @@
|
||||
# Provider 架构迁移计划
|
||||
|
||||
## 目标基线
|
||||
|
||||
本计划依据 2026-09-02 对最新内部配置与 test 环境的只读核验调整 EKP Provider 和数据库
|
||||
选型,同时保留 canonical-data 和 API-first 原则。
|
||||
|
||||
```text
|
||||
OceanBase MySQL canonical Metadata / Chunk / Index State
|
||||
Baidu BOS immutable source and derived artifacts
|
||||
OceanBase Vector co-located rebuildable Dense retrieval index
|
||||
Qianfan bge-large-zh 1024-dimensional Embedding
|
||||
Qianfan BCE Reranker bce-reranker-base
|
||||
```
|
||||
|
||||
Provider、内部地址、数据库账户和凭据不得出现在 API/MCP Contract、日志、镜像或提交中。
|
||||
本地测试继续使用 LocalObjectStorage、FakeVectorStore、FakeEmbeddingProvider 和
|
||||
FakeRerankerProvider。
|
||||
|
||||
## 明确保留的行为
|
||||
|
||||
- OceanBase 关系表保持 canonical database,专用向量表只保存可重建索引;
|
||||
- 原始文件不可变;
|
||||
- Dense Top20 → Reranker Top5;
|
||||
- 当前 PDF/DOCX/TXT/Markdown Parser 和 `structure_v1` Chunk;
|
||||
- `/api/v1` 和 MCP 继续只暴露业务语义;
|
||||
- `index_profile_id` 过滤在召回阶段下推;
|
||||
- 新索引通过 shadow 表/Profile 构建和验证,不原地覆盖旧索引。
|
||||
|
||||
## 明确不随迁移引入
|
||||
|
||||
- 将 OceanBase 向量表当作 canonical source;
|
||||
- Top50、距离阈值、标准答案捷径、双查询和 RRF;
|
||||
- Marker/Pandoc、LLM 摘要/问题/关键词索引;
|
||||
- RBAC/ACL、异步队列、MCP、Hybrid/BM25 或 Admin Console 扩展;
|
||||
- 旧系统 `.env`/镜像配置加载方式或任何现有敏感凭据。
|
||||
|
||||
## Task 与验收
|
||||
|
||||
### ARCH-001 架构和执行基线
|
||||
|
||||
更新 ADR、PRD、Architecture、Demo Scope、Tasks 和 Progress,消除目标 Provider 歧义。
|
||||
|
||||
### ARCH-002 BOS ObjectStorage
|
||||
|
||||
新增 provider-neutral BOS Adapter、配置、脱敏错误和离线单元测试。Local Adapter 保留为
|
||||
测试/开发实现。
|
||||
|
||||
依赖:ARCH-001。
|
||||
|
||||
### ARCH-003 OceanBase VectorStore
|
||||
|
||||
新增 OceanBase Vector Adapter 和专用索引 DDL,覆盖 Upsert、文档/Profile 删除、Dense
|
||||
TopN、Cosine Distance 到统一高分优先 `score` 的转换,以及 profile 过滤。
|
||||
|
||||
依赖:ARCH-002。
|
||||
|
||||
### ARCH-004 BCE Reranker
|
||||
|
||||
将运行基线改为 `bce-reranker-base`,按真实接口约束更新配置、Adapter、单元测试和 RAG
|
||||
Evaluation;保持 Top5 API 行为。
|
||||
|
||||
依赖:ARCH-003。
|
||||
|
||||
### ARCH-005 Runtime Cutover
|
||||
|
||||
将 API 运行时切换到 BOS、OceanBase Vector 和 BCE Reranker;提供从 canonical
|
||||
Chunk 构建新 shadow index 的可重试命令,验证后激活新 Profile,再清理 Mochow 运行配置。
|
||||
|
||||
依赖:ARCH-004。
|
||||
|
||||
### ARCH-006 真实验收
|
||||
|
||||
修正先前错误的 PostgreSQL 前提:将 canonical SQLAlchemy/Alembic 迁移到 OceanBase MySQL,
|
||||
同库保留逻辑隔离的可重建向量表;在受控 test 资源中执行 OceanBase canonical → BOS →
|
||||
Qianfan Embedding → OceanBase Vector → BCE Reranker → canonical Source Result 全链路,并完成
|
||||
30~50 条脱敏 Golden Query 验收。
|
||||
|
||||
依赖:ARCH-005、已授权的 test 运行凭据、数据库/表和 BOS 专用前缀。
|
||||
|
||||
## 安全边界
|
||||
|
||||
内部配置清单中的现有凭据按已暴露处理。经所有者明确要求,本次只允许把 test 凭据临时
|
||||
注入 ARCH-006 进程;不得生成 `.env`、测试快照或调试日志,不得连接 fat/prod,验收后仍应
|
||||
轮换。真实集成测试必须使用专用表/BOS 前缀并清理本次临时对象。
|
||||
@ -1,123 +0,0 @@
|
||||
# MCP 与异步批量入库优先级方案
|
||||
|
||||
**Task:** PLAN-001
|
||||
**Status:** Accepted planning baseline
|
||||
**Date:** 2026-09-02
|
||||
|
||||
## 1. 现状判断
|
||||
|
||||
| 能力 | 当前实现 | 结论 |
|
||||
|---|---|---|
|
||||
| 单文件上传 | `POST /api/v1/documents` 持久化 Source/Document/Job 后返回 `202`,Worker 后台处理 | DEMO-015 已实现 |
|
||||
| 批量上传 | 批次 API、逐文件 Job 状态和 Admin Console 自动轮询 | DEMO-016 已实现 |
|
||||
| 失败补偿 | 阶段内补偿 + 重试前清理全部派生数据,保留不可变源和执行历史 | DEMO-015 已实现 |
|
||||
| 自动重试 | OceanBase Job/Attempt + Celery/RabbitMQ + 有界退避 + Worker 租约恢复 | DEMO-015 已实现 |
|
||||
| 重试前清理 | 重试前删除向量、索引状态、Chunk、解析产物,保留源文件与历史 | DEMO-015 已实现 |
|
||||
| MCP | 独立 Gateway、认证边界、`search_knowledge` 与 WorkBuddy 协议级验收 | DEMO-012 已实现 |
|
||||
|
||||
PLAN-001 的异步执行缺口已由 DEMO-015 补齐,批量提交与状态 UI 已由 DEMO-016 补齐。
|
||||
|
||||
## 2. Task 归并结论
|
||||
|
||||
### DEMO-012:MCP `search_knowledge`(提升为最高优先级)
|
||||
|
||||
保留既有 Task ID,不新建重复任务。范围限定为:
|
||||
|
||||
- 提供可运行的标准 MCP Gateway;
|
||||
- 只暴露 `search_knowledge`;
|
||||
- 复用现有 Search Service 或正式 `/api/v1/search`,不复制检索逻辑;
|
||||
- 建立 WorkBuddy 所需的 user/service identity 边界,不使用共享超级管理员凭据;
|
||||
- 完成协议契约、错误收敛、凭据脱敏和 WorkBuddy 兼容性验证。
|
||||
|
||||
该 Task 是 `EKP-070`、`EKP-071`、`EKP-072` 的最小垂直切片,不能据此把三个完整 V1 Task 标记为完成。
|
||||
|
||||
### DEMO-015:异步入库 Job 与自动重试清理(新增)
|
||||
|
||||
从 `EKP-017` 和 `EKP-023` 抽取 Demo 所需的最小基础能力,并复用 DEMO-003~006 的处理服务:
|
||||
|
||||
- 新增持久化 Ingestion Job/Attempt,记录阶段、状态、尝试次数、安全错误和时间戳;
|
||||
- 源文件成功写入 BOS 且 Document 已持久化后立即排队,API 不等待解析和索引;
|
||||
- 同一 Document 同时最多一个有效执行,任务投递和消费具备幂等保护;
|
||||
- 对可重试的瞬时错误自动执行有界重试和退避;Demo 默认最多 3 次总尝试,允许通过非敏感配置调整;
|
||||
- 格式不支持、文件为空、无可提取内容等确定性错误直接失败,不进行无意义重试;
|
||||
- 每次重新执行前,按顺序清理该 Document 的向量行、`ChunkIndexState`、Chunk 和解析产物,再从不可变原始文件重新处理;
|
||||
- 原始文件、Document 标识和审计所需的 Job/Attempt 历史不得被重试清理删除;
|
||||
- 任一清理步骤失败时禁止启动下一次尝试,并记录为终止失败,避免在未知脏状态上重复执行。
|
||||
- 通过执行器抽象接入 Celery/RabbitMQ,使 API 和 Worker 可独立部署;业务应用服务不得直接
|
||||
依赖 Celery API,确保执行器仍可替换;
|
||||
- Worker 重启、消息重复投递和确认超时必须保持持久化 Job 状态一致,不能产生双重有效
|
||||
Attempt 或重复派生数据。
|
||||
|
||||
该 Task 只完成公共知识库入库所需的 Job/Retry 与队列基线,不扩展到完整审计系统、
|
||||
Compilation、OCR、跨区域高可用或通用企业级 Job 平台。
|
||||
|
||||
### DEMO-016:批量异步上传与状态展示(已完成)
|
||||
|
||||
建立在 DEMO-015 之上,不与 Job 基础合并:
|
||||
|
||||
- 新增批量上传 API,一次接收多个 Demo 支持类型文件;
|
||||
- 每个文件独立生成 Document 和 Ingestion Job,单个文件失败不回滚已成功接收的其他文件;
|
||||
- 返回批次 ID、逐文件 Document ID、Job ID 和接收状态,不等待后台处理完成;
|
||||
- 提供批次/Job 查询能力,并在 Admin Console 展示逐文件阶段、重试次数和最终结果;
|
||||
- 文件数量、单文件大小和请求总大小均受可配置限制;
|
||||
- 不在本 Task 引入目录上传、压缩包解包、断点续传或跨批次去重。
|
||||
|
||||
该 Task 对应 `EKP-021` 的批量异步扩展及 `EKP-080`/`EKP-081` 的最小状态 UI 切片,不能据此把完整 V1 Task 标记为完成。
|
||||
|
||||
## 3. 实施顺序与依赖
|
||||
|
||||
```text
|
||||
DEMO-009 + ARCH-006 → DEMO-012(已完成)
|
||||
|
||||
PLAN-002(已完成)
|
||||
└──→ DEMO-003 → DEMO-004 → DEMO-005 → DEMO-006
|
||||
└──→ DEMO-015 Async Job + Celery/RabbitMQ + Retry Cleanup
|
||||
└──→ DEMO-016 Batch Async Upload + UI
|
||||
└──→ DEPLOY-001 → DEPLOY-002 → DEPLOY-003
|
||||
```
|
||||
|
||||
推荐严格按以下顺序执行并逐项闭环:
|
||||
|
||||
1. `PLAN-002`(已完成)
|
||||
2. `DEMO-015`(已完成)
|
||||
3. `DEMO-016`(已完成)
|
||||
4. `DEPLOY-001`
|
||||
5. `DEPLOY-002`
|
||||
6. `DEPLOY-003`
|
||||
|
||||
MCP 与异步入库没有代码级前置依赖,因此提高 MCP 优先级不会阻塞后续批量上传。`DEMO-016` 必须等待 `DEMO-015`,否则会把同步流水线简单并发放大,缺少可靠的状态、隔离和恢复边界。
|
||||
|
||||
## 4. 验收边界
|
||||
|
||||
### DEMO-012
|
||||
|
||||
- MCP Inspector/标准客户端能发现并调用 `search_knowledge`;
|
||||
- WorkBuddy 使用已映射的 user/service identity 完成一次端到端检索;
|
||||
- REST 与 MCP 对同一查询复用同一业务逻辑并返回一致的来源字段;
|
||||
- 响应与日志不泄露 OceanBase、BOS、模型名、Endpoint 或凭据。
|
||||
|
||||
### DEMO-015
|
||||
|
||||
- 上传接口在源文件和 Job 持久化后返回 `202`,后台完成现有全流水线;
|
||||
- 注入一次可重试故障后可观察到自动重试,并最终成功;
|
||||
- 第二次尝试开始前,旧解析产物、Chunk、索引状态和向量均已清理,原始源文件仍存在;
|
||||
- 永久错误不重试,达到次数上限后稳定进入 `failed`;
|
||||
- 并发或重复投递不会产生双重有效执行和重复派生数据。
|
||||
- RabbitMQ 消息确认超时或 Worker 重启后,Job 可恢复到终态且不会重复生成派生数据。
|
||||
|
||||
### DEMO-016
|
||||
|
||||
- 一次提交多个文件后立即得到逐文件 Document/Job 标识;
|
||||
- 批次内成功、处理中、重试中和失败状态互不影响且可查询;
|
||||
- Admin Console 无需刷新即可轮询展示逐文件进度和最终状态;
|
||||
- 批次边界、大小限制、部分失败和重试清理均有集成测试。
|
||||
|
||||
## 5. 非目标与 ADR 影响
|
||||
|
||||
- PLAN-002 已把 Celery/RabbitMQ 纳入 `DEMO-015`,并以独立 `DEPLOY-*` Task 纳入最小
|
||||
Kubernetes、多副本和滚动升级能力;跨区域容灾、自动云资源编排和完整 Observability
|
||||
仍为非目标。
|
||||
- 不改变 OceanBase/BOS canonical、向量索引可重建、原始文档不可变、Provider 抽象和 MCP 隐藏基础设施等 Accepted ADR。
|
||||
- 公共知识库只是阶段性单一可见域;ADR-007 保持 Accepted,用户/ACL 实现后置且不得用全局
|
||||
Top-K 后过滤替代。
|
||||
- 本次任务拆分和交付边界不需要新增或重开 Accepted ADR。
|
||||
@ -1,426 +1,22 @@
|
||||
# 项目进度
|
||||
# Progress
|
||||
|
||||
当前阶段:生产容器镜像闭环完成,准备实施 Kubernetes/Helm 一键部署
|
||||
Current phase: Knowledge Retrieval Demo v0.1
|
||||
|
||||
已完成:DEMO-001~DEMO-010、DEMO-012、DEMO-014~DEMO-016、ARCH-001~ARCH-006、PLAN-001~PLAN-002、DEPLOY-001
|
||||
进行中:无
|
||||
阻塞:无
|
||||
Completed: DEMO-001
|
||||
In progress: none
|
||||
Blocked: none
|
||||
|
||||
下一任务:`DEPLOY-002`,完成并合并后执行 `DEPLOY-003`。
|
||||
Next: DEMO-002
|
||||
|
||||
Provider 架构迁移按 `ARCH-001 → ARCH-002 → ARCH-003 → ARCH-004 → ARCH-005 → ARCH-006`
|
||||
执行并已完成。DEMO-010 在迁移闭环验收后实施,未扩大为完整 V1 Admin Console。
|
||||
The full V1 backlog is deferred until the Demo scope is complete. See
|
||||
`docs/demo/DEMO_SCOPE.md` for the active task sequence.
|
||||
|
||||
## 任务记录
|
||||
## Task records
|
||||
|
||||
### DEMO-001
|
||||
|
||||
- 建立 Python 3.12/uv 工具链与依赖锁文件,新增 FastAPI 应用、类型化配置、
|
||||
健康检查、OpenAPI 地址和可执行的 Make 命令。
|
||||
- 新增可重复执行的单元测试、集成测试及 Evaluation 种子数据校验。
|
||||
- 已验证格式、Lint、严格类型检查、测试、依赖锁、API 启动、健康检查响应及
|
||||
版本化 OpenAPI 输出。
|
||||
|
||||
### DEMO-002
|
||||
|
||||
- 新增 canonical Document 元数据、简化的处理状态、SQLAlchemy 数据库/Session
|
||||
工厂,以及首个只向前执行的 Alembic 迁移。
|
||||
- 新增与 Provider 无关的 ObjectStorage 接口和本地适配器,支持禁止覆盖、拒绝
|
||||
不安全路径并清理写入中断产生的半文件。
|
||||
- 新增模型、数据库、存储、迁移及 Schema Drift 测试;无需连接生产数据库即可
|
||||
验证 PostgreSQL 方言的迁移 SQL。
|
||||
|
||||
### DEMO-003
|
||||
|
||||
- 新增 PDF、DOCX、TXT、MD 单文件上传应用服务,完成文件名、扩展名、媒体类型、
|
||||
空文件和流式大小限制校验。
|
||||
- 上传使用 UUID 隔离的不可覆盖 storage key,并将原始文件与 `uploaded` 状态的
|
||||
Document 元数据关联保存。
|
||||
- 数据库写入失败时自动回滚并删除已保存文件;新增四类文件、拒绝规则、大小限制
|
||||
和失败补偿集成测试。
|
||||
- Parser 触发与公开上传 REST API 分别留在 DEMO-004 和 DEMO-009,不在本任务
|
||||
提前实现。
|
||||
|
||||
### DEMO-004
|
||||
|
||||
- 新增统一的 `ParsedDocument` / `ParsedBlock` 领域模型和 Parser Provider 接口,
|
||||
使用 PyMuPDF、python-docx 及轻量文本解析器支持 PDF、DOCX、UTF-8 TXT、Markdown。
|
||||
- PDF 保留一基页码,DOCX/Markdown 保留基础标题层级;基础 Cleaner 统一处理多余
|
||||
空白、空行、异常换行和空块。
|
||||
- 新增上传后同步触发解析的应用层编排,将版本化解析 JSON 作为可重建派生文件保存,
|
||||
并通过数据库迁移增加 `parsed_storage_key` 追踪结果;原始文件保持不可变。
|
||||
- 解析完成后文档保持 `processing`,等待 DEMO-005/006 完成后续流水线;解析失败记录
|
||||
`failed` 和有界错误信息,派生结果写库失败时执行文件补偿。
|
||||
- 新增四类格式、页码/章节、清洗、解析持久化、禁止覆盖和失败状态测试。公开 REST API
|
||||
仍按计划留在 DEMO-009。
|
||||
|
||||
### DEMO-005
|
||||
|
||||
- 新增可注入 `TokenCounter` 的 `structure_v1` Chunk 策略,默认使用 450 tokens 上限和
|
||||
75 tokens overlap;标题或章节变化形成硬边界,同章节段落优先组合,超长内容按 token
|
||||
窗口切分。
|
||||
- 默认轻量计数器按 CJK 字符、标点和保守 ASCII 子词估算 token,避免引入模型运行时;
|
||||
精确模型 tokenizer 可通过既有接口注入,留待 DEMO-006 与 bge-large-zh 联调。
|
||||
- 新增 canonical Chunk 元数据与只向前迁移,原子保存确定性 `chunk_id`、文档/文件、
|
||||
一基页码范围、章节路径、顺序、token 数和策略版本;禁止静默覆盖同策略结果。
|
||||
- 新增上传 → 解析/清洗 → Chunk 的同步流水线编排。Chunk 完成后仍保持 `processing`,
|
||||
等待 DEMO-006 Embedding/Mochow;失败时不产生半套 Chunk,并保留原始文件。
|
||||
- 新增完全合成的 Markdown 测试文档和 Golden Dataset Chunk 回归,覆盖章节追踪、
|
||||
token 上限、overlap、页码范围、迁移、持久化、重复执行和失败状态。
|
||||
|
||||
### DEMO-006
|
||||
|
||||
- 新增与厂商无关的 `EmbeddingProvider` / `VectorStore` 契约及确定性 Fake,业务编排
|
||||
不依赖 Mochow SDK 数据结构。
|
||||
- 新增千帆 v2 `bge-large-zh` Adapter,使用 Bearer API Key,支持最多 16 条批量请求,
|
||||
Chunk 同时限制 450 tokens/2000 字符,并严格校验响应顺序、1024 维度和有限数值;
|
||||
凭证使用 `SecretStr`,
|
||||
不进入错误消息或提交记录。
|
||||
- 新增官方 `pymochow` Adapter,支持分批 Upsert 和按 `document_id + index_profile_id`
|
||||
删除;Mochow 只保存向量与检索元数据,PostgreSQL 继续保存 canonical Chunk 和索引状态。
|
||||
- 新增版本化 `IndexProfile` 与 `chunk_index_states` 向前迁移,防止模型/Chunk 策略原地
|
||||
覆盖;所有 Chunk 成功入库后文档才进入 `ready`。
|
||||
- 新增上传 → 解析/清洗 → Chunk → Embedding → Mochow Upsert 同步编排。Mochow 批次
|
||||
或 canonical 状态写入失败时按文档/profile 清理可能的半批索引并记录 `failed`。
|
||||
- 新增 Qianfan、Mochow 映射、配置脱敏、索引状态、失败补偿和 Golden Dataset 全链路
|
||||
回归测试。真实凭证/网络集成未在默认测试中执行;专用 Mochow 表需按
|
||||
`docs/development/MOCHOW_SETUP.md` 预创建。
|
||||
|
||||
### DEMO-007
|
||||
|
||||
- 新增 `DenseSearchRequest` / `DenseSearchCandidate` 与 `DenseRetrievalService`,完成
|
||||
Query 清洗、同模型 Embedding、版本化 `index_profile_id` 过滤和默认 Top20 候选召回;
|
||||
此阶段不加入 Reranker、最终 Source Result 或 REST API。
|
||||
- `FakeVectorStore` 新增离线精确余弦检索,先按 profile 过滤再排序,并使用稳定的 UUID
|
||||
tie-break,支持无网络的单元/集成/Evaluation 回归。
|
||||
- `MochowVectorStore` 新增官方 `VectorTopkSearchRequest` 调用,固定检索 `embedding` 字段,
|
||||
下推 profile 标量过滤,支持可配置 HNSW `ef`,严格校验响应字段、UUID、有限分数和
|
||||
Top-N 数量,异常不泄露 SDK 请求或凭证。
|
||||
- 新增 Golden Dataset Dense Retrieval 回归,完整覆盖上传 → 解析/清洗 → Chunk →
|
||||
Embedding/入库 → Query Embedding → Dense Top20,确认两个合成 Query 均召回预期文件和
|
||||
章节。
|
||||
- 默认自动化测试不连接真实千帆/Mochow;真实环境仍需按
|
||||
`docs/development/MOCHOW_SETUP.md` 注入凭证并验证网络、表 Schema 与索引状态。
|
||||
|
||||
### DEMO-008
|
||||
|
||||
- 新增与厂商无关的 `RerankerProvider` / `RerankResult` 契约及确定性 Fake,使用原始
|
||||
candidate index 映射精排结果,业务服务不依赖千帆响应结构。
|
||||
- 新增千帆 v2 Reranker Adapter(初始为 `qwen3-reranker-0.6b`,ARCH-004 已迁移至
|
||||
`bce-reranker-base`),调用 `/v2/rerank`,严格限制 Query、
|
||||
documents 和 Top-N,校验响应数量、唯一索引、回显文本、有限分数与降序关系;错误不会
|
||||
回显请求内容或凭证。
|
||||
- 新增 `SearchService`,完成 Dense Top20 → Reranker → Top5。Reranker 选中候选后按
|
||||
`chunk_id` 回查 PostgreSQL canonical Chunk,再生成包含 document、filename、page、
|
||||
section、chunk、content、Dense score 和 rerank score 的最终 Source Result。
|
||||
- Source Enrichment 对缺失 Chunk、document_id 不一致和 stale vector content 采用失败
|
||||
关闭,防止 Mochow 副本与 canonical 来源不一致时返回错误证据。
|
||||
- 新增 Provider、配置脱敏、排序映射、canonical 回查、一致性失败和 Golden Dataset
|
||||
全链路回归。默认测试不调用真实千帆;真实联调边界记录在
|
||||
`docs/development/RERANKER_SETUP.md`。公开 Search REST API 留在 DEMO-009。
|
||||
|
||||
### DEMO-009
|
||||
|
||||
- 新增 `/api/v1/documents` 上传/列表、`/api/v1/documents/{document_id}` 详情/直接删除和
|
||||
`/api/v1/search`,通过 OpenAPI 固化 multipart 上传、Document 状态和最小 Source Result
|
||||
契约;API 响应不暴露 storage key、Index Profile 或厂商实现。
|
||||
- 上传接口同步复用 DEMO-003~006 的完整入库流水线,并在完成后回读 PostgreSQL canonical
|
||||
Document,确保返回最终 `ready` 状态和数据库生成时间戳。
|
||||
- 新增文档查询/删除应用服务。删除覆盖当前及历史已记录 Index Profile 的向量行、
|
||||
ChunkIndexState、Chunk、Document、解析 JSON 和原始文件;向量清理失败时保留 canonical
|
||||
数据,来源清理失败返回明确错误并记录 Demo 一致性边界。
|
||||
- 业务依赖延迟创建,因此健康检查和 OpenAPI 启动不要求云端凭证;Provider 配置不完整时
|
||||
业务接口返回 `503`,上游异常使用收敛后的错误响应,避免泄露凭证或厂商正文。
|
||||
- 新增完全离线的 REST 全链路测试,覆盖上传、列表、详情、Dense/Reranker 检索、来源字段、
|
||||
直接删除、OpenAPI 路径、输入校验、404、Provider 延迟加载及删除失败保护。
|
||||
|
||||
### ARCH-001
|
||||
|
||||
- 接受 BOS、OceanBase Vector、千帆 `bge-large-zh` 和 `bce-reranker-base` 作为新的 V1
|
||||
Provider 基线;PostgreSQL 继续保存 canonical 数据,OceanBase 只保存可重建索引。
|
||||
- 新增 ADR-009~ADR-011 和独立迁移计划,将 ARCH-002~ARCH-006 的范围、依赖、验收顺序
|
||||
与安全边界写入任务清单。
|
||||
- 保留 Demo 的 Dense Top20 → Reranker Top5、现有 Parser/Chunk、API/MCP 业务契约;不随迁移
|
||||
引入 Top50、RRF、Marker/Pandoc、LLM 派生索引、ACL 或 Hybrid/BM25 扩展。
|
||||
- 本任务只调整架构和产品文档,不修改运行时代码;旧系统地址、账号和凭据均未写入仓库,
|
||||
真实验收只允许使用轮换后的 Secret 与受控测试资源。
|
||||
|
||||
### ARCH-002
|
||||
|
||||
- 新增官方 `bce-python-sdk` 驱动的 `BosObjectStorage`,保持逻辑 key 与 Bucket/Prefix 分离,
|
||||
覆盖 Put、流式 Open、HEAD Exists 和 Delete。
|
||||
- 上传先使用可回卷临时流确定长度,再通过 `If-None-Match: *` 原子拒绝覆盖;404/412 和
|
||||
其他 SDK 异常映射为不泄露 Endpoint、响应正文或凭据的稳定存储错误。
|
||||
- 新增 `local|bos` 类型化配置和运行时工厂;BOS AK/SK 使用 `SecretStr`,Bucket、Endpoint、
|
||||
Prefix 和超时均由环境注入。Local 仍是测试/开发及 ARCH-005 前的默认实现。
|
||||
- 新增完全离线的 BOS Adapter、配置、Provider wiring 测试和中文配置说明;未使用内部配置
|
||||
清单中的真实地址或凭据,真实 BOS 联调留在 ARCH-006。
|
||||
|
||||
### ARCH-003
|
||||
|
||||
- 新增官方 `pyobvector` 驱动的 `OceanBaseVectorStore`,覆盖批量 Upsert、按 Document/Profile
|
||||
参数化删除和 Profile 条件下推的 Cosine ANN TopN;SDK 类型不进入业务服务。检索固定使用
|
||||
OceanBase 4.6 的 `VECTOR_INDEX(... pre_filter)`,不依赖普通 `WHERE` 的自适应过滤,也不做
|
||||
全局 TopK 后过滤。
|
||||
- 将低值优先的 `cosine_distance` 转为 `score = 1 - distance`,并对向量维度、有限值、
|
||||
非零范数、返回字段、UUID、Profile、距离范围和排序执行失败关闭校验。
|
||||
- 新增 `VECTOR(1024)`、Cosine HNSW、Profile/Document 标量索引的预置 DDL;应用不自动执行
|
||||
外部索引 DDL,PostgreSQL 继续保存 canonical 数据,OceanBase 表可重建。
|
||||
- 新增 `mochow|oceanbase` 类型化配置、SecretStr 密码和运行时工厂;Mochow 保持 ARCH-005
|
||||
前的默认实现。全部 Adapter、配置、Schema 和 wiring 测试均离线运行,真实数据库验收留在
|
||||
ARCH-006。
|
||||
|
||||
### ARCH-004
|
||||
|
||||
- 将千帆 Reranker 运行基线从 `qwen3-reranker-0.6b` 切换为已接受的
|
||||
`bce-reranker-base`;配置使用受限 Literal,业务 API、Provider 抽象和 Dense Top20 →
|
||||
Reranker Top5 行为保持不变。
|
||||
- 按千帆 `POST /v2/rerank` 官方约束复核并保留 Query 1600 字符、64 个候选、单文档
|
||||
4096 字符、Top-N 及响应排序/索引/分数失败关闭校验;真实 API Key 不进入代码和测试。
|
||||
- 更新 Adapter、环境示例、Provider wiring、配置与请求契约测试;Golden Dataset 继续使用
|
||||
确定性 Fake 离线评估,但显式绑定 `bce-reranker-base` 当前模型标识。真实 BCE 质量与延迟
|
||||
统一留在 ARCH-006 使用轮换凭据验收。
|
||||
|
||||
### ARCH-005
|
||||
|
||||
- 将正式运行默认值切换为 BOS、OceanBase Vector 和 `bce-reranker-base`,新 Profile ID 为
|
||||
`demo-bge-large-zh-structure-v1-oceanbase-v1`;Local/Fake 仍用于显式本地和离线测试。
|
||||
- 新增 PostgreSQL canonical `index_profiles` 迁移与 `shadow / active / retired` 生命周期,
|
||||
通过局部唯一索引保证至多一个 Active Profile;API 启动时优先解析数据库 Active Profile,
|
||||
无记录的新部署才使用配置基线。
|
||||
- 新增 `ekp-index-rebuild build|verify|activate`:从 canonical Chunk 分文档重建,确定性 Upsert,
|
||||
重试时跳过完整文档并清理目标 Profile 的部分写入;激活前核对全部 ChunkIndexState,并在
|
||||
PostgreSQL 事务内锁定 Chunk 写入、退役旧 Profile、激活新 Profile。
|
||||
- 移除 Mochow 运行配置、公开 Adapter、测试、配置文档及 `pymochow` 依赖;不自动删除旧
|
||||
外部索引,保留受控回滚和运维退役边界。
|
||||
- 新增重建/重试/完整性失败/单 Active/激活/迁移/CLI 脱敏测试及中文操作文档。真实资源
|
||||
构建、BOS/OceanBase/Qianfan 连通性和 30~50 条脱敏查询验收仍属于 ARCH-006。
|
||||
|
||||
### ARCH-006(已完成)
|
||||
|
||||
- Golden Dataset 已扩展为 32 条完全合成 Query,覆盖退款、技术支持、住宿标准和报销材料;
|
||||
Dense、Reranker、canonical Source Result 的离线回归全部通过。
|
||||
- 新增 `ekp-live-acceptance` / `make live-acceptance`,实际覆盖上传、解析、Chunk、Embedding、
|
||||
OceanBase Dense Top20、BCE Reranker Top5、canonical 来源核对及本次运行数据清理。
|
||||
- live 命令强制要求 `APP_ENV=test`、专用 `live-acceptance/` BOS Prefix、凭据授权确认、资源
|
||||
授权确认和显式运行开关;当前环境下已验证会在创建 Provider 前失败关闭且不泄露细节。
|
||||
- 根据最新配置清单和 test 环境只读探测,确认没有独立 PostgreSQL;目标为 OceanBase
|
||||
4.3.5.4 MySQL 模式,同一数据库承载 canonical 关系数据和逻辑隔离的向量表。ADR-012 已
|
||||
取代先前 PostgreSQL 部署假设,业务事实来源与可重建索引的逻辑边界保持不变。
|
||||
- 已在目标 test 集群只读验证数据库连通、版本与兼容模式,并通过 `EXPLAIN` 确认
|
||||
`VECTOR_INDEX(... pre_filter)` 生成 `VECTOR INDEX ADAPTIVE SCAN (PRE-FILTER)`;未读取
|
||||
业务正文或输出任何连接信息/凭据。
|
||||
- canonical SQLAlchemy 默认已切换为 `mysql+pymysql`,数据库 URL 使用 `SecretStr`,新增
|
||||
有界连接池/超时配置;新增 forward-only migration,以生成列唯一索引保证 OceanBase 与
|
||||
SQLite 下至多一个 Active Profile;PostgreSQL 驱动已移除。
|
||||
- 配置清单中的 test 凭据仅按用户明确授权临时注入本次验收,不保存为 `.env`、不提交、
|
||||
不连接 fat/prod;完成后仍记录为待轮换风险。
|
||||
- 已在 test OceanBase 执行全部 forward-only migration,并对本项目表完成列、类型、默认值、
|
||||
主键的 scoped schema drift 核对;OceanBase 扩展语法导致标准 MySQL 反射器不能可靠比较的
|
||||
二级索引/唯一约束/外键,由 SQLite 全量 drift、真实唯一约束事务和 live 级联删除补充验证。创建
|
||||
`knowledge_chunks_v1` 12 列专用向量表和 Cosine HNSW,确认 PRE-FILTER 执行计划。迁移前
|
||||
已确认所有目标表均不存在,没有覆盖原有 31 张业务表;事务内双 Active 写入实测被唯一
|
||||
索引拒绝并完整回滚。
|
||||
- 已完成 OceanBase canonical → BOS → `bge-large-zh` → OceanBase Vector →
|
||||
`bce-reranker-base` → canonical Source Result 正式验收:32 条查询的 Recall@1/3/5、MRR、
|
||||
NDCG@5 均为 `1.0`,P50 `3787.61 ms`、P95 `4759.21 ms`,通过 P95 `10000 ms` 门槛。
|
||||
- 正式命令退出码为 `0`;两份合成文档、canonical Chunk/State、向量行与 BOS 对象均走随机
|
||||
前缀清理,数据库复核 `live-` Document/Chunk/Vector 残留均为 `0`。首次 live 运行曾出现
|
||||
一次瞬时失败,分阶段诊断及随后完整 32 查询均成功,仍需在后续稳定性任务关注 Provider
|
||||
重试、限流和告警。
|
||||
|
||||
### DEMO-010(已完成)
|
||||
|
||||
- 新增同源轻量 Admin Console,提供 `/admin/upload`、`/admin/documents` 和
|
||||
`/admin/search` 三个直接可访问页面;FastAPI 只负责静态文件路由,页面业务操作全部复用
|
||||
`/api/v1`,没有复制上传、文档管理或检索逻辑。
|
||||
- 上传页支持 PDF、DOCX、TXT、Markdown 文件选择、处理状态和结果摘要;文档页展示文件名、
|
||||
类型、上传时间、状态,支持详情与确认删除;Search Lab 显式展示 Rank、Filename、Page、
|
||||
Section、Chunk Content、Dense Score 和 Rerank Score。
|
||||
- 前端使用安全 DOM 文本节点渲染 API 数据,不注入 HTML,不展示 storage key、Index Profile、
|
||||
Provider Endpoint 或凭据;只实现 Demo 规定的三个页面,没有引入 ACL/RBAC、任务、审计、
|
||||
Compiled Knowledge、MCP、Hybrid/BM25 或额外前端构建工具链。
|
||||
- 新增静态路由、资产、范围和 API First 契约测试;浏览器使用本地确定性 Provider 完成
|
||||
`README.md` 上传 → Ready → 列表/详情 → Top5 检索,并验证桌面和 390px 移动布局无横向
|
||||
溢出、无前端警告或错误。删除的 UI 确认和服务端清理语义由现有 REST 全链路测试覆盖。
|
||||
- 完整验证结果:Ruff Format/Lint、Mypy、JavaScript 语法、Git diff、依赖锁、Alembic Head/
|
||||
离线升级均通过;Unit + Integration `150 passed`,RAG Evaluation `5 passed`。本任务没有
|
||||
修改检索算法或 Provider,不重复调用真实资源验收;ARCH-006 的 32 条真实查询结果继续
|
||||
作为底层链路基线。
|
||||
- 已知非阻塞风险保持不变:配置清单中的历史 test 凭据仍应轮换;真实 Provider 瞬时失败的
|
||||
重试、限流与告警留待后续明确 Task。Demo Console 本身没有新增凭据或生产 Endpoint。
|
||||
|
||||
### DEMO-014(已完成)
|
||||
|
||||
- 将 DEMO-010 通用绿色界面升级为地瓜机器人品牌化 Admin Console,使用唯一品牌强调色
|
||||
`#FF3C00`、黑白高对比、无衬线中文排版、低圆角工业组件和与三步操作对应的计算信号轨;
|
||||
没有增加页面、接口或业务能力。
|
||||
- 品牌标记由内联 SVG 与 CSS 原生绘制,不下载、复制或热链官网资产。页面没有新增外部
|
||||
Runtime、字体请求或第三方脚本,继续保持原生 HTML/CSS/JS 和同源 `/api/v1` 调用。
|
||||
- GitHub 调研确认官方 `openai/skills` 当前没有通用前端设计 Skill;选择并安装了 Apache-2.0
|
||||
的 `anthropics/skills/skills/frontend-design`。该 Skill 只影响视觉策划和自检方法,不进入
|
||||
项目运行依赖或仓库产物。
|
||||
- 浏览器使用本地确定性 Provider 再次完成 README 上传 → Ready → 文档列表 → Top5 检索;
|
||||
1440px 桌面端和移动端均无根页面横向溢出、无前端警告或错误。复验中发现隐藏表头将移动
|
||||
页面撑宽至 503px,已改为 `aria-label`,最终根宽度与 463px 视口一致,表格只在容器内滚动。
|
||||
- 完整验证结果:Ruff Format/Lint、Mypy、JavaScript 语法与 Git diff 通过;Unit、Integration
|
||||
和 RAG Evaluation 合计 `155 passed`;Alembic Head 与离线升级通过。本任务不修改 API、
|
||||
检索算法、Provider 或数据库,不重复调用真实资源验收。
|
||||
|
||||
### PLAN-001(已完成)
|
||||
|
||||
- 审计确认当前 `/api/v1/documents` 仅接受一个文件,并在请求线程同步执行解析、Chunk、
|
||||
Embedding 和索引;API 与 Admin Console 均没有批量提交、Job 查询或后台状态轮询。
|
||||
- 当前上传、解析和向量索引具备阶段内失败补偿,影子索引重建具备目标 Profile 的重试安全
|
||||
清理,但正常上传流水线没有持久化 Job/Attempt、自动调度退避或统一的重试前派生数据清理,
|
||||
因而不能视为已实现自动重试。
|
||||
- 保留既有 `DEMO-012` 并提升为最高优先级;新增 `DEMO-015` 承担 Async Job、自动重试和
|
||||
重试前清理,新增 `DEMO-016` 承担批量异步上传和逐文件状态 UI。严格执行顺序为
|
||||
`DEMO-012 → DEMO-015 → DEMO-016`。
|
||||
- 已更新 PRD、Architecture、API/MCP Contract、Execution Plan、Task List、Demo Scope 与 DoD;
|
||||
本任务只形成规划和待实现契约,没有修改业务代码、数据库 Migration 或 Accepted ADR。
|
||||
- 验证结果:Ruff Format/Lint、Mypy、Git diff 与 Alembic Head/离线升级通过;Unit + Integration
|
||||
`150 passed`,RAG Evaluation `5 passed`。首次受限网络运行因无法解析 PyPI 失败,获准联网
|
||||
获取已锁定的构建依赖后全套复验通过。
|
||||
- 下一任务:`DEMO-012`。详细依赖、范围和验收标准见
|
||||
`docs/plans/ASYNC_INGESTION_MCP_PLAN.md`。
|
||||
|
||||
### DEMO-012(已完成)
|
||||
|
||||
- 新增独立 `ekp-mcp` Streamable HTTP Gateway 和 `make run-mcp` 入口,使用官方 MCP Python
|
||||
SDK v2;默认只监听 `127.0.0.1:8766/mcp`,且只注册 `search_knowledge` 一个业务 Tool。
|
||||
- Gateway 直接复用现有 `SearchService`,没有复制 Dense Retrieval、Reranker 或 Source
|
||||
Enrichment;结构化输出与 REST 保持相同来源字段,不返回 Index Profile 或 Provider 信息。
|
||||
- 新增运行时专用 Bearer Token 验证,将凭据映射为显式 client/service identity 和最小
|
||||
`knowledge:search` scope;Token 至少 32 字符并通过 `SecretStr` 管理,不允许写入仓库。
|
||||
当前身份层是 Demo 边界,不宣称完成 V1 OAuth、Document ACL 或用户委托授权。
|
||||
- 启用 Host/Origin allowlist 和请求体限制;缺失/错误认证、scope、参数和检索异常均按安全
|
||||
边界收敛,响应和日志不回显 Token、内部 Endpoint、数据库、模型或索引实现。
|
||||
- 新增官方 MCP Client 的协议级测试,覆盖 initialize、tools/list、tools/call、唯一 Tool、
|
||||
identity 传递、REST 字段一致性、无结果、参数拒绝、认证失败、Origin 防护和错误脱敏;
|
||||
WorkBuddy 的 HTTP + Bearer + 环境变量配置记录于 `docs/development/MCP_SETUP.md`。
|
||||
- 验证结果:Ruff Format/Lint、Mypy、Git diff、Alembic Head/离线升级全部通过;Unit +
|
||||
Integration `165 passed`,RAG Evaluation `5 passed`。本任务未改变检索算法或数据库结构,
|
||||
不重复消耗真实 Embedding/OceanBase/Reranker 资源。
|
||||
- 2026-09-02 补充完成 WorkBuddy 现场验收:本机 WorkBuddy Desktop `5.3.14` 自带的
|
||||
CodeBuddy CLI `2.115.0` 通过仓库交付的 `deploy/workbuddy-mcp.example.json` 建立
|
||||
Streamable HTTP + Bearer 连接,健康检查为 `Connected`,仅发现
|
||||
`mcp__enterprise-knowledge__search_knowledge`,并以 `query` + `top_k=1` 完成真实 Tool
|
||||
调用、读取结构化来源字段。验收使用真实 Gateway、认证、身份传递、Schema 和 Tool
|
||||
注册代码,检索应用返回确定性数据,以隔离 WorkBuddy 兼容性与外部 Provider 波动。
|
||||
- 非交互客户端调用使用仅对当前会话放行 `DeferExecuteTool` 的权限设置,没有持久化放宽权限;
|
||||
临时客户端配置已删除、Gateway 已停止,仓库和用户配置中均未留下测试 Token。至此
|
||||
`DEMO-012` 约定的 WorkBuddy 客户端验收项闭环;公网 HTTPS 部署验证属于部署环境验收,
|
||||
不阻塞本地 Demo Task 完成。
|
||||
- 下一任务:`DEMO-015`。
|
||||
|
||||
### PLAN-002(已完成)
|
||||
|
||||
- 将当前交付目标明确为企业受控网络内的单一公共知识库:本阶段所有知识共享一个可见域,
|
||||
暂缓用户、角色、知识库空间和 Document ACL,但不废弃 ADR-007,也不把未实现项标记完成。
|
||||
- 保持 Accepted ADR 的模块化单体边界,确定 API/Admin、MCP Gateway 和 Async Worker 独立
|
||||
部署;OceanBase/BOS/AI Provider 使用外部服务,当前云端模型链路不启用 GPU Worker。
|
||||
- 将 Celery/RabbitMQ 正式并入 `DEMO-015`,并新增 `DEPLOY-001~003`,严格顺序为
|
||||
`DEMO-015 → DEMO-016 → DEPLOY-001 → DEPLOY-002 → DEPLOY-003`。
|
||||
- 定义 `make deploy VALUES=...` 的一键部署边界:操作者预先提供集群访问与 Secret,仓库负责
|
||||
生产镜像、串行 Migration、Helm 安装/升级、Ingress/TLS、探针、资源与 Pod 安全配置;不得
|
||||
将凭据写入 Git、镜像、命令行或日志。
|
||||
- 明确现有 `docker-compose.dev.yml` 的 PostgreSQL/MinIO 假设已被 ADR-012 取代,不作为集群
|
||||
交付模板;新增 `docs/plans/PUBLIC_KNOWLEDGE_CLUSTER_PLAN.md` 作为任务范围和验收基线。
|
||||
- 本任务仅修改规划、产品、架构、API/MCP 契约、Demo Scope 和 DoD 文档,没有修改业务代码、
|
||||
数据库 Migration 或 Accepted ADR 状态。
|
||||
- 验证结果:Markdown 相对链接、敏感值模式和 Git diff 检查通过;Ruff Format/Lint、Mypy、
|
||||
Alembic Head/离线升级通过;Unit + Integration `165 passed`,RAG Evaluation `5 passed`。
|
||||
- 下一任务:`DEMO-015`。
|
||||
|
||||
### DEMO-015(已完成)
|
||||
|
||||
- 新增 OceanBase canonical `ingestion_jobs` / `ingestion_attempts` 与 forward-only migration;
|
||||
Document 唯一 Job、Attempt 编号上限和单一运行中 Attempt 均由数据库约束保护。
|
||||
- `POST /api/v1/documents` 改为持久化不可变 Source、Document、Job 后返回 `202`,新增
|
||||
`GET /api/v1/jobs/{job_id}`;活动 Job 的 Document 删除返回 `409`,避免与 Worker 竞态。
|
||||
- 新增队列无关的 Dispatcher 与 Job Runner,首个适配器使用 Celery/RabbitMQ;消息只包含 Job
|
||||
ID,启用 late ack、worker-lost reject、单任务 prefetch 与周期性持久化 Job 恢复扫描。
|
||||
- 瞬时错误按配置执行指数退避和最多三次总 Attempt;Worker 租约有效时重复投递不创建新
|
||||
Attempt,租约过期后关闭旧 Attempt 并恢复。错误记录为固定安全摘要,不保存 Provider 正文。
|
||||
- 第二次及后续 Attempt 前依次清理向量、ChunkIndexState、Chunk 与 parsed JSON,再从不可变
|
||||
源文件重建;清理失败直接终止且不创建新 Attempt。Job 创建失败会补偿 Document 与源对象。
|
||||
- Admin Console 仅将上传提示调整为“已接收并进入队列”;批量选择、Job 轮询和逐文件状态 UI
|
||||
没有提前实现,继续属于 `DEMO-016`。
|
||||
- 验证结果:Ruff Format/Lint、Mypy、JavaScript 语法、依赖锁、Alembic Head/离线 MySQL SQL
|
||||
与 SQLite schema drift 均通过;Unit + Integration `179 passed`,RAG Evaluation `5 passed`;
|
||||
Worker 入口确认注册 `ekp.ingestion.run` 与 `ekp.ingestion.recover`。
|
||||
- 环境限制:当前工作站没有 Docker/RabbitMQ,也没有持久化注入目标 OceanBase URL,因此未做
|
||||
真实 Broker 断连/确认超时和目标库 migration;这些真实环境项必须在 DEPLOY-003 集群验收
|
||||
重跑。本任务以 Celery 配置测试、SQLite 持久化/唯一约束和故障注入集成测试闭环代码语义。
|
||||
- 下一任务:`DEMO-016`。
|
||||
|
||||
### DEMO-016(已完成)
|
||||
|
||||
- 新增 canonical `ingestion_batches` / `ingestion_batch_items` 及 forward-only migration,保存
|
||||
输入顺序、逐文件接收结果与 Document/Job 引用;异步阶段和最终结果仍以 Job 为事实源,
|
||||
批次不是跨文件事务。
|
||||
- 新增 `POST /api/v1/ingestion-batches` 与 `GET /api/v1/ingestion-batches/{batch_id}`;批量
|
||||
请求字段为 `files`,成功返回 `202`、聚合状态和逐文件接收/Job 状态。类型、大小或内容错误
|
||||
只拒绝当前文件,错误映射为固定安全摘要,其他文件继续接收。
|
||||
- 新增 `BATCH_MAX_FILES` 与 `BATCH_MAX_TOTAL_BYTES`,复用 `MAX_UPLOAD_BYTES` 约束单文件;
|
||||
上传流共享总字节预算,不把整批文件读入内存。数量超限在创建批次前返回 `413`。
|
||||
- Admin Console 支持多选提交,显示当前批次的接收数量、完成数量、逐文件阶段、Attempt 与安全
|
||||
错误,并每 1.5 秒自动轮询;进入 `completed`、`partial_failure` 或 `failed` 后停止轮询。
|
||||
DOM 数据继续只通过文本节点渲染。
|
||||
- 集成测试覆盖顺序稳定、部分拒绝不回滚、数量/单文件/批次总大小、全成功/部分失败聚合、
|
||||
未知批次、OpenAPI 和前端契约;迁移 drift 增加两张新表。Ruff Format/Lint、Mypy、
|
||||
JavaScript 语法、依赖锁、Alembic Head/离线升级均通过;Unit + Integration `183 passed`,
|
||||
RAG Evaluation `5 passed`。
|
||||
- 浏览器验收确认桌面布局和 390px 窄屏布局正常、无根页面横向溢出、无控制台警告或错误;
|
||||
本地页面不注入云端凭据,批次结果行为由确定性 API 集成测试验证。
|
||||
- 未扩展目录上传、压缩包解包、断点续传、手动重试、去重、用户或 ACL;自动重试和清理继续
|
||||
复用 DEMO-015,Accepted ADR 未变更。
|
||||
- 下一任务:`DEPLOY-001`。
|
||||
|
||||
### DEPLOY-001(已完成)
|
||||
|
||||
- 新增共享生产 Dockerfile 的 `api`、`mcp`、`worker` 三个 target,使用精确 Python/uv 版本、
|
||||
frozen production dependencies、固定非 root UID/GID、exec-form 启动命令和逐 target 健康检查。
|
||||
- 新增 amd64/arm64 Buildx Bake 计划、镜像构建/渲染/推送 Make 入口和 `.dockerignore`;构建上下文
|
||||
排除凭据、本地状态、缓存、测试与开发产物,镜像不包含环境 Endpoint 或 Secret。
|
||||
- Admin 静态目录新增类型化运行时覆盖,使非 editable 安装后的 API 镜像仍能提供同源页面;
|
||||
源码运行继续使用既有默认目录。
|
||||
- 新增不依赖 Docker daemon 的镜像契约测试与中文运行文档;基础 Python/uv 镜像同时固定精确
|
||||
tag 与 manifest digest,Registry 地址和本项目发布 digest 继续由部署环境提供。
|
||||
- 使用 Docker Buildx `0.36.1` + Colima 分别实际构建并加载全部三个 `linux/arm64` 和三个
|
||||
`linux/amd64` target;最终 digest 固定版本再次构建命中相同业务 layer。没有执行 Registry
|
||||
push,也没有在构建参数、history 或镜像文件系统中发现凭据。
|
||||
- API 镜像以 `10001:10001` 启动并进入 healthy,`/healthz`、`/admin/upload`、OpenAPI 及
|
||||
DEMO-016 批量路径 smoke test 通过;MCP/Worker 镜像以同一非 root 用户完成安装入口导入,
|
||||
默认 CMD 与 HEALTHCHECK 元数据核对通过。真实 Provider/Broker 启动仍属于 DEPLOY-003。
|
||||
- Ruff Format/Lint、Mypy、Docker Bake 渲染、JavaScript/依赖锁/Alembic 与完整测试门禁通过;
|
||||
Unit + Integration `187 passed`,RAG Evaluation `5 passed`。下一任务:`DEPLOY-002`。
|
||||
|
||||
### DEPLOY-002(已完成)
|
||||
|
||||
- 新增 `deploy/helm/knowledge-platform` Helm Chart,部署 API/Admin、MCP Gateway、Async Worker、
|
||||
单实例 Recovery Scheduler、串行 Migration Job、Service、Ingress/TLS 和可选 Demo RabbitMQ;
|
||||
API/MCP 默认各两个副本,Worker 可独立扩缩,Beat 使用 `Recreate` 避免重复调度。
|
||||
- 新增严格 `values.schema.json` 与仓库外环境 Values 示例,拒绝未知顶层字段、`latest` tag、
|
||||
非法 digest、API/MCP 单副本和错误类型;跨字段模板校验继续约束批量大小、重试、Chunk 和
|
||||
Top-K 关系。应用和可选 RabbitMQ 镜像均支持 digest,Demo RabbitMQ 已固定 manifest digest。
|
||||
- 敏感配置只通过预建 `existingSecret` 注入;ConfigMap 仅渲染显式白名单内的非敏感字段,
|
||||
没有任意 `extraEnv` 或 Secret 对象。`scripts/deploy.sh` 在部署前校验 Values、Namespace、运行时
|
||||
Secret/必需键、可选 RabbitMQ Secret/键和启用 Ingress 时的 TLS Secret 引用,不读取 Secret
|
||||
值;Helm 3/4 分别使用等价的原子失败回滚参数。
|
||||
- Migration 使用 `pre-install,pre-upgrade` Hook、固定名称和 `before-hook-creation`,安装/升级在
|
||||
Migration 成功前不更新业务 Pod。应用容器固定非 root UID/GID、只读根文件系统、丢弃全部
|
||||
capabilities、禁用 ServiceAccount Token,并为临时目录、RabbitMQ 日志和 Beat 状态提供
|
||||
`emptyDir`;API/MCP/Worker/Scheduler 均配置启动和存活/就绪探针及资源 requests/limits。
|
||||
- 本机使用 K3s `v1.35.0+k3s1`、Helm `4.2.4` 和 DEPLOY-001 ARM64 镜像完成真实安装;Migration
|
||||
成功,API/MCP 为 `2/2`,Worker、Scheduler 与可选 RabbitMQ 全部 Ready。首次安装暴露的
|
||||
RabbitMQ 只读日志目录和 Beat 调度文件权限问题已修复,随后相同 Values 连续升级至 Revision
|
||||
3~5 均成功;API `/healthz` 和 Admin 上传页 smoke test 通过。
|
||||
- 默认、环境示例和 RabbitMQ 三种 Helm lint 均通过;含 Ingress/TLS/RabbitMQ 的 14 个最终
|
||||
资源通过 kubeconform strict 校验。仓库、Helm manifest、Pod 日志三路 Secret 值扫描均为 `0`;
|
||||
Ruff Format/Lint、Mypy、Alembic、Bash 语法与完整门禁通过,Unit + Integration `195 passed`,
|
||||
RAG Evaluation `5 passed`。
|
||||
- 目标企业集群的稳定 HTTPS、滚动重启、Worker 故障恢复、真实外部依赖监测与远程 WorkBuddy
|
||||
仍属于 `DEPLOY-003`;本任务没有把这些项提前标记完成,也没有改变 Accepted ADR 或公共知识库
|
||||
范围。下一任务:`DEPLOY-003`。
|
||||
- Added the Python 3.12/uv toolchain, dependency lock, FastAPI application,
|
||||
typed settings, health/OpenAPI endpoints, and runnable Make targets.
|
||||
- Added deterministic unit, integration, and seed evaluation tests.
|
||||
- Verified format, lint, strict type checking, tests, dependency lock, API
|
||||
startup, health response, and versioned OpenAPI output.
|
||||
|
||||
@ -1,123 +0,0 @@
|
||||
# 公共知识库集群交付计划
|
||||
|
||||
**Task:** PLAN-002
|
||||
**Status:** Accepted planning baseline
|
||||
**Date:** 2026-09-02
|
||||
|
||||
## 1. 阶段目标与边界
|
||||
|
||||
当前阶段只交付一个面向企业内部受控网络的公共知识库,先保证服务持续可用、可重复部署,
|
||||
并让使用者打开 Admin Console 或连接 WorkBuddy 后即可使用,无需感知后端进程启停。
|
||||
|
||||
“公共”表示本阶段所有已接收知识处于同一个可见域,不代表面向公网匿名开放,也不代表
|
||||
ADR-007 被废弃。用户、用户组、角色、知识库多空间、Document ACL、SSO、OAuth 用户委托等
|
||||
完整 V1 能力继续后置;一旦需要接收非公共知识,必须先实现 Permission Resolver 和检索前/
|
||||
检索中 ACL 下推,禁止先做全局 Top-K 再过滤。
|
||||
|
||||
Admin 写操作只部署在企业受控网络或统一接入网关后。MCP 继续使用运行时注入、只具备
|
||||
`knowledge:search` scope 的服务凭据;静态 Demo credential 不得演变为共享超级管理员 Token。
|
||||
|
||||
## 2. 目标部署拓扑
|
||||
|
||||
```text
|
||||
Ingress / TLS
|
||||
│
|
||||
┌──────────────┴──────────────┐
|
||||
│ │
|
||||
Admin Console + Knowledge API MCP Gateway
|
||||
replicas replicas
|
||||
│ │
|
||||
└──────── Application Services ────────┐
|
||||
│
|
||||
┌─────────────────────────────┼─────────────┐
|
||||
│ │ │
|
||||
OceanBase MySQL BOS RabbitMQ
|
||||
canonical + vector │
|
||||
▼
|
||||
Async Worker(s)
|
||||
```
|
||||
|
||||
沿用 ADR-002:业务核心仍为模块化单体,不拆分额外微服务;API、MCP Gateway 和异步 Worker
|
||||
分别作为可独立扩缩容的进程/容器交付。当前 Parser 与云端 Embedding/Reranker 不需要本地 GPU,
|
||||
因此 GPU Worker 不进入本轮最小部署,未来引入本地 OCR/模型时再按既有架构角色启用。
|
||||
|
||||
OceanBase、BOS 和模型服务使用集群外已有服务。RabbitMQ 默认使用外部地址;Helm Chart 可以
|
||||
提供显式开关安装开发/演示依赖,但生产值不得把队列数据生命周期绑定到应用 Pod。
|
||||
|
||||
## 3. “一键部署”的定义
|
||||
|
||||
在操作者已具备 Kubernetes 访问权限、目标 Namespace 和 Secret 的前提下,仓库提供单一入口:
|
||||
|
||||
```text
|
||||
make deploy VALUES=<environment-values.yaml>
|
||||
```
|
||||
|
||||
该入口负责:
|
||||
|
||||
- 校验非敏感配置和 `existingSecret` 引用,不把凭据写入 Git、镜像、命令行或日志;
|
||||
- 运行一次性、可追踪且串行的数据库 Migration Job;
|
||||
- 安装或升级 API/Admin、MCP Gateway、Async Worker、Service 和可选 RabbitMQ 依赖;
|
||||
- 配置 Ingress/TLS、资源 requests/limits、非 root、只读根文件系统和最小 ServiceAccount;
|
||||
- 配置 readiness/liveness probe、副本数和滚动升级策略;
|
||||
- 支持重复执行和幂等升级,并提供对应卸载/回滚说明。
|
||||
|
||||
本定义不承诺自动创建 OceanBase、BOS、域名证书或云厂商账号,也不允许在仓库内保存这些
|
||||
资源的敏感凭据。
|
||||
|
||||
## 4. Task 顺序与依赖
|
||||
|
||||
```text
|
||||
PLAN-002
|
||||
└──→ DEMO-015 Celery/RabbitMQ Async Job + Retry Cleanup
|
||||
└──→ DEMO-016 Batch Async Upload + Status UI
|
||||
└──→ DEPLOY-001 Production Images
|
||||
└──→ DEPLOY-002 Kubernetes/Helm One-click Deploy
|
||||
└──→ DEPLOY-003 Availability + Remote Acceptance
|
||||
```
|
||||
|
||||
### DEMO-015
|
||||
|
||||
实现持久化 Job/Attempt、重试前清理、幂等消费和有界自动重试,并把执行器正式接入
|
||||
Celery/RabbitMQ。队列调用保持在抽象边界后,不允许业务服务直接依赖 Celery API。
|
||||
|
||||
### DEMO-016
|
||||
|
||||
**已完成。** 基于 DEMO-015 提供逐文件隔离的批量异步提交、批次/Job 查询和 Admin Console
|
||||
状态展示;下一交付门槛为 DEPLOY-001 的生产镜像。
|
||||
|
||||
### DEPLOY-001
|
||||
|
||||
**已完成。** 交付 API/Admin、MCP Gateway、Async Worker 的生产容器镜像与启动约定;镜像使用固定依赖、
|
||||
非 root 用户、健康检查和多架构可构建基线,不内置凭据。
|
||||
|
||||
### DEPLOY-002
|
||||
|
||||
**已完成。** 交付 Kubernetes/Helm Chart、严格环境 Values Schema、`existingSecret`、串行
|
||||
Migration Hook、Ingress/TLS、Pod 安全、资源限制、探针和 `make deploy` 入口;Worker 与单实例
|
||||
恢复调度器已分离。现有 `docker-compose.dev.yml` 仍仅作为旧的本地开发文件,因其
|
||||
PostgreSQL/MinIO 假设与 ADR-012 不一致,不得作为集群交付模板。下一交付门槛为 DEPLOY-003
|
||||
的目标集群可用性与远程验收。
|
||||
|
||||
### DEPLOY-003
|
||||
|
||||
在目标集群完成安装、幂等升级、滚动重启、Worker 故障恢复和远程 WorkBuddy 验收;形成运行
|
||||
手册、备份/恢复责任边界和最小日志/指标检查。灾备演练与完整 Observability 平台仍属后置项。
|
||||
|
||||
## 5. 最小验收标准
|
||||
|
||||
- 新 Namespace 可由单一命令完成安装,重复执行不会破坏已有数据;
|
||||
- Migration 在应用升级前串行成功,失败时新版本不继续接管流量;
|
||||
- Admin/API 与 MCP 使用稳定 HTTPS 地址,重启或滚动升级不要求最终用户重新注入配置;
|
||||
- API/MCP 支持至少两个副本,readiness 未通过的 Pod 不接收流量;
|
||||
- Worker 重启后持久化 Job 可继续恢复,重复消息不产生双重有效执行或重复派生数据;
|
||||
- 远程 WorkBuddy 仅发现批准的 `search_knowledge` Tool,并完成一次真实检索;
|
||||
- 清单、镜像、日志和验收产物不包含凭据,敏感值均来自 Secret;
|
||||
- 关键请求、Job 和外部依赖失败具备可定位的结构化日志和最小指标。
|
||||
|
||||
## 6. 明确后置能力
|
||||
|
||||
- 用户、用户组、角色、多租户、知识库空间隔离和 Document ACL;
|
||||
- LDAP/SSO、OAuth 用户委托、公共互联网匿名访问;
|
||||
- GPU Worker、本地模型、OCR/MinerU;
|
||||
- 跨区域容灾、自动云资源编排和完整 Observability 平台;
|
||||
- 未被既有 Demo/V1 明确批准的业务能力。
|
||||
@ -1,52 +1,5 @@
|
||||
# V1 Tasks
|
||||
|
||||
## Architecture provider migration
|
||||
|
||||
- [x] ARCH-001 Architecture decision, PRD, Scope and execution baseline
|
||||
- [x] ARCH-002 BOS ObjectStorage adapter and runtime configuration
|
||||
- [x] ARCH-003 OceanBase VectorStore adapter and index schema
|
||||
- [x] ARCH-004 Qianfan `bce-reranker-base` alignment
|
||||
- [x] ARCH-005 Runtime cutover, shadow rebuild workflow and legacy configuration cleanup
|
||||
- [x] ARCH-006 OceanBase canonical cutover, controlled live integration and RAG acceptance
|
||||
|
||||
Dependencies:
|
||||
|
||||
```text
|
||||
ARCH-001
|
||||
→ ARCH-002
|
||||
→ ARCH-003
|
||||
→ ARCH-004
|
||||
→ ARCH-005
|
||||
→ ARCH-006
|
||||
```
|
||||
|
||||
Each architecture task uses an independent branch and is merged into `main` only after its own
|
||||
validation. Provider credentials and internal endpoints are runtime inputs and never task artifacts.
|
||||
|
||||
## Post-Demo priority plan
|
||||
|
||||
- [x] PLAN-001 MCP 与异步批量入库现状审计、任务拆分和契约基线
|
||||
- [x] DEMO-012 MCP `search_knowledge` Gateway + identity + WorkBuddy verification
|
||||
- [x] PLAN-002 公共知识库集群交付范围、任务拆分和一键部署基线
|
||||
- [x] DEMO-015 Async ingestion Job + Celery/RabbitMQ + bounded retry + pre-retry cleanup
|
||||
- [x] DEMO-016 Batch async upload API + per-file Job status + Admin Console
|
||||
- [x] DEPLOY-001 Production images for API/Admin, MCP Gateway and Async Worker
|
||||
- [x] DEPLOY-002 Kubernetes/Helm one-click deployment + Secret/Migration/Ingress baseline
|
||||
- [ ] DEPLOY-003 Availability, rolling upgrade and remote WorkBuddy acceptance
|
||||
|
||||
Dependencies and acceptance boundaries are defined in
|
||||
[`ASYNC_INGESTION_MCP_PLAN.md`](ASYNC_INGESTION_MCP_PLAN.md) and
|
||||
[`PUBLIC_KNOWLEDGE_CLUSTER_PLAN.md`](PUBLIC_KNOWLEDGE_CLUSTER_PLAN.md). These Demo/Deploy tasks are
|
||||
vertical slices of the V1 backlog below; completing a slice does not automatically complete its
|
||||
corresponding `EKP-*` task. The current public-knowledge phase defers, but does not complete or
|
||||
supersede, the unchecked user/ACL tasks.
|
||||
|
||||
```text
|
||||
DEMO-009 + ARCH-006 → DEMO-012
|
||||
PLAN-002
|
||||
└→ DEMO-003~006 → DEMO-015 → DEMO-016 → DEPLOY-001 → DEPLOY-002 → DEPLOY-003
|
||||
```
|
||||
|
||||
## Phase 0
|
||||
- [ ] EKP-001 Repository bootstrap
|
||||
- [ ] EKP-002 Backend Python toolchain
|
||||
@ -58,14 +11,14 @@ PLAN-002
|
||||
|
||||
## Canonical domain / ingestion
|
||||
- [ ] EKP-010 Core domain package
|
||||
- [ ] EKP-011 OceanBase MySQL schema and migrations
|
||||
- [ ] EKP-011 PostgreSQL schema and migrations
|
||||
- [ ] EKP-012 User/group/role model
|
||||
- [ ] EKP-013 Knowledge Base model
|
||||
- [ ] EKP-014 Folder model
|
||||
- [ ] EKP-015 Document + DocumentVersion model
|
||||
- [ ] EKP-016 ACL model
|
||||
- [ ] EKP-017 Job + audit models
|
||||
- [ ] EKP-020 ObjectStorage abstraction + BOS adapter
|
||||
- [ ] EKP-020 ObjectStorage abstraction + S3 adapter
|
||||
- [ ] EKP-021 Document upload API
|
||||
- [ ] EKP-022 Source hash/dedup/version activation
|
||||
- [ ] EKP-023 Async job framework
|
||||
@ -82,13 +35,13 @@ PLAN-002
|
||||
- [ ] EKP-042 Qwen embedding adapter
|
||||
- [ ] EKP-043 IndexProfile model
|
||||
- [ ] EKP-044 VectorStore + fake
|
||||
- [ ] EKP-045 OceanBase Vector adapter
|
||||
- [ ] EKP-045 Mochow adapter
|
||||
- [ ] EKP-046 Index build + shadow index lifecycle
|
||||
- [ ] EKP-050 Permission resolver + ACL retrieval filters
|
||||
- [ ] EKP-051 Dense retrieval
|
||||
- [ ] EKP-052 BM25/sparse retrieval
|
||||
- [ ] EKP-053 Hybrid fusion
|
||||
- [ ] EKP-054 RerankerProvider + fake + Qianfan BCE adapter
|
||||
- [ ] EKP-054 RerankerProvider + fake + Qwen adapter
|
||||
- [ ] EKP-055 Source enrichment + Search service/API
|
||||
- [ ] EKP-056 Retrieval diagnostics / Search Lab backend
|
||||
|
||||
|
||||
@ -3,32 +3,9 @@
|
||||
1. Repository / developer harness
|
||||
2. Canonical domain and ACL
|
||||
3. Document ingestion and parsing
|
||||
4. Embedding / Index Profile / OceanBase Vector adapter
|
||||
4. Embedding / Index Profile / Mochow adapter
|
||||
5. ACL-aware hybrid retrieval + reranker
|
||||
6. MCP gateway and WorkBuddy verification
|
||||
7. Admin Console
|
||||
8. Knowledge Compilation
|
||||
9. Evaluation, security and release hardening
|
||||
|
||||
## 当前增量实施顺序(2026-09-02)
|
||||
|
||||
`DEMO-012` 已完成 MCP `search_knowledge` 与本地 WorkBuddy 验收。根据 PLAN-002,当前阶段按
|
||||
“单一公共知识可见域、企业受控网络、持续可用和一键集群部署”推进,不提前实现用户/权限隔离:
|
||||
|
||||
1. `PLAN-002`:确定公共知识库交付边界、目标拓扑、Task 拆分与验收门槛;
|
||||
2. `DEMO-015`(已完成):持久化异步 Job、Celery/RabbitMQ 执行器、自动重试和重试前清理;
|
||||
3. `DEMO-016`(已完成):批量异步上传、批次/Job 查询和 Admin Console 状态展示;
|
||||
4. `DEPLOY-001`(已完成):API/Admin、MCP Gateway、Async Worker 生产镜像;
|
||||
5. `DEPLOY-002`(已完成):Kubernetes/Helm、Secret、Migration、Ingress/TLS 和一键部署入口;
|
||||
6. `DEPLOY-003`:多副本可用性、滚动升级、Worker 恢复和远程 WorkBuddy 验收。
|
||||
|
||||
严格依赖为:
|
||||
|
||||
```text
|
||||
PLAN-002 → DEMO-015 → DEMO-016 → DEPLOY-001 → DEPLOY-002 → DEPLOY-003
|
||||
```
|
||||
|
||||
`DEMO-016` 不得在缺少持久化状态、幂等消费和恢复边界时直接并发调用同步流水线;集群交付也
|
||||
不得继续使用 `docker-compose.dev.yml` 中已被 ADR-012 取代的 PostgreSQL/MinIO 假设。详细
|
||||
范围和验收标准见 [`ASYNC_INGESTION_MCP_PLAN.md`](ASYNC_INGESTION_MCP_PLAN.md) 与
|
||||
[`PUBLIC_KNOWLEDGE_CLUSTER_PLAN.md`](PUBLIC_KNOWLEDGE_CLUSTER_PLAN.md)。
|
||||
|
||||
@ -10,21 +10,6 @@
|
||||
|
||||
---
|
||||
|
||||
## 当前阶段交付覆盖(PLAN-002)
|
||||
|
||||
为优先达成服务可用和一键集群部署,当前阶段只交付企业受控网络内的单一公共知识库:所有
|
||||
已接收知识处于同一个可见域,用户打开 Admin Console 或连接 WorkBuddy 即可使用,无需感知
|
||||
后端进程启停。用户、用户组、角色、知识库多空间、Document ACL、LDAP/SSO 和 OAuth 用户
|
||||
委托仍是本 PRD 的后续 V1 要求,本阶段不实现,也不得标记为完成。
|
||||
|
||||
该阶段覆盖不会取代安全原则:公共知识库不等于匿名公网开放;Admin 写操作必须位于受控网络
|
||||
或统一接入网关后,MCP 使用运行时注入的最小 `knowledge:search` 服务凭据。一旦平台需要接收
|
||||
非公共知识,必须先完成 Permission Resolver 与检索前/检索中 ACL 下推,禁止全局 Top-K 后置
|
||||
过滤。实施顺序和集群验收边界见
|
||||
[`PUBLIC_KNOWLEDGE_CLUSTER_PLAN.md`](../plans/PUBLIC_KNOWLEDGE_CLUSTER_PLAN.md)。
|
||||
|
||||
---
|
||||
|
||||
# 1. 项目背景
|
||||
|
||||
企业内部存在大量分散于 PDF、Word、PPT、Excel、Markdown、图片、扫描件及其他文件中的知识资产。
|
||||
@ -127,8 +112,7 @@ Admin Console 是官方管理客户端,主要解决:
|
||||
|
||||
WorkBuddy、Cursor、Claude Code 或其他 Agent 不应该知道:
|
||||
|
||||
- OceanBase Vector;
|
||||
- BOS;
|
||||
- Mochow;
|
||||
- Qdrant;
|
||||
- Milvus;
|
||||
- bge-large-zh;
|
||||
@ -462,19 +446,6 @@ V1 必须支持:
|
||||
|
||||
Document processing 必须异步执行。
|
||||
|
||||
源文件与 Document/Job 成功持久化后返回 `202 Accepted`。批量上传中的每个文件拥有独立的
|
||||
Document ID 和 Ingestion Job;单个文件处理失败不得阻塞或回滚批次内其他已接收文件。
|
||||
|
||||
当前 Demo 已实现多文件 `202` 接收、批次聚合查询和逐文件 Job 状态;目录上传、压缩包解包、
|
||||
断点续传以及完整 V1 的 Hash/Version/ACL 能力仍按各自 Task 后置。
|
||||
|
||||
公共知识库集群交付使用分别可扩缩的 API/Admin、MCP Gateway 和 Async Worker 生产镜像;
|
||||
镜像只包含锁定代码与非敏感默认值,环境凭据必须在部署时注入。
|
||||
|
||||
集群编排由 Helm Chart 统一管理:API/MCP 默认多副本,Worker 与单实例恢复调度器分离;敏感值
|
||||
只引用预建 Kubernetes Secret,数据库 Migration 在安装/升级前串行完成。Ingress/TLS、探针、
|
||||
资源限制和 Pod 安全默认值属于部署契约,不改变 REST/MCP 业务接口。
|
||||
|
||||
**REQ-INGEST-004**
|
||||
|
||||
展示处理阶段:
|
||||
@ -496,18 +467,10 @@ Failed
|
||||
失败必须展示:
|
||||
|
||||
- Failed Stage;
|
||||
- Attempt Count;
|
||||
- 是否仍会自动重试;
|
||||
- Error Code;
|
||||
- Error Message;
|
||||
- Retry。
|
||||
|
||||
**REQ-INGEST-006**
|
||||
|
||||
可重试的瞬时执行失败必须自动进行有界重试。每次重新执行前必须清理该 Document 的旧解析
|
||||
产物、Chunk、Chunk Index State 和向量/稀疏索引数据,再从不可变原始文件开始处理。清理不得
|
||||
删除原始文件、Document 标识或 Job/Attempt 历史;清理失败时不得继续重试。
|
||||
|
||||
---
|
||||
|
||||
# 11. 文件版本和去重
|
||||
@ -837,24 +800,21 @@ VectorStoreProvider
|
||||
|
||||
抽象。
|
||||
|
||||
当前默认:
|
||||
默认首选:
|
||||
|
||||
```text
|
||||
OceanBase Vector
|
||||
Mochow
|
||||
```
|
||||
|
||||
原因:
|
||||
|
||||
公司现有知识平台主检索链已经使用 OceanBase Vector,并可与既有 OceanBase 运维体系复用。
|
||||
V1 通过标量条件下推满足 Profile 与 ACL 过滤;BM25/Hybrid 能力必须继续通过统一检索契约
|
||||
演进,不把特定数据库查询结构写入业务层。
|
||||
公司已有使用经验,同时 Mochow 当前官方能力已经支持向量检索、BM25、Hybrid Search 和标量过滤。
|
||||
|
||||
系统不得使用 OceanBase/`pyobvector`-specific data model 污染业务 Domain Model。
|
||||
但系统不得使用 Mochow-specific data model 污染业务 Domain Model。
|
||||
|
||||
未来可以新增:
|
||||
|
||||
```text
|
||||
Mochow
|
||||
Qdrant
|
||||
Milvus
|
||||
pgvector
|
||||
@ -953,13 +913,15 @@ Provider:
|
||||
RerankerProvider
|
||||
```
|
||||
|
||||
当前基线与后续候选:
|
||||
初始候选:
|
||||
|
||||
- bce-reranker-base(当前基线);
|
||||
- Qwen3-Reranker 系列(后续 Benchmark 候选);
|
||||
- Qwen3-Reranker-0.6B;
|
||||
- Qwen3-Reranker-4B;
|
||||
- Qwen3-Reranker-8B;
|
||||
- bce-reranker-base;
|
||||
- Local Reranker。
|
||||
|
||||
模型与参数由版本化 Index/Profile 配置记录,业务 API 不暴露模型名。
|
||||
千帆当前已经提供 Qwen3 Reranker 系列,最大输入支持 32K。
|
||||
|
||||
---
|
||||
|
||||
@ -1850,9 +1812,6 @@ Cancelled
|
||||
Retry
|
||||
```
|
||||
|
||||
Retry 必须记录 Attempt 历史并区分可重试的瞬时错误与确定性永久错误;重试执行前遵循
|
||||
`REQ-INGEST-006` 的派生数据清理规则。
|
||||
|
||||
---
|
||||
|
||||
# 57. 删除语义
|
||||
@ -1912,15 +1871,15 @@ Embedding × Retrieval × Reranker
|
||||
|
||||
```text
|
||||
bge-large-zh
|
||||
+ OceanBase Vector Dense
|
||||
+ Mochow Dense
|
||||
+ No Reranker
|
||||
|
||||
bge-large-zh
|
||||
+ OceanBase Vector Dense
|
||||
+ bce-reranker-base
|
||||
+ Mochow Hybrid
|
||||
+ Qwen3 Reranker
|
||||
|
||||
Qwen3 Embedding
|
||||
+ OceanBase Vector Hybrid(能力验证后)
|
||||
+ Mochow Hybrid
|
||||
+ Qwen3 Reranker
|
||||
```
|
||||
|
||||
@ -2209,7 +2168,7 @@ MCP Error Rate
|
||||
10. Document Version;
|
||||
11. Hash 去重;
|
||||
12. Embedding Provider;
|
||||
13. OceanBase Vector Provider;
|
||||
13. Mochow Provider;
|
||||
14. Dense Search;
|
||||
15. BM25;
|
||||
16. Hybrid Retrieval;
|
||||
@ -2332,9 +2291,9 @@ Embedding 和 Reranker 可以通过 Provider 配置替换。
|
||||
|
||||
## Vector DB
|
||||
|
||||
OceanBase Vector 作为当前默认生产 Vector Store。
|
||||
Mochow 可以作为默认生产 Vector Store。
|
||||
|
||||
业务代码不存在必须依赖 OceanBase/`pyobvector` SDK 数据结构的 API Contract。
|
||||
业务代码不存在必须依赖 Mochow SDK 数据结构的 API Contract。
|
||||
|
||||
---
|
||||
|
||||
@ -2379,14 +2338,11 @@ get_source
|
||||
|
||||
# 73. 当前技术基线
|
||||
|
||||
当前 V1 架构基线:
|
||||
PRD 阶段暂定但不最终锁定:
|
||||
|
||||
```text
|
||||
Primary Vector Store:
|
||||
OceanBase Vector
|
||||
|
||||
Primary Object Storage:
|
||||
Baidu BOS
|
||||
Mochow
|
||||
|
||||
Embedding Baseline:
|
||||
bge-large-zh
|
||||
@ -2395,7 +2351,7 @@ Embedding Benchmark:
|
||||
Qwen3 Embedding
|
||||
|
||||
Reranker:
|
||||
bce-reranker-base
|
||||
Qwen3 Reranker / equivalent
|
||||
|
||||
Document Parsing:
|
||||
Benchmark MinerU / Docling
|
||||
@ -2413,10 +2369,6 @@ Backend:
|
||||
API First
|
||||
```
|
||||
|
||||
运行时切换约束:索引 Provider 或 Profile 变更必须以新 Profile 构建 Shadow Index;Profile
|
||||
生命周期和激活状态保存在 OceanBase canonical 关系表。只有 canonical Chunk 与 ChunkIndexState 完整核对
|
||||
通过后才能原子激活,禁止复用旧 Profile ID 或原地覆盖当前 Active Index。
|
||||
|
||||
---
|
||||
|
||||
# 74. 已确认产品决策
|
||||
@ -2435,8 +2387,7 @@ API First + Admin Console
|
||||
|
||||
**DEC-002**
|
||||
|
||||
OceanBase Vector 作为当前生产主检索索引,同时必须保留 Vector Store 替换能力。
|
||||
OceanBase 关系表保存 canonical 数据,同库 Vector Store 表只保存可重建索引。
|
||||
Mochow 继续作为生产首选候选,同时必须保留 Vector Store 替换能力。
|
||||
|
||||
---
|
||||
|
||||
@ -2474,7 +2425,7 @@ RAG + Knowledge Compilation
|
||||
|
||||
**DEC-007**
|
||||
|
||||
MCP 只能暴露业务能力,禁止泄露 OceanBase Vector、BOS、Embedding Model 等底层实现。
|
||||
MCP 只能暴露业务能力,禁止泄露 Mochow、Embedding Model 等底层实现。
|
||||
|
||||
---
|
||||
|
||||
@ -2492,13 +2443,13 @@ PRD 评审通过后进入《总体技术架构设计》。
|
||||
|
||||
1. Backend 技术栈;
|
||||
2. Admin Console 技术栈;
|
||||
3. OceanBase MySQL canonical 数据模型;
|
||||
3. PostgreSQL 数据模型;
|
||||
4. Object Storage;
|
||||
5. Task Queue;
|
||||
6. MinerU vs Docling;
|
||||
7. OCR Pipeline;
|
||||
8. Chunk Strategy;
|
||||
9. OceanBase Vector Index Schema;
|
||||
9. Mochow Schema;
|
||||
10. VectorStore Adapter;
|
||||
11. Embedding Provider;
|
||||
12. Reranker Serving;
|
||||
@ -2522,9 +2473,7 @@ PRD 评审通过后进入《总体技术架构设计》。
|
||||
```text
|
||||
Qianfan
|
||||
+
|
||||
BOS
|
||||
+
|
||||
OceanBase Vector
|
||||
Mochow
|
||||
+
|
||||
Local Parsing
|
||||
```
|
||||
@ -2562,9 +2511,7 @@ Replaceable Embedding
|
||||
+
|
||||
Replaceable Reranker
|
||||
+
|
||||
BOS Object Storage
|
||||
+
|
||||
OceanBase Vector Primary
|
||||
Mochow Primary
|
||||
+
|
||||
Cloud LLM
|
||||
```
|
||||
@ -2577,12 +2524,9 @@ Cloud LLM
|
||||
|
||||
# 77. 调研依据备注
|
||||
|
||||
内部最新配置基线确认现有知识平台主链使用千帆 `bge-large-zh`、`bce-reranker-base`、
|
||||
BOS 与 OceanBase MySQL 模式;OceanBase 同时承载 canonical 关系表和逻辑隔离的可重建
|
||||
向量表。连接地址、账号和密钥仍只能作为运行时 Secret,不进入产品文档或提交记录。
|
||||
当前百度千帆模型列表表明 bge-large-zh 与 Qwen3 Embedding、Qwen3 Reranker 可以作为同平台 Benchmark 候选,因此早期无需为模型实验同时引入多个云厂商。
|
||||
|
||||
OceanBase Vector 的生产接入必须验证向量检索、标量 Filter、索引参数和版本兼容性;
|
||||
BM25/Hybrid 在相应任务验收前仍按产品能力要求保留,不在本轮 Provider 迁移中提前扩展。
|
||||
Mochow 当前官方 API 已支持向量检索与 BM25 全文检索组合的 Hybrid Search,同时可施加标量 Filter,与企业知识库的 Dense + Sparse + ACL Retrieval 需求相匹配。
|
||||
|
||||
WorkBuddy Enterprise 当前公开 MCP 文档支持自定义 MCP Server,并提供传输、权限、安全、远程 API Server 以及 iWiki 配置示例,因此“企业知识服务 → MCP → WorkBuddy”的产品路线具备正式产品支持基础。
|
||||
|
||||
@ -2590,4 +2534,4 @@ Karpathy 的 LLM Wiki 原始思路强调 immutable raw sources 与由 LLM 持续
|
||||
|
||||
---
|
||||
|
||||
**PRD v0.1 END**
|
||||
**PRD v0.1 END**
|
||||
@ -1,6 +1,3 @@
|
||||
# Test Documents
|
||||
|
||||
Only commit synthetic, public, or properly anonymized fixtures. Never commit real internal enterprise documents, HR data, customer data, contracts, credentials, or confidential material.
|
||||
|
||||
The `sample-*.md` files are entirely synthetic and provide stable parser/chunker
|
||||
inputs for the Golden Dataset regression checks.
|
||||
|
||||
@ -1,11 +0,0 @@
|
||||
# 示例产品 A 服务政策
|
||||
|
||||
> 本文件完全由测试内容构造,不描述任何真实产品或客户承诺。
|
||||
|
||||
## 退款
|
||||
|
||||
示例产品 A 的测试退款周期为提交申请后的七个自然日。本规则只用于检索效果验证。
|
||||
|
||||
## 技术支持
|
||||
|
||||
测试用户可以通过虚构的支持入口提交问题,本文不包含真实联系方式。
|
||||
@ -1,12 +0,0 @@
|
||||
# 示例出差管理制度
|
||||
|
||||
> 本文件完全由测试内容构造,不代表任何真实企业政策。
|
||||
|
||||
## 住宿标准
|
||||
|
||||
员工出差时应在批准的预算范围内选择安全、交通便利的住宿。示例标准仅用于验证知识检索,
|
||||
不作为实际报销依据。
|
||||
|
||||
## 报销材料
|
||||
|
||||
提交报销时需附上示例行程单和合规票据。本段内容同样只用于自动化测试。
|
||||
@ -1,32 +1,2 @@
|
||||
{"id":"refund-001","query":"产品A退款周期","expected_documents":["sample-product-a-policy.md"],"expected_sections":["退款"],"tags":["产品","短查询"]}
|
||||
{"id":"refund-002","query":"示例产品 A 的退款需要几个自然日?","expected_documents":["sample-product-a-policy.md"],"expected_sections":["退款"],"tags":["产品","自然语言"]}
|
||||
{"id":"refund-003","query":"申请产品A退款后多久完成?","expected_documents":["sample-product-a-policy.md"],"expected_sections":["退款"],"tags":["产品","同义表达"]}
|
||||
{"id":"refund-004","query":"退款规则里写的处理周期是什么?","expected_documents":["sample-product-a-policy.md"],"expected_sections":["退款"],"tags":["产品","规则"]}
|
||||
{"id":"refund-005","query":"产品 A 提交退款申请后的等待时间","expected_documents":["sample-product-a-policy.md"],"expected_sections":["退款"],"tags":["产品","长查询"]}
|
||||
{"id":"refund-006","query":"退款是否按自然日计算?","expected_documents":["sample-product-a-policy.md"],"expected_sections":["退款"],"tags":["产品","精确术语"]}
|
||||
{"id":"refund-007","query":"请查找产品A有关退款时限的说明","expected_documents":["sample-product-a-policy.md"],"expected_sections":["退款"],"tags":["产品","指令式"]}
|
||||
{"id":"refund-008","query":"示例服务政策中的退款周期是多少天?","expected_documents":["sample-product-a-policy.md"],"expected_sections":["退款"],"tags":["产品","文档定位"]}
|
||||
{"id":"support-001","query":"产品A遇到问题如何联系技术支持?","expected_documents":["sample-product-a-policy.md"],"expected_sections":["技术支持"],"tags":["支持","自然语言"]}
|
||||
{"id":"support-002","query":"测试用户在哪里提交支持问题?","expected_documents":["sample-product-a-policy.md"],"expected_sections":["技术支持"],"tags":["支持","入口"]}
|
||||
{"id":"support-003","query":"技术支持入口是否提供真实联系方式?","expected_documents":["sample-product-a-policy.md"],"expected_sections":["技术支持"],"tags":["支持","边界"]}
|
||||
{"id":"support-004","query":"我想提交产品问题,支持说明是什么?","expected_documents":["sample-product-a-policy.md"],"expected_sections":["技术支持"],"tags":["支持","同义表达"]}
|
||||
{"id":"support-005","query":"测试用户的虚构支持入口","expected_documents":["sample-product-a-policy.md"],"expected_sections":["技术支持"],"tags":["支持","短查询"]}
|
||||
{"id":"support-006","query":"支持入口能否用于测试用户反馈问题?","expected_documents":["sample-product-a-policy.md"],"expected_sections":["技术支持"],"tags":["支持","多条件"]}
|
||||
{"id":"support-007","query":"请定位服务政策里关于提交问题的段落","expected_documents":["sample-product-a-policy.md"],"expected_sections":["技术支持"],"tags":["支持","文档定位"]}
|
||||
{"id":"support-008","query":"产品技术支持部分有没有联系电话?","expected_documents":["sample-product-a-policy.md"],"expected_sections":["技术支持"],"tags":["支持","否定事实"]}
|
||||
{"id":"lodging-001","query":"员工出差住宿标准是多少?","expected_documents":["sample-travel-policy.md"],"expected_sections":["住宿标准"],"tags":["制度","自然语言"]}
|
||||
{"id":"lodging-002","query":"出差时应如何选择住宿?","expected_documents":["sample-travel-policy.md"],"expected_sections":["住宿标准"],"tags":["制度","行为"]}
|
||||
{"id":"lodging-003","query":"住宿地点需要满足哪些安全和交通条件?","expected_documents":["sample-travel-policy.md"],"expected_sections":["住宿标准"],"tags":["制度","多条件"]}
|
||||
{"id":"lodging-004","query":"出差住宿是否必须在批准预算内?","expected_documents":["sample-travel-policy.md"],"expected_sections":["住宿标准"],"tags":["制度","预算"]}
|
||||
{"id":"lodging-005","query":"批准预算范围内的安全住宿","expected_documents":["sample-travel-policy.md"],"expected_sections":["住宿标准"],"tags":["制度","短查询"]}
|
||||
{"id":"lodging-006","query":"员工出差选酒店时要考虑什么?","expected_documents":["sample-travel-policy.md"],"expected_sections":["住宿标准"],"tags":["制度","同义表达"]}
|
||||
{"id":"lodging-007","query":"请找出关于安全且交通便利住宿的说明","expected_documents":["sample-travel-policy.md"],"expected_sections":["住宿标准"],"tags":["制度","文档定位"]}
|
||||
{"id":"lodging-008","query":"出差住宿示例标准能否作为实际报销依据?","expected_documents":["sample-travel-policy.md"],"expected_sections":["住宿标准"],"tags":["制度","边界"]}
|
||||
{"id":"expense-001","query":"出差报销需要提交哪些材料?","expected_documents":["sample-travel-policy.md"],"expected_sections":["报销材料"],"tags":["报销","自然语言"]}
|
||||
{"id":"expense-002","query":"报销时是否需要行程单?","expected_documents":["sample-travel-policy.md"],"expected_sections":["报销材料"],"tags":["报销","材料"]}
|
||||
{"id":"expense-003","query":"合规票据是不是报销必需材料?","expected_documents":["sample-travel-policy.md"],"expected_sections":["报销材料"],"tags":["报销","票据"]}
|
||||
{"id":"expense-004","query":"提交报销要附上什么凭证?","expected_documents":["sample-travel-policy.md"],"expected_sections":["报销材料"],"tags":["报销","同义表达"]}
|
||||
{"id":"expense-005","query":"提交报销的示例行程单和合规票据","expected_documents":["sample-travel-policy.md"],"expected_sections":["报销材料"],"tags":["报销","短查询"]}
|
||||
{"id":"expense-006","query":"行程单和票据应在什么时候提交?","expected_documents":["sample-travel-policy.md"],"expected_sections":["报销材料"],"tags":["报销","多条件"]}
|
||||
{"id":"expense-007","query":"请定位管理制度中有关合规票据的段落","expected_documents":["sample-travel-policy.md"],"expected_sections":["报销材料"],"tags":["报销","文档定位"]}
|
||||
{"id":"expense-008","query":"报销材料段落是否只用于自动化测试?","expected_documents":["sample-travel-policy.md"],"expected_sections":["报销材料"],"tags":["报销","边界"]}
|
||||
{"id":"sample-001","query":"员工出差住宿标准是多少?","expected_documents":["sample-travel-policy.md"],"expected_sections":["住宿标准"],"tags":["制度","自然语言"]}
|
||||
{"id":"sample-002","query":"产品A退款周期","expected_documents":["sample-product-a-policy.md"],"expected_sections":["退款"],"tags":["产品","短查询"]}
|
||||
|
||||
@ -1,27 +0,0 @@
|
||||
# Database migrations
|
||||
|
||||
Migrations are forward-only and operate on OceanBase MySQL canonical metadata. Run
|
||||
`make migration-check` to render the full upgrade path without changing a
|
||||
database. Integration tests also apply the migration to a temporary database
|
||||
and compare it with SQLAlchemy metadata.
|
||||
|
||||
The target OceanBase database is shared. Alembic autogenerate and drift checks
|
||||
are intentionally scoped to tables present in EKP `Base.metadata`; unrelated
|
||||
application tables and the separately provisioned vector-index table are never
|
||||
reflected or treated as migration targets.
|
||||
|
||||
OceanBase adds `BLOCK_SIZE ... LOCAL` syntax that SQLAlchemy's stock MySQL
|
||||
reflector does not recognize for secondary indexes, unique constraints, and
|
||||
foreign keys. Real OceanBase drift checks therefore compare columns, types,
|
||||
defaults, and primary keys while excluding those three falsely reported object
|
||||
types. Full constraint comparison still runs against SQLite in integration
|
||||
tests; target-cluster checks separately verify HNSW/PRE-FILTER and the
|
||||
single-active generated unique index.
|
||||
|
||||
DEMO-015 adds `ingestion_jobs` and `ingestion_attempts`. The former is the canonical execution state;
|
||||
the latter preserves numbered history and uses a generated nullable active marker plus a unique index
|
||||
to reject two running Attempts for the same Job on both OceanBase MySQL and SQLite.
|
||||
|
||||
DEMO-016 adds `ingestion_batches` and `ingestion_batch_items`. They retain input order and per-file
|
||||
acceptance references while Job rows remain the processing state source of truth; a batch is an aggregate,
|
||||
not a cross-file transaction.
|
||||
@ -1,101 +0,0 @@
|
||||
"""Alembic environment for canonical document metadata migrations."""
|
||||
|
||||
from logging.config import fileConfig
|
||||
from typing import Any
|
||||
|
||||
from alembic import context
|
||||
from sqlalchemy import engine_from_config, pool
|
||||
|
||||
from ekp_core.config import get_settings
|
||||
from ekp_domain import Base
|
||||
|
||||
config = context.config
|
||||
if config.config_file_name is not None:
|
||||
fileConfig(config.config_file_name)
|
||||
|
||||
target_metadata = Base.metadata
|
||||
TARGET_TABLE_NAMES = frozenset(target_metadata.tables)
|
||||
|
||||
|
||||
def _include_name(
|
||||
name: str | None,
|
||||
type_: str,
|
||||
parent_names: dict[str, str | None],
|
||||
) -> bool:
|
||||
"""Keep Alembic reflection inside EKP-owned canonical tables.
|
||||
|
||||
The target OceanBase database is shared with another application and
|
||||
contains provider-specific VECTOR columns. Excluding unrelated tables
|
||||
before reflection prevents schema checks from inspecting or proposing
|
||||
changes to objects outside this repository's ownership boundary.
|
||||
"""
|
||||
del parent_names
|
||||
return type_ != "table" or name in TARGET_TABLE_NAMES
|
||||
|
||||
|
||||
def _include_oceanbase_object(
|
||||
object_: Any,
|
||||
name: str | None,
|
||||
type_: str,
|
||||
reflected: bool,
|
||||
compare_to: Any,
|
||||
) -> bool:
|
||||
"""Skip constraints the stock MySQL reflector cannot parse from OceanBase.
|
||||
|
||||
OceanBase appends `BLOCK_SIZE ... LOCAL` to secondary constraints. SQLAlchemy
|
||||
consequently reports model constraints as missing even when they exist.
|
||||
Columns, types, defaults and primary keys remain in the real drift check;
|
||||
constraints are covered by migration tests and targeted live invariants.
|
||||
"""
|
||||
del object_, name, reflected, compare_to
|
||||
return type_ not in {"index", "unique_constraint", "foreign_key_constraint"}
|
||||
|
||||
|
||||
def _database_url() -> str:
|
||||
"""Allow tests to override the URL without changing runtime settings."""
|
||||
override = config.attributes.get("database_url")
|
||||
return override if isinstance(override, str) else get_settings().database_url.get_secret_value()
|
||||
|
||||
|
||||
def run_migrations_offline() -> None:
|
||||
"""Render migration SQL without creating an engine or connection."""
|
||||
context.configure(
|
||||
url=_database_url(),
|
||||
target_metadata=target_metadata,
|
||||
literal_binds=True,
|
||||
dialect_opts={"paramstyle": "named"},
|
||||
compare_type=True,
|
||||
include_name=_include_name,
|
||||
)
|
||||
|
||||
with context.begin_transaction():
|
||||
context.run_migrations()
|
||||
|
||||
|
||||
def run_migrations_online() -> None:
|
||||
"""Run migrations through a short-lived SQLAlchemy engine."""
|
||||
connectable = engine_from_config(
|
||||
{"sqlalchemy.url": _database_url()},
|
||||
prefix="sqlalchemy.",
|
||||
poolclass=pool.NullPool,
|
||||
)
|
||||
|
||||
with connectable.connect() as connection:
|
||||
context.configure(
|
||||
connection=connection,
|
||||
target_metadata=target_metadata,
|
||||
compare_type=True,
|
||||
include_name=_include_name,
|
||||
include_object=(
|
||||
_include_oceanbase_object if connection.dialect.name == "mysql" else None
|
||||
),
|
||||
)
|
||||
|
||||
with context.begin_transaction():
|
||||
context.run_migrations()
|
||||
|
||||
|
||||
if context.is_offline_mode():
|
||||
run_migrations_offline()
|
||||
else:
|
||||
run_migrations_online()
|
||||
@ -1,27 +0,0 @@
|
||||
"""${message}
|
||||
|
||||
Revision ID: ${up_revision}
|
||||
Revises: ${down_revision | comma,n}
|
||||
Create Date: ${create_date}
|
||||
"""
|
||||
|
||||
from collections.abc import Sequence
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
${imports if imports else ""}
|
||||
|
||||
revision: str = ${repr(up_revision)}
|
||||
down_revision: str | Sequence[str] | None = ${repr(down_revision)}
|
||||
branch_labels: str | Sequence[str] | None = ${repr(branch_labels)}
|
||||
depends_on: str | Sequence[str] | None = ${repr(depends_on)}
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Apply the forward migration."""
|
||||
${upgrades if upgrades else "pass"}
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Reject downgrade because project migrations are forward-only."""
|
||||
raise NotImplementedError("Database migrations are forward-only")
|
||||
@ -1,75 +0,0 @@
|
||||
"""Create canonical documents table.
|
||||
|
||||
Revision ID: 20260901_0001
|
||||
Revises: None
|
||||
Create Date: 2026-09-01
|
||||
"""
|
||||
|
||||
from collections.abc import Sequence
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "20260901_0001"
|
||||
down_revision: str | None = None
|
||||
branch_labels: str | Sequence[str] | None = None
|
||||
depends_on: str | Sequence[str] | None = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Create document metadata and processing-status index."""
|
||||
op.create_table(
|
||||
"documents",
|
||||
sa.Column("id", sa.Uuid(), nullable=False),
|
||||
sa.Column("filename", sa.String(length=255), nullable=False),
|
||||
sa.Column("media_type", sa.String(length=127), nullable=False),
|
||||
sa.Column("size_bytes", sa.BigInteger(), nullable=False),
|
||||
sa.Column("storage_key", sa.String(length=512), nullable=False),
|
||||
sa.Column(
|
||||
"status",
|
||||
sa.Enum(
|
||||
"uploaded",
|
||||
"processing",
|
||||
"ready",
|
||||
"failed",
|
||||
name="document_status",
|
||||
native_enum=False,
|
||||
create_constraint=False,
|
||||
),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column("error_message", sa.Text(), nullable=True),
|
||||
sa.Column(
|
||||
"created_at",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("CURRENT_TIMESTAMP"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"updated_at",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("CURRENT_TIMESTAMP"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.CheckConstraint("length(filename) > 0", name="filename_not_empty"),
|
||||
sa.CheckConstraint(
|
||||
"length(storage_key) > 0",
|
||||
name="storage_key_not_empty",
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"size_bytes >= 0",
|
||||
name="size_bytes_non_negative",
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"status IN ('uploaded', 'processing', 'ready', 'failed')",
|
||||
name="status_valid",
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name="pk_documents"),
|
||||
sa.UniqueConstraint("storage_key", name="uq_documents_storage_key"),
|
||||
)
|
||||
op.create_index("ix_documents_status", "documents", ["status"], unique=False)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Reject downgrade because project migrations are forward-only."""
|
||||
raise NotImplementedError("Database migrations are forward-only")
|
||||
@ -1,29 +0,0 @@
|
||||
"""Add parsed-result object reference to documents.
|
||||
|
||||
Revision ID: 20260901_0002
|
||||
Revises: 20260901_0001
|
||||
Create Date: 2026-09-01
|
||||
"""
|
||||
|
||||
from collections.abc import Sequence
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "20260901_0002"
|
||||
down_revision: str | None = "20260901_0001"
|
||||
branch_labels: str | Sequence[str] | None = None
|
||||
depends_on: str | Sequence[str] | None = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Reference the rebuildable parsed JSON object after parsing succeeds."""
|
||||
op.add_column(
|
||||
"documents",
|
||||
sa.Column("parsed_storage_key", sa.String(length=512), nullable=True),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Reject downgrade because project migrations are forward-only."""
|
||||
raise NotImplementedError("Database migrations are forward-only")
|
||||
@ -1,73 +0,0 @@
|
||||
"""Create canonical chunk metadata table.
|
||||
|
||||
Revision ID: 20260901_0003
|
||||
Revises: 20260901_0002
|
||||
Create Date: 2026-09-01
|
||||
"""
|
||||
|
||||
from collections.abc import Sequence
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "20260901_0003"
|
||||
down_revision: str | None = "20260901_0002"
|
||||
branch_labels: str | Sequence[str] | None = None
|
||||
depends_on: str | Sequence[str] | None = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Persist traceable chunks without introducing embedding/index fields."""
|
||||
op.create_table(
|
||||
"chunks",
|
||||
sa.Column("chunk_id", sa.Uuid(), nullable=False),
|
||||
sa.Column("document_id", sa.Uuid(), nullable=False),
|
||||
sa.Column("filename", sa.String(length=255), nullable=False),
|
||||
sa.Column("page", sa.Integer(), nullable=True),
|
||||
sa.Column("page_end", sa.Integer(), nullable=True),
|
||||
sa.Column("section", sa.Text(), nullable=True),
|
||||
sa.Column("section_path", sa.JSON(), nullable=False),
|
||||
sa.Column("chunk_index", sa.Integer(), nullable=False),
|
||||
sa.Column("content", sa.Text(), nullable=False),
|
||||
sa.Column("token_count", sa.Integer(), nullable=False),
|
||||
sa.Column("strategy_version", sa.String(length=64), nullable=False),
|
||||
sa.Column(
|
||||
"created_at",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("CURRENT_TIMESTAMP"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.CheckConstraint("length(filename) > 0", name="filename_not_empty"),
|
||||
sa.CheckConstraint("length(content) > 0", name="content_not_empty"),
|
||||
sa.CheckConstraint(
|
||||
"length(strategy_version) > 0",
|
||||
name="strategy_version_not_empty",
|
||||
),
|
||||
sa.CheckConstraint("chunk_index >= 0", name="chunk_index_non_negative"),
|
||||
sa.CheckConstraint("token_count > 0", name="token_count_positive"),
|
||||
sa.CheckConstraint("page IS NULL OR page > 0", name="page_positive"),
|
||||
sa.CheckConstraint("page_end IS NULL OR page_end > 0", name="page_end_positive"),
|
||||
sa.CheckConstraint(
|
||||
"page IS NULL OR page_end IS NULL OR page_end >= page",
|
||||
name="page_range_valid",
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["document_id"],
|
||||
["documents.id"],
|
||||
name="fk_chunks_document_id_documents",
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
sa.PrimaryKeyConstraint("chunk_id", name="pk_chunks"),
|
||||
sa.UniqueConstraint(
|
||||
"document_id",
|
||||
"strategy_version",
|
||||
"chunk_index",
|
||||
name="uq_chunks_document_strategy_index",
|
||||
),
|
||||
)
|
||||
op.create_index("ix_chunks_document_id", "chunks", ["document_id"], unique=False)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Reject downgrade because project migrations are forward-only."""
|
||||
raise NotImplementedError("Database migrations are forward-only")
|
||||
@ -1,58 +0,0 @@
|
||||
"""Create versioned canonical chunk-index state.
|
||||
|
||||
Revision ID: 20260901_0004
|
||||
Revises: 20260901_0003
|
||||
Create Date: 2026-09-01
|
||||
"""
|
||||
|
||||
from collections.abc import Sequence
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "20260901_0004"
|
||||
down_revision: str | None = "20260901_0003"
|
||||
branch_labels: str | Sequence[str] | None = None
|
||||
depends_on: str | Sequence[str] | None = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Track successful index writes without storing canonical vectors in Mochow."""
|
||||
op.create_table(
|
||||
"chunk_index_states",
|
||||
sa.Column("chunk_id", sa.Uuid(), nullable=False),
|
||||
sa.Column("index_profile_id", sa.String(length=128), nullable=False),
|
||||
sa.Column("embedding_model", sa.String(length=128), nullable=False),
|
||||
sa.Column("vector_dimension", sa.Integer(), nullable=False),
|
||||
sa.Column(
|
||||
"indexed_at",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("CURRENT_TIMESTAMP"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"length(index_profile_id) > 0",
|
||||
name="profile_not_empty",
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"length(embedding_model) > 0",
|
||||
name="model_not_empty",
|
||||
),
|
||||
sa.CheckConstraint("vector_dimension > 0", name="dimension_positive"),
|
||||
sa.ForeignKeyConstraint(
|
||||
["chunk_id"],
|
||||
["chunks.chunk_id"],
|
||||
name="fk_chunk_index_states_chunk_id_chunks",
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
sa.PrimaryKeyConstraint(
|
||||
"chunk_id",
|
||||
"index_profile_id",
|
||||
name="pk_chunk_index_states",
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Reject downgrade because project migrations are forward-only."""
|
||||
raise NotImplementedError("Database migrations are forward-only")
|
||||
@ -1,77 +0,0 @@
|
||||
"""Create canonical index-profile lifecycle records.
|
||||
|
||||
Revision ID: 20260902_0005
|
||||
Revises: 20260901_0004
|
||||
Create Date: 2026-09-02
|
||||
"""
|
||||
|
||||
from collections.abc import Sequence
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "20260902_0005"
|
||||
down_revision: str | None = "20260901_0004"
|
||||
branch_labels: str | Sequence[str] | None = None
|
||||
depends_on: str | Sequence[str] | None = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Persist shadow/active/retired profiles without storing vector data."""
|
||||
op.create_table(
|
||||
"index_profiles",
|
||||
sa.Column("profile_id", sa.String(length=128), nullable=False),
|
||||
sa.Column("chunk_strategy_version", sa.String(length=64), nullable=False),
|
||||
sa.Column("embedding_model", sa.String(length=128), nullable=False),
|
||||
sa.Column("vector_dimension", sa.Integer(), nullable=False),
|
||||
sa.Column(
|
||||
"status",
|
||||
sa.Enum(
|
||||
"shadow",
|
||||
"active",
|
||||
"retired",
|
||||
name="index_profile_status",
|
||||
native_enum=False,
|
||||
create_constraint=False,
|
||||
),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"created_at",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("CURRENT_TIMESTAMP"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column("activated_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.CheckConstraint("length(profile_id) > 0", name="profile_id_not_empty"),
|
||||
sa.CheckConstraint(
|
||||
"length(chunk_strategy_version) > 0",
|
||||
name="profile_chunk_strategy_not_empty",
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"length(embedding_model) > 0",
|
||||
name="profile_embedding_model_not_empty",
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"vector_dimension > 0",
|
||||
name="profile_dimension_positive",
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"status IN ('shadow', 'active', 'retired')",
|
||||
name="profile_status_valid",
|
||||
),
|
||||
sa.PrimaryKeyConstraint("profile_id", name="pk_index_profiles"),
|
||||
)
|
||||
op.create_index(
|
||||
"uq_index_profiles_single_active",
|
||||
"index_profiles",
|
||||
["status"],
|
||||
unique=True,
|
||||
postgresql_where=sa.text("status = 'active'"),
|
||||
sqlite_where=sa.text("status = 'active'"),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Reject downgrade because project migrations are forward-only."""
|
||||
raise NotImplementedError("Database migrations are forward-only")
|
||||
@ -1,41 +0,0 @@
|
||||
"""Make single-active profile enforcement portable to OceanBase MySQL.
|
||||
|
||||
Revision ID: 20260902_0006
|
||||
Revises: 20260902_0005
|
||||
Create Date: 2026-09-02
|
||||
"""
|
||||
|
||||
from collections.abc import Sequence
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "20260902_0006"
|
||||
down_revision: str | None = "20260902_0005"
|
||||
branch_labels: str | Sequence[str] | None = None
|
||||
depends_on: str | Sequence[str] | None = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Replace dialect-specific partial uniqueness with a generated marker."""
|
||||
op.drop_index("uq_index_profiles_single_active", table_name="index_profiles")
|
||||
op.add_column(
|
||||
"index_profiles",
|
||||
sa.Column(
|
||||
"active_marker",
|
||||
sa.String(length=6),
|
||||
sa.Computed("CASE WHEN status = 'active' THEN 'active' ELSE NULL END"),
|
||||
nullable=True,
|
||||
),
|
||||
)
|
||||
op.create_index(
|
||||
"uq_index_profiles_single_active",
|
||||
"index_profiles",
|
||||
["active_marker"],
|
||||
unique=True,
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Reject downgrade because project migrations are forward-only."""
|
||||
raise NotImplementedError("Database migrations are forward-only")
|
||||
@ -1,174 +0,0 @@
|
||||
"""Create durable ingestion jobs and attempt history.
|
||||
|
||||
Revision ID: 20260902_0007
|
||||
Revises: 20260902_0006
|
||||
Create Date: 2026-09-02
|
||||
"""
|
||||
|
||||
from collections.abc import Sequence
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "20260902_0007"
|
||||
down_revision: str | None = "20260902_0006"
|
||||
branch_labels: str | Sequence[str] | None = None
|
||||
depends_on: str | Sequence[str] | None = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Persist queue-independent Job state and bounded execution attempts."""
|
||||
op.create_table(
|
||||
"ingestion_jobs",
|
||||
sa.Column("id", sa.Uuid(), nullable=False),
|
||||
sa.Column("document_id", sa.Uuid(), nullable=False),
|
||||
sa.Column(
|
||||
"status",
|
||||
sa.Enum(
|
||||
"queued",
|
||||
"running",
|
||||
"retrying",
|
||||
"succeeded",
|
||||
"failed",
|
||||
"cancelled",
|
||||
name="ingestion_job_status",
|
||||
native_enum=False,
|
||||
create_constraint=False,
|
||||
),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"stage",
|
||||
sa.Enum(
|
||||
"parsing",
|
||||
"cleaning",
|
||||
"chunking",
|
||||
"embedding",
|
||||
"indexing",
|
||||
name="ingestion_stage",
|
||||
native_enum=False,
|
||||
create_constraint=False,
|
||||
),
|
||||
nullable=True,
|
||||
),
|
||||
sa.Column("attempt_count", sa.Integer(), nullable=False),
|
||||
sa.Column("max_attempts", sa.Integer(), nullable=False),
|
||||
sa.Column("error_message", sa.Text(), nullable=True),
|
||||
sa.Column("next_run_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("lease_expires_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column(
|
||||
"created_at",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("CURRENT_TIMESTAMP"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"updated_at",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("CURRENT_TIMESTAMP"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column("completed_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.CheckConstraint("attempt_count >= 0", name="attempt_count_non_negative"),
|
||||
sa.CheckConstraint("max_attempts > 0", name="max_attempts_positive"),
|
||||
sa.CheckConstraint("attempt_count <= max_attempts", name="attempt_count_bounded"),
|
||||
sa.CheckConstraint(
|
||||
"status IN ('queued', 'running', 'retrying', 'succeeded', 'failed', 'cancelled')",
|
||||
name="status_valid",
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"stage IS NULL OR stage IN "
|
||||
"('parsing', 'cleaning', 'chunking', 'embedding', 'indexing')",
|
||||
name="stage_valid",
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["document_id"],
|
||||
["documents.id"],
|
||||
name="fk_ingestion_jobs_document_id_documents",
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name="pk_ingestion_jobs"),
|
||||
sa.UniqueConstraint("document_id", name="uq_ingestion_jobs_document_id"),
|
||||
)
|
||||
op.create_index(
|
||||
"ix_ingestion_jobs_status_next_run",
|
||||
"ingestion_jobs",
|
||||
["status", "next_run_at"],
|
||||
unique=False,
|
||||
)
|
||||
op.create_table(
|
||||
"ingestion_attempts",
|
||||
sa.Column("id", sa.Uuid(), nullable=False),
|
||||
sa.Column("job_id", sa.Uuid(), nullable=False),
|
||||
sa.Column("attempt_number", sa.Integer(), nullable=False),
|
||||
sa.Column(
|
||||
"status",
|
||||
sa.Enum(
|
||||
"running",
|
||||
"succeeded",
|
||||
"failed",
|
||||
name="ingestion_attempt_status",
|
||||
native_enum=False,
|
||||
create_constraint=False,
|
||||
),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"stage",
|
||||
sa.Enum(
|
||||
"parsing",
|
||||
"cleaning",
|
||||
"chunking",
|
||||
"embedding",
|
||||
"indexing",
|
||||
name="ingestion_attempt_stage",
|
||||
native_enum=False,
|
||||
create_constraint=False,
|
||||
),
|
||||
nullable=True,
|
||||
),
|
||||
sa.Column("retryable", sa.Boolean(), nullable=True),
|
||||
sa.Column("error_message", sa.Text(), nullable=True),
|
||||
sa.Column(
|
||||
"active_marker",
|
||||
sa.Integer(),
|
||||
sa.Computed("CASE WHEN status = 'running' THEN 1 ELSE NULL END"),
|
||||
nullable=True,
|
||||
),
|
||||
sa.Column(
|
||||
"started_at",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("CURRENT_TIMESTAMP"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.CheckConstraint("attempt_number > 0", name="attempt_number_positive"),
|
||||
sa.CheckConstraint(
|
||||
"status IN ('running', 'succeeded', 'failed')",
|
||||
name="status_valid",
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"stage IS NULL OR stage IN "
|
||||
"('parsing', 'cleaning', 'chunking', 'embedding', 'indexing')",
|
||||
name="stage_valid",
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["job_id"],
|
||||
["ingestion_jobs.id"],
|
||||
name="fk_ingestion_attempts_job_id_ingestion_jobs",
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name="pk_ingestion_attempts"),
|
||||
sa.UniqueConstraint("job_id", "attempt_number", name="uq_ingestion_attempts_job_number"),
|
||||
)
|
||||
op.create_index(
|
||||
"uq_ingestion_attempts_active",
|
||||
"ingestion_attempts",
|
||||
["job_id", "active_marker"],
|
||||
unique=True,
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Reject downgrade because project migrations are forward-only."""
|
||||
raise NotImplementedError("Database migrations are forward-only")
|
||||
@ -1,109 +0,0 @@
|
||||
"""Create ingestion batch aggregation metadata.
|
||||
|
||||
Revision ID: 20260902_0008
|
||||
Revises: 20260902_0007
|
||||
Create Date: 2026-09-02
|
||||
"""
|
||||
|
||||
from collections.abc import Sequence
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "20260902_0008"
|
||||
down_revision: str | None = "20260902_0007"
|
||||
branch_labels: str | Sequence[str] | None = None
|
||||
depends_on: str | Sequence[str] | None = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Persist ordered per-file acceptance without making a batch transactional."""
|
||||
op.create_table(
|
||||
"ingestion_batches",
|
||||
sa.Column("id", sa.Uuid(), nullable=False),
|
||||
sa.Column("total_files", sa.Integer(), nullable=False),
|
||||
sa.Column(
|
||||
"created_at",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("CURRENT_TIMESTAMP"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.CheckConstraint("total_files > 0", name="total_files_positive"),
|
||||
sa.PrimaryKeyConstraint("id", name="pk_ingestion_batches"),
|
||||
)
|
||||
op.create_table(
|
||||
"ingestion_batch_items",
|
||||
sa.Column("id", sa.Uuid(), nullable=False),
|
||||
sa.Column("batch_id", sa.Uuid(), nullable=False),
|
||||
sa.Column("item_index", sa.Integer(), nullable=False),
|
||||
sa.Column("filename", sa.String(length=255), nullable=False),
|
||||
sa.Column(
|
||||
"acceptance_status",
|
||||
sa.Enum(
|
||||
"pending",
|
||||
"accepted",
|
||||
"rejected",
|
||||
name="batch_item_acceptance_status",
|
||||
native_enum=False,
|
||||
create_constraint=False,
|
||||
),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column("document_id", sa.Uuid(), nullable=True),
|
||||
sa.Column("job_id", sa.Uuid(), nullable=True),
|
||||
sa.Column("error_message", sa.Text(), nullable=True),
|
||||
sa.Column(
|
||||
"created_at",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("CURRENT_TIMESTAMP"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"updated_at",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("CURRENT_TIMESTAMP"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.CheckConstraint("item_index >= 0", name="item_index_non_negative"),
|
||||
sa.CheckConstraint("length(filename) > 0", name="filename_not_empty"),
|
||||
sa.CheckConstraint(
|
||||
"acceptance_status IN ('pending', 'accepted', 'rejected')",
|
||||
name="acceptance_status_valid",
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["batch_id"],
|
||||
["ingestion_batches.id"],
|
||||
name="fk_ingestion_batch_items_batch_id_ingestion_batches",
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["document_id"],
|
||||
["documents.id"],
|
||||
name="fk_ingestion_batch_items_document_id_documents",
|
||||
ondelete="SET NULL",
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["job_id"],
|
||||
["ingestion_jobs.id"],
|
||||
name="fk_ingestion_batch_items_job_id_ingestion_jobs",
|
||||
ondelete="SET NULL",
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name="pk_ingestion_batch_items"),
|
||||
)
|
||||
op.create_index(
|
||||
"uq_ingestion_batch_items_order",
|
||||
"ingestion_batch_items",
|
||||
["batch_id", "item_index"],
|
||||
unique=True,
|
||||
)
|
||||
op.create_index(
|
||||
"uq_ingestion_batch_items_job_id",
|
||||
"ingestion_batch_items",
|
||||
["job_id"],
|
||||
unique=True,
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Reject downgrade because project migrations are forward-only."""
|
||||
raise NotImplementedError("Database migrations are forward-only")
|
||||
@ -1,26 +1 @@
|
||||
"""Shared core utilities for the Enterprise Knowledge Platform."""
|
||||
|
||||
from ekp_core.dense_retrieval import DenseRetrievalResult, DenseRetrievalService
|
||||
from ekp_core.document_chunking import DocumentChunkingService, DocumentPipelineService
|
||||
from ekp_core.document_cleaner import clean_parsed_document
|
||||
from ekp_core.document_indexing import (
|
||||
DocumentIndexingPipelineService,
|
||||
DocumentIndexingService,
|
||||
)
|
||||
from ekp_core.document_parsing import DocumentIngestionService, DocumentParsingService
|
||||
from ekp_core.search import SearchResult, SearchService, SourceResult
|
||||
|
||||
__all__ = [
|
||||
"DocumentChunkingService",
|
||||
"DenseRetrievalResult",
|
||||
"DenseRetrievalService",
|
||||
"DocumentIngestionService",
|
||||
"DocumentIndexingPipelineService",
|
||||
"DocumentIndexingService",
|
||||
"DocumentPipelineService",
|
||||
"DocumentParsingService",
|
||||
"SearchResult",
|
||||
"SearchService",
|
||||
"SourceResult",
|
||||
"clean_parsed_document",
|
||||
]
|
||||
|
||||
@ -1,223 +0,0 @@
|
||||
"""Structure-aware, token-window chunking for the retrieval Demo."""
|
||||
|
||||
import re
|
||||
from collections.abc import Iterable
|
||||
from dataclasses import dataclass
|
||||
from typing import Protocol
|
||||
|
||||
from ekp_domain import ParsedBlock, ParsedBlockType, ParsedDocument
|
||||
|
||||
DEFAULT_CHUNK_STRATEGY_VERSION = "structure_v1"
|
||||
TOKEN_PATTERN = re.compile(
|
||||
r"[\u3400-\u4dbf\u4e00-\u9fff\u3040-\u30ff\uac00-\ud7af]"
|
||||
r"|[A-Za-z0-9_]+"
|
||||
r"|[^\s]"
|
||||
)
|
||||
ASCII_SUBTOKEN_CHARACTERS = 4
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class TokenSpan:
|
||||
"""Character offsets for one conservatively estimated model token."""
|
||||
|
||||
start: int
|
||||
end: int
|
||||
|
||||
|
||||
class TokenCounter(Protocol):
|
||||
"""Injectable token boundary contract for future model-specific counters."""
|
||||
|
||||
def spans(self, text: str) -> tuple[TokenSpan, ...]:
|
||||
"""Return ordered, non-overlapping token character spans."""
|
||||
|
||||
def count(self, text: str) -> int:
|
||||
"""Return the token estimate for text."""
|
||||
|
||||
|
||||
class ApproximateTokenCounter:
|
||||
"""Conservatively estimate Chinese/BGE-sized token windows without a model SDK.
|
||||
|
||||
CJK characters and punctuation count individually. ASCII words are divided
|
||||
into four-character pieces so unusually long terms cannot bypass the limit.
|
||||
DEMO-006 may inject a model-specific counter without changing chunk policy.
|
||||
"""
|
||||
|
||||
def spans(self, text: str) -> tuple[TokenSpan, ...]:
|
||||
token_spans: list[TokenSpan] = []
|
||||
for match in TOKEN_PATTERN.finditer(text):
|
||||
value = match.group()
|
||||
if value.isascii() and (value.isalnum() or "_" in value):
|
||||
token_spans.extend(
|
||||
TokenSpan(start, min(start + ASCII_SUBTOKEN_CHARACTERS, match.end()))
|
||||
for start in range(match.start(), match.end(), ASCII_SUBTOKEN_CHARACTERS)
|
||||
)
|
||||
else:
|
||||
token_spans.append(TokenSpan(match.start(), match.end()))
|
||||
return tuple(token_spans)
|
||||
|
||||
def count(self, text: str) -> int:
|
||||
"""Count the same boundaries used to create chunk windows."""
|
||||
return len(self.spans(text))
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ChunkDraft:
|
||||
"""Persistence-neutral chunk content produced by one strategy version."""
|
||||
|
||||
chunk_index: int
|
||||
content: str
|
||||
token_count: int
|
||||
page: int | None
|
||||
page_end: int | None
|
||||
section_path: tuple[str, ...]
|
||||
|
||||
@property
|
||||
def section(self) -> str | None:
|
||||
"""Return a compact section label for Demo retrieval results."""
|
||||
return " / ".join(self.section_path) or None
|
||||
|
||||
|
||||
class ChunkingStrategy(Protocol):
|
||||
"""Provider-neutral strategy contract consumed by the pipeline service."""
|
||||
|
||||
@property
|
||||
def version(self) -> str:
|
||||
"""Return a stable strategy identifier for rebuilds and shadow indexes."""
|
||||
|
||||
def chunk(self, document: ParsedDocument) -> tuple[ChunkDraft, ...]:
|
||||
"""Build ordered chunks from a cleaned parsed document."""
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _BlockRange:
|
||||
start: int
|
||||
end: int
|
||||
page_number: int | None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _StructuralGroup:
|
||||
content: str
|
||||
section_path: tuple[str, ...]
|
||||
block_ranges: tuple[_BlockRange, ...]
|
||||
|
||||
|
||||
class StructureAwareChunker:
|
||||
"""Combine paragraphs inside structural boundaries, then apply token overlap."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
max_tokens: int = 450,
|
||||
overlap_tokens: int = 75,
|
||||
max_characters: int = 2000,
|
||||
token_counter: TokenCounter | None = None,
|
||||
strategy_version: str = DEFAULT_CHUNK_STRATEGY_VERSION,
|
||||
) -> None:
|
||||
if max_tokens <= 0:
|
||||
raise ValueError("max_tokens must be positive")
|
||||
if overlap_tokens < 0 or overlap_tokens >= max_tokens:
|
||||
raise ValueError("overlap_tokens must be non-negative and less than max_tokens")
|
||||
if max_characters < ASCII_SUBTOKEN_CHARACTERS:
|
||||
raise ValueError("max_characters must be at least four")
|
||||
if not strategy_version:
|
||||
raise ValueError("strategy_version must not be empty")
|
||||
self._max_tokens = max_tokens
|
||||
self._overlap_tokens = overlap_tokens
|
||||
self._max_characters = max_characters
|
||||
self._token_counter = token_counter or ApproximateTokenCounter()
|
||||
self._version = strategy_version
|
||||
|
||||
@property
|
||||
def version(self) -> str:
|
||||
"""Identify the chunk policy used by persisted rows."""
|
||||
return self._version
|
||||
|
||||
def chunk(self, document: ParsedDocument) -> tuple[ChunkDraft, ...]:
|
||||
"""Create deterministic chunks without crossing heading/section boundaries."""
|
||||
drafts: list[ChunkDraft] = []
|
||||
for group in _structural_groups(document.blocks):
|
||||
for content, token_count, page, page_end in self._windows(group):
|
||||
drafts.append(
|
||||
ChunkDraft(
|
||||
chunk_index=len(drafts),
|
||||
content=content,
|
||||
token_count=token_count,
|
||||
page=page,
|
||||
page_end=page_end,
|
||||
section_path=group.section_path,
|
||||
)
|
||||
)
|
||||
return tuple(drafts)
|
||||
|
||||
def _windows(
|
||||
self,
|
||||
group: _StructuralGroup,
|
||||
) -> Iterable[tuple[str, int, int | None, int | None]]:
|
||||
spans = self._token_counter.spans(group.content)
|
||||
start_token = 0
|
||||
while start_token < len(spans):
|
||||
end_token = min(start_token + self._max_tokens, len(spans))
|
||||
char_start = spans[start_token].start
|
||||
while (
|
||||
end_token > start_token + 1
|
||||
and spans[end_token - 1].end - char_start > self._max_characters
|
||||
):
|
||||
end_token -= 1
|
||||
char_end = spans[end_token - 1].end
|
||||
content = group.content[char_start:char_end].strip()
|
||||
pages = [
|
||||
block_range.page_number
|
||||
for block_range in group.block_ranges
|
||||
if block_range.page_number is not None
|
||||
and block_range.end > char_start
|
||||
and block_range.start < char_end
|
||||
]
|
||||
yield (
|
||||
content,
|
||||
end_token - start_token,
|
||||
min(pages, default=None),
|
||||
max(pages, default=None),
|
||||
)
|
||||
if end_token == len(spans):
|
||||
break
|
||||
start_token = end_token - self._overlap_tokens
|
||||
|
||||
|
||||
def _structural_groups(blocks: tuple[ParsedBlock, ...]) -> Iterable[_StructuralGroup]:
|
||||
"""Group content without crossing explicit headings or section changes."""
|
||||
current: list[ParsedBlock] = []
|
||||
current_section: tuple[str, ...] | None = None
|
||||
|
||||
for block in blocks:
|
||||
is_new_boundary = bool(current) and (
|
||||
block.block_type is ParsedBlockType.HEADING or block.section_path != current_section
|
||||
)
|
||||
if is_new_boundary:
|
||||
yield _build_group(current)
|
||||
current = []
|
||||
if not current:
|
||||
current_section = block.section_path
|
||||
current.append(block)
|
||||
|
||||
if current:
|
||||
yield _build_group(current)
|
||||
|
||||
|
||||
def _build_group(blocks: list[ParsedBlock]) -> _StructuralGroup:
|
||||
"""Join blocks with readable separators and retain their character ranges."""
|
||||
parts: list[str] = []
|
||||
block_ranges: list[_BlockRange] = []
|
||||
offset = 0
|
||||
for block in blocks:
|
||||
if parts:
|
||||
parts.append("\n\n")
|
||||
offset += 2
|
||||
start = offset
|
||||
parts.append(block.text)
|
||||
offset += len(block.text)
|
||||
block_ranges.append(_BlockRange(start=start, end=offset, page_number=block.page_number))
|
||||
return _StructuralGroup(
|
||||
content="".join(parts),
|
||||
section_path=blocks[0].section_path,
|
||||
block_ranges=tuple(block_ranges),
|
||||
)
|
||||
@ -1,12 +1,10 @@
|
||||
"""Application configuration loaded from environment variables."""
|
||||
|
||||
from functools import lru_cache
|
||||
from pathlib import Path
|
||||
from typing import Literal
|
||||
|
||||
from pydantic import AnyHttpUrl, Field, SecretStr, model_validator
|
||||
from pydantic import Field
|
||||
from pydantic_settings import BaseSettings, SettingsConfigDict
|
||||
from sqlalchemy.engine import make_url
|
||||
|
||||
|
||||
class Settings(BaseSettings):
|
||||
@ -25,105 +23,6 @@ class Settings(BaseSettings):
|
||||
api_host: str = "0.0.0.0"
|
||||
api_port: int = Field(default=8000, ge=1, le=65535)
|
||||
api_prefix: str = "/api/v1"
|
||||
admin_web_root: Path | None = None
|
||||
# SQLAlchemy URLs commonly embed a password, so configuration rendering
|
||||
# must mask this value and unwrap it only at the database driver boundary.
|
||||
database_url: SecretStr = SecretStr("mysql+pymysql://localhost:2881/ekp")
|
||||
database_pool_size: int = Field(default=10, ge=1, le=100)
|
||||
database_max_overflow: int = Field(default=10, ge=0, le=100)
|
||||
database_pool_recycle_seconds: int = Field(default=600, ge=1)
|
||||
database_pool_timeout_seconds: float = Field(default=30.0, gt=0)
|
||||
database_connect_timeout_seconds: int = Field(default=30, ge=1)
|
||||
rabbitmq_url: SecretStr | None = None
|
||||
ingestion_max_attempts: int = Field(default=3, ge=1, le=10)
|
||||
ingestion_retry_backoff_seconds: int = Field(default=2, ge=1, le=3600)
|
||||
ingestion_retry_backoff_max_seconds: int = Field(default=60, ge=1, le=86400)
|
||||
ingestion_worker_lease_seconds: int = Field(default=300, ge=30, le=86400)
|
||||
ingestion_recovery_interval_seconds: int = Field(default=30, ge=5, le=3600)
|
||||
batch_max_files: int = Field(default=20, ge=1, le=100)
|
||||
batch_max_total_bytes: int = Field(default=200 * 1024 * 1024, gt=0)
|
||||
object_storage_provider: Literal["local", "bos"] = "bos"
|
||||
object_storage_path: Path = Path("storage")
|
||||
bos_endpoint: str | None = None
|
||||
bos_access_key_id: SecretStr | None = None
|
||||
bos_secret_access_key: SecretStr | None = None
|
||||
bos_bucket: str | None = None
|
||||
bos_key_prefix: str = "knowledge-platform"
|
||||
bos_timeout_seconds: float = Field(default=30.0, gt=0)
|
||||
max_upload_bytes: int = Field(default=50 * 1024 * 1024, gt=0)
|
||||
chunk_max_tokens: int = Field(default=450, gt=0)
|
||||
chunk_overlap_tokens: int = Field(default=75, ge=0)
|
||||
chunk_max_characters: int = Field(default=2000, ge=4)
|
||||
embedding_provider: Literal["qianfan"] = "qianfan"
|
||||
embedding_model: Literal["bge-large-zh"] = "bge-large-zh"
|
||||
embedding_endpoint: str = "https://qianfan.baidubce.com/v2/embeddings"
|
||||
baidu_qianfan_api_key: SecretStr | None = None
|
||||
embedding_timeout_seconds: float = Field(default=30.0, gt=0)
|
||||
vector_store_provider: Literal["oceanbase"] = "oceanbase"
|
||||
oceanbase_vector_uri: str | None = None
|
||||
oceanbase_vector_user: str | None = None
|
||||
oceanbase_vector_password: SecretStr | None = None
|
||||
oceanbase_vector_database: str | None = None
|
||||
oceanbase_vector_table: str = "knowledge_chunks_v1"
|
||||
oceanbase_vector_index: str = "vidx_knowledge_chunks_embedding"
|
||||
oceanbase_vector_batch_size: int = Field(default=100, ge=1, le=1000)
|
||||
oceanbase_vector_timeout_seconds: float = Field(default=30.0, gt=0)
|
||||
index_profile_id: str = "demo-bge-large-zh-structure-v1-oceanbase-v1"
|
||||
dense_top_n: int = Field(default=20, ge=1, le=64)
|
||||
reranker_provider: Literal["qianfan"] = "qianfan"
|
||||
reranker_model: Literal["bce-reranker-base"] = "bce-reranker-base"
|
||||
reranker_endpoint: str = "https://qianfan.baidubce.com/v2/rerank"
|
||||
reranker_api_key: SecretStr | None = None
|
||||
reranker_timeout_seconds: float = Field(default=30.0, gt=0)
|
||||
search_top_k: int = Field(default=5, ge=1, le=64)
|
||||
mcp_host: str = "127.0.0.1"
|
||||
mcp_port: int = Field(default=8766, ge=1, le=65535)
|
||||
mcp_path: str = "/mcp"
|
||||
mcp_public_url: AnyHttpUrl = AnyHttpUrl("http://127.0.0.1:8766/mcp")
|
||||
mcp_issuer_url: AnyHttpUrl = AnyHttpUrl("http://127.0.0.1:8766")
|
||||
mcp_auth_token: SecretStr | None = None
|
||||
mcp_client_id: str = Field(default="workbuddy-enterprise", min_length=1, max_length=200)
|
||||
mcp_service_identity: str = Field(default="workbuddy-demo", min_length=1, max_length=200)
|
||||
mcp_required_scope: str = Field(default="knowledge:search", min_length=1, max_length=200)
|
||||
mcp_allowed_hosts: list[str] = ["127.0.0.1:*", "localhost:*", "[::1]:*", "testserver"]
|
||||
mcp_allowed_origins: list[str] = [
|
||||
"http://127.0.0.1:*",
|
||||
"http://localhost:*",
|
||||
"http://[::1]:*",
|
||||
]
|
||||
mcp_max_request_body_bytes: int = Field(default=1024 * 1024, ge=1024, le=4 * 1024 * 1024)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_chunk_window(self) -> "Settings":
|
||||
"""Require forward progress for every overlapping chunk window."""
|
||||
if self.chunk_overlap_tokens >= self.chunk_max_tokens:
|
||||
raise ValueError("chunk_overlap_tokens must be less than chunk_max_tokens")
|
||||
if self.search_top_k > self.dense_top_n:
|
||||
raise ValueError("search_top_k must not exceed dense_top_n")
|
||||
if self.ingestion_retry_backoff_max_seconds < self.ingestion_retry_backoff_seconds:
|
||||
raise ValueError(
|
||||
"ingestion_retry_backoff_max_seconds must be at least the initial backoff"
|
||||
)
|
||||
longest_provider_timeout = max(
|
||||
self.bos_timeout_seconds,
|
||||
self.embedding_timeout_seconds,
|
||||
self.oceanbase_vector_timeout_seconds,
|
||||
)
|
||||
if self.ingestion_worker_lease_seconds <= longest_provider_timeout:
|
||||
raise ValueError("ingestion_worker_lease_seconds must exceed provider timeouts")
|
||||
if self.batch_max_total_bytes < self.max_upload_bytes:
|
||||
raise ValueError("batch_max_total_bytes must be at least max_upload_bytes")
|
||||
if not self.mcp_path.startswith("/") or self.mcp_path.endswith("/"):
|
||||
raise ValueError("mcp_path must start with '/' and must not end with '/'")
|
||||
if not self.mcp_allowed_hosts:
|
||||
raise ValueError("mcp_allowed_hosts must not be empty")
|
||||
database_url = make_url(self.database_url.get_secret_value())
|
||||
if database_url.get_backend_name() == "sqlite":
|
||||
if self.app_env != "test":
|
||||
raise ValueError("SQLite database URLs are restricted to APP_ENV=test")
|
||||
elif database_url.drivername != "mysql+pymysql":
|
||||
raise ValueError("canonical database must use OceanBase through mysql+pymysql")
|
||||
return self
|
||||
|
||||
|
||||
@lru_cache
|
||||
|
||||
@ -1,37 +0,0 @@
|
||||
"""SQLAlchemy engine and session construction for canonical metadata."""
|
||||
|
||||
from sqlalchemy import Engine, create_engine
|
||||
from sqlalchemy.engine import make_url
|
||||
from sqlalchemy.orm import Session, sessionmaker
|
||||
|
||||
from ekp_core.config import Settings
|
||||
|
||||
|
||||
def create_database_engine(settings: Settings) -> Engine:
|
||||
"""Create the canonical engine without opening a connection eagerly.
|
||||
|
||||
OceanBase uses the synchronous PyMySQL driver in this Demo. Pool and
|
||||
socket timeouts are explicit so an outage cannot hold API workers
|
||||
indefinitely. SQLite remains available only for isolated tests.
|
||||
"""
|
||||
database_url = settings.database_url.get_secret_value()
|
||||
url = make_url(database_url)
|
||||
options: dict[str, object] = {"pool_pre_ping": True}
|
||||
if url.get_backend_name() == "mysql":
|
||||
options.update(
|
||||
pool_size=settings.database_pool_size,
|
||||
max_overflow=settings.database_max_overflow,
|
||||
pool_recycle=settings.database_pool_recycle_seconds,
|
||||
pool_timeout=settings.database_pool_timeout_seconds,
|
||||
connect_args={
|
||||
"connect_timeout": settings.database_connect_timeout_seconds,
|
||||
"read_timeout": settings.database_connect_timeout_seconds,
|
||||
"write_timeout": settings.database_connect_timeout_seconds,
|
||||
},
|
||||
)
|
||||
return create_engine(database_url, **options)
|
||||
|
||||
|
||||
def create_session_factory(engine: Engine) -> sessionmaker[Session]:
|
||||
"""Create sessions that retain loaded attributes after a commit."""
|
||||
return sessionmaker(bind=engine, expire_on_commit=False)
|
||||
@ -1,90 +0,0 @@
|
||||
"""Query embedding and profile-scoped dense candidate retrieval."""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from math import isfinite
|
||||
|
||||
from ekp_domain import IndexProfile
|
||||
from ekp_embedding import EmbeddingProvider
|
||||
from ekp_vector_store import DenseSearchCandidate, DenseSearchRequest, VectorStore
|
||||
|
||||
DEFAULT_DENSE_TOP_N = 20
|
||||
|
||||
|
||||
class InvalidRetrievalQueryError(ValueError):
|
||||
"""Raised when a query cannot be embedded safely."""
|
||||
|
||||
|
||||
class DenseRetrievalContractError(RuntimeError):
|
||||
"""Raised when an embedding or vector provider violates its contract."""
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class DenseRetrievalResult:
|
||||
"""Dense candidates before DEMO-008 reranking and source-result assembly."""
|
||||
|
||||
query: str
|
||||
index_profile_id: str
|
||||
candidates: tuple[DenseSearchCandidate, ...]
|
||||
|
||||
|
||||
class DenseRetrievalService:
|
||||
"""Embed one query and request Top-N from the active versioned index profile."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
embedding_provider: EmbeddingProvider,
|
||||
vector_store: VectorStore,
|
||||
index_profile: IndexProfile,
|
||||
default_top_n: int = DEFAULT_DENSE_TOP_N,
|
||||
) -> None:
|
||||
if embedding_provider.model_name != index_profile.embedding_model:
|
||||
raise ValueError("embedding provider model does not match index profile")
|
||||
if embedding_provider.dimensions != index_profile.vector_dimension:
|
||||
raise ValueError("embedding dimensions do not match index profile")
|
||||
if not 1 <= default_top_n <= 1000:
|
||||
raise ValueError("default_top_n must be between 1 and 1000")
|
||||
self._embedding_provider = embedding_provider
|
||||
self._vector_store = vector_store
|
||||
self._index_profile = index_profile
|
||||
self._default_top_n = default_top_n
|
||||
|
||||
def search(self, query: str, top_n: int | None = None) -> DenseRetrievalResult:
|
||||
"""Return raw dense candidates without reranking or final API shaping."""
|
||||
normalized_query = query.strip()
|
||||
if not normalized_query:
|
||||
raise InvalidRetrievalQueryError("retrieval query must not be empty")
|
||||
requested_top_n = self._default_top_n if top_n is None else top_n
|
||||
if not 1 <= requested_top_n <= 1000:
|
||||
raise InvalidRetrievalQueryError("top_n must be between 1 and 1000")
|
||||
|
||||
embedded = self._embedding_provider.embed((normalized_query,))
|
||||
if len(embedded) != 1:
|
||||
raise DenseRetrievalContractError("embedding provider returned the wrong vector count")
|
||||
query_vector = embedded[0]
|
||||
if len(query_vector) != self._index_profile.vector_dimension:
|
||||
raise DenseRetrievalContractError("embedding provider returned the wrong dimensions")
|
||||
if any(not isfinite(value) for value in query_vector):
|
||||
raise DenseRetrievalContractError("embedding provider returned non-finite values")
|
||||
if not any(value != 0 for value in query_vector):
|
||||
raise DenseRetrievalContractError("embedding provider returned a zero vector")
|
||||
|
||||
candidates = self._vector_store.dense_search(
|
||||
DenseSearchRequest(
|
||||
query_vector=query_vector,
|
||||
index_profile_id=self._index_profile.profile_id,
|
||||
top_n=requested_top_n,
|
||||
)
|
||||
)
|
||||
if len(candidates) > requested_top_n:
|
||||
raise DenseRetrievalContractError("vector store returned too many candidates")
|
||||
if any(
|
||||
candidate.index_profile_id != self._index_profile.profile_id for candidate in candidates
|
||||
):
|
||||
raise DenseRetrievalContractError("vector store crossed the active index profile")
|
||||
if any(not isfinite(candidate.score) for candidate in candidates):
|
||||
raise DenseRetrievalContractError("vector store returned a non-finite score")
|
||||
return DenseRetrievalResult(
|
||||
query=normalized_query,
|
||||
index_profile_id=self._index_profile.profile_id,
|
||||
candidates=candidates,
|
||||
)
|
||||
@ -1,194 +0,0 @@
|
||||
"""Application services for building and persisting document chunks."""
|
||||
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
from hashlib import sha256
|
||||
from uuid import UUID, uuid5
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session, sessionmaker
|
||||
|
||||
from ekp_core.chunking import ChunkDraft, ChunkingStrategy
|
||||
from ekp_core.document_parsing import DocumentIngestionService, DocumentParsingService
|
||||
from ekp_core.document_upload import DocumentUploadService
|
||||
from ekp_core.parsed_document_artifact import deserialize_parsed_document
|
||||
from ekp_domain import Chunk, Document, DocumentStatus, IngestionStage, ParsedDocument
|
||||
from ekp_object_storage import BinarySource, ObjectStorage
|
||||
|
||||
MAX_CHUNKING_ERROR_LENGTH = 2000
|
||||
|
||||
|
||||
class DocumentNotFoundError(LookupError):
|
||||
"""Raised when chunking is requested for an unknown document id."""
|
||||
|
||||
|
||||
class DocumentNotParsedError(RuntimeError):
|
||||
"""Raised when chunking is requested before a parsed artifact exists."""
|
||||
|
||||
|
||||
class DocumentAlreadyChunkedError(RuntimeError):
|
||||
"""Raised instead of silently replacing an existing strategy result."""
|
||||
|
||||
|
||||
class ParsedDocumentMismatchError(ValueError):
|
||||
"""Raised when a parsed artifact does not belong to its Document metadata."""
|
||||
|
||||
|
||||
class EmptyChunkSetError(ValueError):
|
||||
"""Raised when a parsed document produces no usable chunks."""
|
||||
|
||||
|
||||
class ChunkingStatePersistenceError(RuntimeError):
|
||||
"""Raised when a chunking failure cannot be recorded in metadata."""
|
||||
|
||||
|
||||
class DocumentChunkingService:
|
||||
"""Load a parsed artifact, build chunks, and atomically persist their metadata."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
session_factory: sessionmaker[Session],
|
||||
storage: ObjectStorage,
|
||||
strategy: ChunkingStrategy,
|
||||
) -> None:
|
||||
self._session_factory = session_factory
|
||||
self._storage = storage
|
||||
self._strategy = strategy
|
||||
|
||||
def process(
|
||||
self,
|
||||
document_id: UUID,
|
||||
stage_callback: Callable[[IngestionStage], None] | None = None,
|
||||
) -> tuple[Chunk, ...]:
|
||||
"""Build chunks while optionally reporting the durable Job stage."""
|
||||
filename, parsed_key = self._start_chunking(document_id)
|
||||
try:
|
||||
if stage_callback is not None:
|
||||
stage_callback(IngestionStage.CHUNKING)
|
||||
with self._storage.open(parsed_key) as source:
|
||||
parsed = deserialize_parsed_document(source.read())
|
||||
if parsed.document_id != document_id or parsed.filename != filename:
|
||||
raise ParsedDocumentMismatchError(
|
||||
f"parsed artifact identity does not match document {document_id}"
|
||||
)
|
||||
|
||||
drafts = self._strategy.chunk(parsed)
|
||||
if not drafts:
|
||||
raise EmptyChunkSetError(f"{filename!r} produced no chunks")
|
||||
return self._persist_chunks(document_id, filename, drafts)
|
||||
except Exception as error:
|
||||
self._record_failure(document_id, error)
|
||||
raise
|
||||
|
||||
def _start_chunking(self, document_id: UUID) -> tuple[str, str]:
|
||||
"""Validate prerequisites and guard against in-place strategy replacement."""
|
||||
with self._session_factory() as session:
|
||||
document = session.get(Document, document_id)
|
||||
if document is None:
|
||||
raise DocumentNotFoundError(str(document_id))
|
||||
if document.parsed_storage_key is None:
|
||||
raise DocumentNotParsedError(str(document_id))
|
||||
existing_chunk_id = session.scalar(
|
||||
select(Chunk.chunk_id)
|
||||
.where(
|
||||
Chunk.document_id == document_id,
|
||||
Chunk.strategy_version == self._strategy.version,
|
||||
)
|
||||
.limit(1)
|
||||
)
|
||||
if existing_chunk_id is not None:
|
||||
raise DocumentAlreadyChunkedError(str(document_id))
|
||||
|
||||
document.status = DocumentStatus.PROCESSING
|
||||
document.error_message = None
|
||||
session.commit()
|
||||
return document.filename, document.parsed_storage_key
|
||||
|
||||
def _persist_chunks(
|
||||
self,
|
||||
document_id: UUID,
|
||||
filename: str,
|
||||
drafts: tuple[ChunkDraft, ...],
|
||||
) -> tuple[Chunk, ...]:
|
||||
"""Persist all chunks in one transaction so partial sets are never visible."""
|
||||
chunks: list[Chunk] = []
|
||||
for draft in drafts:
|
||||
content_digest = sha256(draft.content.encode("utf-8")).hexdigest()
|
||||
chunks.append(
|
||||
Chunk(
|
||||
chunk_id=uuid5(
|
||||
document_id,
|
||||
f"{self._strategy.version}:{draft.chunk_index}:{content_digest}",
|
||||
),
|
||||
document_id=document_id,
|
||||
filename=filename,
|
||||
page=draft.page,
|
||||
page_end=draft.page_end,
|
||||
section=draft.section,
|
||||
section_path=list(draft.section_path),
|
||||
chunk_index=draft.chunk_index,
|
||||
content=draft.content,
|
||||
token_count=draft.token_count,
|
||||
strategy_version=self._strategy.version,
|
||||
)
|
||||
)
|
||||
|
||||
with self._session_factory() as session:
|
||||
document = session.get(Document, document_id)
|
||||
if document is None:
|
||||
raise DocumentNotFoundError(str(document_id))
|
||||
session.add_all(chunks)
|
||||
document.status = DocumentStatus.PROCESSING
|
||||
document.error_message = None
|
||||
session.commit()
|
||||
return tuple(chunks)
|
||||
|
||||
def _record_failure(self, document_id: UUID, error: Exception) -> None:
|
||||
"""Persist a bounded chunking diagnostic after transaction rollback."""
|
||||
try:
|
||||
with self._session_factory() as session:
|
||||
document = session.get(Document, document_id)
|
||||
if document is None:
|
||||
return
|
||||
document.status = DocumentStatus.FAILED
|
||||
message = str(error).strip() or type(error).__name__
|
||||
document.error_message = message[:MAX_CHUNKING_ERROR_LENGTH]
|
||||
session.commit()
|
||||
except Exception as state_error:
|
||||
raise ChunkingStatePersistenceError(
|
||||
f"could not record chunking failure for {document_id}"
|
||||
) from state_error
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class DocumentPipelineResult:
|
||||
"""Current synchronous upload-to-chunk result before embedding is added."""
|
||||
|
||||
document: Document
|
||||
parsed_document: ParsedDocument
|
||||
chunks: tuple[Chunk, ...]
|
||||
|
||||
|
||||
class DocumentPipelineService:
|
||||
"""Compose upload, parse/clean, and chunk stages for the current Demo scope."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
upload_service: DocumentUploadService,
|
||||
parsing_service: DocumentParsingService,
|
||||
chunking_service: DocumentChunkingService,
|
||||
) -> None:
|
||||
self._ingestion_service = DocumentIngestionService(upload_service, parsing_service)
|
||||
self._chunking_service = chunking_service
|
||||
|
||||
def ingest(
|
||||
self,
|
||||
filename: str,
|
||||
media_type: str | None,
|
||||
source: BinarySource,
|
||||
) -> DocumentPipelineResult:
|
||||
"""Run every completed Demo ingestion stage in dependency order."""
|
||||
document, parsed = self._ingestion_service.ingest(filename, media_type, source)
|
||||
chunks = self._chunking_service.process(document.id)
|
||||
document.status = DocumentStatus.PROCESSING
|
||||
return DocumentPipelineResult(document=document, parsed_document=parsed, chunks=chunks)
|
||||
@ -1,34 +0,0 @@
|
||||
"""Basic deterministic cleanup for provider-neutral parsed documents."""
|
||||
|
||||
from ekp_domain import ParsedBlock, ParsedDocument
|
||||
|
||||
|
||||
def clean_parsed_document(document: ParsedDocument) -> ParsedDocument:
|
||||
"""Normalize whitespace, remove empty blocks, and clean section labels."""
|
||||
cleaned_blocks: list[ParsedBlock] = []
|
||||
for block in document.blocks:
|
||||
text = _normalize_whitespace(block.text)
|
||||
if not text:
|
||||
continue
|
||||
section_path = tuple(
|
||||
cleaned for section in block.section_path if (cleaned := _normalize_whitespace(section))
|
||||
)
|
||||
cleaned_blocks.append(
|
||||
ParsedBlock(
|
||||
text=text,
|
||||
block_type=block.block_type,
|
||||
page_number=block.page_number,
|
||||
section_path=section_path,
|
||||
)
|
||||
)
|
||||
|
||||
return ParsedDocument(
|
||||
document_id=document.document_id,
|
||||
filename=document.filename,
|
||||
blocks=tuple(cleaned_blocks),
|
||||
)
|
||||
|
||||
|
||||
def _normalize_whitespace(value: str) -> str:
|
||||
"""Collapse Unicode whitespace, including abnormal intra-block line breaks."""
|
||||
return " ".join(value.split())
|
||||
@ -1,258 +0,0 @@
|
||||
"""Embedding and vector-index orchestration for canonical document chunks."""
|
||||
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
from uuid import UUID, uuid5
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session, sessionmaker
|
||||
|
||||
from ekp_core.document_chunking import DocumentPipelineResult, DocumentPipelineService
|
||||
from ekp_domain import (
|
||||
Chunk,
|
||||
ChunkIndexState,
|
||||
Document,
|
||||
DocumentStatus,
|
||||
IndexProfile,
|
||||
IngestionStage,
|
||||
)
|
||||
from ekp_embedding import EmbeddingProvider
|
||||
from ekp_object_storage import BinarySource
|
||||
from ekp_vector_store import VectorRecord, VectorStore
|
||||
|
||||
MAX_INDEXING_ERROR_LENGTH = 2000
|
||||
|
||||
|
||||
class DocumentNotFoundError(LookupError):
|
||||
"""Raised when indexing is requested for an unknown document id."""
|
||||
|
||||
|
||||
class DocumentHasNoChunksError(RuntimeError):
|
||||
"""Raised when the selected chunk strategy has no canonical rows."""
|
||||
|
||||
|
||||
class DocumentAlreadyIndexedError(RuntimeError):
|
||||
"""Raised instead of overwriting an existing versioned index profile."""
|
||||
|
||||
|
||||
class EmbeddingResultError(RuntimeError):
|
||||
"""Raised when provider output violates count or dimension contracts."""
|
||||
|
||||
|
||||
class VectorIndexRollbackError(RuntimeError):
|
||||
"""Raised when failed indexing also cannot delete partial vector rows."""
|
||||
|
||||
|
||||
class IndexingStatePersistenceError(RuntimeError):
|
||||
"""Raised when an indexing failure cannot be stored in canonical metadata."""
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class DocumentIndexingResult:
|
||||
"""Summary of one completed versioned index build."""
|
||||
|
||||
document_id: UUID
|
||||
index_profile_id: str
|
||||
indexed_chunks: int
|
||||
|
||||
|
||||
class DocumentIndexingService:
|
||||
"""Embed canonical chunks and upsert a rebuildable VectorStore index."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
session_factory: sessionmaker[Session],
|
||||
embedding_provider: EmbeddingProvider,
|
||||
vector_store: VectorStore,
|
||||
index_profile: IndexProfile,
|
||||
) -> None:
|
||||
if embedding_provider.model_name != index_profile.embedding_model:
|
||||
raise ValueError("embedding provider model does not match index profile")
|
||||
if embedding_provider.dimensions != index_profile.vector_dimension:
|
||||
raise ValueError("embedding dimensions do not match index profile")
|
||||
self._session_factory = session_factory
|
||||
self._embedding_provider = embedding_provider
|
||||
self._vector_store = vector_store
|
||||
self._index_profile = index_profile
|
||||
|
||||
def process(
|
||||
self,
|
||||
document_id: UUID,
|
||||
stage_callback: Callable[[IngestionStage], None] | None = None,
|
||||
) -> DocumentIndexingResult:
|
||||
"""Embed/index chunks while optionally reporting durable Job stages."""
|
||||
chunks = self._start_indexing(document_id)
|
||||
upsert_attempted = False
|
||||
try:
|
||||
vectors = self._embed_chunks(chunks, stage_callback)
|
||||
records = tuple(
|
||||
VectorRecord(
|
||||
index_record_id=uuid5(
|
||||
chunk.chunk_id,
|
||||
self._index_profile.profile_id,
|
||||
),
|
||||
chunk_id=chunk.chunk_id,
|
||||
document_id=chunk.document_id,
|
||||
filename=chunk.filename,
|
||||
page=chunk.page,
|
||||
page_end=chunk.page_end,
|
||||
section=chunk.section,
|
||||
chunk_index=chunk.chunk_index,
|
||||
content=chunk.content,
|
||||
embedding=vector,
|
||||
embedding_model=self._index_profile.embedding_model,
|
||||
index_profile_id=self._index_profile.profile_id,
|
||||
)
|
||||
for chunk, vector in zip(chunks, vectors, strict=True)
|
||||
)
|
||||
if stage_callback is not None:
|
||||
stage_callback(IngestionStage.INDEXING)
|
||||
upsert_attempted = True
|
||||
self._vector_store.upsert(records)
|
||||
self._record_success(document_id, chunks)
|
||||
return DocumentIndexingResult(
|
||||
document_id=document_id,
|
||||
index_profile_id=self._index_profile.profile_id,
|
||||
indexed_chunks=len(chunks),
|
||||
)
|
||||
except Exception as error:
|
||||
cleanup_error: Exception | None = None
|
||||
if upsert_attempted:
|
||||
try:
|
||||
self._vector_store.delete_document(
|
||||
document_id,
|
||||
self._index_profile.profile_id,
|
||||
)
|
||||
except Exception as caught_cleanup_error:
|
||||
cleanup_error = caught_cleanup_error
|
||||
self._record_failure(document_id, error)
|
||||
if cleanup_error is not None:
|
||||
raise VectorIndexRollbackError(
|
||||
f"indexing failed and vector cleanup failed for {document_id}"
|
||||
) from cleanup_error
|
||||
raise
|
||||
|
||||
def _start_indexing(self, document_id: UUID) -> tuple[Chunk, ...]:
|
||||
"""Load the selected chunk version and reject in-place profile replacement."""
|
||||
with self._session_factory() as session:
|
||||
document = session.get(Document, document_id)
|
||||
if document is None:
|
||||
raise DocumentNotFoundError(str(document_id))
|
||||
chunks = tuple(
|
||||
session.scalars(
|
||||
select(Chunk)
|
||||
.where(
|
||||
Chunk.document_id == document_id,
|
||||
Chunk.strategy_version == self._index_profile.chunk_strategy_version,
|
||||
)
|
||||
.order_by(Chunk.chunk_index)
|
||||
).all()
|
||||
)
|
||||
if not chunks:
|
||||
raise DocumentHasNoChunksError(str(document_id))
|
||||
chunk_ids = [chunk.chunk_id for chunk in chunks]
|
||||
existing_state = session.scalar(
|
||||
select(ChunkIndexState.chunk_id)
|
||||
.where(
|
||||
ChunkIndexState.index_profile_id == self._index_profile.profile_id,
|
||||
ChunkIndexState.chunk_id.in_(chunk_ids),
|
||||
)
|
||||
.limit(1)
|
||||
)
|
||||
if existing_state is not None:
|
||||
raise DocumentAlreadyIndexedError(str(document_id))
|
||||
document.status = DocumentStatus.PROCESSING
|
||||
document.error_message = None
|
||||
session.commit()
|
||||
return chunks
|
||||
|
||||
def _embed_chunks(
|
||||
self,
|
||||
chunks: tuple[Chunk, ...],
|
||||
stage_callback: Callable[[IngestionStage], None] | None,
|
||||
) -> tuple[tuple[float, ...], ...]:
|
||||
"""Embed bounded batches and renew an optional Job lease before each call."""
|
||||
vectors: list[tuple[float, ...]] = []
|
||||
batch_size = self._embedding_provider.max_batch_size
|
||||
if batch_size <= 0:
|
||||
raise EmbeddingResultError("embedding provider has an invalid batch size")
|
||||
for start in range(0, len(chunks), batch_size):
|
||||
if stage_callback is not None:
|
||||
stage_callback(IngestionStage.EMBEDDING)
|
||||
batch = chunks[start : start + batch_size]
|
||||
embedded = self._embedding_provider.embed([chunk.content for chunk in batch])
|
||||
if len(embedded) != len(batch):
|
||||
raise EmbeddingResultError("embedding provider returned the wrong vector count")
|
||||
if any(len(vector) != self._index_profile.vector_dimension for vector in embedded):
|
||||
raise EmbeddingResultError("embedding provider returned the wrong dimensions")
|
||||
vectors.extend(embedded)
|
||||
return tuple(vectors)
|
||||
|
||||
def _record_success(self, document_id: UUID, chunks: tuple[Chunk, ...]) -> None:
|
||||
"""Atomically record every indexed chunk before marking the document ready."""
|
||||
with self._session_factory() as session:
|
||||
document = session.get(Document, document_id)
|
||||
if document is None:
|
||||
raise DocumentNotFoundError(str(document_id))
|
||||
session.add_all(
|
||||
[
|
||||
ChunkIndexState(
|
||||
chunk_id=chunk.chunk_id,
|
||||
index_profile_id=self._index_profile.profile_id,
|
||||
embedding_model=self._index_profile.embedding_model,
|
||||
vector_dimension=self._index_profile.vector_dimension,
|
||||
)
|
||||
for chunk in chunks
|
||||
]
|
||||
)
|
||||
document.status = DocumentStatus.READY
|
||||
document.error_message = None
|
||||
session.commit()
|
||||
|
||||
def _record_failure(self, document_id: UUID, error: Exception) -> None:
|
||||
"""Store a bounded error without exposing credentials or request bodies."""
|
||||
try:
|
||||
with self._session_factory() as session:
|
||||
document = session.get(Document, document_id)
|
||||
if document is None:
|
||||
return
|
||||
document.status = DocumentStatus.FAILED
|
||||
message = str(error).strip() or type(error).__name__
|
||||
document.error_message = message[:MAX_INDEXING_ERROR_LENGTH]
|
||||
session.commit()
|
||||
except Exception as state_error:
|
||||
raise IndexingStatePersistenceError(
|
||||
f"could not record indexing failure for {document_id}"
|
||||
) from state_error
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class IndexedDocumentPipelineResult:
|
||||
"""Upload-to-index result for all Demo stages completed through DEMO-006."""
|
||||
|
||||
ingestion: DocumentPipelineResult
|
||||
indexing: DocumentIndexingResult
|
||||
|
||||
|
||||
class DocumentIndexingPipelineService:
|
||||
"""Compose upload, parsing, chunking, embedding, and vector upsert."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
document_pipeline: DocumentPipelineService,
|
||||
indexing_service: DocumentIndexingService,
|
||||
) -> None:
|
||||
self._document_pipeline = document_pipeline
|
||||
self._indexing_service = indexing_service
|
||||
|
||||
def ingest(
|
||||
self,
|
||||
filename: str,
|
||||
media_type: str | None,
|
||||
source: BinarySource,
|
||||
) -> IndexedDocumentPipelineResult:
|
||||
"""Run the complete current ingestion pipeline and return trace metadata."""
|
||||
ingestion = self._document_pipeline.ingest(filename, media_type, source)
|
||||
indexing = self._indexing_service.process(ingestion.document.id)
|
||||
ingestion.document.status = DocumentStatus.READY
|
||||
return IndexedDocumentPipelineResult(ingestion=ingestion, indexing=indexing)
|
||||
@ -1,119 +0,0 @@
|
||||
"""Document query and direct-delete services for the Demo REST API."""
|
||||
|
||||
from uuid import UUID
|
||||
|
||||
from sqlalchemy import delete, select
|
||||
from sqlalchemy.orm import Session, sessionmaker
|
||||
|
||||
from ekp_domain import Chunk, ChunkIndexState, Document, IngestionJob, IngestionJobStatus
|
||||
from ekp_object_storage import ObjectStorage
|
||||
from ekp_vector_store import VectorStore
|
||||
|
||||
|
||||
class DocumentNotFoundError(LookupError):
|
||||
"""Raised when a requested document does not exist."""
|
||||
|
||||
|
||||
class DocumentArtifactCleanupError(RuntimeError):
|
||||
"""Raised when deleted metadata leaves an external storage artifact behind."""
|
||||
|
||||
|
||||
class DocumentIngestionActiveError(RuntimeError):
|
||||
"""Raised when direct deletion would race an active asynchronous Job."""
|
||||
|
||||
|
||||
class DocumentManagementService:
|
||||
"""Read canonical documents and coordinate the Demo's direct deletion.
|
||||
|
||||
Vector rows are removed before canonical metadata. Object bytes are removed
|
||||
after the metadata commit, so a storage failure can only leave an unreachable
|
||||
orphan rather than expose a searchable row without canonical provenance.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
session_factory: sessionmaker[Session],
|
||||
storage: ObjectStorage,
|
||||
vector_store: VectorStore,
|
||||
active_index_profile_id: str,
|
||||
) -> None:
|
||||
if not active_index_profile_id:
|
||||
raise ValueError("active_index_profile_id must not be empty")
|
||||
self._session_factory = session_factory
|
||||
self._storage = storage
|
||||
self._vector_store = vector_store
|
||||
self._active_index_profile_id = active_index_profile_id
|
||||
|
||||
def list_documents(self) -> tuple[Document, ...]:
|
||||
"""Return documents newest first with a deterministic UUID tie-break."""
|
||||
with self._session_factory() as session:
|
||||
return tuple(
|
||||
session.scalars(
|
||||
select(Document).order_by(Document.created_at.desc(), Document.id.desc())
|
||||
).all()
|
||||
)
|
||||
|
||||
def get_document(self, document_id: UUID) -> Document:
|
||||
"""Return one canonical document or raise a stable not-found error."""
|
||||
with self._session_factory() as session:
|
||||
document = session.get(Document, document_id)
|
||||
if document is None:
|
||||
raise DocumentNotFoundError(str(document_id))
|
||||
return document
|
||||
|
||||
def delete_document(self, document_id: UUID) -> None:
|
||||
"""Delete vector rows, canonical metadata, and immutable source artifacts."""
|
||||
with self._session_factory() as session:
|
||||
document = session.get(Document, document_id)
|
||||
if document is None:
|
||||
raise DocumentNotFoundError(str(document_id))
|
||||
active_job_id = session.scalar(
|
||||
select(IngestionJob.id)
|
||||
.where(
|
||||
IngestionJob.document_id == document_id,
|
||||
IngestionJob.status.in_(
|
||||
(
|
||||
IngestionJobStatus.QUEUED,
|
||||
IngestionJobStatus.RUNNING,
|
||||
IngestionJobStatus.RETRYING,
|
||||
)
|
||||
),
|
||||
)
|
||||
.limit(1)
|
||||
)
|
||||
if active_job_id is not None:
|
||||
raise DocumentIngestionActiveError(str(document_id))
|
||||
|
||||
source_keys = tuple(
|
||||
key
|
||||
for key in (document.parsed_storage_key, document.storage_key)
|
||||
if key is not None
|
||||
)
|
||||
profile_ids = set(
|
||||
session.scalars(
|
||||
select(ChunkIndexState.index_profile_id)
|
||||
.join(Chunk, Chunk.chunk_id == ChunkIndexState.chunk_id)
|
||||
.where(Chunk.document_id == document_id)
|
||||
.distinct()
|
||||
).all()
|
||||
)
|
||||
# Also clean the active profile when an interrupted index write did
|
||||
# not reach canonical ChunkIndexState persistence.
|
||||
profile_ids.add(self._active_index_profile_id)
|
||||
for profile_id in sorted(profile_ids):
|
||||
self._vector_store.delete_document(document_id, profile_id)
|
||||
|
||||
chunk_ids = select(Chunk.chunk_id).where(Chunk.document_id == document_id)
|
||||
session.execute(delete(ChunkIndexState).where(ChunkIndexState.chunk_id.in_(chunk_ids)))
|
||||
session.execute(delete(Chunk).where(Chunk.document_id == document_id))
|
||||
session.execute(delete(Document).where(Document.id == document_id))
|
||||
session.commit()
|
||||
|
||||
try:
|
||||
for source_key in source_keys:
|
||||
if self._storage.exists(source_key):
|
||||
self._storage.delete(source_key)
|
||||
except Exception as error:
|
||||
raise DocumentArtifactCleanupError(
|
||||
f"external artifact cleanup failed for document {document_id}"
|
||||
) from error
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
x
Reference in New Issue
Block a user