Compare commits
No commits in common. "main" and "codex/arch-005" have entirely different histories.
main
...
codex/arch
@ -1,21 +0,0 @@
|
|||||||
# Keep credentials, local state, caches, and developer artifacts out of every image context.
|
|
||||||
.env
|
|
||||||
.env.*
|
|
||||||
!.env.example
|
|
||||||
.git
|
|
||||||
.github
|
|
||||||
.idea
|
|
||||||
.vscode
|
|
||||||
.DS_Store
|
|
||||||
.mypy_cache
|
|
||||||
.pytest_cache
|
|
||||||
.ruff_cache
|
|
||||||
.venv
|
|
||||||
__pycache__
|
|
||||||
*.py[cod]
|
|
||||||
*.sqlite*
|
|
||||||
dist
|
|
||||||
storage
|
|
||||||
tests
|
|
||||||
docs
|
|
||||||
docker-compose.dev.yml
|
|
||||||
44
.env.example
44
.env.example
@ -4,24 +4,10 @@ LOG_LEVEL=INFO
|
|||||||
API_HOST=0.0.0.0
|
API_HOST=0.0.0.0
|
||||||
API_PORT=8000
|
API_PORT=8000
|
||||||
API_PREFIX=/api/v1
|
API_PREFIX=/api/v1
|
||||||
# Optional outside source checkouts; production image fixes this to /app/apps/admin-web.
|
|
||||||
# ADMIN_WEB_ROOT=/absolute/path/to/apps/admin-web
|
|
||||||
|
|
||||||
DATABASE_URL=mysql+pymysql://ekp:change-me@localhost:2881/ekp
|
DATABASE_URL=postgresql+psycopg://ekp:ekp@localhost:5432/ekp
|
||||||
DATABASE_POOL_SIZE=10
|
REDIS_URL=redis://localhost:6379/0
|
||||||
DATABASE_MAX_OVERFLOW=10
|
RABBITMQ_URL=amqp://guest:guest@localhost:5672//
|
||||||
DATABASE_POOL_RECYCLE_SECONDS=600
|
|
||||||
DATABASE_POOL_TIMEOUT_SECONDS=30
|
|
||||||
DATABASE_CONNECT_TIMEOUT_SECONDS=30
|
|
||||||
# DEMO-015 queue. Inject the real credential at runtime; never commit its value.
|
|
||||||
RABBITMQ_URL=
|
|
||||||
INGESTION_MAX_ATTEMPTS=3
|
|
||||||
INGESTION_RETRY_BACKOFF_SECONDS=2
|
|
||||||
INGESTION_RETRY_BACKOFF_MAX_SECONDS=60
|
|
||||||
INGESTION_WORKER_LEASE_SECONDS=300
|
|
||||||
INGESTION_RECOVERY_INTERVAL_SECONDS=30
|
|
||||||
BATCH_MAX_FILES=20
|
|
||||||
BATCH_MAX_TOTAL_BYTES=209715200
|
|
||||||
|
|
||||||
OBJECT_STORAGE_PROVIDER=bos
|
OBJECT_STORAGE_PROVIDER=bos
|
||||||
OBJECT_STORAGE_PATH=storage
|
OBJECT_STORAGE_PATH=storage
|
||||||
@ -61,30 +47,6 @@ RERANKER_API_KEY=
|
|||||||
RERANKER_TIMEOUT_SECONDS=30
|
RERANKER_TIMEOUT_SECONDS=30
|
||||||
SEARCH_TOP_K=5
|
SEARCH_TOP_K=5
|
||||||
|
|
||||||
# DEMO-012 MCP gateway. Inject MCP_AUTH_TOKEN at runtime; never commit its value.
|
|
||||||
MCP_HOST=127.0.0.1
|
|
||||||
MCP_PORT=8766
|
|
||||||
MCP_PATH=/mcp
|
|
||||||
MCP_PUBLIC_URL=http://127.0.0.1:8766/mcp
|
|
||||||
MCP_ISSUER_URL=http://127.0.0.1:8766
|
|
||||||
MCP_AUTH_TOKEN=
|
|
||||||
MCP_CLIENT_ID=workbuddy-enterprise
|
|
||||||
MCP_SERVICE_IDENTITY=workbuddy-demo
|
|
||||||
MCP_REQUIRED_SCOPE=knowledge:search
|
|
||||||
MCP_ALLOWED_HOSTS='["127.0.0.1:*","localhost:*","[::1]:*","testserver"]'
|
|
||||||
MCP_ALLOWED_ORIGINS='["http://127.0.0.1:*","http://localhost:*","http://[::1]:*"]'
|
|
||||||
MCP_MAX_REQUEST_BODY_BYTES=1048576
|
|
||||||
|
|
||||||
# Controlled ARCH-006 acceptance; keep disabled outside an authorized test run.
|
|
||||||
EKP_RUN_LIVE_TESTS=0
|
|
||||||
EKP_CREDENTIALS_AUTHORIZED=0
|
|
||||||
EKP_LIVE_RESOURCE_ACK=0
|
|
||||||
EKP_LIVE_MIN_RECALL_AT_1=0.90
|
|
||||||
EKP_LIVE_MIN_RECALL_AT_5=1.0
|
|
||||||
EKP_LIVE_MIN_MRR=0.90
|
|
||||||
EKP_LIVE_MIN_NDCG_AT_5=0.90
|
|
||||||
EKP_LIVE_MAX_P95_MS=10000
|
|
||||||
|
|
||||||
LLM_PROVIDER=qianfan
|
LLM_PROVIDER=qianfan
|
||||||
LLM_MODEL=
|
LLM_MODEL=
|
||||||
LLM_ENDPOINT=
|
LLM_ENDPOINT=
|
||||||
|
|||||||
@ -19,8 +19,8 @@ Do not redesign accepted architecture unless the task explicitly asks for an arc
|
|||||||
|
|
||||||
1. API First.
|
1. API First.
|
||||||
2. V1 uses a modular monolith plus independent async/GPU workers and an MCP gateway.
|
2. V1 uses a modular monolith plus independent async/GPU workers and an MCP gateway.
|
||||||
3. OceanBase relational tables + object storage hold canonical data.
|
3. PostgreSQL + object storage hold canonical data.
|
||||||
4. OceanBase vector tables are rebuildable indexes, not sources of truth, even when co-located with canonical relational tables.
|
4. Vector databases are rebuildable indexes, not sources of truth.
|
||||||
5. Business services depend on `VectorStore` abstractions, never directly on Mochow.
|
5. Business services depend on `VectorStore` abstractions, never directly on Mochow.
|
||||||
6. Embedding, reranking, parsing, OCR, object storage, and LLMs use provider interfaces.
|
6. Embedding, reranking, parsing, OCR, object storage, and LLMs use provider interfaces.
|
||||||
7. Original source documents are immutable.
|
7. Original source documents are immutable.
|
||||||
|
|||||||
33
Makefile
33
Makefile
@ -1,4 +1,4 @@
|
|||||||
.PHONY: help install run run-mcp run-worker images images-print images-push deploy deploy-lint deploy-render format format-check lint typecheck test test-unit test-integration migration-check eval live-acceptance up down
|
.PHONY: help install run format format-check lint typecheck test test-unit test-integration migration-check eval up down
|
||||||
|
|
||||||
UV_CACHE_DIR ?= /tmp/ekp-uv-cache
|
UV_CACHE_DIR ?= /tmp/ekp-uv-cache
|
||||||
export UV_CACHE_DIR
|
export UV_CACHE_DIR
|
||||||
@ -7,14 +7,6 @@ help:
|
|||||||
@echo "Enterprise Knowledge Platform"
|
@echo "Enterprise Knowledge Platform"
|
||||||
@echo " install Install development dependencies"
|
@echo " install Install development dependencies"
|
||||||
@echo " run Start the development API"
|
@echo " run Start the development API"
|
||||||
@echo " run-mcp Start the authenticated MCP gateway"
|
|
||||||
@echo " run-worker Start the ingestion worker and recovery scheduler"
|
|
||||||
@echo " images Build production images for the current builder"
|
|
||||||
@echo " images-print Render the multi-architecture image build plan"
|
|
||||||
@echo " images-push Build and push amd64/arm64 production images"
|
|
||||||
@echo " deploy Install or upgrade Kubernetes with VALUES=<file>"
|
|
||||||
@echo " deploy-lint Validate Helm values and templates"
|
|
||||||
@echo " deploy-render Render non-secret Kubernetes manifests"
|
|
||||||
@echo " format Format code"
|
@echo " format Format code"
|
||||||
@echo " format-check Check formatting without modifying files"
|
@echo " format-check Check formatting without modifying files"
|
||||||
@echo " lint Run linters"
|
@echo " lint Run linters"
|
||||||
@ -24,7 +16,6 @@ help:
|
|||||||
@echo " test-integration Run integration tests"
|
@echo " test-integration Run integration tests"
|
||||||
@echo " migration-check Validate the forward migration path"
|
@echo " migration-check Validate the forward migration path"
|
||||||
@echo " eval Run RAG evaluation"
|
@echo " eval Run RAG evaluation"
|
||||||
@echo " live-acceptance Run controlled target-provider acceptance"
|
|
||||||
@echo " up Start local dependencies"
|
@echo " up Start local dependencies"
|
||||||
@echo " down Stop local dependencies"
|
@echo " down Stop local dependencies"
|
||||||
|
|
||||||
@ -32,24 +23,6 @@ install:
|
|||||||
uv sync --all-groups
|
uv sync --all-groups
|
||||||
run:
|
run:
|
||||||
uv run uvicorn ekp_api.main:app --host "$${API_HOST:-0.0.0.0}" --port "$${API_PORT:-8000}"
|
uv run uvicorn ekp_api.main:app --host "$${API_HOST:-0.0.0.0}" --port "$${API_PORT:-8000}"
|
||||||
run-mcp:
|
|
||||||
uv run ekp-mcp
|
|
||||||
run-worker:
|
|
||||||
uv run ekp-worker
|
|
||||||
images:
|
|
||||||
docker buildx bake -f deploy/docker/docker-bake.hcl
|
|
||||||
images-print:
|
|
||||||
docker buildx bake -f deploy/docker/docker-bake.hcl --print
|
|
||||||
images-push:
|
|
||||||
docker buildx bake -f deploy/docker/docker-bake.hcl --push
|
|
||||||
deploy:
|
|
||||||
./scripts/deploy.sh "$(VALUES)"
|
|
||||||
deploy-lint:
|
|
||||||
@test -n "$(VALUES)" || (echo "VALUES must point to an environment values file" >&2; exit 2)
|
|
||||||
helm lint deploy/helm/knowledge-platform --values "$(VALUES)"
|
|
||||||
deploy-render:
|
|
||||||
@test -n "$(VALUES)" || (echo "VALUES must point to an environment values file" >&2; exit 2)
|
|
||||||
helm template "$${RELEASE:-knowledge-platform}" deploy/helm/knowledge-platform --namespace "$${NAMESPACE:-knowledge-platform}" --values "$(VALUES)"
|
|
||||||
format:
|
format:
|
||||||
uv run ruff format .
|
uv run ruff format .
|
||||||
format-check:
|
format-check:
|
||||||
@ -57,7 +30,7 @@ format-check:
|
|||||||
lint:
|
lint:
|
||||||
uv run ruff check .
|
uv run ruff check .
|
||||||
typecheck:
|
typecheck:
|
||||||
uv run mypy apps/api/src apps/mcp-server/src packages tests
|
uv run mypy apps/api/src packages tests
|
||||||
test:
|
test:
|
||||||
uv run pytest tests/unit tests/integration
|
uv run pytest tests/unit tests/integration
|
||||||
test-unit:
|
test-unit:
|
||||||
@ -69,8 +42,6 @@ migration-check:
|
|||||||
uv run alembic upgrade head --sql >/dev/null
|
uv run alembic upgrade head --sql >/dev/null
|
||||||
eval:
|
eval:
|
||||||
uv run pytest tests/evaluation
|
uv run pytest tests/evaluation
|
||||||
live-acceptance:
|
|
||||||
uv run ekp-live-acceptance
|
|
||||||
up:
|
up:
|
||||||
docker compose -f docker-compose.dev.yml up -d
|
docker compose -f docker-compose.dev.yml up -d
|
||||||
down:
|
down:
|
||||||
|
|||||||
59
README.md
59
README.md
@ -19,8 +19,8 @@ Enterprise internal knowledge infrastructure.
|
|||||||
|
|
||||||
- API First + Admin Console
|
- API First + Admin Console
|
||||||
- Modular Monolith + async workers + MCP gateway
|
- Modular Monolith + async workers + MCP gateway
|
||||||
- OceanBase MySQL relational tables + object storage as canonical source of truth
|
- PostgreSQL + object storage as canonical source of truth
|
||||||
- Co-located OceanBase Vector tables as rebuildable retrieval indexes behind an adapter
|
- OceanBase Vector as the target retrieval index, behind a replaceable adapter
|
||||||
- Baidu BOS as target object storage; local storage remains for tests/development
|
- Baidu BOS as target object storage; local storage remains for tests/development
|
||||||
- Embedding / reranker / parser / LLM providers are replaceable
|
- Embedding / reranker / parser / LLM providers are replaceable
|
||||||
- Raw source is immutable
|
- Raw source is immutable
|
||||||
@ -46,40 +46,19 @@ The current implementation target is the retrieval demo defined in
|
|||||||
Prerequisites:
|
Prerequisites:
|
||||||
|
|
||||||
- `uv`
|
- `uv`
|
||||||
- Docker (for optional local integration dependencies)
|
- Docker (for PostgreSQL and later integration dependencies)
|
||||||
|
|
||||||
Install dependencies and run the API:
|
Install dependencies and run the API:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
cp .env.example .env
|
cp .env.example .env
|
||||||
make install
|
make install
|
||||||
make run-worker
|
|
||||||
make run
|
make run
|
||||||
```
|
```
|
||||||
|
|
||||||
Run the independent authenticated MCP Gateway after injecting a dedicated
|
|
||||||
runtime token of at least 32 characters:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
export MCP_AUTH_TOKEN="<runtime-secret-with-at-least-32-characters>"
|
|
||||||
make run-mcp
|
|
||||||
```
|
|
||||||
|
|
||||||
The Streamable HTTP endpoint defaults to `http://127.0.0.1:8766/mcp` and exposes
|
|
||||||
only `search_knowledge`. See `docs/development/MCP_SETUP.md` for WorkBuddy configuration,
|
|
||||||
identity boundaries, remote allowlists, and protocol verification.
|
|
||||||
|
|
||||||
The health endpoint is available at `http://localhost:8000/healthz`, and the
|
The health endpoint is available at `http://localhost:8000/healthz`, and the
|
||||||
versioned OpenAPI document is available at
|
versioned OpenAPI document is available at
|
||||||
`http://localhost:8000/api/v1/openapi.json`.
|
`http://localhost:8000/api/v1/openapi.json`.
|
||||||
The Demo Admin Console is available at `http://localhost:8000/admin` and contains
|
|
||||||
only document upload, document management, and Search Lab pages. See
|
|
||||||
`docs/development/ADMIN_CONSOLE.md` for the operation and scope boundaries.
|
|
||||||
|
|
||||||
DEPLOY-001 provides locked, non-root `api`, `mcp`, and `worker` production image targets with an
|
|
||||||
amd64/arm64 Buildx Bake plan. DEPLOY-002 adds the Kubernetes/Helm Chart, runtime Secret references,
|
|
||||||
serialized Migration, Ingress/TLS and the `make deploy VALUES=<file>` entrypoint. See
|
|
||||||
`docs/development/CONTAINER_IMAGES.md` and `docs/development/KUBERNETES_DEPLOYMENT.md`.
|
|
||||||
|
|
||||||
Run the development checks with:
|
Run the development checks with:
|
||||||
|
|
||||||
@ -87,12 +66,7 @@ Run the development checks with:
|
|||||||
make format-check lint typecheck test migration-check eval
|
make format-check lint typecheck test migration-check eval
|
||||||
```
|
```
|
||||||
|
|
||||||
Controlled target-provider acceptance uses 32 synthetic queries and is disabled by
|
Document metadata is canonical in PostgreSQL. Demo source files use the local
|
||||||
default. It requires rotated secrets, dedicated test resources, and all safety
|
|
||||||
acknowledgements described in `docs/development/LIVE_ACCEPTANCE.md`; only then run
|
|
||||||
`make live-acceptance`.
|
|
||||||
|
|
||||||
Document metadata is canonical in OceanBase MySQL relational tables. Demo source files use the local
|
|
||||||
`storage/` directory through the provider-neutral `ObjectStorage` interface;
|
`storage/` directory through the provider-neutral `ObjectStorage` interface;
|
||||||
existing objects cannot be overwritten. The upload service accepts PDF, DOCX,
|
existing objects cannot be overwritten. The upload service accepts PDF, DOCX,
|
||||||
TXT, and MD sources, streams at most `MAX_UPLOAD_BYTES`, and compensates stored
|
TXT, and MD sources, streams at most `MAX_UPLOAD_BYTES`, and compensates stored
|
||||||
@ -103,26 +77,23 @@ versioned parsed JSON artifact under `parsed/{document_id}/document.json`.
|
|||||||
The `structure_v1` chunk strategy keeps heading/section boundaries, combines
|
The `structure_v1` chunk strategy keeps heading/section boundaries, combines
|
||||||
paragraphs, and creates configurable 450-token windows with a 75-token overlap.
|
paragraphs, and creates configurable 450-token windows with a 75-token overlap.
|
||||||
Chunk metadata, including deterministic IDs, filename, page range, section,
|
Chunk metadata, including deterministic IDs, filename, page range, section,
|
||||||
index, and strategy version, is stored atomically in canonical relational tables. The lightweight
|
index, and strategy version, is stored atomically in PostgreSQL. The lightweight
|
||||||
token counter is replaceable so DEMO-006 can inject model-specific behavior.
|
token counter is replaceable so DEMO-006 can inject model-specific behavior.
|
||||||
Documents remain `processing` until embedding/indexing completes; pipeline
|
Documents remain `processing` until embedding/indexing completes; pipeline
|
||||||
failures become `failed` without deleting the immutable original. DEMO-015 now
|
failures become `failed` without deleting the immutable original. DEMO-009
|
||||||
accepts a source plus durable Job with HTTP `202`, while a Celery/RabbitMQ worker
|
exposes this pipeline through the versioned upload endpoint.
|
||||||
runs the pipeline. Bounded transient retries clean every derived artifact before
|
|
||||||
starting the next Attempt; broker redelivery and expired worker leases converge on
|
|
||||||
OceanBase Job state. See `docs/development/ASYNC_INGESTION_SETUP.md`.
|
|
||||||
|
|
||||||
DEMO-006 added a provider-neutral `EmbeddingProvider` and `VectorStore`, plus a
|
DEMO-006 added a provider-neutral `EmbeddingProvider` and `VectorStore`, plus a
|
||||||
validated Qianfan `bge-large-zh` adapter. ARCH-005 now wires that boundary to OceanBase.
|
validated Qianfan `bge-large-zh` adapter. ARCH-005 now wires that boundary to OceanBase.
|
||||||
Chunk vectors are upserted with source metadata under a versioned index profile;
|
Chunk vectors are upserted with source metadata under a versioned index profile;
|
||||||
OceanBase relational tables record canonical index state while vector rows remain rebuildable. A
|
PostgreSQL records canonical index state while OceanBase remains rebuildable. A
|
||||||
document becomes `ready` only after every chunk is indexed. See
|
document becomes `ready` only after every chunk is indexed. See
|
||||||
`docs/development/OCEANBASE_VECTOR_SETUP.md` for the required environment variables
|
`docs/development/OCEANBASE_VECTOR_SETUP.md` for the required environment variables
|
||||||
and pre-provisioned table schema.
|
and pre-provisioned table schema.
|
||||||
|
|
||||||
The accepted provider migration is tracked by `ARCH-001` through `ARCH-006`:
|
The accepted provider migration is tracked by `ARCH-001` through `ARCH-006`:
|
||||||
Baidu BOS, OceanBase Vector, and Qianfan `bce-reranker-base` replace the initial
|
Baidu BOS, OceanBase Vector, and Qianfan `bce-reranker-base` replace the initial
|
||||||
demo providers, and ARCH-006 aligns canonical ownership with the verified OceanBase target. BOS and OceanBase
|
demo providers without changing PostgreSQL canonical ownership. BOS and OceanBase
|
||||||
are now the runtime defaults; LocalObjectStorage and FakeVectorStore remain available
|
are now the runtime defaults; LocalObjectStorage and FakeVectorStore remain available
|
||||||
for explicit local/offline tests. ARCH-005 also adds the controlled shadow rebuild
|
for explicit local/offline tests. ARCH-005 also adds the controlled shadow rebuild
|
||||||
workflow documented in `docs/development/INDEX_REBUILD.md`.
|
workflow documented in `docs/development/INDEX_REBUILD.md`.
|
||||||
@ -134,17 +105,11 @@ raw dense candidates only.
|
|||||||
|
|
||||||
DEMO-008 adds a provider-neutral reranker, and ARCH-004 aligns its validated Qianfan
|
DEMO-008 adds a provider-neutral reranker, and ARCH-004 aligns its validated Qianfan
|
||||||
adapter with the `bce-reranker-base` baseline. The final Top20-to-Top5 pipeline and
|
adapter with the `bce-reranker-base` baseline. The final Top20-to-Top5 pipeline and
|
||||||
source results are rehydrated from canonical relational chunks and retain both
|
source results are rehydrated from canonical PostgreSQL chunks and retain both
|
||||||
Dense and reranker scores; stale vector content fails closed. See
|
Dense and reranker scores; stale vector content fails closed. See
|
||||||
`docs/development/RERANKER_SETUP.md` for configuration and validation limits.
|
`docs/development/RERANKER_SETUP.md` for configuration and validation limits.
|
||||||
DEMO-009 exposes document and search REST endpoints under `/api/v1`; DEMO-015
|
DEMO-009 exposes the completed synchronous pipeline through versioned document
|
||||||
changes upload to asynchronous acceptance and adds Job status. DEMO-016 adds bounded multi-file
|
and search REST endpoints under `/api/v1`. Upload, list, detail, direct delete,
|
||||||
submission, batch/per-file status, and automatic Admin Console polling. Upload, list, detail, direct delete,
|
|
||||||
and reranked source search are documented in
|
and reranked source search are documented in
|
||||||
`docs/development/REST_API_SETUP.md`; provider and storage internals remain
|
`docs/development/REST_API_SETUP.md`; provider and storage internals remain
|
||||||
outside the public response contract.
|
outside the public response contract.
|
||||||
|
|
||||||
DEMO-012 adds the independent authenticated MCP Gateway. It uses the official
|
|
||||||
MCP Python SDK, maps a runtime-only Bearer credential to a least-privilege service
|
|
||||||
identity, and delegates `search_knowledge` to the same `SearchService` used by REST.
|
|
||||||
No provider, storage, database, model, or index-profile details are exposed.
|
|
||||||
|
|||||||
@ -1,411 +0,0 @@
|
|||||||
// The Admin Console owns presentation state only. Batch acceptance, Job state,
|
|
||||||
// and retrieval remain behind the versioned public REST contract.
|
|
||||||
const API_BASE = "/api/v1";
|
|
||||||
const BATCH_POLL_INTERVAL_MS = 1500;
|
|
||||||
const terminalBatchStatuses = new Set(["completed", "partial_failure", "failed"]);
|
|
||||||
let batchPollTimer;
|
|
||||||
let activeBatchId;
|
|
||||||
|
|
||||||
const statusLabels = {
|
|
||||||
uploaded: "Uploaded",
|
|
||||||
processing: "Processing",
|
|
||||||
ready: "Ready",
|
|
||||||
failed: "Failed",
|
|
||||||
pending: "Pending",
|
|
||||||
accepted: "Accepted",
|
|
||||||
rejected: "Rejected",
|
|
||||||
queued: "Queued",
|
|
||||||
running: "Running",
|
|
||||||
retrying: "Retrying",
|
|
||||||
succeeded: "Succeeded",
|
|
||||||
completed: "Completed",
|
|
||||||
partial_failure: "Partial failure",
|
|
||||||
};
|
|
||||||
|
|
||||||
const stageLabels = {
|
|
||||||
parsing: "解析",
|
|
||||||
cleaning: "清理",
|
|
||||||
chunking: "切分",
|
|
||||||
embedding: "向量化",
|
|
||||||
indexing: "索引",
|
|
||||||
};
|
|
||||||
|
|
||||||
const pageTitles = {
|
|
||||||
upload: "上传文档",
|
|
||||||
documents: "文档列表",
|
|
||||||
search: "Search Lab",
|
|
||||||
};
|
|
||||||
|
|
||||||
function currentPage() {
|
|
||||||
const route = window.location.pathname.split("/").filter(Boolean).at(-1);
|
|
||||||
return Object.hasOwn(pageTitles, route) ? route : "upload";
|
|
||||||
}
|
|
||||||
|
|
||||||
function showPage(page) {
|
|
||||||
document.querySelectorAll("[data-page]").forEach((section) => {
|
|
||||||
section.hidden = section.dataset.page !== page;
|
|
||||||
});
|
|
||||||
document.querySelectorAll("[data-nav]").forEach((link) => {
|
|
||||||
const active = link.dataset.nav === page;
|
|
||||||
link.classList.toggle("active", active);
|
|
||||||
if (active) link.setAttribute("aria-current", "page");
|
|
||||||
else link.removeAttribute("aria-current");
|
|
||||||
});
|
|
||||||
document.querySelector("#page-title").textContent = pageTitles[page];
|
|
||||||
}
|
|
||||||
|
|
||||||
function showNotice(message, kind = "success") {
|
|
||||||
const notice = document.querySelector("#notice");
|
|
||||||
notice.textContent = message;
|
|
||||||
notice.className = `notice ${kind}`;
|
|
||||||
notice.hidden = false;
|
|
||||||
window.clearTimeout(showNotice.timeout);
|
|
||||||
showNotice.timeout = window.setTimeout(() => {
|
|
||||||
notice.hidden = true;
|
|
||||||
}, 6000);
|
|
||||||
}
|
|
||||||
|
|
||||||
async function requestJson(path, options = {}) {
|
|
||||||
const response = await fetch(`${API_BASE}${path}`, options);
|
|
||||||
if (!response.ok) {
|
|
||||||
let message = `请求失败(HTTP ${response.status})`;
|
|
||||||
try {
|
|
||||||
const body = await response.json();
|
|
||||||
if (body.detail) message = body.detail;
|
|
||||||
} catch {
|
|
||||||
// A non-JSON upstream error is represented by the stable HTTP message above.
|
|
||||||
}
|
|
||||||
throw new Error(message);
|
|
||||||
}
|
|
||||||
return response.status === 204 ? null : response.json();
|
|
||||||
}
|
|
||||||
|
|
||||||
function formatDate(value) {
|
|
||||||
if (!value) return "—";
|
|
||||||
return new Intl.DateTimeFormat("zh-CN", {
|
|
||||||
dateStyle: "medium",
|
|
||||||
timeStyle: "short",
|
|
||||||
}).format(new Date(value));
|
|
||||||
}
|
|
||||||
|
|
||||||
function formatSize(bytes) {
|
|
||||||
if (bytes < 1024) return `${bytes} B`;
|
|
||||||
if (bytes < 1024 ** 2) return `${(bytes / 1024).toFixed(1)} KB`;
|
|
||||||
return `${(bytes / 1024 ** 2).toFixed(1)} MB`;
|
|
||||||
}
|
|
||||||
|
|
||||||
function statusPill(status) {
|
|
||||||
const pill = document.createElement("span");
|
|
||||||
pill.className = `status-pill status-${status}`;
|
|
||||||
pill.textContent = statusLabels[status] ?? status;
|
|
||||||
return pill;
|
|
||||||
}
|
|
||||||
|
|
||||||
function setButtonBusy(button, busy, busyText) {
|
|
||||||
if (!button.dataset.label) button.dataset.label = button.textContent;
|
|
||||||
button.disabled = busy;
|
|
||||||
button.textContent = busy ? busyText : button.dataset.label;
|
|
||||||
}
|
|
||||||
|
|
||||||
function initializeUpload() {
|
|
||||||
const form = document.querySelector("#upload-form");
|
|
||||||
const input = document.querySelector("#file-input");
|
|
||||||
const selected = document.querySelector("#selected-file");
|
|
||||||
const button = document.querySelector("#upload-button");
|
|
||||||
|
|
||||||
const updateFilename = () => {
|
|
||||||
const files = Array.from(input.files);
|
|
||||||
if (!files.length) {
|
|
||||||
selected.textContent = "尚未选择文件";
|
|
||||||
} else if (files.length === 1) {
|
|
||||||
selected.textContent = files[0].name;
|
|
||||||
} else {
|
|
||||||
selected.textContent = `已选择 ${files.length} 个文件`;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
input.addEventListener("change", updateFilename);
|
|
||||||
|
|
||||||
form.addEventListener("submit", async (event) => {
|
|
||||||
event.preventDefault();
|
|
||||||
if (!input.files.length) return;
|
|
||||||
window.clearTimeout(batchPollTimer);
|
|
||||||
activeBatchId = undefined;
|
|
||||||
setButtonBusy(button, true, "提交中…");
|
|
||||||
try {
|
|
||||||
const formData = new FormData();
|
|
||||||
Array.from(input.files).forEach((file) => formData.append("files", file));
|
|
||||||
const batch = await requestJson("/ingestion-batches", {
|
|
||||||
method: "POST",
|
|
||||||
body: formData,
|
|
||||||
});
|
|
||||||
activeBatchId = batch.batch_id;
|
|
||||||
renderBatchResult(batch);
|
|
||||||
scheduleBatchPoll(batch);
|
|
||||||
showNotice(
|
|
||||||
`批次已接收:${batch.accepted_files} 个进入后台队列,${batch.rejected_files} 个被拒绝。`,
|
|
||||||
batch.rejected_files ? "error" : "success",
|
|
||||||
);
|
|
||||||
form.reset();
|
|
||||||
updateFilename();
|
|
||||||
} catch (error) {
|
|
||||||
showNotice(`上传失败:${error.message}`, "error");
|
|
||||||
} finally {
|
|
||||||
setButtonBusy(button, false, "");
|
|
||||||
}
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
function scheduleBatchPoll(batch) {
|
|
||||||
window.clearTimeout(batchPollTimer);
|
|
||||||
if (batch.batch_id !== activeBatchId) return;
|
|
||||||
if (terminalBatchStatuses.has(batch.status)) return;
|
|
||||||
batchPollTimer = window.setTimeout(async () => {
|
|
||||||
try {
|
|
||||||
const latest = await requestJson(`/ingestion-batches/${batch.batch_id}`);
|
|
||||||
renderBatchResult(latest);
|
|
||||||
scheduleBatchPoll(latest);
|
|
||||||
} catch (error) {
|
|
||||||
showNotice(`批次状态刷新失败:${error.message}`, "error");
|
|
||||||
scheduleBatchPoll(batch);
|
|
||||||
}
|
|
||||||
}, BATCH_POLL_INTERVAL_MS);
|
|
||||||
}
|
|
||||||
|
|
||||||
function renderBatchResult(batch) {
|
|
||||||
const card = document.querySelector("#upload-result");
|
|
||||||
document.querySelector("#upload-batch-title").textContent =
|
|
||||||
`批次 ${batch.batch_id.slice(0, 8)} · ${formatDate(batch.created_at)}`;
|
|
||||||
document.querySelector("#upload-total").textContent = `${batch.total_files} 个`;
|
|
||||||
document.querySelector("#upload-completed").textContent =
|
|
||||||
`${batch.completed_files} / ${batch.total_files}`;
|
|
||||||
document.querySelector("#upload-accepted").textContent =
|
|
||||||
`${batch.accepted_files} / ${batch.rejected_files}`;
|
|
||||||
document.querySelector("#upload-status").replaceWith(statusPill(batch.status));
|
|
||||||
const status = card.querySelector(".status-pill");
|
|
||||||
status.id = "upload-status";
|
|
||||||
const items = document.querySelector("#batch-items");
|
|
||||||
items.replaceChildren(...batch.items.map(batchItem));
|
|
||||||
card.hidden = false;
|
|
||||||
}
|
|
||||||
|
|
||||||
function batchItem(item) {
|
|
||||||
const row = document.createElement("article");
|
|
||||||
row.className = "batch-item";
|
|
||||||
|
|
||||||
const order = document.createElement("span");
|
|
||||||
order.className = "batch-item-order";
|
|
||||||
order.textContent = String(item.item_index + 1).padStart(2, "0");
|
|
||||||
|
|
||||||
const body = document.createElement("div");
|
|
||||||
body.className = "batch-item-body";
|
|
||||||
const name = document.createElement("strong");
|
|
||||||
name.textContent = item.filename;
|
|
||||||
const detail = document.createElement("span");
|
|
||||||
const stage = item.stage ? ` · ${stageLabels[item.stage] ?? item.stage}` : "";
|
|
||||||
const attempts = item.max_attempts == null ? "" : ` · 尝试 ${item.attempt}/${item.max_attempts}`;
|
|
||||||
detail.textContent = item.error_message ?? `后台任务${stage}${attempts}`;
|
|
||||||
body.append(name, detail);
|
|
||||||
|
|
||||||
const state = item.job_status ?? item.acceptance_status;
|
|
||||||
row.append(order, body, statusPill(state));
|
|
||||||
return row;
|
|
||||||
}
|
|
||||||
|
|
||||||
async function loadDocuments() {
|
|
||||||
const refresh = document.querySelector("#refresh-documents");
|
|
||||||
setButtonBusy(refresh, true, "加载中…");
|
|
||||||
try {
|
|
||||||
const documents = await requestJson("/documents");
|
|
||||||
renderDocuments(documents);
|
|
||||||
} catch (error) {
|
|
||||||
showNotice(`文档列表加载失败:${error.message}`, "error");
|
|
||||||
} finally {
|
|
||||||
setButtonBusy(refresh, false, "");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
function renderDocuments(documents) {
|
|
||||||
const rows = document.querySelector("#document-rows");
|
|
||||||
const empty = document.querySelector("#document-empty");
|
|
||||||
rows.replaceChildren();
|
|
||||||
empty.hidden = documents.length !== 0;
|
|
||||||
document.querySelector("#document-count").textContent = `${documents.length} 个文档`;
|
|
||||||
document.querySelector("#document-refresh-time").textContent =
|
|
||||||
`更新于 ${new Intl.DateTimeFormat("zh-CN", { timeStyle: "short" }).format(new Date())}`;
|
|
||||||
|
|
||||||
documents.forEach((documentData) => {
|
|
||||||
const row = document.createElement("tr");
|
|
||||||
row.append(
|
|
||||||
tableCell(documentData.filename, "filename-cell"),
|
|
||||||
tableCell(documentData.media_type),
|
|
||||||
tableCell(formatDate(documentData.created_at)),
|
|
||||||
);
|
|
||||||
const statusCell = document.createElement("td");
|
|
||||||
statusCell.append(statusPill(documentData.status));
|
|
||||||
row.append(statusCell);
|
|
||||||
|
|
||||||
const actions = document.createElement("td");
|
|
||||||
actions.className = "row-actions";
|
|
||||||
const view = actionButton("查看", () => showDocumentDetail(documentData));
|
|
||||||
const remove = actionButton("删除", () => deleteDocument(documentData), "danger-link");
|
|
||||||
actions.append(view, remove);
|
|
||||||
row.append(actions);
|
|
||||||
rows.append(row);
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
function tableCell(value, className = "") {
|
|
||||||
const cell = document.createElement("td");
|
|
||||||
cell.textContent = value;
|
|
||||||
cell.className = className;
|
|
||||||
return cell;
|
|
||||||
}
|
|
||||||
|
|
||||||
function actionButton(label, handler, className = "") {
|
|
||||||
const button = document.createElement("button");
|
|
||||||
button.type = "button";
|
|
||||||
button.className = `table-action ${className}`;
|
|
||||||
button.textContent = label;
|
|
||||||
button.addEventListener("click", handler);
|
|
||||||
return button;
|
|
||||||
}
|
|
||||||
|
|
||||||
function showDocumentDetail(documentData) {
|
|
||||||
const dialog = document.querySelector("#document-dialog");
|
|
||||||
document.querySelector("#dialog-filename").textContent = documentData.filename;
|
|
||||||
const metadata = document.querySelector("#dialog-metadata");
|
|
||||||
metadata.replaceChildren();
|
|
||||||
[
|
|
||||||
["文档 ID", documentData.id],
|
|
||||||
["文件类型", documentData.media_type],
|
|
||||||
["文件大小", formatSize(documentData.size_bytes)],
|
|
||||||
["当前状态", statusLabels[documentData.status] ?? documentData.status],
|
|
||||||
["上传时间", formatDate(documentData.created_at)],
|
|
||||||
["更新时间", formatDate(documentData.updated_at)],
|
|
||||||
["错误信息", documentData.error_message ?? "无"],
|
|
||||||
].forEach(([term, value]) => {
|
|
||||||
const wrapper = document.createElement("div");
|
|
||||||
const dt = document.createElement("dt");
|
|
||||||
const dd = document.createElement("dd");
|
|
||||||
dt.textContent = term;
|
|
||||||
dd.textContent = value;
|
|
||||||
wrapper.append(dt, dd);
|
|
||||||
metadata.append(wrapper);
|
|
||||||
});
|
|
||||||
dialog.showModal();
|
|
||||||
}
|
|
||||||
|
|
||||||
async function deleteDocument(documentData) {
|
|
||||||
if (!window.confirm(`确认删除“${documentData.filename}”及其索引数据?`)) return;
|
|
||||||
try {
|
|
||||||
await requestJson(`/documents/${documentData.id}`, { method: "DELETE" });
|
|
||||||
showNotice(`已删除 ${documentData.filename}`, "success");
|
|
||||||
await loadDocuments();
|
|
||||||
} catch (error) {
|
|
||||||
showNotice(`删除失败:${error.message}`, "error");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
function initializeSearch() {
|
|
||||||
const form = document.querySelector("#search-form");
|
|
||||||
const button = document.querySelector("#search-button");
|
|
||||||
form.addEventListener("submit", async (event) => {
|
|
||||||
event.preventDefault();
|
|
||||||
const query = document.querySelector("#search-query").value.trim();
|
|
||||||
const topK = Number(document.querySelector("#search-top-k").value);
|
|
||||||
if (!query) return;
|
|
||||||
setButtonBusy(button, true, "检索中…");
|
|
||||||
try {
|
|
||||||
const startedAt = performance.now();
|
|
||||||
const response = await requestJson("/search", {
|
|
||||||
method: "POST",
|
|
||||||
headers: { "Content-Type": "application/json" },
|
|
||||||
body: JSON.stringify({ query, top_k: topK }),
|
|
||||||
});
|
|
||||||
renderSearchResults(response.results, performance.now() - startedAt);
|
|
||||||
} catch (error) {
|
|
||||||
showNotice(`检索失败:${error.message}`, "error");
|
|
||||||
} finally {
|
|
||||||
setButtonBusy(button, false, "");
|
|
||||||
}
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
function renderSearchResults(results, elapsedMs) {
|
|
||||||
const container = document.querySelector("#search-results");
|
|
||||||
container.replaceChildren();
|
|
||||||
document.querySelector("#search-summary").textContent =
|
|
||||||
`${results.length} 条结果 · ${(elapsedMs / 1000).toFixed(2)} 秒`;
|
|
||||||
if (!results.length) {
|
|
||||||
const empty = document.createElement("div");
|
|
||||||
empty.className = "empty-state search-empty";
|
|
||||||
const strong = document.createElement("strong");
|
|
||||||
strong.textContent = "没有找到匹配内容";
|
|
||||||
const detail = document.createElement("span");
|
|
||||||
detail.textContent = "可以换一种问法,或先确认文档状态为 Ready。";
|
|
||||||
empty.append(strong, detail);
|
|
||||||
container.append(empty);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
results.forEach((result, index) => {
|
|
||||||
const card = document.createElement("article");
|
|
||||||
card.className = "search-result-card";
|
|
||||||
const rank = document.createElement("span");
|
|
||||||
rank.className = "rank";
|
|
||||||
rank.textContent = String(index + 1).padStart(2, "0");
|
|
||||||
const body = document.createElement("div");
|
|
||||||
body.className = "result-body";
|
|
||||||
const heading = document.createElement("div");
|
|
||||||
heading.className = "result-title-row";
|
|
||||||
const title = document.createElement("h4");
|
|
||||||
title.textContent = result.filename;
|
|
||||||
const sourceMeta = document.createElement("div");
|
|
||||||
sourceMeta.className = "source-meta";
|
|
||||||
sourceMeta.append(
|
|
||||||
metadataItem("Page", result.page == null ? "—" : String(result.page)),
|
|
||||||
metadataItem("Section", result.section ?? "—"),
|
|
||||||
);
|
|
||||||
heading.append(title, sourceMeta);
|
|
||||||
const content = document.createElement("p");
|
|
||||||
content.className = "chunk-content";
|
|
||||||
content.textContent = result.content;
|
|
||||||
const scores = document.createElement("div");
|
|
||||||
scores.className = "score-row";
|
|
||||||
scores.append(
|
|
||||||
scoreItem("Dense Score", result.score),
|
|
||||||
scoreItem("Rerank Score", result.rerank_score),
|
|
||||||
);
|
|
||||||
body.append(heading, content, scores);
|
|
||||||
card.append(rank, body);
|
|
||||||
container.append(card);
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
function scoreItem(label, value) {
|
|
||||||
const item = document.createElement("span");
|
|
||||||
const labelNode = document.createElement("small");
|
|
||||||
const valueNode = document.createElement("strong");
|
|
||||||
labelNode.textContent = label;
|
|
||||||
valueNode.textContent = Number(value).toFixed(4);
|
|
||||||
item.append(labelNode, valueNode);
|
|
||||||
return item;
|
|
||||||
}
|
|
||||||
|
|
||||||
function metadataItem(label, value) {
|
|
||||||
const item = document.createElement("span");
|
|
||||||
const labelNode = document.createElement("small");
|
|
||||||
const valueNode = document.createElement("strong");
|
|
||||||
labelNode.textContent = label;
|
|
||||||
valueNode.textContent = value;
|
|
||||||
item.append(labelNode, valueNode);
|
|
||||||
return item;
|
|
||||||
}
|
|
||||||
|
|
||||||
const page = currentPage();
|
|
||||||
showPage(page);
|
|
||||||
initializeUpload();
|
|
||||||
initializeSearch();
|
|
||||||
document.querySelector("#refresh-documents").addEventListener("click", loadDocuments);
|
|
||||||
if (page === "documents") loadDocuments();
|
|
||||||
window.addEventListener("pagehide", () => window.clearTimeout(batchPollTimer));
|
|
||||||
File diff suppressed because it is too large
Load Diff
@ -1,201 +0,0 @@
|
|||||||
<!doctype html>
|
|
||||||
<html lang="zh-CN">
|
|
||||||
<head>
|
|
||||||
<meta charset="utf-8" />
|
|
||||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
|
||||||
<meta
|
|
||||||
name="description"
|
|
||||||
content="Enterprise Knowledge Platform Demo 管理控制台"
|
|
||||||
/>
|
|
||||||
<title>Enterprise Knowledge Platform</title>
|
|
||||||
<link rel="stylesheet" href="/admin/assets/styles.css" />
|
|
||||||
<script type="module" src="/admin/assets/app.js"></script>
|
|
||||||
</head>
|
|
||||||
<body>
|
|
||||||
<div class="app-shell">
|
|
||||||
<aside class="sidebar" aria-label="管理控制台导航">
|
|
||||||
<a class="brand" href="/admin/upload" aria-label="Enterprise Knowledge Platform 首页">
|
|
||||||
<svg class="brand-mark" viewBox="0 0 48 36" aria-hidden="true">
|
|
||||||
<path d="M15 3h17a15 15 0 0 1 0 30H15v-9h17a6 6 0 0 0 0-12H15z" />
|
|
||||||
<circle cx="11" cy="18" r="8" />
|
|
||||||
</svg>
|
|
||||||
<span>
|
|
||||||
<strong>地瓜机器人</strong>
|
|
||||||
<small>Knowledge Platform</small>
|
|
||||||
</span>
|
|
||||||
</a>
|
|
||||||
<div class="workspace-label">
|
|
||||||
<span>CONTROL DESK</span>
|
|
||||||
<strong>企业知识运营台</strong>
|
|
||||||
</div>
|
|
||||||
<nav class="primary-nav">
|
|
||||||
<a data-nav="upload" href="/admin/upload">
|
|
||||||
<span aria-hidden="true">01</span>上传文档
|
|
||||||
</a>
|
|
||||||
<a data-nav="documents" href="/admin/documents">
|
|
||||||
<span aria-hidden="true">02</span>文档列表
|
|
||||||
</a>
|
|
||||||
<a data-nav="search" href="/admin/search">
|
|
||||||
<span aria-hidden="true">03</span>Search Lab
|
|
||||||
</a>
|
|
||||||
</nav>
|
|
||||||
<div class="sidebar-foot">
|
|
||||||
<span class="environment-dot" aria-hidden="true"></span>
|
|
||||||
DEMO ENVIRONMENT
|
|
||||||
</div>
|
|
||||||
</aside>
|
|
||||||
|
|
||||||
<main class="workspace">
|
|
||||||
<header class="topbar">
|
|
||||||
<div>
|
|
||||||
<p class="eyebrow">KNOWLEDGE OPERATIONS</p>
|
|
||||||
<h1 id="page-title">上传文档</h1>
|
|
||||||
</div>
|
|
||||||
<div class="topbar-actions">
|
|
||||||
<span class="system-state"><i aria-hidden="true"></i>API FIRST · V1</span>
|
|
||||||
<a class="api-link" href="/api/v1/docs" target="_blank" rel="noreferrer">
|
|
||||||
API 文档 <span aria-hidden="true">↗</span>
|
|
||||||
</a>
|
|
||||||
</div>
|
|
||||||
</header>
|
|
||||||
<div class="signal-line" aria-hidden="true"><span></span><span></span><span></span></div>
|
|
||||||
|
|
||||||
<div id="notice" class="notice" role="status" aria-live="polite" hidden></div>
|
|
||||||
|
|
||||||
<section id="upload-page" class="page" data-page="upload" aria-labelledby="page-title">
|
|
||||||
<div class="page-heading">
|
|
||||||
<div>
|
|
||||||
<p class="section-label">01 / INGESTION</p>
|
|
||||||
<h2>让知识进入检索链路</h2>
|
|
||||||
<p>批量提交 PDF、DOCX、TXT 或 Markdown 文件,系统将逐个接收并在后台独立完成解析、切分与索引。</p>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
<form id="upload-form" class="upload-panel">
|
|
||||||
<label id="drop-zone" class="drop-zone" for="file-input">
|
|
||||||
<input
|
|
||||||
id="file-input"
|
|
||||||
name="files"
|
|
||||||
type="file"
|
|
||||||
multiple
|
|
||||||
accept=".pdf,.docx,.txt,.md,application/pdf,application/vnd.openxmlformats-officedocument.wordprocessingml.document,text/plain,text/markdown"
|
|
||||||
required
|
|
||||||
/>
|
|
||||||
<span class="upload-icon" aria-hidden="true">↑</span>
|
|
||||||
<strong>选择一批知识文件</strong>
|
|
||||||
<span>支持 PDF、DOCX、TXT、Markdown · 每个文件独立处理</span>
|
|
||||||
<span id="selected-file" class="selected-file">尚未选择文件</span>
|
|
||||||
</label>
|
|
||||||
<button id="upload-button" class="primary-button" type="submit">上传并处理</button>
|
|
||||||
</form>
|
|
||||||
|
|
||||||
<article id="upload-result" class="result-card" hidden>
|
|
||||||
<div>
|
|
||||||
<p class="section-label">LATEST BATCH</p>
|
|
||||||
<h3 id="upload-batch-title"></h3>
|
|
||||||
</div>
|
|
||||||
<span id="upload-status" class="status-pill"></span>
|
|
||||||
<dl class="metadata-grid">
|
|
||||||
<div><dt>总文件</dt><dd id="upload-total"></dd></div>
|
|
||||||
<div><dt>已完成</dt><dd id="upload-completed"></dd></div>
|
|
||||||
<div><dt>接收 / 拒绝</dt><dd id="upload-accepted"></dd></div>
|
|
||||||
</dl>
|
|
||||||
<div id="batch-items" class="batch-items" aria-live="polite"></div>
|
|
||||||
<a class="text-link" href="/admin/search">前往 Search Lab 检索 <span>→</span></a>
|
|
||||||
</article>
|
|
||||||
</section>
|
|
||||||
|
|
||||||
<section id="documents-page" class="page" data-page="documents" aria-labelledby="page-title" hidden>
|
|
||||||
<div class="page-heading heading-row">
|
|
||||||
<div>
|
|
||||||
<p class="section-label">02 / DOCUMENTS</p>
|
|
||||||
<h2>文档列表</h2>
|
|
||||||
<p>查看知识文档的状态、元数据并管理 Demo 数据。</p>
|
|
||||||
</div>
|
|
||||||
<button id="refresh-documents" class="secondary-button" type="button">刷新列表</button>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
<div class="table-card">
|
|
||||||
<div class="table-meta">
|
|
||||||
<span id="document-count">0 个文档</span>
|
|
||||||
<span id="document-refresh-time">尚未加载</span>
|
|
||||||
</div>
|
|
||||||
<div class="table-scroll">
|
|
||||||
<table>
|
|
||||||
<thead>
|
|
||||||
<tr>
|
|
||||||
<th>文件名</th>
|
|
||||||
<th>类型</th>
|
|
||||||
<th>上传时间</th>
|
|
||||||
<th>状态</th>
|
|
||||||
<th aria-label="操作"></th>
|
|
||||||
</tr>
|
|
||||||
</thead>
|
|
||||||
<tbody id="document-rows"></tbody>
|
|
||||||
</table>
|
|
||||||
</div>
|
|
||||||
<div id="document-empty" class="empty-state" hidden>
|
|
||||||
<strong>还没有文档</strong>
|
|
||||||
<span>上传第一个文件后,它会出现在这里。</span>
|
|
||||||
<a class="primary-button compact" href="/admin/upload">上传文档</a>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
</section>
|
|
||||||
|
|
||||||
<section id="search-page" class="page" data-page="search" aria-labelledby="page-title" hidden>
|
|
||||||
<div class="page-heading">
|
|
||||||
<div>
|
|
||||||
<p class="section-label">03 / RETRIEVAL</p>
|
|
||||||
<h2>Search Lab</h2>
|
|
||||||
<p>输入问题,检查 Dense 召回与 Reranker 精排后的 Top-K 来源。</p>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
<form id="search-form" class="search-panel">
|
|
||||||
<label for="search-query">检索问题</label>
|
|
||||||
<div class="search-controls">
|
|
||||||
<textarea
|
|
||||||
id="search-query"
|
|
||||||
name="query"
|
|
||||||
rows="3"
|
|
||||||
maxlength="1600"
|
|
||||||
placeholder="例如:产品 A 的退款周期是多少?"
|
|
||||||
required
|
|
||||||
></textarea>
|
|
||||||
<label class="top-k-control" for="search-top-k">
|
|
||||||
Top-K
|
|
||||||
<select id="search-top-k" name="top_k">
|
|
||||||
<option value="3">3</option>
|
|
||||||
<option value="5" selected>5</option>
|
|
||||||
<option value="10">10</option>
|
|
||||||
</select>
|
|
||||||
</label>
|
|
||||||
<button id="search-button" class="primary-button" type="submit">开始检索</button>
|
|
||||||
</div>
|
|
||||||
</form>
|
|
||||||
|
|
||||||
<div class="results-heading">
|
|
||||||
<div>
|
|
||||||
<p class="section-label">RESULTS</p>
|
|
||||||
<h3>来源结果</h3>
|
|
||||||
</div>
|
|
||||||
<span id="search-summary">等待检索</span>
|
|
||||||
</div>
|
|
||||||
<div id="search-results" class="search-results" aria-live="polite">
|
|
||||||
<div class="empty-state search-empty">
|
|
||||||
<strong>检索结果会显示在这里</strong>
|
|
||||||
<span>结果包含排名、文件、页码、章节、Chunk 内容和两阶段分数。</span>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
</section>
|
|
||||||
</main>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
<dialog id="document-dialog" class="document-dialog">
|
|
||||||
<form method="dialog"><button class="icon-button" aria-label="关闭详情">×</button></form>
|
|
||||||
<p class="section-label">DOCUMENT DETAIL</p>
|
|
||||||
<h2 id="dialog-filename"></h2>
|
|
||||||
<dl id="dialog-metadata" class="dialog-metadata"></dl>
|
|
||||||
</dialog>
|
|
||||||
</body>
|
|
||||||
</html>
|
|
||||||
@ -1,55 +0,0 @@
|
|||||||
"""Celery/RabbitMQ adapter behind the queue-neutral ingestion dispatcher."""
|
|
||||||
|
|
||||||
from uuid import UUID
|
|
||||||
|
|
||||||
from celery import Celery # type: ignore[import-untyped]
|
|
||||||
|
|
||||||
from ekp_core.config import Settings
|
|
||||||
|
|
||||||
INGESTION_TASK_NAME = "ekp.ingestion.run"
|
|
||||||
RECOVERY_TASK_NAME = "ekp.ingestion.recover"
|
|
||||||
|
|
||||||
|
|
||||||
def create_celery_app(settings: Settings) -> Celery:
|
|
||||||
"""Create a secret-safe broker client without opening a connection eagerly."""
|
|
||||||
if settings.rabbitmq_url is None or not settings.rabbitmq_url.get_secret_value().strip():
|
|
||||||
raise ValueError("RABBITMQ_URL is required for asynchronous ingestion")
|
|
||||||
app = Celery(
|
|
||||||
"enterprise-knowledge-platform",
|
|
||||||
broker=settings.rabbitmq_url.get_secret_value(),
|
|
||||||
)
|
|
||||||
app.conf.update(
|
|
||||||
task_ignore_result=True,
|
|
||||||
task_acks_late=True,
|
|
||||||
task_reject_on_worker_lost=True,
|
|
||||||
worker_prefetch_multiplier=1,
|
|
||||||
broker_connection_retry_on_startup=True,
|
|
||||||
timezone="UTC",
|
|
||||||
enable_utc=True,
|
|
||||||
task_routes={
|
|
||||||
INGESTION_TASK_NAME: {"queue": "ingestion"},
|
|
||||||
RECOVERY_TASK_NAME: {"queue": "ingestion"},
|
|
||||||
},
|
|
||||||
beat_schedule={
|
|
||||||
"recover-durable-ingestion-jobs": {
|
|
||||||
"task": RECOVERY_TASK_NAME,
|
|
||||||
"schedule": float(settings.ingestion_recovery_interval_seconds),
|
|
||||||
}
|
|
||||||
},
|
|
||||||
)
|
|
||||||
return app
|
|
||||||
|
|
||||||
|
|
||||||
class CeleryIngestionDispatcher:
|
|
||||||
"""Publish only a durable Job id; source bytes and state stay canonical."""
|
|
||||||
|
|
||||||
def __init__(self, app: Celery) -> None:
|
|
||||||
self._app = app
|
|
||||||
|
|
||||||
def enqueue(self, job_id: UUID) -> None:
|
|
||||||
"""Deliver one idempotent execution hint to the ingestion queue."""
|
|
||||||
self._app.send_task(
|
|
||||||
INGESTION_TASK_NAME,
|
|
||||||
args=[str(job_id)],
|
|
||||||
queue="ingestion",
|
|
||||||
)
|
|
||||||
@ -1,279 +0,0 @@
|
|||||||
"""Controlled end-to-end acceptance for the target provider chain."""
|
|
||||||
|
|
||||||
import json
|
|
||||||
import math
|
|
||||||
import os
|
|
||||||
import sys
|
|
||||||
from collections.abc import Sequence
|
|
||||||
from dataclasses import asdict, dataclass
|
|
||||||
from io import BytesIO
|
|
||||||
from pathlib import Path
|
|
||||||
from time import perf_counter
|
|
||||||
from typing import cast
|
|
||||||
from uuid import uuid4
|
|
||||||
|
|
||||||
from ekp_api.providers import ApiServices, create_api_services
|
|
||||||
from ekp_core.config import Settings
|
|
||||||
from ekp_core.search import SourceResult
|
|
||||||
|
|
||||||
MIN_GOLDEN_QUERIES = 30
|
|
||||||
MAX_GOLDEN_QUERIES = 50
|
|
||||||
DEFAULT_MIN_RECALL_AT_1 = 0.90
|
|
||||||
DEFAULT_MIN_RECALL_AT_5 = 1.0
|
|
||||||
DEFAULT_MIN_MRR = 0.90
|
|
||||||
DEFAULT_MIN_NDCG_AT_5 = 0.90
|
|
||||||
DEFAULT_MAX_P95_MS = 10_000.0
|
|
||||||
|
|
||||||
|
|
||||||
class LiveAcceptanceError(RuntimeError):
|
|
||||||
"""Raised for a safe precondition or acceptance failure."""
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True, slots=True)
|
|
||||||
class GoldenQuery:
|
|
||||||
"""One anonymized expected source and section."""
|
|
||||||
|
|
||||||
query_id: str
|
|
||||||
query: str
|
|
||||||
expected_document: str
|
|
||||||
expected_sections: tuple[str, ...]
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True, slots=True)
|
|
||||||
class AcceptanceReport:
|
|
||||||
"""Provider-neutral quality and latency metrics without retrieved content."""
|
|
||||||
|
|
||||||
query_count: int
|
|
||||||
recall_at_1: float
|
|
||||||
recall_at_3: float
|
|
||||||
recall_at_5: float
|
|
||||||
mrr: float
|
|
||||||
ndcg_at_5: float
|
|
||||||
latency_p50_ms: float
|
|
||||||
latency_p95_ms: float
|
|
||||||
passed: bool
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True, slots=True)
|
|
||||||
class AcceptanceThresholds:
|
|
||||||
"""Explicit gates for controlled live acceptance."""
|
|
||||||
|
|
||||||
min_recall_at_1: float = DEFAULT_MIN_RECALL_AT_1
|
|
||||||
min_recall_at_5: float = DEFAULT_MIN_RECALL_AT_5
|
|
||||||
min_mrr: float = DEFAULT_MIN_MRR
|
|
||||||
min_ndcg_at_5: float = DEFAULT_MIN_NDCG_AT_5
|
|
||||||
max_p95_ms: float = DEFAULT_MAX_P95_MS
|
|
||||||
|
|
||||||
def __post_init__(self) -> None:
|
|
||||||
quality_values = (
|
|
||||||
self.min_recall_at_1,
|
|
||||||
self.min_recall_at_5,
|
|
||||||
self.min_mrr,
|
|
||||||
self.min_ndcg_at_5,
|
|
||||||
)
|
|
||||||
if any(not 0 <= value <= 1 for value in quality_values):
|
|
||||||
raise ValueError("live quality thresholds must be between zero and one")
|
|
||||||
if self.max_p95_ms <= 0:
|
|
||||||
raise ValueError("live latency threshold must be positive")
|
|
||||||
|
|
||||||
|
|
||||||
def _project_root() -> Path:
|
|
||||||
return Path(__file__).resolve().parents[4]
|
|
||||||
|
|
||||||
|
|
||||||
def _load_golden_queries(path: Path) -> tuple[GoldenQuery, ...]:
|
|
||||||
"""Load only the minimal fields needed by the live evaluator."""
|
|
||||||
records: list[GoldenQuery] = []
|
|
||||||
for line in path.read_text(encoding="utf-8").splitlines():
|
|
||||||
if not line:
|
|
||||||
continue
|
|
||||||
raw: object = json.loads(line)
|
|
||||||
if not isinstance(raw, dict):
|
|
||||||
raise LiveAcceptanceError("Golden Query record must be an object")
|
|
||||||
documents = raw.get("expected_documents")
|
|
||||||
sections = raw.get("expected_sections")
|
|
||||||
if (
|
|
||||||
not isinstance(raw.get("id"), str)
|
|
||||||
or not isinstance(raw.get("query"), str)
|
|
||||||
or not isinstance(documents, list)
|
|
||||||
or len(documents) != 1
|
|
||||||
or not isinstance(documents[0], str)
|
|
||||||
or not isinstance(sections, list)
|
|
||||||
or not sections
|
|
||||||
or any(not isinstance(section, str) for section in sections)
|
|
||||||
):
|
|
||||||
raise LiveAcceptanceError("Golden Query record violates the live contract")
|
|
||||||
records.append(
|
|
||||||
GoldenQuery(
|
|
||||||
query_id=cast(str, raw["id"]),
|
|
||||||
query=cast(str, raw["query"]),
|
|
||||||
expected_document=documents[0],
|
|
||||||
expected_sections=tuple(cast(list[str], sections)),
|
|
||||||
)
|
|
||||||
)
|
|
||||||
if not MIN_GOLDEN_QUERIES <= len(records) <= MAX_GOLDEN_QUERIES:
|
|
||||||
raise LiveAcceptanceError("live Golden Dataset must contain 30 to 50 queries")
|
|
||||||
if len({record.query_id for record in records}) != len(records):
|
|
||||||
raise LiveAcceptanceError("live Golden Query IDs must be unique")
|
|
||||||
return tuple(records)
|
|
||||||
|
|
||||||
|
|
||||||
def _matching_rank(
|
|
||||||
results: Sequence[SourceResult],
|
|
||||||
*,
|
|
||||||
expected_filename: str,
|
|
||||||
expected_sections: tuple[str, ...],
|
|
||||||
) -> int | None:
|
|
||||||
"""Return the one-based rank of the first matching canonical source."""
|
|
||||||
for rank, result in enumerate(results, start=1):
|
|
||||||
if result.filename != expected_filename or result.section is None:
|
|
||||||
continue
|
|
||||||
if any(section in result.section for section in expected_sections):
|
|
||||||
return rank
|
|
||||||
return None
|
|
||||||
|
|
||||||
|
|
||||||
def _percentile(values: Sequence[float], percentile: float) -> float:
|
|
||||||
"""Use nearest-rank percentiles for a small, deterministic acceptance set."""
|
|
||||||
if not values:
|
|
||||||
raise ValueError("latency sample must not be empty")
|
|
||||||
ordered = sorted(values)
|
|
||||||
index = max(0, math.ceil(percentile * len(ordered)) - 1)
|
|
||||||
return ordered[index]
|
|
||||||
|
|
||||||
|
|
||||||
def _build_report(
|
|
||||||
ranks: Sequence[int | None],
|
|
||||||
latencies_ms: Sequence[float],
|
|
||||||
thresholds: AcceptanceThresholds,
|
|
||||||
) -> AcceptanceReport:
|
|
||||||
"""Calculate binary-relevance metrics and apply explicit acceptance gates."""
|
|
||||||
if not ranks or len(ranks) != len(latencies_ms):
|
|
||||||
raise ValueError("live ranks and latencies must have equal non-zero length")
|
|
||||||
query_count = len(ranks)
|
|
||||||
|
|
||||||
def recall_at(limit: int) -> float:
|
|
||||||
return sum(rank is not None and rank <= limit for rank in ranks) / query_count
|
|
||||||
|
|
||||||
reciprocal_ranks = [0.0 if rank is None else 1.0 / rank for rank in ranks]
|
|
||||||
discounted_gains = [
|
|
||||||
0.0 if rank is None or rank > 5 else 1.0 / math.log2(rank + 1) for rank in ranks
|
|
||||||
]
|
|
||||||
recall_at_1 = recall_at(1)
|
|
||||||
recall_at_3 = recall_at(3)
|
|
||||||
recall_at_5 = recall_at(5)
|
|
||||||
mrr = sum(reciprocal_ranks) / query_count
|
|
||||||
ndcg_at_5 = sum(discounted_gains) / query_count
|
|
||||||
p50 = _percentile(latencies_ms, 0.50)
|
|
||||||
p95 = _percentile(latencies_ms, 0.95)
|
|
||||||
passed = (
|
|
||||||
recall_at_1 >= thresholds.min_recall_at_1
|
|
||||||
and recall_at_5 >= thresholds.min_recall_at_5
|
|
||||||
and mrr >= thresholds.min_mrr
|
|
||||||
and ndcg_at_5 >= thresholds.min_ndcg_at_5
|
|
||||||
and p95 <= thresholds.max_p95_ms
|
|
||||||
)
|
|
||||||
return AcceptanceReport(
|
|
||||||
query_count=query_count,
|
|
||||||
recall_at_1=recall_at_1,
|
|
||||||
recall_at_3=recall_at_3,
|
|
||||||
recall_at_5=recall_at_5,
|
|
||||||
mrr=mrr,
|
|
||||||
ndcg_at_5=ndcg_at_5,
|
|
||||||
latency_p50_ms=p50,
|
|
||||||
latency_p95_ms=p95,
|
|
||||||
passed=passed,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def _thresholds_from_environment() -> AcceptanceThresholds:
|
|
||||||
return AcceptanceThresholds(
|
|
||||||
min_recall_at_1=float(os.getenv("EKP_LIVE_MIN_RECALL_AT_1", str(DEFAULT_MIN_RECALL_AT_1))),
|
|
||||||
min_recall_at_5=float(os.getenv("EKP_LIVE_MIN_RECALL_AT_5", str(DEFAULT_MIN_RECALL_AT_5))),
|
|
||||||
min_mrr=float(os.getenv("EKP_LIVE_MIN_MRR", str(DEFAULT_MIN_MRR))),
|
|
||||||
min_ndcg_at_5=float(os.getenv("EKP_LIVE_MIN_NDCG_AT_5", str(DEFAULT_MIN_NDCG_AT_5))),
|
|
||||||
max_p95_ms=float(os.getenv("EKP_LIVE_MAX_P95_MS", str(DEFAULT_MAX_P95_MS))),
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def _require_safe_live_settings(settings: Settings) -> None:
|
|
||||||
"""Refuse external writes unless rotation and resource authorization are explicit."""
|
|
||||||
required_flags = (
|
|
||||||
"EKP_RUN_LIVE_TESTS",
|
|
||||||
"EKP_CREDENTIALS_AUTHORIZED",
|
|
||||||
"EKP_LIVE_RESOURCE_ACK",
|
|
||||||
)
|
|
||||||
if any(os.getenv(name) != "1" for name in required_flags):
|
|
||||||
raise LiveAcceptanceError("live acceptance safety flags are not enabled")
|
|
||||||
if settings.app_env != "test":
|
|
||||||
raise LiveAcceptanceError("live acceptance requires APP_ENV=test")
|
|
||||||
if settings.object_storage_provider != "bos" or settings.vector_store_provider != "oceanbase":
|
|
||||||
raise LiveAcceptanceError("live acceptance requires the target provider baseline")
|
|
||||||
if not settings.bos_key_prefix.startswith("live-acceptance/"):
|
|
||||||
raise LiveAcceptanceError("BOS_KEY_PREFIX must use a dedicated live-acceptance prefix")
|
|
||||||
|
|
||||||
|
|
||||||
def _cleanup_run_documents(services: ApiServices, filename_prefix: str) -> None:
|
|
||||||
"""Delete only documents created under this random acceptance-run prefix."""
|
|
||||||
documents = tuple(
|
|
||||||
document
|
|
||||||
for document in services.documents.list_documents()
|
|
||||||
if document.filename.startswith(filename_prefix)
|
|
||||||
)
|
|
||||||
for document in documents:
|
|
||||||
services.documents.delete_document(document.id)
|
|
||||||
|
|
||||||
|
|
||||||
def _run_live(settings: Settings) -> AcceptanceReport:
|
|
||||||
"""Upload synthetic sources, execute every query, and clean run-owned data."""
|
|
||||||
_require_safe_live_settings(settings)
|
|
||||||
project_root = _project_root()
|
|
||||||
golden = _load_golden_queries(project_root / "fixtures/evaluation/golden.jsonl")
|
|
||||||
services = create_api_services(settings)
|
|
||||||
filename_prefix = f"live-{uuid4().hex[:12]}-"
|
|
||||||
uploaded_names: dict[str, str] = {}
|
|
||||||
cleanup_error: Exception | None = None
|
|
||||||
try:
|
|
||||||
source_names = sorted({record.expected_document for record in golden})
|
|
||||||
for source_name in source_names:
|
|
||||||
live_name = f"{filename_prefix}{source_name}"
|
|
||||||
payload = (project_root / "fixtures/documents" / source_name).read_bytes()
|
|
||||||
services.ingestion.ingest(live_name, "text/markdown", BytesIO(payload))
|
|
||||||
uploaded_names[source_name] = live_name
|
|
||||||
|
|
||||||
ranks: list[int | None] = []
|
|
||||||
latencies_ms: list[float] = []
|
|
||||||
for record in golden:
|
|
||||||
started = perf_counter()
|
|
||||||
result = services.search.search(record.query, top_k=5)
|
|
||||||
latencies_ms.append((perf_counter() - started) * 1000)
|
|
||||||
ranks.append(
|
|
||||||
_matching_rank(
|
|
||||||
result.results,
|
|
||||||
expected_filename=uploaded_names[record.expected_document],
|
|
||||||
expected_sections=record.expected_sections,
|
|
||||||
)
|
|
||||||
)
|
|
||||||
return _build_report(ranks, latencies_ms, _thresholds_from_environment())
|
|
||||||
finally:
|
|
||||||
try:
|
|
||||||
_cleanup_run_documents(services, filename_prefix)
|
|
||||||
except Exception as error:
|
|
||||||
cleanup_error = error
|
|
||||||
if cleanup_error is not None:
|
|
||||||
raise LiveAcceptanceError("live acceptance cleanup failed") from cleanup_error
|
|
||||||
|
|
||||||
|
|
||||||
def main() -> int:
|
|
||||||
"""Print only aggregate metrics and return non-zero for unsafe or failed runs."""
|
|
||||||
try:
|
|
||||||
report = _run_live(Settings())
|
|
||||||
except Exception:
|
|
||||||
print("live acceptance failed; no provider details were emitted", file=sys.stderr)
|
|
||||||
return 1
|
|
||||||
print(json.dumps(asdict(report), sort_keys=True, separators=(",", ":")))
|
|
||||||
return 0 if report.passed else 2
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
raise SystemExit(main())
|
|
||||||
@ -1,18 +1,12 @@
|
|||||||
"""FastAPI application entry point."""
|
"""FastAPI application entry point."""
|
||||||
|
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
from fastapi import FastAPI
|
from fastapi import FastAPI
|
||||||
from fastapi.responses import FileResponse
|
|
||||||
from fastapi.staticfiles import StaticFiles
|
|
||||||
from pydantic import BaseModel
|
from pydantic import BaseModel
|
||||||
|
|
||||||
from ekp_api.providers import ApiServices
|
from ekp_api.providers import ApiServices
|
||||||
from ekp_api.routes import router
|
from ekp_api.routes import router
|
||||||
from ekp_core.config import Settings, get_settings
|
from ekp_core.config import Settings, get_settings
|
||||||
|
|
||||||
ADMIN_WEB_ROOT = Path(__file__).resolve().parents[3] / "admin-web"
|
|
||||||
|
|
||||||
|
|
||||||
class HealthResponse(BaseModel):
|
class HealthResponse(BaseModel):
|
||||||
"""Health endpoint response."""
|
"""Health endpoint response."""
|
||||||
@ -37,23 +31,6 @@ def create_app(
|
|||||||
application.state.settings = resolved_settings
|
application.state.settings = resolved_settings
|
||||||
application.state.api_services = api_services
|
application.state.api_services = api_services
|
||||||
application.include_router(router, prefix=resolved_settings.api_prefix)
|
application.include_router(router, prefix=resolved_settings.api_prefix)
|
||||||
admin_web_root = (resolved_settings.admin_web_root or ADMIN_WEB_ROOT).resolve()
|
|
||||||
|
|
||||||
# The Demo console is a deliberately thin, same-origin client. It contains
|
|
||||||
# no business logic and calls only the versioned public REST API.
|
|
||||||
application.mount(
|
|
||||||
"/admin/assets",
|
|
||||||
StaticFiles(directory=admin_web_root / "assets"),
|
|
||||||
name="admin-assets",
|
|
||||||
)
|
|
||||||
|
|
||||||
@application.get("/admin", include_in_schema=False)
|
|
||||||
@application.get("/admin/upload", include_in_schema=False)
|
|
||||||
@application.get("/admin/documents", include_in_schema=False)
|
|
||||||
@application.get("/admin/search", include_in_schema=False)
|
|
||||||
async def admin_console() -> FileResponse:
|
|
||||||
"""Serve the three-page Demo Admin Console shell."""
|
|
||||||
return FileResponse(admin_web_root / "index.html")
|
|
||||||
|
|
||||||
@application.get("/healthz", response_model=HealthResponse, tags=["system"])
|
@application.get("/healthz", response_model=HealthResponse, tags=["system"])
|
||||||
async def health() -> HealthResponse:
|
async def health() -> HealthResponse:
|
||||||
|
|||||||
@ -4,7 +4,6 @@ from dataclasses import dataclass
|
|||||||
|
|
||||||
from sqlalchemy.orm import Session, sessionmaker
|
from sqlalchemy.orm import Session, sessionmaker
|
||||||
|
|
||||||
from ekp_api.celery_queue import CeleryIngestionDispatcher, create_celery_app
|
|
||||||
from ekp_core.chunking import StructureAwareChunker
|
from ekp_core.chunking import StructureAwareChunker
|
||||||
from ekp_core.config import Settings
|
from ekp_core.config import Settings
|
||||||
from ekp_core.database import create_database_engine, create_session_factory
|
from ekp_core.database import create_database_engine, create_session_factory
|
||||||
@ -18,13 +17,6 @@ from ekp_core.document_management import DocumentManagementService
|
|||||||
from ekp_core.document_parsing import DocumentParsingService
|
from ekp_core.document_parsing import DocumentParsingService
|
||||||
from ekp_core.document_upload import DocumentUploadService
|
from ekp_core.document_upload import DocumentUploadService
|
||||||
from ekp_core.index_rebuild import resolve_active_index_profile
|
from ekp_core.index_rebuild import resolve_active_index_profile
|
||||||
from ekp_core.ingestion_batches import IngestionBatchService
|
|
||||||
from ekp_core.ingestion_jobs import (
|
|
||||||
AsyncIngestionSubmissionService,
|
|
||||||
DerivedDocumentCleanupService,
|
|
||||||
IngestionJobRunner,
|
|
||||||
IngestionJobService,
|
|
||||||
)
|
|
||||||
from ekp_core.search import SearchService
|
from ekp_core.search import SearchService
|
||||||
from ekp_domain import IndexProfile
|
from ekp_domain import IndexProfile
|
||||||
from ekp_embedding import EmbeddingProvider, QianfanEmbeddingProvider
|
from ekp_embedding import EmbeddingProvider, QianfanEmbeddingProvider
|
||||||
@ -50,39 +42,10 @@ class ApiServices:
|
|||||||
"""Business services injected into REST routes without provider leakage."""
|
"""Business services injected into REST routes without provider leakage."""
|
||||||
|
|
||||||
ingestion: DocumentIndexingPipelineService
|
ingestion: DocumentIndexingPipelineService
|
||||||
submissions: AsyncIngestionSubmissionService
|
|
||||||
batches: IngestionBatchService
|
|
||||||
jobs: IngestionJobService
|
|
||||||
documents: DocumentManagementService
|
documents: DocumentManagementService
|
||||||
search: SearchService
|
search: SearchService
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True, slots=True)
|
|
||||||
class WorkerServices:
|
|
||||||
"""Provider graph required by Celery without REST search-only services."""
|
|
||||||
|
|
||||||
runner: IngestionJobRunner
|
|
||||||
jobs: IngestionJobService
|
|
||||||
dispatcher: CeleryIngestionDispatcher
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True, slots=True)
|
|
||||||
class _IngestionComponents:
|
|
||||||
"""Shared composition result used by API and independent worker processes."""
|
|
||||||
|
|
||||||
session_factory: sessionmaker[Session]
|
|
||||||
storage: ObjectStorage
|
|
||||||
vector_store: VectorStore
|
|
||||||
embedding_provider: EmbeddingProvider
|
|
||||||
index_profile: IndexProfile
|
|
||||||
upload: DocumentUploadService
|
|
||||||
parsing: DocumentParsingService
|
|
||||||
chunking: DocumentChunkingService
|
|
||||||
indexing: DocumentIndexingService
|
|
||||||
jobs: IngestionJobService
|
|
||||||
dispatcher: CeleryIngestionDispatcher
|
|
||||||
|
|
||||||
|
|
||||||
def create_index_profile(
|
def create_index_profile(
|
||||||
settings: Settings,
|
settings: Settings,
|
||||||
session_factory: sessionmaker[Session] | None = None,
|
session_factory: sessionmaker[Session] | None = None,
|
||||||
@ -198,31 +161,8 @@ def create_search_service(
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def create_runtime_search_service(settings: Settings) -> SearchService:
|
def create_api_services(settings: Settings) -> ApiServices:
|
||||||
"""Compose only read-path providers for the independent MCP Gateway."""
|
"""Compose the synchronous Demo runtime once for all business endpoints."""
|
||||||
engine = create_database_engine(settings)
|
|
||||||
session_factory = create_session_factory(engine)
|
|
||||||
index_profile = create_index_profile(settings, session_factory)
|
|
||||||
embedding_provider = create_embedding_provider(settings)
|
|
||||||
vector_store = create_vector_store(settings)
|
|
||||||
dense_retrieval = create_dense_retrieval_service(
|
|
||||||
settings,
|
|
||||||
embedding_provider,
|
|
||||||
vector_store,
|
|
||||||
index_profile,
|
|
||||||
)
|
|
||||||
return create_search_service(
|
|
||||||
settings,
|
|
||||||
session_factory,
|
|
||||||
dense_retrieval,
|
|
||||||
create_reranker_provider(settings),
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def _create_ingestion_components(settings: Settings) -> _IngestionComponents:
|
|
||||||
"""Compose provider-neutral ingestion stages and the Celery delivery adapter."""
|
|
||||||
if settings.rabbitmq_url is None or not settings.rabbitmq_url.get_secret_value().strip():
|
|
||||||
raise ProviderConfigurationError("RABBITMQ_URL is required")
|
|
||||||
engine = create_database_engine(settings)
|
engine = create_database_engine(settings)
|
||||||
session_factory = create_session_factory(engine)
|
session_factory = create_session_factory(engine)
|
||||||
storage = create_object_storage(settings)
|
storage = create_object_storage(settings)
|
||||||
@ -245,103 +185,35 @@ def _create_ingestion_components(settings: Settings) -> _IngestionComponents:
|
|||||||
max_characters=settings.chunk_max_characters,
|
max_characters=settings.chunk_max_characters,
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
document_pipeline = DocumentPipelineService(
|
||||||
|
upload_service,
|
||||||
|
parsing_service,
|
||||||
|
chunking_service,
|
||||||
|
)
|
||||||
indexing_service = DocumentIndexingService(
|
indexing_service = DocumentIndexingService(
|
||||||
session_factory,
|
session_factory,
|
||||||
embedding_provider,
|
embedding_provider,
|
||||||
vector_store,
|
vector_store,
|
||||||
index_profile,
|
index_profile,
|
||||||
)
|
)
|
||||||
jobs = IngestionJobService(session_factory, settings.ingestion_max_attempts)
|
ingestion = DocumentIndexingPipelineService(document_pipeline, indexing_service)
|
||||||
dispatcher = CeleryIngestionDispatcher(create_celery_app(settings))
|
|
||||||
return _IngestionComponents(
|
|
||||||
session_factory=session_factory,
|
|
||||||
storage=storage,
|
|
||||||
vector_store=vector_store,
|
|
||||||
embedding_provider=embedding_provider,
|
|
||||||
index_profile=index_profile,
|
|
||||||
upload=upload_service,
|
|
||||||
parsing=parsing_service,
|
|
||||||
chunking=chunking_service,
|
|
||||||
indexing=indexing_service,
|
|
||||||
jobs=jobs,
|
|
||||||
dispatcher=dispatcher,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def _create_runner(settings: Settings, components: _IngestionComponents) -> IngestionJobRunner:
|
|
||||||
"""Compose retry cleanup and execution policy from non-sensitive settings."""
|
|
||||||
cleanup = DerivedDocumentCleanupService(
|
|
||||||
components.session_factory,
|
|
||||||
components.storage,
|
|
||||||
components.vector_store,
|
|
||||||
components.index_profile.profile_id,
|
|
||||||
)
|
|
||||||
return IngestionJobRunner(
|
|
||||||
components.session_factory,
|
|
||||||
components.parsing,
|
|
||||||
components.chunking,
|
|
||||||
components.indexing,
|
|
||||||
cleanup,
|
|
||||||
retry_backoff_seconds=settings.ingestion_retry_backoff_seconds,
|
|
||||||
retry_backoff_max_seconds=settings.ingestion_retry_backoff_max_seconds,
|
|
||||||
lease_seconds=settings.ingestion_worker_lease_seconds,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def create_worker_services(settings: Settings) -> WorkerServices:
|
|
||||||
"""Build the independent worker runtime from the same application boundaries."""
|
|
||||||
components = _create_ingestion_components(settings)
|
|
||||||
return WorkerServices(
|
|
||||||
runner=_create_runner(settings, components),
|
|
||||||
jobs=components.jobs,
|
|
||||||
dispatcher=components.dispatcher,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def create_api_services(settings: Settings) -> ApiServices:
|
|
||||||
"""Compose REST services while ingestion execution remains in workers."""
|
|
||||||
components = _create_ingestion_components(settings)
|
|
||||||
document_pipeline = DocumentPipelineService(
|
|
||||||
components.upload,
|
|
||||||
components.parsing,
|
|
||||||
components.chunking,
|
|
||||||
)
|
|
||||||
ingestion = DocumentIndexingPipelineService(document_pipeline, components.indexing)
|
|
||||||
submissions = AsyncIngestionSubmissionService(
|
|
||||||
components.upload,
|
|
||||||
components.jobs,
|
|
||||||
components.dispatcher,
|
|
||||||
)
|
|
||||||
batches = IngestionBatchService(
|
|
||||||
components.session_factory,
|
|
||||||
submissions,
|
|
||||||
max_files=settings.batch_max_files,
|
|
||||||
max_total_bytes=settings.batch_max_total_bytes,
|
|
||||||
)
|
|
||||||
dense_retrieval = create_dense_retrieval_service(
|
dense_retrieval = create_dense_retrieval_service(
|
||||||
settings,
|
settings,
|
||||||
components.embedding_provider,
|
embedding_provider,
|
||||||
components.vector_store,
|
vector_store,
|
||||||
components.index_profile,
|
index_profile,
|
||||||
)
|
)
|
||||||
reranker_provider = create_reranker_provider(settings)
|
reranker_provider = create_reranker_provider(settings)
|
||||||
search = create_search_service(
|
search = create_search_service(
|
||||||
settings,
|
settings,
|
||||||
components.session_factory,
|
session_factory,
|
||||||
dense_retrieval,
|
dense_retrieval,
|
||||||
reranker_provider,
|
reranker_provider,
|
||||||
)
|
)
|
||||||
documents = DocumentManagementService(
|
documents = DocumentManagementService(
|
||||||
components.session_factory,
|
session_factory,
|
||||||
components.storage,
|
storage,
|
||||||
components.vector_store,
|
vector_store,
|
||||||
active_index_profile_id=components.index_profile.profile_id,
|
active_index_profile_id=index_profile.profile_id,
|
||||||
)
|
|
||||||
return ApiServices(
|
|
||||||
ingestion=ingestion,
|
|
||||||
submissions=submissions,
|
|
||||||
batches=batches,
|
|
||||||
jobs=components.jobs,
|
|
||||||
documents=documents,
|
|
||||||
search=search,
|
|
||||||
)
|
)
|
||||||
|
return ApiServices(ingestion=ingestion, documents=documents, search=search)
|
||||||
|
|||||||
@ -6,18 +6,10 @@ from uuid import UUID
|
|||||||
from fastapi import APIRouter, Depends, File, HTTPException, Request, Response, UploadFile, status
|
from fastapi import APIRouter, Depends, File, HTTPException, Request, Response, UploadFile, status
|
||||||
|
|
||||||
from ekp_api.providers import ApiServices, ProviderConfigurationError, create_api_services
|
from ekp_api.providers import ApiServices, ProviderConfigurationError, create_api_services
|
||||||
from ekp_api.schemas import (
|
from ekp_api.schemas import DocumentResponse, SearchRequest, SearchResponse
|
||||||
DocumentResponse,
|
|
||||||
IngestionAcceptedResponse,
|
|
||||||
IngestionBatchResponse,
|
|
||||||
IngestionJobResponse,
|
|
||||||
SearchRequest,
|
|
||||||
SearchResponse,
|
|
||||||
)
|
|
||||||
from ekp_core.dense_retrieval import InvalidRetrievalQueryError
|
from ekp_core.dense_retrieval import InvalidRetrievalQueryError
|
||||||
from ekp_core.document_management import (
|
from ekp_core.document_management import (
|
||||||
DocumentArtifactCleanupError,
|
DocumentArtifactCleanupError,
|
||||||
DocumentIngestionActiveError,
|
|
||||||
DocumentNotFoundError,
|
DocumentNotFoundError,
|
||||||
)
|
)
|
||||||
from ekp_core.document_upload import (
|
from ekp_core.document_upload import (
|
||||||
@ -26,8 +18,7 @@ from ekp_core.document_upload import (
|
|||||||
MediaTypeMismatchError,
|
MediaTypeMismatchError,
|
||||||
UnsupportedDocumentTypeError,
|
UnsupportedDocumentTypeError,
|
||||||
)
|
)
|
||||||
from ekp_core.ingestion_batches import BatchFileCountLimitError, BatchNotFoundError, BatchSource
|
from ekp_parser import DocumentParseError
|
||||||
from ekp_core.ingestion_jobs import IngestionJobNotFoundError
|
|
||||||
|
|
||||||
router = APIRouter()
|
router = APIRouter()
|
||||||
|
|
||||||
@ -53,17 +44,17 @@ Services = Annotated[ApiServices, Depends(get_api_services)]
|
|||||||
|
|
||||||
@router.post(
|
@router.post(
|
||||||
"/documents",
|
"/documents",
|
||||||
response_model=IngestionAcceptedResponse,
|
response_model=DocumentResponse,
|
||||||
status_code=status.HTTP_202_ACCEPTED,
|
status_code=status.HTTP_201_CREATED,
|
||||||
tags=["documents"],
|
tags=["documents"],
|
||||||
)
|
)
|
||||||
def upload_document(
|
def upload_document(
|
||||||
services: Services,
|
services: Services,
|
||||||
file: Annotated[UploadFile, File(description="One PDF, DOCX, TXT, or Markdown file")],
|
file: Annotated[UploadFile, File(description="One PDF, DOCX, TXT, or Markdown file")],
|
||||||
) -> IngestionAcceptedResponse:
|
) -> DocumentResponse:
|
||||||
"""Persist one immutable source and durable Job, then return immediately."""
|
"""Upload one file and synchronously run the complete Demo ingestion pipeline."""
|
||||||
try:
|
try:
|
||||||
submission = services.submissions.submit(file.filename or "", file.content_type, file.file)
|
result = services.ingestion.ingest(file.filename or "", file.content_type, file.file)
|
||||||
except DocumentTooLargeError as error:
|
except DocumentTooLargeError as error:
|
||||||
raise HTTPException(
|
raise HTTPException(
|
||||||
status_code=status.HTTP_413_CONTENT_TOO_LARGE,
|
status_code=status.HTTP_413_CONTENT_TOO_LARGE,
|
||||||
@ -74,85 +65,21 @@ def upload_document(
|
|||||||
status_code=status.HTTP_415_UNSUPPORTED_MEDIA_TYPE,
|
status_code=status.HTTP_415_UNSUPPORTED_MEDIA_TYPE,
|
||||||
detail="unsupported document type or media type",
|
detail="unsupported document type or media type",
|
||||||
) from error
|
) from error
|
||||||
except DocumentUploadError as error:
|
except (DocumentUploadError, DocumentParseError) as error:
|
||||||
raise HTTPException(
|
raise HTTPException(
|
||||||
status_code=status.HTTP_422_UNPROCESSABLE_CONTENT,
|
status_code=status.HTTP_422_UNPROCESSABLE_CONTENT,
|
||||||
detail="document could not be accepted",
|
detail="document could not be accepted or parsed",
|
||||||
) from error
|
) from error
|
||||||
except Exception as error:
|
except Exception as error:
|
||||||
# Provider and persistence exceptions are intentionally not reflected to clients.
|
# Provider and persistence exceptions are intentionally not reflected to clients.
|
||||||
raise HTTPException(
|
raise HTTPException(
|
||||||
status_code=status.HTTP_502_BAD_GATEWAY,
|
status_code=status.HTTP_502_BAD_GATEWAY,
|
||||||
detail="document acceptance failed",
|
detail="document processing failed",
|
||||||
) from error
|
) from error
|
||||||
return IngestionAcceptedResponse.from_submission(submission)
|
# Reload after every pipeline stage so server-generated timestamps and the
|
||||||
|
# final persisted status, not a detached intermediate ORM object, are returned.
|
||||||
|
document = services.documents.get_document(result.ingestion.document.id)
|
||||||
@router.get("/jobs/{job_id}", response_model=IngestionJobResponse, tags=["jobs"])
|
return DocumentResponse.from_document(document)
|
||||||
def get_ingestion_job(job_id: UUID, services: Services) -> IngestionJobResponse:
|
|
||||||
"""Return durable stage, attempt and safe failure state for one Job."""
|
|
||||||
try:
|
|
||||||
job = services.jobs.get(job_id)
|
|
||||||
except IngestionJobNotFoundError as error:
|
|
||||||
raise HTTPException(
|
|
||||||
status_code=status.HTTP_404_NOT_FOUND, detail="job not found"
|
|
||||||
) from error
|
|
||||||
return IngestionJobResponse.from_job(job)
|
|
||||||
|
|
||||||
|
|
||||||
@router.post(
|
|
||||||
"/ingestion-batches",
|
|
||||||
response_model=IngestionBatchResponse,
|
|
||||||
status_code=status.HTTP_202_ACCEPTED,
|
|
||||||
tags=["jobs"],
|
|
||||||
)
|
|
||||||
def upload_ingestion_batch(
|
|
||||||
services: Services,
|
|
||||||
files: Annotated[
|
|
||||||
list[UploadFile],
|
|
||||||
File(description="One or more PDF, DOCX, TXT, or Markdown files"),
|
|
||||||
],
|
|
||||||
) -> IngestionBatchResponse:
|
|
||||||
"""Accept a bounded batch while isolating every file's result."""
|
|
||||||
try:
|
|
||||||
view = services.batches.submit(
|
|
||||||
tuple(
|
|
||||||
BatchSource(
|
|
||||||
filename=file.filename or "",
|
|
||||||
media_type=file.content_type,
|
|
||||||
source=file.file,
|
|
||||||
)
|
|
||||||
for file in files
|
|
||||||
)
|
|
||||||
)
|
|
||||||
except BatchFileCountLimitError as error:
|
|
||||||
raise HTTPException(
|
|
||||||
status_code=status.HTTP_413_CONTENT_TOO_LARGE,
|
|
||||||
detail="batch file count exceeds the configured limit",
|
|
||||||
) from error
|
|
||||||
except Exception as error:
|
|
||||||
raise HTTPException(
|
|
||||||
status_code=status.HTTP_502_BAD_GATEWAY,
|
|
||||||
detail="batch could not be accepted",
|
|
||||||
) from error
|
|
||||||
return IngestionBatchResponse.from_view(view)
|
|
||||||
|
|
||||||
|
|
||||||
@router.get(
|
|
||||||
"/ingestion-batches/{batch_id}",
|
|
||||||
response_model=IngestionBatchResponse,
|
|
||||||
tags=["jobs"],
|
|
||||||
)
|
|
||||||
def get_ingestion_batch(batch_id: UUID, services: Services) -> IngestionBatchResponse:
|
|
||||||
"""Return aggregate and per-file status for Admin Console polling."""
|
|
||||||
try:
|
|
||||||
view = services.batches.get(batch_id)
|
|
||||||
except BatchNotFoundError as error:
|
|
||||||
raise HTTPException(
|
|
||||||
status_code=status.HTTP_404_NOT_FOUND,
|
|
||||||
detail="ingestion batch not found",
|
|
||||||
) from error
|
|
||||||
return IngestionBatchResponse.from_view(view)
|
|
||||||
|
|
||||||
|
|
||||||
@router.get("/documents", response_model=list[DocumentResponse], tags=["documents"])
|
@router.get("/documents", response_model=list[DocumentResponse], tags=["documents"])
|
||||||
@ -188,11 +115,6 @@ def delete_document(document_id: UUID, services: Services) -> Response:
|
|||||||
raise HTTPException(
|
raise HTTPException(
|
||||||
status_code=status.HTTP_404_NOT_FOUND, detail="document not found"
|
status_code=status.HTTP_404_NOT_FOUND, detail="document not found"
|
||||||
) from error
|
) from error
|
||||||
except DocumentIngestionActiveError as error:
|
|
||||||
raise HTTPException(
|
|
||||||
status_code=status.HTTP_409_CONFLICT,
|
|
||||||
detail="document ingestion is still active",
|
|
||||||
) from error
|
|
||||||
except DocumentArtifactCleanupError as error:
|
except DocumentArtifactCleanupError as error:
|
||||||
raise HTTPException(
|
raise HTTPException(
|
||||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||||
|
|||||||
@ -5,17 +5,8 @@ from uuid import UUID
|
|||||||
|
|
||||||
from pydantic import BaseModel, ConfigDict, Field
|
from pydantic import BaseModel, ConfigDict, Field
|
||||||
|
|
||||||
from ekp_core.ingestion_batches import BatchAggregateStatus, BatchItemView, BatchView
|
|
||||||
from ekp_core.ingestion_jobs import IngestionSubmission
|
|
||||||
from ekp_core.search import SearchResult
|
from ekp_core.search import SearchResult
|
||||||
from ekp_domain import (
|
from ekp_domain import Document, DocumentStatus
|
||||||
BatchItemAcceptanceStatus,
|
|
||||||
Document,
|
|
||||||
DocumentStatus,
|
|
||||||
IngestionJob,
|
|
||||||
IngestionJobStatus,
|
|
||||||
IngestionStage,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
class DocumentResponse(BaseModel):
|
class DocumentResponse(BaseModel):
|
||||||
@ -38,138 +29,6 @@ class DocumentResponse(BaseModel):
|
|||||||
return cls.model_validate(document)
|
return cls.model_validate(document)
|
||||||
|
|
||||||
|
|
||||||
class IngestionAcceptedResponse(DocumentResponse):
|
|
||||||
"""Single-file acceptance with the durable Job needed for status polling."""
|
|
||||||
|
|
||||||
job_id: UUID
|
|
||||||
job_status: IngestionJobStatus
|
|
||||||
max_attempts: int
|
|
||||||
|
|
||||||
@classmethod
|
|
||||||
def from_submission(cls, submission: IngestionSubmission) -> "IngestionAcceptedResponse":
|
|
||||||
"""Flatten identifiers while retaining the existing document fields."""
|
|
||||||
document = submission.document
|
|
||||||
job = submission.job
|
|
||||||
return cls(
|
|
||||||
id=document.id,
|
|
||||||
filename=document.filename,
|
|
||||||
media_type=document.media_type,
|
|
||||||
size_bytes=document.size_bytes,
|
|
||||||
status=document.status,
|
|
||||||
error_message=document.error_message,
|
|
||||||
created_at=document.created_at,
|
|
||||||
updated_at=document.updated_at,
|
|
||||||
job_id=job.id,
|
|
||||||
job_status=job.status,
|
|
||||||
max_attempts=job.max_attempts,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
class IngestionJobResponse(BaseModel):
|
|
||||||
"""Public queue-independent Job status without provider or broker details."""
|
|
||||||
|
|
||||||
id: UUID
|
|
||||||
document_id: UUID
|
|
||||||
status: IngestionJobStatus
|
|
||||||
stage: IngestionStage | None
|
|
||||||
attempt: int
|
|
||||||
max_attempts: int
|
|
||||||
error_message: str | None
|
|
||||||
created_at: datetime
|
|
||||||
updated_at: datetime
|
|
||||||
completed_at: datetime | None
|
|
||||||
|
|
||||||
@classmethod
|
|
||||||
def from_job(cls, job: IngestionJob) -> "IngestionJobResponse":
|
|
||||||
"""Serialize only client-actionable state and safe diagnostics."""
|
|
||||||
return cls(
|
|
||||||
id=job.id,
|
|
||||||
document_id=job.document_id,
|
|
||||||
status=job.status,
|
|
||||||
stage=job.stage,
|
|
||||||
attempt=job.attempt_count,
|
|
||||||
max_attempts=job.max_attempts,
|
|
||||||
error_message=job.error_message,
|
|
||||||
created_at=job.created_at,
|
|
||||||
updated_at=job.updated_at,
|
|
||||||
completed_at=job.completed_at,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
class IngestionBatchItemResponse(BaseModel):
|
|
||||||
"""Per-file acceptance and latest asynchronous processing state."""
|
|
||||||
|
|
||||||
item_index: int
|
|
||||||
filename: str
|
|
||||||
acceptance_status: BatchItemAcceptanceStatus
|
|
||||||
document_id: UUID | None
|
|
||||||
job_id: UUID | None
|
|
||||||
job_status: IngestionJobStatus | None
|
|
||||||
stage: IngestionStage | None
|
|
||||||
attempt: int
|
|
||||||
max_attempts: int | None
|
|
||||||
error_message: str | None
|
|
||||||
|
|
||||||
@classmethod
|
|
||||||
def from_view(cls, view: BatchItemView) -> "IngestionBatchItemResponse":
|
|
||||||
job = view.job
|
|
||||||
return cls(
|
|
||||||
item_index=view.item.item_index,
|
|
||||||
filename=view.item.filename,
|
|
||||||
acceptance_status=view.item.acceptance_status,
|
|
||||||
document_id=view.item.document_id,
|
|
||||||
job_id=view.item.job_id,
|
|
||||||
job_status=job.status if job is not None else None,
|
|
||||||
stage=job.stage if job is not None else None,
|
|
||||||
attempt=job.attempt_count if job is not None else 0,
|
|
||||||
max_attempts=job.max_attempts if job is not None else None,
|
|
||||||
error_message=(job.error_message if job is not None else view.item.error_message),
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
class IngestionBatchResponse(BaseModel):
|
|
||||||
"""Batch aggregate with ordered independently processed files."""
|
|
||||||
|
|
||||||
batch_id: UUID
|
|
||||||
status: BatchAggregateStatus
|
|
||||||
total_files: int
|
|
||||||
accepted_files: int
|
|
||||||
rejected_files: int
|
|
||||||
completed_files: int
|
|
||||||
created_at: datetime
|
|
||||||
items: list[IngestionBatchItemResponse]
|
|
||||||
|
|
||||||
@classmethod
|
|
||||||
def from_view(cls, view: BatchView) -> "IngestionBatchResponse":
|
|
||||||
items = [IngestionBatchItemResponse.from_view(item) for item in view.items]
|
|
||||||
accepted = sum(
|
|
||||||
item.acceptance_status == BatchItemAcceptanceStatus.ACCEPTED for item in items
|
|
||||||
)
|
|
||||||
rejected = sum(
|
|
||||||
item.acceptance_status == BatchItemAcceptanceStatus.REJECTED for item in items
|
|
||||||
)
|
|
||||||
completed = sum(
|
|
||||||
item.acceptance_status == BatchItemAcceptanceStatus.REJECTED
|
|
||||||
or item.job_status
|
|
||||||
in {
|
|
||||||
IngestionJobStatus.SUCCEEDED,
|
|
||||||
IngestionJobStatus.FAILED,
|
|
||||||
IngestionJobStatus.CANCELLED,
|
|
||||||
}
|
|
||||||
for item in items
|
|
||||||
)
|
|
||||||
return cls(
|
|
||||||
batch_id=view.batch.id,
|
|
||||||
status=view.status,
|
|
||||||
total_files=view.batch.total_files,
|
|
||||||
accepted_files=accepted,
|
|
||||||
rejected_files=rejected,
|
|
||||||
completed_files=completed,
|
|
||||||
created_at=view.batch.created_at,
|
|
||||||
items=items,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
class SearchRequest(BaseModel):
|
class SearchRequest(BaseModel):
|
||||||
"""Validated public search input; normalization remains in the core service."""
|
"""Validated public search input; normalization remains in the core service."""
|
||||||
|
|
||||||
|
|||||||
@ -1,51 +0,0 @@
|
|||||||
"""Celery worker entry point for durable ingestion Jobs."""
|
|
||||||
|
|
||||||
from functools import lru_cache
|
|
||||||
from typing import Any
|
|
||||||
from uuid import UUID
|
|
||||||
|
|
||||||
from ekp_api.celery_queue import (
|
|
||||||
INGESTION_TASK_NAME,
|
|
||||||
RECOVERY_TASK_NAME,
|
|
||||||
create_celery_app,
|
|
||||||
)
|
|
||||||
from ekp_api.providers import WorkerServices, create_worker_services
|
|
||||||
from ekp_core.config import get_settings
|
|
||||||
from ekp_core.ingestion_jobs import JobRunAction, dispatch_pending_jobs
|
|
||||||
|
|
||||||
_settings = get_settings()
|
|
||||||
celery_app = create_celery_app(_settings)
|
|
||||||
|
|
||||||
|
|
||||||
@lru_cache
|
|
||||||
def _worker_services() -> WorkerServices:
|
|
||||||
"""Build one provider/service graph per worker process."""
|
|
||||||
return create_worker_services(_settings)
|
|
||||||
|
|
||||||
|
|
||||||
@celery_app.task( # type: ignore[untyped-decorator]
|
|
||||||
bind=True,
|
|
||||||
name=INGESTION_TASK_NAME,
|
|
||||||
max_retries=None,
|
|
||||||
)
|
|
||||||
def run_ingestion(self: Any, job_id: str) -> None:
|
|
||||||
"""Execute one delivery and ask Celery to redeliver only when required.
|
|
||||||
|
|
||||||
Celery delivery retries are unbounded because valid-lease deferrals are not
|
|
||||||
business Attempts; the durable runner independently enforces max_attempts.
|
|
||||||
"""
|
|
||||||
result = _worker_services().runner.run(UUID(job_id))
|
|
||||||
if result.action == JobRunAction.RETRY:
|
|
||||||
raise self.retry(countdown=result.retry_delay_seconds or 1, max_retries=None)
|
|
||||||
|
|
||||||
|
|
||||||
@celery_app.task(name=RECOVERY_TASK_NAME) # type: ignore[untyped-decorator]
|
|
||||||
def recover_ingestion_jobs() -> int:
|
|
||||||
"""Redeliver due durable Jobs after API/broker or worker interruptions."""
|
|
||||||
services = _worker_services()
|
|
||||||
return dispatch_pending_jobs(services.jobs, services.dispatcher)
|
|
||||||
|
|
||||||
|
|
||||||
def main() -> None:
|
|
||||||
"""Run one worker with the embedded Demo recovery scheduler."""
|
|
||||||
celery_app.worker_main(["worker", "--loglevel=INFO", "--queues=ingestion", "--beat"])
|
|
||||||
@ -1,5 +0,0 @@
|
|||||||
"""Enterprise Knowledge MCP gateway."""
|
|
||||||
|
|
||||||
from ekp_mcp.server import create_mcp_http_app, create_mcp_server
|
|
||||||
|
|
||||||
__all__ = ["create_mcp_http_app", "create_mcp_server"]
|
|
||||||
@ -1,76 +0,0 @@
|
|||||||
"""Request-scoped MCP service identity and bearer-token verification."""
|
|
||||||
|
|
||||||
from dataclasses import dataclass
|
|
||||||
from secrets import compare_digest
|
|
||||||
|
|
||||||
from mcp.server.auth.middleware.auth_context import get_access_token
|
|
||||||
from mcp.server.auth.provider import AccessToken
|
|
||||||
from mcp.server.mcpserver.exceptions import ToolError
|
|
||||||
from pydantic import SecretStr
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True, slots=True)
|
|
||||||
class McpIdentity:
|
|
||||||
"""Authenticated client/service identity passed across the application boundary."""
|
|
||||||
|
|
||||||
client_id: str
|
|
||||||
subject: str
|
|
||||||
scopes: frozenset[str]
|
|
||||||
|
|
||||||
|
|
||||||
class StaticServiceTokenVerifier:
|
|
||||||
"""Validate one runtime-injected, least-privilege Demo service credential.
|
|
||||||
|
|
||||||
The verifier is intentionally small for DEMO-012. It establishes the MCP
|
|
||||||
identity boundary without pretending to be the V1 OAuth or ACL solution.
|
|
||||||
"""
|
|
||||||
|
|
||||||
__slots__ = (
|
|
||||||
"_client_id",
|
|
||||||
"_expected_token",
|
|
||||||
"_issuer",
|
|
||||||
"_scope",
|
|
||||||
"_subject",
|
|
||||||
)
|
|
||||||
|
|
||||||
def __init__(
|
|
||||||
self,
|
|
||||||
expected_token: SecretStr,
|
|
||||||
*,
|
|
||||||
client_id: str,
|
|
||||||
subject: str,
|
|
||||||
scope: str,
|
|
||||||
issuer: str,
|
|
||||||
) -> None:
|
|
||||||
self._expected_token = expected_token
|
|
||||||
self._client_id = client_id
|
|
||||||
self._subject = subject
|
|
||||||
self._scope = scope
|
|
||||||
self._issuer = issuer
|
|
||||||
|
|
||||||
async def verify_token(self, token: str) -> AccessToken | None:
|
|
||||||
"""Return scoped identity metadata only for a constant-time token match."""
|
|
||||||
if not compare_digest(token, self._expected_token.get_secret_value()):
|
|
||||||
return None
|
|
||||||
return AccessToken(
|
|
||||||
token=token,
|
|
||||||
client_id=self._client_id,
|
|
||||||
subject=self._subject,
|
|
||||||
scopes=[self._scope],
|
|
||||||
claims={"iss": self._issuer},
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def current_mcp_identity() -> McpIdentity:
|
|
||||||
"""Resolve the SDK-authenticated principal for the active tool request."""
|
|
||||||
access_token = get_access_token()
|
|
||||||
if access_token is None:
|
|
||||||
# HTTP middleware normally rejects this earlier; retaining the guard
|
|
||||||
# prevents accidental unauthenticated in-process tool invocation.
|
|
||||||
raise ToolError("authentication required")
|
|
||||||
subject = access_token.subject or access_token.client_id
|
|
||||||
return McpIdentity(
|
|
||||||
client_id=access_token.client_id,
|
|
||||||
subject=subject,
|
|
||||||
scopes=frozenset(access_token.scopes),
|
|
||||||
)
|
|
||||||
@ -1,30 +0,0 @@
|
|||||||
"""Command-line entry point for the independent MCP gateway process."""
|
|
||||||
|
|
||||||
from mcp.server.transport_security import TransportSecuritySettings
|
|
||||||
|
|
||||||
from ekp_core.config import get_settings
|
|
||||||
from ekp_mcp.server import create_mcp_server
|
|
||||||
|
|
||||||
|
|
||||||
def main() -> None:
|
|
||||||
"""Run the authenticated Streamable HTTP server from environment settings."""
|
|
||||||
settings = get_settings()
|
|
||||||
server = create_mcp_server(settings)
|
|
||||||
server.run(
|
|
||||||
transport="streamable-http",
|
|
||||||
host=settings.mcp_host,
|
|
||||||
port=settings.mcp_port,
|
|
||||||
streamable_http_path=settings.mcp_path,
|
|
||||||
json_response=True,
|
|
||||||
stateless_http=True,
|
|
||||||
max_request_body_size=settings.mcp_max_request_body_bytes,
|
|
||||||
transport_security=TransportSecuritySettings(
|
|
||||||
enable_dns_rebinding_protection=True,
|
|
||||||
allowed_hosts=settings.mcp_allowed_hosts,
|
|
||||||
allowed_origins=settings.mcp_allowed_origins,
|
|
||||||
),
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__": # pragma: no cover - exercised through the console script.
|
|
||||||
main()
|
|
||||||
@ -1,77 +0,0 @@
|
|||||||
"""MCP-specific application boundary over the provider-neutral SearchService."""
|
|
||||||
|
|
||||||
from dataclasses import dataclass
|
|
||||||
from typing import Protocol
|
|
||||||
from uuid import UUID
|
|
||||||
|
|
||||||
from pydantic import BaseModel
|
|
||||||
|
|
||||||
from ekp_core.search import SearchResult
|
|
||||||
from ekp_mcp.auth import McpIdentity
|
|
||||||
|
|
||||||
|
|
||||||
class KnowledgeSearcher(Protocol):
|
|
||||||
"""Subset of SearchService required by the MCP gateway."""
|
|
||||||
|
|
||||||
def search(self, query: str, top_k: int | None = None) -> SearchResult:
|
|
||||||
"""Return reranked canonical sources for a normalized query."""
|
|
||||||
|
|
||||||
|
|
||||||
class McpSourceResponse(BaseModel):
|
|
||||||
"""Provider-neutral source shape exposed by `search_knowledge`."""
|
|
||||||
|
|
||||||
document_id: UUID
|
|
||||||
filename: str
|
|
||||||
page: int | None
|
|
||||||
section: str | None
|
|
||||||
chunk_id: UUID
|
|
||||||
content: str
|
|
||||||
score: float
|
|
||||||
rerank_score: float
|
|
||||||
|
|
||||||
|
|
||||||
class McpSearchResponse(BaseModel):
|
|
||||||
"""Structured MCP result aligned with the REST search source fields."""
|
|
||||||
|
|
||||||
results: list[McpSourceResponse]
|
|
||||||
|
|
||||||
|
|
||||||
class McpKnowledgeSearch(Protocol):
|
|
||||||
"""Identity-aware search boundary consumed by the protocol server."""
|
|
||||||
|
|
||||||
def search(self, identity: McpIdentity, query: str, top_k: int) -> McpSearchResponse:
|
|
||||||
"""Return authorized, structured knowledge sources."""
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True, slots=True)
|
|
||||||
class McpSearchApplication:
|
|
||||||
"""Require an identity before delegating to the shared SearchService.
|
|
||||||
|
|
||||||
Document ACL is outside the current Demo scope. Keeping identity explicit
|
|
||||||
here prevents the gateway from becoming an anonymous bypass when the
|
|
||||||
permission resolver is added in the V1 tasks.
|
|
||||||
"""
|
|
||||||
|
|
||||||
searcher: KnowledgeSearcher
|
|
||||||
required_scope: str
|
|
||||||
|
|
||||||
def search(self, identity: McpIdentity, query: str, top_k: int) -> McpSearchResponse:
|
|
||||||
"""Authorize the Demo scope and serialize canonical search results."""
|
|
||||||
if self.required_scope not in identity.scopes:
|
|
||||||
raise PermissionError("MCP identity lacks the knowledge search scope")
|
|
||||||
result = self.searcher.search(query, top_k)
|
|
||||||
return McpSearchResponse(
|
|
||||||
results=[
|
|
||||||
McpSourceResponse(
|
|
||||||
document_id=source.document_id,
|
|
||||||
filename=source.filename,
|
|
||||||
page=source.page,
|
|
||||||
section=source.section,
|
|
||||||
chunk_id=source.chunk_id,
|
|
||||||
content=source.content,
|
|
||||||
score=source.score,
|
|
||||||
rerank_score=source.rerank_score,
|
|
||||||
)
|
|
||||||
for source in result.results
|
|
||||||
]
|
|
||||||
)
|
|
||||||
@ -1,133 +0,0 @@
|
|||||||
"""Authenticated Streamable HTTP MCP server exposing Demo business tools."""
|
|
||||||
|
|
||||||
import logging
|
|
||||||
from functools import partial
|
|
||||||
from typing import Annotated
|
|
||||||
|
|
||||||
import anyio
|
|
||||||
from mcp.server import MCPServer
|
|
||||||
from mcp.server.auth.settings import AuthSettings
|
|
||||||
from mcp.server.mcpserver.exceptions import ToolError
|
|
||||||
from mcp.server.transport_security import TransportSecuritySettings
|
|
||||||
from pydantic import Field
|
|
||||||
from starlette.applications import Starlette
|
|
||||||
|
|
||||||
from ekp_api.providers import ApiServices, create_runtime_search_service
|
|
||||||
from ekp_core.config import Settings, get_settings
|
|
||||||
from ekp_core.dense_retrieval import InvalidRetrievalQueryError
|
|
||||||
from ekp_mcp.auth import StaticServiceTokenVerifier, current_mcp_identity
|
|
||||||
from ekp_mcp.search import McpKnowledgeSearch, McpSearchApplication, McpSearchResponse
|
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
|
||||||
|
|
||||||
|
|
||||||
class McpConfigurationError(ValueError):
|
|
||||||
"""Raised when the gateway lacks its dedicated runtime credential."""
|
|
||||||
|
|
||||||
|
|
||||||
def _create_search_application(
|
|
||||||
settings: Settings,
|
|
||||||
api_services: ApiServices | None,
|
|
||||||
) -> McpSearchApplication:
|
|
||||||
searcher = (
|
|
||||||
api_services.search if api_services is not None else create_runtime_search_service(settings)
|
|
||||||
)
|
|
||||||
return McpSearchApplication(searcher, settings.mcp_required_scope)
|
|
||||||
|
|
||||||
|
|
||||||
def create_mcp_server(
|
|
||||||
settings: Settings | None = None,
|
|
||||||
*,
|
|
||||||
api_services: ApiServices | None = None,
|
|
||||||
search_application: McpKnowledgeSearch | None = None,
|
|
||||||
) -> MCPServer[dict[str, object]]:
|
|
||||||
"""Create the authenticated MCP gateway with only `search_knowledge`."""
|
|
||||||
resolved_settings = settings or get_settings()
|
|
||||||
if (
|
|
||||||
resolved_settings.mcp_auth_token is None
|
|
||||||
or len(resolved_settings.mcp_auth_token.get_secret_value()) < 32
|
|
||||||
):
|
|
||||||
raise McpConfigurationError("MCP_AUTH_TOKEN must contain at least 32 characters")
|
|
||||||
|
|
||||||
application = search_application or _create_search_application(
|
|
||||||
resolved_settings,
|
|
||||||
api_services,
|
|
||||||
)
|
|
||||||
token_verifier = StaticServiceTokenVerifier(
|
|
||||||
resolved_settings.mcp_auth_token,
|
|
||||||
client_id=resolved_settings.mcp_client_id,
|
|
||||||
subject=resolved_settings.mcp_service_identity,
|
|
||||||
scope=resolved_settings.mcp_required_scope,
|
|
||||||
issuer=str(resolved_settings.mcp_issuer_url),
|
|
||||||
)
|
|
||||||
server: MCPServer[dict[str, object]] = MCPServer(
|
|
||||||
name="enterprise-knowledge",
|
|
||||||
title="Enterprise Knowledge MCP Server",
|
|
||||||
description="Authenticated enterprise knowledge search backed by canonical sources.",
|
|
||||||
version="0.1.0",
|
|
||||||
token_verifier=token_verifier,
|
|
||||||
auth=AuthSettings(
|
|
||||||
issuer_url=resolved_settings.mcp_issuer_url,
|
|
||||||
resource_server_url=resolved_settings.mcp_public_url,
|
|
||||||
required_scopes=[resolved_settings.mcp_required_scope],
|
|
||||||
),
|
|
||||||
log_level=resolved_settings.log_level,
|
|
||||||
)
|
|
||||||
|
|
||||||
@server.tool(
|
|
||||||
name="search_knowledge",
|
|
||||||
description=(
|
|
||||||
"Search authorized enterprise knowledge and return traceable source passages. "
|
|
||||||
"Use for factual questions that should be grounded in indexed company documents."
|
|
||||||
),
|
|
||||||
structured_output=True,
|
|
||||||
)
|
|
||||||
async def search_knowledge(
|
|
||||||
query: Annotated[str, Field(min_length=1, max_length=1600)],
|
|
||||||
top_k: Annotated[int, Field(ge=1, le=64, strict=True)] = 5,
|
|
||||||
) -> McpSearchResponse:
|
|
||||||
"""Run the shared synchronous search pipeline outside the ASGI event loop."""
|
|
||||||
identity = current_mcp_identity()
|
|
||||||
try:
|
|
||||||
return await anyio.to_thread.run_sync(
|
|
||||||
partial(application.search, identity, query, top_k)
|
|
||||||
)
|
|
||||||
except InvalidRetrievalQueryError as error:
|
|
||||||
raise ToolError("search query must contain non-whitespace text") from error
|
|
||||||
except PermissionError as error:
|
|
||||||
raise ToolError("knowledge search is not authorized") from error
|
|
||||||
except Exception:
|
|
||||||
# Do not reflect provider, database, endpoint, or credential details
|
|
||||||
# into MCP content or logs. Operations can correlate this generic
|
|
||||||
# event with upstream sanitized service telemetry.
|
|
||||||
logger.error("MCP knowledge search failed")
|
|
||||||
raise ToolError("knowledge search failed") from None
|
|
||||||
|
|
||||||
return server
|
|
||||||
|
|
||||||
|
|
||||||
def create_mcp_http_app(
|
|
||||||
settings: Settings | None = None,
|
|
||||||
*,
|
|
||||||
api_services: ApiServices | None = None,
|
|
||||||
search_application: McpKnowledgeSearch | None = None,
|
|
||||||
) -> Starlette:
|
|
||||||
"""Build the stateless Streamable HTTP app used by WorkBuddy and tests."""
|
|
||||||
resolved_settings = settings or get_settings()
|
|
||||||
server = create_mcp_server(
|
|
||||||
resolved_settings,
|
|
||||||
api_services=api_services,
|
|
||||||
search_application=search_application,
|
|
||||||
)
|
|
||||||
return server.streamable_http_app(
|
|
||||||
streamable_http_path=resolved_settings.mcp_path,
|
|
||||||
json_response=True,
|
|
||||||
stateless_http=True,
|
|
||||||
max_request_body_size=resolved_settings.mcp_max_request_body_bytes,
|
|
||||||
host=resolved_settings.mcp_host,
|
|
||||||
transport_security=TransportSecuritySettings(
|
|
||||||
enable_dns_rebinding_protection=True,
|
|
||||||
allowed_hosts=resolved_settings.mcp_allowed_hosts,
|
|
||||||
allowed_origins=resolved_settings.mcp_allowed_origins,
|
|
||||||
),
|
|
||||||
)
|
|
||||||
@ -1,61 +0,0 @@
|
|||||||
# syntax=docker/dockerfile:1.7
|
|
||||||
# One locked runtime produces three independently deployable process images.
|
|
||||||
ARG PYTHON_IMAGE=python:3.12.14-slim-bookworm@sha256:782412e85d0f0984994c290652577d4018aff08145c85b262bb63dc0c7522254
|
|
||||||
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.12.5@sha256:e85be844203885286c60ffad8a858d48afb6c5a5c237ca0e67f12e74b8f174b1
|
|
||||||
|
|
||||||
FROM ${UV_IMAGE} AS uv-bin
|
|
||||||
|
|
||||||
FROM ${PYTHON_IMAGE} AS runtime
|
|
||||||
|
|
||||||
ENV PYTHONUNBUFFERED=1 \
|
|
||||||
PYTHONDONTWRITEBYTECODE=1 \
|
|
||||||
UV_COMPILE_BYTECODE=1 \
|
|
||||||
UV_LINK_MODE=copy \
|
|
||||||
VIRTUAL_ENV=/app/.venv \
|
|
||||||
PATH=/app/.venv/bin:${PATH} \
|
|
||||||
HOME=/tmp/ekp \
|
|
||||||
XDG_CACHE_HOME=/tmp/ekp/cache \
|
|
||||||
APP_ENV=production \
|
|
||||||
API_HOST=0.0.0.0 \
|
|
||||||
MCP_HOST=0.0.0.0 \
|
|
||||||
ADMIN_WEB_ROOT=/app/apps/admin-web
|
|
||||||
|
|
||||||
WORKDIR /app
|
|
||||||
|
|
||||||
COPY --from=uv-bin /uv /uvx /usr/local/bin/
|
|
||||||
COPY pyproject.toml uv.lock README.md alembic.ini ./
|
|
||||||
COPY apps ./apps
|
|
||||||
COPY packages ./packages
|
|
||||||
COPY migrations ./migrations
|
|
||||||
|
|
||||||
# --frozen prevents lock changes; --no-dev and --no-editable keep the runtime
|
|
||||||
# independent from source-tree import hooks while retaining migration sources.
|
|
||||||
RUN uv sync --frozen --no-dev --no-editable \
|
|
||||||
&& test -f /app/apps/admin-web/index.html \
|
|
||||||
&& groupadd --gid 10001 ekp \
|
|
||||||
&& useradd --uid 10001 --gid 10001 --no-create-home --shell /usr/sbin/nologin ekp \
|
|
||||||
&& mkdir -p /tmp/ekp/cache \
|
|
||||||
&& chown -R 10001:10001 /tmp/ekp \
|
|
||||||
&& rm -rf /root/.cache/uv
|
|
||||||
|
|
||||||
USER 10001:10001
|
|
||||||
|
|
||||||
FROM runtime AS api
|
|
||||||
EXPOSE 8000
|
|
||||||
# This endpoint is intentionally credential-free and suitable for container liveness.
|
|
||||||
HEALTHCHECK --interval=30s --timeout=3s --start-period=10s --retries=3 \
|
|
||||||
CMD ["python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/healthz', timeout=2)"]
|
|
||||||
CMD ["uvicorn", "ekp_api.main:app", "--host", "0.0.0.0", "--port", "8000", "--no-access-log"]
|
|
||||||
|
|
||||||
FROM runtime AS mcp
|
|
||||||
EXPOSE 8766
|
|
||||||
# MCP requests require authentication, so the image-level check only verifies PID 1.
|
|
||||||
HEALTHCHECK --interval=30s --timeout=3s --start-period=10s --retries=3 \
|
|
||||||
CMD ["python", "-c", "import os; os.kill(1, 0)"]
|
|
||||||
CMD ["ekp-mcp"]
|
|
||||||
|
|
||||||
FROM runtime AS worker
|
|
||||||
# Worker health is process-level here; durable Job recovery is validated by the service tests.
|
|
||||||
HEALTHCHECK --interval=30s --timeout=3s --start-period=10s --retries=3 \
|
|
||||||
CMD ["python", "-c", "import os; os.kill(1, 0)"]
|
|
||||||
CMD ["ekp-worker"]
|
|
||||||
@ -1,36 +0,0 @@
|
|||||||
# Reproducible target names consumed by local and CI buildx invocations.
|
|
||||||
variable "IMAGE_REGISTRY" {
|
|
||||||
default = "knowledge-platform"
|
|
||||||
}
|
|
||||||
|
|
||||||
variable "IMAGE_TAG" {
|
|
||||||
default = "dev"
|
|
||||||
}
|
|
||||||
|
|
||||||
group "default" {
|
|
||||||
targets = ["api", "mcp", "worker"]
|
|
||||||
}
|
|
||||||
|
|
||||||
target "common" {
|
|
||||||
context = "."
|
|
||||||
dockerfile = "deploy/docker/Dockerfile"
|
|
||||||
platforms = ["linux/amd64", "linux/arm64"]
|
|
||||||
}
|
|
||||||
|
|
||||||
target "api" {
|
|
||||||
inherits = ["common"]
|
|
||||||
target = "api"
|
|
||||||
tags = ["${IMAGE_REGISTRY}/enterprise-knowledge-api:${IMAGE_TAG}"]
|
|
||||||
}
|
|
||||||
|
|
||||||
target "mcp" {
|
|
||||||
inherits = ["common"]
|
|
||||||
target = "mcp"
|
|
||||||
tags = ["${IMAGE_REGISTRY}/enterprise-knowledge-mcp:${IMAGE_TAG}"]
|
|
||||||
}
|
|
||||||
|
|
||||||
target "worker" {
|
|
||||||
inherits = ["common"]
|
|
||||||
target = "worker"
|
|
||||||
tags = ["${IMAGE_REGISTRY}/enterprise-knowledge-worker:${IMAGE_TAG}"]
|
|
||||||
}
|
|
||||||
@ -1,50 +0,0 @@
|
|||||||
# Copy this file outside the repository for one environment. It contains only
|
|
||||||
# non-sensitive settings and Kubernetes Secret references; never add values for
|
|
||||||
# DATABASE_URL, RABBITMQ_URL, provider keys, passwords or MCP_AUTH_TOKEN here.
|
|
||||||
existingSecret: knowledge-platform-runtime
|
|
||||||
|
|
||||||
images:
|
|
||||||
api:
|
|
||||||
repository: registry.example.internal/knowledge/enterprise-knowledge-api
|
|
||||||
tag: "0.1.0"
|
|
||||||
digest: ""
|
|
||||||
mcp:
|
|
||||||
repository: registry.example.internal/knowledge/enterprise-knowledge-mcp
|
|
||||||
tag: "0.1.0"
|
|
||||||
digest: ""
|
|
||||||
worker:
|
|
||||||
repository: registry.example.internal/knowledge/enterprise-knowledge-worker
|
|
||||||
tag: "0.1.0"
|
|
||||||
digest: ""
|
|
||||||
|
|
||||||
ingress:
|
|
||||||
enabled: true
|
|
||||||
className: nginx
|
|
||||||
api:
|
|
||||||
host: knowledge.example.internal
|
|
||||||
tlsSecretName: knowledge-platform-api-tls
|
|
||||||
mcp:
|
|
||||||
host: mcp.example.internal
|
|
||||||
tlsSecretName: knowledge-platform-mcp-tls
|
|
||||||
|
|
||||||
config:
|
|
||||||
objectStorage:
|
|
||||||
endpoint: https://s3.example.internal
|
|
||||||
bucket: knowledge-platform
|
|
||||||
vectorStore:
|
|
||||||
uri: oceanbase.example.internal:2881
|
|
||||||
user: knowledge-platform
|
|
||||||
database: knowledge_platform
|
|
||||||
mcp:
|
|
||||||
publicUrl: https://mcp.example.internal/mcp
|
|
||||||
issuerUrl: https://mcp.example.internal
|
|
||||||
allowedHosts:
|
|
||||||
- mcp.example.internal
|
|
||||||
allowedOrigins:
|
|
||||||
- https://mcp.example.internal
|
|
||||||
|
|
||||||
# External RabbitMQ is the production default. If this local dependency is
|
|
||||||
# explicitly enabled for a demo cluster, create its Secret before deployment.
|
|
||||||
rabbitmq:
|
|
||||||
enabled: false
|
|
||||||
existingSecret: knowledge-platform-rabbitmq
|
|
||||||
@ -1,15 +0,0 @@
|
|||||||
apiVersion: v2
|
|
||||||
name: knowledge-platform
|
|
||||||
description: Enterprise Knowledge Platform public-knowledge deployment
|
|
||||||
type: application
|
|
||||||
version: 0.1.0
|
|
||||||
appVersion: "0.1.0"
|
|
||||||
|
|
||||||
# This local dependency is disabled by default. Production deployments should
|
|
||||||
# use an externally managed RabbitMQ service with an independently owned data
|
|
||||||
# lifecycle; the subchart exists only for explicit development/demo use.
|
|
||||||
dependencies:
|
|
||||||
- name: rabbitmq
|
|
||||||
version: 0.1.0
|
|
||||||
repository: file://charts/rabbitmq
|
|
||||||
condition: rabbitmq.enabled
|
|
||||||
@ -1,6 +0,0 @@
|
|||||||
apiVersion: v2
|
|
||||||
name: rabbitmq
|
|
||||||
description: Minimal single-node RabbitMQ dependency for local demo clusters only
|
|
||||||
type: application
|
|
||||||
version: 0.1.0
|
|
||||||
appVersion: "4.1.4"
|
|
||||||
@ -1,11 +0,0 @@
|
|||||||
{{- define "rabbitmq.fullname" -}}
|
|
||||||
{{- printf "%s-rabbitmq" .Release.Name | trunc 63 | trimSuffix "-" -}}
|
|
||||||
{{- end -}}
|
|
||||||
|
|
||||||
{{- define "rabbitmq.image" -}}
|
|
||||||
{{- if .Values.image.digest -}}
|
|
||||||
{{- printf "%s@%s" .Values.image.repository .Values.image.digest -}}
|
|
||||||
{{- else -}}
|
|
||||||
{{- printf "%s:%s" .Values.image.repository .Values.image.tag -}}
|
|
||||||
{{- end -}}
|
|
||||||
{{- end -}}
|
|
||||||
@ -1,16 +0,0 @@
|
|||||||
apiVersion: v1
|
|
||||||
kind: Service
|
|
||||||
metadata:
|
|
||||||
name: {{ include "rabbitmq.fullname" . }}
|
|
||||||
labels:
|
|
||||||
app.kubernetes.io/name: rabbitmq
|
|
||||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
|
||||||
spec:
|
|
||||||
clusterIP: None
|
|
||||||
ports:
|
|
||||||
- name: amqp
|
|
||||||
port: {{ .Values.service.port }}
|
|
||||||
targetPort: amqp
|
|
||||||
selector:
|
|
||||||
app.kubernetes.io/name: rabbitmq
|
|
||||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
|
||||||
@ -1,8 +0,0 @@
|
|||||||
apiVersion: v1
|
|
||||||
kind: ServiceAccount
|
|
||||||
metadata:
|
|
||||||
name: {{ include "rabbitmq.fullname" . }}
|
|
||||||
labels:
|
|
||||||
app.kubernetes.io/name: rabbitmq
|
|
||||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
|
||||||
automountServiceAccountToken: false
|
|
||||||
@ -1,109 +0,0 @@
|
|||||||
apiVersion: apps/v1
|
|
||||||
kind: StatefulSet
|
|
||||||
metadata:
|
|
||||||
name: {{ include "rabbitmq.fullname" . }}
|
|
||||||
labels:
|
|
||||||
app.kubernetes.io/name: rabbitmq
|
|
||||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
|
||||||
spec:
|
|
||||||
serviceName: {{ include "rabbitmq.fullname" . }}
|
|
||||||
replicas: 1
|
|
||||||
selector:
|
|
||||||
matchLabels:
|
|
||||||
app.kubernetes.io/name: rabbitmq
|
|
||||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
|
||||||
template:
|
|
||||||
metadata:
|
|
||||||
labels:
|
|
||||||
app.kubernetes.io/name: rabbitmq
|
|
||||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
|
||||||
spec:
|
|
||||||
serviceAccountName: {{ include "rabbitmq.fullname" . }}
|
|
||||||
automountServiceAccountToken: false
|
|
||||||
securityContext:
|
|
||||||
runAsNonRoot: true
|
|
||||||
runAsUser: 999
|
|
||||||
runAsGroup: 999
|
|
||||||
fsGroup: 999
|
|
||||||
seccompProfile:
|
|
||||||
type: RuntimeDefault
|
|
||||||
containers:
|
|
||||||
- name: rabbitmq
|
|
||||||
image: {{ include "rabbitmq.image" . | quote }}
|
|
||||||
imagePullPolicy: {{ .Values.image.pullPolicy }}
|
|
||||||
securityContext:
|
|
||||||
allowPrivilegeEscalation: false
|
|
||||||
readOnlyRootFilesystem: true
|
|
||||||
capabilities:
|
|
||||||
drop: ["ALL"]
|
|
||||||
ports:
|
|
||||||
- name: amqp
|
|
||||||
containerPort: 5672
|
|
||||||
env:
|
|
||||||
- name: RABBITMQ_DEFAULT_USER
|
|
||||||
valueFrom:
|
|
||||||
secretKeyRef:
|
|
||||||
name: {{ .Values.existingSecret }}
|
|
||||||
key: {{ .Values.secretKeys.username }}
|
|
||||||
- name: RABBITMQ_DEFAULT_PASS
|
|
||||||
valueFrom:
|
|
||||||
secretKeyRef:
|
|
||||||
name: {{ .Values.existingSecret }}
|
|
||||||
key: {{ .Values.secretKeys.password }}
|
|
||||||
- name: RABBITMQ_ERLANG_COOKIE
|
|
||||||
valueFrom:
|
|
||||||
secretKeyRef:
|
|
||||||
name: {{ .Values.existingSecret }}
|
|
||||||
key: {{ .Values.secretKeys.erlangCookie }}
|
|
||||||
startupProbe:
|
|
||||||
exec:
|
|
||||||
command: ["rabbitmq-diagnostics", "-q", "ping"]
|
|
||||||
periodSeconds: 5
|
|
||||||
failureThreshold: 24
|
|
||||||
readinessProbe:
|
|
||||||
exec:
|
|
||||||
command: ["rabbitmq-diagnostics", "-q", "check_running"]
|
|
||||||
periodSeconds: 10
|
|
||||||
timeoutSeconds: 5
|
|
||||||
failureThreshold: 3
|
|
||||||
livenessProbe:
|
|
||||||
exec:
|
|
||||||
command: ["rabbitmq-diagnostics", "-q", "ping"]
|
|
||||||
periodSeconds: 20
|
|
||||||
timeoutSeconds: 5
|
|
||||||
failureThreshold: 3
|
|
||||||
resources:
|
|
||||||
{{- toYaml .Values.resources | nindent 12 }}
|
|
||||||
volumeMounts:
|
|
||||||
- name: data
|
|
||||||
mountPath: /var/lib/rabbitmq
|
|
||||||
- name: config
|
|
||||||
mountPath: /etc/rabbitmq/conf.d
|
|
||||||
- name: log
|
|
||||||
mountPath: /var/log/rabbitmq
|
|
||||||
- name: tmp
|
|
||||||
mountPath: /tmp
|
|
||||||
volumes:
|
|
||||||
- name: config
|
|
||||||
emptyDir: {}
|
|
||||||
- name: log
|
|
||||||
emptyDir: {}
|
|
||||||
- name: tmp
|
|
||||||
emptyDir: {}
|
|
||||||
{{- if not .Values.persistence.enabled }}
|
|
||||||
- name: data
|
|
||||||
emptyDir: {}
|
|
||||||
{{- end }}
|
|
||||||
{{- if .Values.persistence.enabled }}
|
|
||||||
volumeClaimTemplates:
|
|
||||||
- metadata:
|
|
||||||
name: data
|
|
||||||
spec:
|
|
||||||
accessModes: [{{ .Values.persistence.accessMode | quote }}]
|
|
||||||
{{- if .Values.persistence.storageClass }}
|
|
||||||
storageClassName: {{ .Values.persistence.storageClass | quote }}
|
|
||||||
{{- end }}
|
|
||||||
resources:
|
|
||||||
requests:
|
|
||||||
storage: {{ .Values.persistence.size }}
|
|
||||||
{{- end }}
|
|
||||||
@ -1,69 +0,0 @@
|
|||||||
{
|
|
||||||
"$schema": "https://json-schema.org/draft-07/schema#",
|
|
||||||
"type": "object",
|
|
||||||
"additionalProperties": false,
|
|
||||||
"required": ["enabled", "existingSecret", "secretKeys", "image", "service", "persistence", "resources"],
|
|
||||||
"properties": {
|
|
||||||
"global": {"type": "object"},
|
|
||||||
"enabled": {"type": "boolean"},
|
|
||||||
"existingSecret": {"type": "string", "minLength": 1},
|
|
||||||
"secretKeys": {
|
|
||||||
"type": "object",
|
|
||||||
"additionalProperties": false,
|
|
||||||
"required": ["username", "password", "erlangCookie"],
|
|
||||||
"properties": {
|
|
||||||
"username": {"type": "string", "minLength": 1},
|
|
||||||
"password": {"type": "string", "minLength": 1},
|
|
||||||
"erlangCookie": {"type": "string", "minLength": 1}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"image": {
|
|
||||||
"type": "object",
|
|
||||||
"additionalProperties": false,
|
|
||||||
"required": ["repository", "tag", "digest", "pullPolicy"],
|
|
||||||
"properties": {
|
|
||||||
"repository": {"type": "string", "minLength": 1},
|
|
||||||
"tag": {"type": "string", "minLength": 1, "not": {"enum": ["latest"]}},
|
|
||||||
"digest": {"type": "string", "pattern": "^$|^sha256:[a-f0-9]{64}$"},
|
|
||||||
"pullPolicy": {"enum": ["Always", "IfNotPresent", "Never"]}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"service": {
|
|
||||||
"type": "object",
|
|
||||||
"additionalProperties": false,
|
|
||||||
"required": ["port"],
|
|
||||||
"properties": {"port": {"type": "integer", "minimum": 1, "maximum": 65535}}
|
|
||||||
},
|
|
||||||
"persistence": {
|
|
||||||
"type": "object",
|
|
||||||
"additionalProperties": false,
|
|
||||||
"required": ["enabled", "storageClass", "accessMode", "size"],
|
|
||||||
"properties": {
|
|
||||||
"enabled": {"type": "boolean"},
|
|
||||||
"storageClass": {"type": "string"},
|
|
||||||
"accessMode": {"enum": ["ReadWriteOnce", "ReadWriteMany"]},
|
|
||||||
"size": {"type": "string", "minLength": 2}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"resources": {
|
|
||||||
"type": "object",
|
|
||||||
"additionalProperties": false,
|
|
||||||
"required": ["requests", "limits"],
|
|
||||||
"properties": {
|
|
||||||
"requests": {"$ref": "#/definitions/resourcePair"},
|
|
||||||
"limits": {"$ref": "#/definitions/resourcePair"}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"definitions": {
|
|
||||||
"resourcePair": {
|
|
||||||
"type": "object",
|
|
||||||
"additionalProperties": false,
|
|
||||||
"required": ["cpu", "memory"],
|
|
||||||
"properties": {
|
|
||||||
"cpu": {"type": ["string", "number"]},
|
|
||||||
"memory": {"type": "string", "minLength": 2}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@ -1,25 +0,0 @@
|
|||||||
enabled: false
|
|
||||||
existingSecret: knowledge-platform-rabbitmq
|
|
||||||
secretKeys:
|
|
||||||
username: username
|
|
||||||
password: password
|
|
||||||
erlangCookie: erlang-cookie
|
|
||||||
image:
|
|
||||||
repository: rabbitmq
|
|
||||||
tag: 4.1.4-alpine
|
|
||||||
digest: sha256:b736d649308e1b3e1a116c3f36986b605ee3d03e88f10166be2900083d2e63f2
|
|
||||||
pullPolicy: IfNotPresent
|
|
||||||
service:
|
|
||||||
port: 5672
|
|
||||||
persistence:
|
|
||||||
enabled: true
|
|
||||||
storageClass: ""
|
|
||||||
accessMode: ReadWriteOnce
|
|
||||||
size: 8Gi
|
|
||||||
resources:
|
|
||||||
requests:
|
|
||||||
cpu: 250m
|
|
||||||
memory: 512Mi
|
|
||||||
limits:
|
|
||||||
cpu: "1"
|
|
||||||
memory: 1Gi
|
|
||||||
@ -1,12 +0,0 @@
|
|||||||
Enterprise Knowledge Platform {{ .Chart.AppVersion }} is managed by Helm release {{ .Release.Name }}.
|
|
||||||
|
|
||||||
Runtime credentials are read only from Secret {{ .Values.existingSecret }}.
|
|
||||||
{{- if .Values.ingress.enabled }}
|
|
||||||
Admin/API: https://{{ .Values.ingress.api.host }}{{ .Values.ingress.api.path }}
|
|
||||||
MCP: https://{{ .Values.ingress.mcp.host }}{{ .Values.ingress.mcp.path }}
|
|
||||||
{{- else }}
|
|
||||||
Ingress is disabled. Enable it in the environment values file before remote use.
|
|
||||||
{{- end }}
|
|
||||||
|
|
||||||
Inspect rollout state without printing environment variables:
|
|
||||||
kubectl -n {{ .Release.Namespace }} get deploy,job,pod
|
|
||||||
@ -1,106 +0,0 @@
|
|||||||
{{/* Stable resource name helpers keep every workload under one release identity. */}}
|
|
||||||
{{- define "knowledge-platform.name" -}}
|
|
||||||
{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
|
|
||||||
{{- end -}}
|
|
||||||
|
|
||||||
{{- define "knowledge-platform.fullname" -}}
|
|
||||||
{{- if .Values.fullnameOverride -}}
|
|
||||||
{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
|
|
||||||
{{- else -}}
|
|
||||||
{{- $name := include "knowledge-platform.name" . -}}
|
|
||||||
{{- if contains $name .Release.Name -}}
|
|
||||||
{{- .Release.Name | trunc 63 | trimSuffix "-" -}}
|
|
||||||
{{- else -}}
|
|
||||||
{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
|
|
||||||
{{- end -}}
|
|
||||||
{{- end -}}
|
|
||||||
{{- end -}}
|
|
||||||
|
|
||||||
{{- define "knowledge-platform.labels" -}}
|
|
||||||
helm.sh/chart: {{ printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | quote }}
|
|
||||||
app.kubernetes.io/name: {{ include "knowledge-platform.name" . }}
|
|
||||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
|
||||||
app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
|
|
||||||
app.kubernetes.io/managed-by: {{ .Release.Service }}
|
|
||||||
{{- end -}}
|
|
||||||
|
|
||||||
{{- define "knowledge-platform.selectorLabels" -}}
|
|
||||||
app.kubernetes.io/name: {{ include "knowledge-platform.name" . }}
|
|
||||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
|
||||||
{{- end -}}
|
|
||||||
|
|
||||||
{{- define "knowledge-platform.serviceAccountName" -}}
|
|
||||||
{{- if .Values.serviceAccount.create -}}
|
|
||||||
{{- default (include "knowledge-platform.fullname" .) .Values.serviceAccount.name -}}
|
|
||||||
{{- else -}}
|
|
||||||
{{- required "serviceAccount.name is required when serviceAccount.create=false" .Values.serviceAccount.name -}}
|
|
||||||
{{- end -}}
|
|
||||||
{{- end -}}
|
|
||||||
|
|
||||||
{{/* A supplied digest takes precedence over the immutable release tag. */}}
|
|
||||||
{{- define "knowledge-platform.image" -}}
|
|
||||||
{{- if .digest -}}
|
|
||||||
{{- printf "%s@%s" .repository .digest -}}
|
|
||||||
{{- else -}}
|
|
||||||
{{- printf "%s:%s" .repository .tag -}}
|
|
||||||
{{- end -}}
|
|
||||||
{{- end -}}
|
|
||||||
|
|
||||||
{{- define "knowledge-platform.configMapName" -}}
|
|
||||||
{{- printf "%s-runtime" (include "knowledge-platform.fullname" .) -}}
|
|
||||||
{{- end -}}
|
|
||||||
|
|
||||||
{{/* Cross-field rules complement values.schema.json's structural checks. */}}
|
|
||||||
{{- define "knowledge-platform.validateValues" -}}
|
|
||||||
{{- if lt (float64 .Values.config.ingestion.retryBackoffMaxSeconds) (float64 .Values.config.ingestion.retryBackoffSeconds) -}}
|
|
||||||
{{- fail "config.ingestion.retryBackoffMaxSeconds must be >= retryBackoffSeconds" -}}
|
|
||||||
{{- end -}}
|
|
||||||
{{- if lt (float64 .Values.config.ingestion.batchMaxTotalBytes) (float64 .Values.config.ingestion.maxUploadBytes) -}}
|
|
||||||
{{- fail "config.ingestion.batchMaxTotalBytes must be >= maxUploadBytes" -}}
|
|
||||||
{{- end -}}
|
|
||||||
{{- if ge (float64 .Values.config.chunking.overlapTokens) (float64 .Values.config.chunking.maxTokens) -}}
|
|
||||||
{{- fail "config.chunking.overlapTokens must be < maxTokens" -}}
|
|
||||||
{{- end -}}
|
|
||||||
{{- if gt (float64 .Values.config.retrieval.searchTopK) (float64 .Values.config.retrieval.denseTopN) -}}
|
|
||||||
{{- fail "config.retrieval.searchTopK must be <= denseTopN" -}}
|
|
||||||
{{- end -}}
|
|
||||||
{{- end -}}
|
|
||||||
|
|
||||||
{{- define "knowledge-platform.podSecurityContext" -}}
|
|
||||||
runAsNonRoot: true
|
|
||||||
runAsUser: 10001
|
|
||||||
runAsGroup: 10001
|
|
||||||
fsGroup: 10001
|
|
||||||
fsGroupChangePolicy: OnRootMismatch
|
|
||||||
seccompProfile:
|
|
||||||
type: RuntimeDefault
|
|
||||||
{{- end -}}
|
|
||||||
|
|
||||||
{{- define "knowledge-platform.containerSecurityContext" -}}
|
|
||||||
allowPrivilegeEscalation: false
|
|
||||||
readOnlyRootFilesystem: true
|
|
||||||
runAsNonRoot: true
|
|
||||||
runAsUser: 10001
|
|
||||||
runAsGroup: 10001
|
|
||||||
capabilities:
|
|
||||||
drop:
|
|
||||||
- ALL
|
|
||||||
{{- end -}}
|
|
||||||
|
|
||||||
{{- define "knowledge-platform.envFrom" -}}
|
|
||||||
- configMapRef:
|
|
||||||
name: {{ include "knowledge-platform.configMapName" . }}
|
|
||||||
- secretRef:
|
|
||||||
name: {{ .Values.existingSecret }}
|
|
||||||
{{- end -}}
|
|
||||||
|
|
||||||
{{- define "knowledge-platform.tmpVolume" -}}
|
|
||||||
- name: tmp
|
|
||||||
emptyDir:
|
|
||||||
sizeLimit: 1Gi
|
|
||||||
{{- end -}}
|
|
||||||
|
|
||||||
{{- define "knowledge-platform.tmpMount" -}}
|
|
||||||
- name: tmp
|
|
||||||
mountPath: /tmp
|
|
||||||
{{- end -}}
|
|
||||||
@ -1,75 +0,0 @@
|
|||||||
apiVersion: apps/v1
|
|
||||||
kind: Deployment
|
|
||||||
metadata:
|
|
||||||
name: {{ include "knowledge-platform.fullname" . }}-api
|
|
||||||
labels:
|
|
||||||
{{- include "knowledge-platform.labels" . | nindent 4 }}
|
|
||||||
app.kubernetes.io/component: api
|
|
||||||
spec:
|
|
||||||
replicas: {{ .Values.api.replicaCount }}
|
|
||||||
minReadySeconds: 5
|
|
||||||
strategy:
|
|
||||||
type: RollingUpdate
|
|
||||||
rollingUpdate:
|
|
||||||
maxUnavailable: 0
|
|
||||||
maxSurge: 1
|
|
||||||
selector:
|
|
||||||
matchLabels:
|
|
||||||
{{- include "knowledge-platform.selectorLabels" . | nindent 6 }}
|
|
||||||
app.kubernetes.io/component: api
|
|
||||||
template:
|
|
||||||
metadata:
|
|
||||||
labels:
|
|
||||||
{{- include "knowledge-platform.selectorLabels" . | nindent 8 }}
|
|
||||||
app.kubernetes.io/component: api
|
|
||||||
{{- with .Values.podLabels }}
|
|
||||||
{{- toYaml . | nindent 8 }}
|
|
||||||
{{- end }}
|
|
||||||
annotations:
|
|
||||||
checksum/runtime-config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }}
|
|
||||||
checksum/runtime-secret-reference: {{ .Values.existingSecret | sha256sum }}
|
|
||||||
{{- with .Values.podAnnotations }}
|
|
||||||
{{- toYaml . | nindent 8 }}
|
|
||||||
{{- end }}
|
|
||||||
spec:
|
|
||||||
serviceAccountName: {{ include "knowledge-platform.serviceAccountName" . }}
|
|
||||||
automountServiceAccountToken: false
|
|
||||||
securityContext:
|
|
||||||
{{- include "knowledge-platform.podSecurityContext" . | nindent 8 }}
|
|
||||||
{{- with .Values.imagePullSecrets }}
|
|
||||||
imagePullSecrets:
|
|
||||||
{{- toYaml . | nindent 8 }}
|
|
||||||
{{- end }}
|
|
||||||
containers:
|
|
||||||
- name: api
|
|
||||||
image: {{ include "knowledge-platform.image" .Values.images.api | quote }}
|
|
||||||
imagePullPolicy: {{ .Values.images.api.pullPolicy }}
|
|
||||||
securityContext:
|
|
||||||
{{- include "knowledge-platform.containerSecurityContext" . | nindent 12 }}
|
|
||||||
ports:
|
|
||||||
- name: http
|
|
||||||
containerPort: 8000
|
|
||||||
protocol: TCP
|
|
||||||
envFrom:
|
|
||||||
{{- include "knowledge-platform.envFrom" . | nindent 12 }}
|
|
||||||
startupProbe:
|
|
||||||
httpGet:
|
|
||||||
path: /healthz
|
|
||||||
port: http
|
|
||||||
{{- toYaml .Values.api.probes.startup | nindent 12 }}
|
|
||||||
readinessProbe:
|
|
||||||
httpGet:
|
|
||||||
path: /healthz
|
|
||||||
port: http
|
|
||||||
{{- toYaml .Values.api.probes.readiness | nindent 12 }}
|
|
||||||
livenessProbe:
|
|
||||||
httpGet:
|
|
||||||
path: /healthz
|
|
||||||
port: http
|
|
||||||
{{- toYaml .Values.api.probes.liveness | nindent 12 }}
|
|
||||||
resources:
|
|
||||||
{{- toYaml .Values.api.resources | nindent 12 }}
|
|
||||||
volumeMounts:
|
|
||||||
{{- include "knowledge-platform.tmpMount" . | nindent 12 }}
|
|
||||||
volumes:
|
|
||||||
{{- include "knowledge-platform.tmpVolume" . | nindent 8 }}
|
|
||||||
@ -1,17 +0,0 @@
|
|||||||
apiVersion: v1
|
|
||||||
kind: Service
|
|
||||||
metadata:
|
|
||||||
name: {{ include "knowledge-platform.fullname" . }}-api
|
|
||||||
labels:
|
|
||||||
{{- include "knowledge-platform.labels" . | nindent 4 }}
|
|
||||||
app.kubernetes.io/component: api
|
|
||||||
spec:
|
|
||||||
type: {{ .Values.api.service.type }}
|
|
||||||
ports:
|
|
||||||
- name: http
|
|
||||||
port: {{ .Values.api.service.port }}
|
|
||||||
targetPort: http
|
|
||||||
protocol: TCP
|
|
||||||
selector:
|
|
||||||
{{- include "knowledge-platform.selectorLabels" . | nindent 4 }}
|
|
||||||
app.kubernetes.io/component: api
|
|
||||||
@ -1,66 +0,0 @@
|
|||||||
{{- include "knowledge-platform.validateValues" . }}
|
|
||||||
apiVersion: v1
|
|
||||||
kind: ConfigMap
|
|
||||||
metadata:
|
|
||||||
name: {{ include "knowledge-platform.configMapName" . }}
|
|
||||||
labels:
|
|
||||||
{{- include "knowledge-platform.labels" . | nindent 4 }}
|
|
||||||
data:
|
|
||||||
# Only the explicitly enumerated non-sensitive settings are rendered here.
|
|
||||||
APP_ENV: "production"
|
|
||||||
APP_NAME: {{ .Values.config.appName | quote }}
|
|
||||||
LOG_LEVEL: {{ .Values.config.logLevel | quote }}
|
|
||||||
API_HOST: "0.0.0.0"
|
|
||||||
API_PORT: "8000"
|
|
||||||
API_PREFIX: "/api/v1"
|
|
||||||
DATABASE_POOL_SIZE: {{ .Values.config.database.poolSize | quote }}
|
|
||||||
DATABASE_MAX_OVERFLOW: {{ .Values.config.database.maxOverflow | quote }}
|
|
||||||
DATABASE_POOL_RECYCLE_SECONDS: {{ .Values.config.database.poolRecycleSeconds | quote }}
|
|
||||||
DATABASE_POOL_TIMEOUT_SECONDS: {{ .Values.config.database.poolTimeoutSeconds | quote }}
|
|
||||||
DATABASE_CONNECT_TIMEOUT_SECONDS: {{ .Values.config.database.connectTimeoutSeconds | quote }}
|
|
||||||
INGESTION_MAX_ATTEMPTS: {{ .Values.config.ingestion.maxAttempts | quote }}
|
|
||||||
INGESTION_RETRY_BACKOFF_SECONDS: {{ .Values.config.ingestion.retryBackoffSeconds | quote }}
|
|
||||||
INGESTION_RETRY_BACKOFF_MAX_SECONDS: {{ .Values.config.ingestion.retryBackoffMaxSeconds | quote }}
|
|
||||||
INGESTION_WORKER_LEASE_SECONDS: {{ .Values.config.ingestion.workerLeaseSeconds | quote }}
|
|
||||||
INGESTION_RECOVERY_INTERVAL_SECONDS: {{ .Values.config.ingestion.recoveryIntervalSeconds | quote }}
|
|
||||||
BATCH_MAX_FILES: {{ .Values.config.ingestion.batchMaxFiles | quote }}
|
|
||||||
BATCH_MAX_TOTAL_BYTES: {{ printf "%.0f" .Values.config.ingestion.batchMaxTotalBytes | quote }}
|
|
||||||
MAX_UPLOAD_BYTES: {{ printf "%.0f" .Values.config.ingestion.maxUploadBytes | quote }}
|
|
||||||
OBJECT_STORAGE_PROVIDER: {{ .Values.config.objectStorage.provider | quote }}
|
|
||||||
BOS_ENDPOINT: {{ .Values.config.objectStorage.endpoint | quote }}
|
|
||||||
BOS_BUCKET: {{ .Values.config.objectStorage.bucket | quote }}
|
|
||||||
BOS_KEY_PREFIX: {{ .Values.config.objectStorage.keyPrefix | quote }}
|
|
||||||
BOS_TIMEOUT_SECONDS: {{ .Values.config.objectStorage.timeoutSeconds | quote }}
|
|
||||||
CHUNK_MAX_TOKENS: {{ .Values.config.chunking.maxTokens | quote }}
|
|
||||||
CHUNK_OVERLAP_TOKENS: {{ .Values.config.chunking.overlapTokens | quote }}
|
|
||||||
CHUNK_MAX_CHARACTERS: {{ .Values.config.chunking.maxCharacters | quote }}
|
|
||||||
EMBEDDING_PROVIDER: {{ .Values.config.embedding.provider | quote }}
|
|
||||||
EMBEDDING_MODEL: {{ .Values.config.embedding.model | quote }}
|
|
||||||
EMBEDDING_ENDPOINT: {{ .Values.config.embedding.endpoint | quote }}
|
|
||||||
EMBEDDING_TIMEOUT_SECONDS: {{ .Values.config.embedding.timeoutSeconds | quote }}
|
|
||||||
VECTOR_STORE_PROVIDER: {{ .Values.config.vectorStore.provider | quote }}
|
|
||||||
OCEANBASE_VECTOR_URI: {{ .Values.config.vectorStore.uri | quote }}
|
|
||||||
OCEANBASE_VECTOR_USER: {{ .Values.config.vectorStore.user | quote }}
|
|
||||||
OCEANBASE_VECTOR_DATABASE: {{ .Values.config.vectorStore.database | quote }}
|
|
||||||
OCEANBASE_VECTOR_TABLE: {{ .Values.config.vectorStore.table | quote }}
|
|
||||||
OCEANBASE_VECTOR_INDEX: {{ .Values.config.vectorStore.index | quote }}
|
|
||||||
OCEANBASE_VECTOR_BATCH_SIZE: {{ .Values.config.vectorStore.batchSize | quote }}
|
|
||||||
OCEANBASE_VECTOR_TIMEOUT_SECONDS: {{ .Values.config.vectorStore.timeoutSeconds | quote }}
|
|
||||||
INDEX_PROFILE_ID: {{ .Values.config.retrieval.indexProfileId | quote }}
|
|
||||||
DENSE_TOP_N: {{ .Values.config.retrieval.denseTopN | quote }}
|
|
||||||
SEARCH_TOP_K: {{ .Values.config.retrieval.searchTopK | quote }}
|
|
||||||
RERANKER_PROVIDER: {{ .Values.config.reranker.provider | quote }}
|
|
||||||
RERANKER_MODEL: {{ .Values.config.reranker.model | quote }}
|
|
||||||
RERANKER_ENDPOINT: {{ .Values.config.reranker.endpoint | quote }}
|
|
||||||
RERANKER_TIMEOUT_SECONDS: {{ .Values.config.reranker.timeoutSeconds | quote }}
|
|
||||||
MCP_HOST: "0.0.0.0"
|
|
||||||
MCP_PORT: "8766"
|
|
||||||
MCP_PATH: "/mcp"
|
|
||||||
MCP_PUBLIC_URL: {{ .Values.config.mcp.publicUrl | quote }}
|
|
||||||
MCP_ISSUER_URL: {{ .Values.config.mcp.issuerUrl | quote }}
|
|
||||||
MCP_CLIENT_ID: {{ .Values.config.mcp.clientId | quote }}
|
|
||||||
MCP_SERVICE_IDENTITY: {{ .Values.config.mcp.serviceIdentity | quote }}
|
|
||||||
MCP_REQUIRED_SCOPE: {{ .Values.config.mcp.requiredScope | quote }}
|
|
||||||
MCP_ALLOWED_HOSTS: {{ .Values.config.mcp.allowedHosts | toJson | quote }}
|
|
||||||
MCP_ALLOWED_ORIGINS: {{ .Values.config.mcp.allowedOrigins | toJson | quote }}
|
|
||||||
MCP_MAX_REQUEST_BODY_BYTES: {{ printf "%.0f" .Values.config.mcp.maxRequestBodyBytes | quote }}
|
|
||||||
@ -1,42 +0,0 @@
|
|||||||
{{- if .Values.ingress.enabled }}
|
|
||||||
apiVersion: networking.k8s.io/v1
|
|
||||||
kind: Ingress
|
|
||||||
metadata:
|
|
||||||
name: {{ include "knowledge-platform.fullname" . }}
|
|
||||||
labels:
|
|
||||||
{{- include "knowledge-platform.labels" . | nindent 4 }}
|
|
||||||
{{- with .Values.ingress.annotations }}
|
|
||||||
annotations:
|
|
||||||
{{- toYaml . | nindent 4 }}
|
|
||||||
{{- end }}
|
|
||||||
spec:
|
|
||||||
ingressClassName: {{ .Values.ingress.className }}
|
|
||||||
tls:
|
|
||||||
- hosts:
|
|
||||||
- {{ .Values.ingress.api.host | quote }}
|
|
||||||
secretName: {{ .Values.ingress.api.tlsSecretName }}
|
|
||||||
- hosts:
|
|
||||||
- {{ .Values.ingress.mcp.host | quote }}
|
|
||||||
secretName: {{ .Values.ingress.mcp.tlsSecretName }}
|
|
||||||
rules:
|
|
||||||
- host: {{ .Values.ingress.api.host | quote }}
|
|
||||||
http:
|
|
||||||
paths:
|
|
||||||
- path: {{ .Values.ingress.api.path }}
|
|
||||||
pathType: Prefix
|
|
||||||
backend:
|
|
||||||
service:
|
|
||||||
name: {{ include "knowledge-platform.fullname" . }}-api
|
|
||||||
port:
|
|
||||||
number: {{ .Values.api.service.port }}
|
|
||||||
- host: {{ .Values.ingress.mcp.host | quote }}
|
|
||||||
http:
|
|
||||||
paths:
|
|
||||||
- path: {{ .Values.ingress.mcp.path }}
|
|
||||||
pathType: Prefix
|
|
||||||
backend:
|
|
||||||
service:
|
|
||||||
name: {{ include "knowledge-platform.fullname" . }}-mcp
|
|
||||||
port:
|
|
||||||
number: {{ .Values.mcp.service.port }}
|
|
||||||
{{- end }}
|
|
||||||
@ -1,72 +0,0 @@
|
|||||||
apiVersion: apps/v1
|
|
||||||
kind: Deployment
|
|
||||||
metadata:
|
|
||||||
name: {{ include "knowledge-platform.fullname" . }}-mcp
|
|
||||||
labels:
|
|
||||||
{{- include "knowledge-platform.labels" . | nindent 4 }}
|
|
||||||
app.kubernetes.io/component: mcp
|
|
||||||
spec:
|
|
||||||
replicas: {{ .Values.mcp.replicaCount }}
|
|
||||||
minReadySeconds: 5
|
|
||||||
strategy:
|
|
||||||
type: RollingUpdate
|
|
||||||
rollingUpdate:
|
|
||||||
maxUnavailable: 0
|
|
||||||
maxSurge: 1
|
|
||||||
selector:
|
|
||||||
matchLabels:
|
|
||||||
{{- include "knowledge-platform.selectorLabels" . | nindent 6 }}
|
|
||||||
app.kubernetes.io/component: mcp
|
|
||||||
template:
|
|
||||||
metadata:
|
|
||||||
labels:
|
|
||||||
{{- include "knowledge-platform.selectorLabels" . | nindent 8 }}
|
|
||||||
app.kubernetes.io/component: mcp
|
|
||||||
{{- with .Values.podLabels }}
|
|
||||||
{{- toYaml . | nindent 8 }}
|
|
||||||
{{- end }}
|
|
||||||
annotations:
|
|
||||||
checksum/runtime-config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }}
|
|
||||||
checksum/runtime-secret-reference: {{ .Values.existingSecret | sha256sum }}
|
|
||||||
{{- with .Values.podAnnotations }}
|
|
||||||
{{- toYaml . | nindent 8 }}
|
|
||||||
{{- end }}
|
|
||||||
spec:
|
|
||||||
serviceAccountName: {{ include "knowledge-platform.serviceAccountName" . }}
|
|
||||||
automountServiceAccountToken: false
|
|
||||||
securityContext:
|
|
||||||
{{- include "knowledge-platform.podSecurityContext" . | nindent 8 }}
|
|
||||||
{{- with .Values.imagePullSecrets }}
|
|
||||||
imagePullSecrets:
|
|
||||||
{{- toYaml . | nindent 8 }}
|
|
||||||
{{- end }}
|
|
||||||
containers:
|
|
||||||
- name: mcp
|
|
||||||
image: {{ include "knowledge-platform.image" .Values.images.mcp | quote }}
|
|
||||||
imagePullPolicy: {{ .Values.images.mcp.pullPolicy }}
|
|
||||||
securityContext:
|
|
||||||
{{- include "knowledge-platform.containerSecurityContext" . | nindent 12 }}
|
|
||||||
ports:
|
|
||||||
- name: mcp
|
|
||||||
containerPort: 8766
|
|
||||||
protocol: TCP
|
|
||||||
envFrom:
|
|
||||||
{{- include "knowledge-platform.envFrom" . | nindent 12 }}
|
|
||||||
startupProbe:
|
|
||||||
tcpSocket:
|
|
||||||
port: mcp
|
|
||||||
{{- toYaml .Values.mcp.probes.startup | nindent 12 }}
|
|
||||||
readinessProbe:
|
|
||||||
tcpSocket:
|
|
||||||
port: mcp
|
|
||||||
{{- toYaml .Values.mcp.probes.readiness | nindent 12 }}
|
|
||||||
livenessProbe:
|
|
||||||
tcpSocket:
|
|
||||||
port: mcp
|
|
||||||
{{- toYaml .Values.mcp.probes.liveness | nindent 12 }}
|
|
||||||
resources:
|
|
||||||
{{- toYaml .Values.mcp.resources | nindent 12 }}
|
|
||||||
volumeMounts:
|
|
||||||
{{- include "knowledge-platform.tmpMount" . | nindent 12 }}
|
|
||||||
volumes:
|
|
||||||
{{- include "knowledge-platform.tmpVolume" . | nindent 8 }}
|
|
||||||
@ -1,17 +0,0 @@
|
|||||||
apiVersion: v1
|
|
||||||
kind: Service
|
|
||||||
metadata:
|
|
||||||
name: {{ include "knowledge-platform.fullname" . }}-mcp
|
|
||||||
labels:
|
|
||||||
{{- include "knowledge-platform.labels" . | nindent 4 }}
|
|
||||||
app.kubernetes.io/component: mcp
|
|
||||||
spec:
|
|
||||||
type: {{ .Values.mcp.service.type }}
|
|
||||||
ports:
|
|
||||||
- name: mcp
|
|
||||||
port: {{ .Values.mcp.service.port }}
|
|
||||||
targetPort: mcp
|
|
||||||
protocol: TCP
|
|
||||||
selector:
|
|
||||||
{{- include "knowledge-platform.selectorLabels" . | nindent 4 }}
|
|
||||||
app.kubernetes.io/component: mcp
|
|
||||||
@ -1,59 +0,0 @@
|
|||||||
{{- if .Values.migration.enabled }}
|
|
||||||
apiVersion: batch/v1
|
|
||||||
kind: Job
|
|
||||||
metadata:
|
|
||||||
name: {{ include "knowledge-platform.fullname" . }}-migration
|
|
||||||
labels:
|
|
||||||
{{- include "knowledge-platform.labels" . | nindent 4 }}
|
|
||||||
app.kubernetes.io/component: migration
|
|
||||||
annotations:
|
|
||||||
# Helm waits for pre-install/pre-upgrade hooks. A failed migration blocks
|
|
||||||
# the application rollout, and the previous Job remains until the next run.
|
|
||||||
helm.sh/hook: pre-install,pre-upgrade
|
|
||||||
helm.sh/hook-weight: "-10"
|
|
||||||
helm.sh/hook-delete-policy: before-hook-creation
|
|
||||||
spec:
|
|
||||||
backoffLimit: {{ .Values.migration.backoffLimit }}
|
|
||||||
activeDeadlineSeconds: {{ .Values.migration.activeDeadlineSeconds }}
|
|
||||||
ttlSecondsAfterFinished: {{ .Values.migration.ttlSecondsAfterFinished }}
|
|
||||||
template:
|
|
||||||
metadata:
|
|
||||||
labels:
|
|
||||||
{{- include "knowledge-platform.selectorLabels" . | nindent 8 }}
|
|
||||||
app.kubernetes.io/component: migration
|
|
||||||
spec:
|
|
||||||
restartPolicy: Never
|
|
||||||
# Pre-install hooks run before the release ServiceAccount exists. The
|
|
||||||
# namespace default account is used with token mounting disabled; the
|
|
||||||
# migration needs no Kubernetes API permissions.
|
|
||||||
serviceAccountName: default
|
|
||||||
automountServiceAccountToken: false
|
|
||||||
securityContext:
|
|
||||||
{{- include "knowledge-platform.podSecurityContext" . | nindent 8 }}
|
|
||||||
{{- with .Values.imagePullSecrets }}
|
|
||||||
imagePullSecrets:
|
|
||||||
{{- toYaml . | nindent 8 }}
|
|
||||||
{{- end }}
|
|
||||||
containers:
|
|
||||||
- name: migration
|
|
||||||
image: {{ include "knowledge-platform.image" .Values.images.api | quote }}
|
|
||||||
imagePullPolicy: {{ .Values.images.api.pullPolicy }}
|
|
||||||
command: ["alembic"]
|
|
||||||
args: ["upgrade", "head"]
|
|
||||||
securityContext:
|
|
||||||
{{- include "knowledge-platform.containerSecurityContext" . | nindent 12 }}
|
|
||||||
env:
|
|
||||||
- name: APP_ENV
|
|
||||||
value: production
|
|
||||||
- name: DATABASE_URL
|
|
||||||
valueFrom:
|
|
||||||
secretKeyRef:
|
|
||||||
name: {{ .Values.existingSecret }}
|
|
||||||
key: DATABASE_URL
|
|
||||||
resources:
|
|
||||||
{{- toYaml .Values.migration.resources | nindent 12 }}
|
|
||||||
volumeMounts:
|
|
||||||
{{- include "knowledge-platform.tmpMount" . | nindent 12 }}
|
|
||||||
volumes:
|
|
||||||
{{- include "knowledge-platform.tmpVolume" . | nindent 8 }}
|
|
||||||
{{- end }}
|
|
||||||
@ -1,71 +0,0 @@
|
|||||||
{{- if .Values.scheduler.enabled }}
|
|
||||||
apiVersion: apps/v1
|
|
||||||
kind: Deployment
|
|
||||||
metadata:
|
|
||||||
name: {{ include "knowledge-platform.fullname" . }}-scheduler
|
|
||||||
labels:
|
|
||||||
{{- include "knowledge-platform.labels" . | nindent 4 }}
|
|
||||||
app.kubernetes.io/component: scheduler
|
|
||||||
spec:
|
|
||||||
# A single Celery Beat owns periodic durable-Job recovery. Recreate prevents
|
|
||||||
# two schedulers from overlapping during upgrades.
|
|
||||||
replicas: 1
|
|
||||||
strategy:
|
|
||||||
type: Recreate
|
|
||||||
selector:
|
|
||||||
matchLabels:
|
|
||||||
{{- include "knowledge-platform.selectorLabels" . | nindent 6 }}
|
|
||||||
app.kubernetes.io/component: scheduler
|
|
||||||
template:
|
|
||||||
metadata:
|
|
||||||
labels:
|
|
||||||
{{- include "knowledge-platform.selectorLabels" . | nindent 8 }}
|
|
||||||
app.kubernetes.io/component: scheduler
|
|
||||||
{{- with .Values.podLabels }}
|
|
||||||
{{- toYaml . | nindent 8 }}
|
|
||||||
{{- end }}
|
|
||||||
annotations:
|
|
||||||
checksum/runtime-config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }}
|
|
||||||
checksum/runtime-secret-reference: {{ .Values.existingSecret | sha256sum }}
|
|
||||||
{{- with .Values.podAnnotations }}
|
|
||||||
{{- toYaml . | nindent 8 }}
|
|
||||||
{{- end }}
|
|
||||||
spec:
|
|
||||||
serviceAccountName: {{ include "knowledge-platform.serviceAccountName" . }}
|
|
||||||
automountServiceAccountToken: false
|
|
||||||
securityContext:
|
|
||||||
{{- include "knowledge-platform.podSecurityContext" . | nindent 8 }}
|
|
||||||
{{- with .Values.imagePullSecrets }}
|
|
||||||
imagePullSecrets:
|
|
||||||
{{- toYaml . | nindent 8 }}
|
|
||||||
{{- end }}
|
|
||||||
containers:
|
|
||||||
- name: scheduler
|
|
||||||
image: {{ include "knowledge-platform.image" .Values.images.worker | quote }}
|
|
||||||
imagePullPolicy: {{ .Values.images.worker.pullPolicy }}
|
|
||||||
workingDir: /tmp
|
|
||||||
command: ["celery"]
|
|
||||||
args:
|
|
||||||
- --app=ekp_api.worker:celery_app
|
|
||||||
- beat
|
|
||||||
- --loglevel=INFO
|
|
||||||
- --schedule=/tmp/celerybeat-schedule
|
|
||||||
securityContext:
|
|
||||||
{{- include "knowledge-platform.containerSecurityContext" . | nindent 12 }}
|
|
||||||
envFrom:
|
|
||||||
{{- include "knowledge-platform.envFrom" . | nindent 12 }}
|
|
||||||
startupProbe:
|
|
||||||
exec:
|
|
||||||
command: ["python", "-c", "import os; os.kill(1, 0)"]
|
|
||||||
{{- toYaml .Values.scheduler.probes.startup | nindent 12 }}
|
|
||||||
livenessProbe:
|
|
||||||
exec:
|
|
||||||
command: ["python", "-c", "import os; os.kill(1, 0)"]
|
|
||||||
{{- toYaml .Values.scheduler.probes.liveness | nindent 12 }}
|
|
||||||
resources:
|
|
||||||
{{- toYaml .Values.scheduler.resources | nindent 12 }}
|
|
||||||
volumeMounts:
|
|
||||||
{{- include "knowledge-platform.tmpMount" . | nindent 12 }}
|
|
||||||
volumes:
|
|
||||||
{{- include "knowledge-platform.tmpVolume" . | nindent 8 }}
|
|
||||||
{{- end }}
|
|
||||||
@ -1,13 +0,0 @@
|
|||||||
{{- if .Values.serviceAccount.create }}
|
|
||||||
apiVersion: v1
|
|
||||||
kind: ServiceAccount
|
|
||||||
metadata:
|
|
||||||
name: {{ include "knowledge-platform.serviceAccountName" . }}
|
|
||||||
labels:
|
|
||||||
{{- include "knowledge-platform.labels" . | nindent 4 }}
|
|
||||||
{{- with .Values.serviceAccount.annotations }}
|
|
||||||
annotations:
|
|
||||||
{{- toYaml . | nindent 4 }}
|
|
||||||
{{- end }}
|
|
||||||
automountServiceAccountToken: false
|
|
||||||
{{- end }}
|
|
||||||
@ -1,17 +0,0 @@
|
|||||||
apiVersion: v1
|
|
||||||
kind: ConfigMap
|
|
||||||
metadata:
|
|
||||||
name: {{ include "knowledge-platform.fullname" . }}-preflight-references
|
|
||||||
labels:
|
|
||||||
{{- include "knowledge-platform.labels" . | nindent 4 }}
|
|
||||||
annotations:
|
|
||||||
# The Make preflight renders this test hook to discover only resource
|
|
||||||
# names. It never renders or reads Secret data.
|
|
||||||
helm.sh/hook: test
|
|
||||||
data:
|
|
||||||
runtimeSecret: {{ .Values.existingSecret | quote }}
|
|
||||||
ingressEnabled: {{ .Values.ingress.enabled | quote }}
|
|
||||||
apiTlsSecret: {{ .Values.ingress.api.tlsSecretName | quote }}
|
|
||||||
mcpTlsSecret: {{ .Values.ingress.mcp.tlsSecretName | quote }}
|
|
||||||
rabbitmqEnabled: {{ .Values.rabbitmq.enabled | quote }}
|
|
||||||
rabbitmqSecret: {{ .Values.rabbitmq.existingSecret | quote }}
|
|
||||||
@ -1,71 +0,0 @@
|
|||||||
apiVersion: apps/v1
|
|
||||||
kind: Deployment
|
|
||||||
metadata:
|
|
||||||
name: {{ include "knowledge-platform.fullname" . }}-worker
|
|
||||||
labels:
|
|
||||||
{{- include "knowledge-platform.labels" . | nindent 4 }}
|
|
||||||
app.kubernetes.io/component: worker
|
|
||||||
spec:
|
|
||||||
replicas: {{ .Values.worker.replicaCount }}
|
|
||||||
strategy:
|
|
||||||
type: RollingUpdate
|
|
||||||
rollingUpdate:
|
|
||||||
maxUnavailable: 1
|
|
||||||
maxSurge: 1
|
|
||||||
selector:
|
|
||||||
matchLabels:
|
|
||||||
{{- include "knowledge-platform.selectorLabels" . | nindent 6 }}
|
|
||||||
app.kubernetes.io/component: worker
|
|
||||||
template:
|
|
||||||
metadata:
|
|
||||||
labels:
|
|
||||||
{{- include "knowledge-platform.selectorLabels" . | nindent 8 }}
|
|
||||||
app.kubernetes.io/component: worker
|
|
||||||
{{- with .Values.podLabels }}
|
|
||||||
{{- toYaml . | nindent 8 }}
|
|
||||||
{{- end }}
|
|
||||||
annotations:
|
|
||||||
checksum/runtime-config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }}
|
|
||||||
checksum/runtime-secret-reference: {{ .Values.existingSecret | sha256sum }}
|
|
||||||
{{- with .Values.podAnnotations }}
|
|
||||||
{{- toYaml . | nindent 8 }}
|
|
||||||
{{- end }}
|
|
||||||
spec:
|
|
||||||
serviceAccountName: {{ include "knowledge-platform.serviceAccountName" . }}
|
|
||||||
automountServiceAccountToken: false
|
|
||||||
terminationGracePeriodSeconds: {{ .Values.worker.terminationGracePeriodSeconds }}
|
|
||||||
securityContext:
|
|
||||||
{{- include "knowledge-platform.podSecurityContext" . | nindent 8 }}
|
|
||||||
{{- with .Values.imagePullSecrets }}
|
|
||||||
imagePullSecrets:
|
|
||||||
{{- toYaml . | nindent 8 }}
|
|
||||||
{{- end }}
|
|
||||||
containers:
|
|
||||||
- name: worker
|
|
||||||
image: {{ include "knowledge-platform.image" .Values.images.worker | quote }}
|
|
||||||
imagePullPolicy: {{ .Values.images.worker.pullPolicy }}
|
|
||||||
workingDir: /tmp
|
|
||||||
command: ["celery"]
|
|
||||||
args:
|
|
||||||
- --app=ekp_api.worker:celery_app
|
|
||||||
- worker
|
|
||||||
- --loglevel=INFO
|
|
||||||
- --queues=ingestion
|
|
||||||
securityContext:
|
|
||||||
{{- include "knowledge-platform.containerSecurityContext" . | nindent 12 }}
|
|
||||||
envFrom:
|
|
||||||
{{- include "knowledge-platform.envFrom" . | nindent 12 }}
|
|
||||||
startupProbe:
|
|
||||||
exec:
|
|
||||||
command: ["python", "-c", "import os; os.kill(1, 0)"]
|
|
||||||
{{- toYaml .Values.worker.probes.startup | nindent 12 }}
|
|
||||||
livenessProbe:
|
|
||||||
exec:
|
|
||||||
command: ["python", "-c", "import os; os.kill(1, 0)"]
|
|
||||||
{{- toYaml .Values.worker.probes.liveness | nindent 12 }}
|
|
||||||
resources:
|
|
||||||
{{- toYaml .Values.worker.resources | nindent 12 }}
|
|
||||||
volumeMounts:
|
|
||||||
{{- include "knowledge-platform.tmpMount" . | nindent 12 }}
|
|
||||||
volumes:
|
|
||||||
{{- include "knowledge-platform.tmpVolume" . | nindent 8 }}
|
|
||||||
@ -1,338 +0,0 @@
|
|||||||
{
|
|
||||||
"$schema": "https://json-schema.org/draft-07/schema#",
|
|
||||||
"title": "Enterprise Knowledge Platform Helm values",
|
|
||||||
"type": "object",
|
|
||||||
"additionalProperties": false,
|
|
||||||
"required": [
|
|
||||||
"existingSecret",
|
|
||||||
"images",
|
|
||||||
"serviceAccount",
|
|
||||||
"api",
|
|
||||||
"mcp",
|
|
||||||
"worker",
|
|
||||||
"scheduler",
|
|
||||||
"migration",
|
|
||||||
"ingress",
|
|
||||||
"config",
|
|
||||||
"rabbitmq"
|
|
||||||
],
|
|
||||||
"properties": {
|
|
||||||
"nameOverride": {"type": "string"},
|
|
||||||
"fullnameOverride": {"type": "string"},
|
|
||||||
"existingSecret": {"$ref": "#/definitions/kubernetesName"},
|
|
||||||
"imagePullSecrets": {
|
|
||||||
"type": "array",
|
|
||||||
"items": {
|
|
||||||
"type": "object",
|
|
||||||
"additionalProperties": false,
|
|
||||||
"required": ["name"],
|
|
||||||
"properties": {"name": {"$ref": "#/definitions/kubernetesName"}}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"images": {
|
|
||||||
"type": "object",
|
|
||||||
"additionalProperties": false,
|
|
||||||
"required": ["api", "mcp", "worker"],
|
|
||||||
"properties": {
|
|
||||||
"api": {"$ref": "#/definitions/image"},
|
|
||||||
"mcp": {"$ref": "#/definitions/image"},
|
|
||||||
"worker": {"$ref": "#/definitions/image"}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"serviceAccount": {
|
|
||||||
"type": "object",
|
|
||||||
"additionalProperties": false,
|
|
||||||
"required": ["create", "name", "annotations"],
|
|
||||||
"properties": {
|
|
||||||
"create": {"type": "boolean"},
|
|
||||||
"name": {"type": "string"},
|
|
||||||
"annotations": {"$ref": "#/definitions/stringMap"}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"podAnnotations": {"$ref": "#/definitions/stringMap"},
|
|
||||||
"podLabels": {"$ref": "#/definitions/stringMap"},
|
|
||||||
"api": {"$ref": "#/definitions/httpWorkload"},
|
|
||||||
"mcp": {"$ref": "#/definitions/httpWorkload"},
|
|
||||||
"worker": {
|
|
||||||
"type": "object",
|
|
||||||
"additionalProperties": false,
|
|
||||||
"required": ["replicaCount", "terminationGracePeriodSeconds", "resources", "probes"],
|
|
||||||
"properties": {
|
|
||||||
"replicaCount": {"type": "integer", "minimum": 1, "maximum": 100},
|
|
||||||
"terminationGracePeriodSeconds": {"type": "integer", "minimum": 30, "maximum": 3600},
|
|
||||||
"resources": {"$ref": "#/definitions/resources"},
|
|
||||||
"probes": {"$ref": "#/definitions/processProbes"}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"scheduler": {
|
|
||||||
"type": "object",
|
|
||||||
"additionalProperties": false,
|
|
||||||
"required": ["enabled", "resources", "probes"],
|
|
||||||
"properties": {
|
|
||||||
"enabled": {"type": "boolean"},
|
|
||||||
"resources": {"$ref": "#/definitions/resources"},
|
|
||||||
"probes": {"$ref": "#/definitions/processProbes"}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"migration": {
|
|
||||||
"type": "object",
|
|
||||||
"additionalProperties": false,
|
|
||||||
"required": ["enabled", "activeDeadlineSeconds", "backoffLimit", "ttlSecondsAfterFinished", "resources"],
|
|
||||||
"properties": {
|
|
||||||
"enabled": {"type": "boolean"},
|
|
||||||
"activeDeadlineSeconds": {"type": "integer", "minimum": 60, "maximum": 3600},
|
|
||||||
"backoffLimit": {"type": "integer", "minimum": 0, "maximum": 3},
|
|
||||||
"ttlSecondsAfterFinished": {"type": "integer", "minimum": 0},
|
|
||||||
"resources": {"$ref": "#/definitions/resources"}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"ingress": {
|
|
||||||
"type": "object",
|
|
||||||
"additionalProperties": false,
|
|
||||||
"required": ["enabled", "className", "annotations", "api", "mcp"],
|
|
||||||
"properties": {
|
|
||||||
"enabled": {"type": "boolean"},
|
|
||||||
"className": {"type": "string", "minLength": 1},
|
|
||||||
"annotations": {"$ref": "#/definitions/stringMap"},
|
|
||||||
"api": {"$ref": "#/definitions/ingressRoute"},
|
|
||||||
"mcp": {"$ref": "#/definitions/ingressRoute"}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"config": {
|
|
||||||
"type": "object",
|
|
||||||
"additionalProperties": false,
|
|
||||||
"required": ["appName", "logLevel", "database", "ingestion", "objectStorage", "chunking", "embedding", "vectorStore", "retrieval", "reranker", "mcp"],
|
|
||||||
"properties": {
|
|
||||||
"appName": {"type": "string", "minLength": 1},
|
|
||||||
"logLevel": {"enum": ["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"]},
|
|
||||||
"database": {"$ref": "#/definitions/databaseConfig"},
|
|
||||||
"ingestion": {"$ref": "#/definitions/ingestionConfig"},
|
|
||||||
"objectStorage": {"$ref": "#/definitions/objectStorageConfig"},
|
|
||||||
"chunking": {"$ref": "#/definitions/chunkingConfig"},
|
|
||||||
"embedding": {"$ref": "#/definitions/embeddingConfig"},
|
|
||||||
"vectorStore": {"$ref": "#/definitions/vectorStoreConfig"},
|
|
||||||
"retrieval": {"$ref": "#/definitions/retrievalConfig"},
|
|
||||||
"reranker": {"$ref": "#/definitions/rerankerConfig"},
|
|
||||||
"mcp": {"$ref": "#/definitions/mcpConfig"}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"rabbitmq": {
|
|
||||||
"type": "object",
|
|
||||||
"required": ["enabled", "existingSecret"],
|
|
||||||
"properties": {
|
|
||||||
"enabled": {"type": "boolean"},
|
|
||||||
"existingSecret": {"$ref": "#/definitions/kubernetesName"}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"definitions": {
|
|
||||||
"kubernetesName": {
|
|
||||||
"type": "string",
|
|
||||||
"minLength": 1,
|
|
||||||
"maxLength": 253,
|
|
||||||
"pattern": "^[a-z0-9]([-a-z0-9.]*[a-z0-9])?$"
|
|
||||||
},
|
|
||||||
"stringMap": {
|
|
||||||
"type": "object",
|
|
||||||
"additionalProperties": {"type": "string"}
|
|
||||||
},
|
|
||||||
"image": {
|
|
||||||
"type": "object",
|
|
||||||
"additionalProperties": false,
|
|
||||||
"required": ["repository", "tag", "digest", "pullPolicy"],
|
|
||||||
"properties": {
|
|
||||||
"repository": {"type": "string", "minLength": 1},
|
|
||||||
"tag": {"type": "string", "minLength": 1, "not": {"enum": ["latest"]}},
|
|
||||||
"digest": {"type": "string", "pattern": "^$|^sha256:[a-f0-9]{64}$"},
|
|
||||||
"pullPolicy": {"enum": ["Always", "IfNotPresent", "Never"]}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"probeTiming": {
|
|
||||||
"type": "object",
|
|
||||||
"additionalProperties": false,
|
|
||||||
"required": ["failureThreshold", "periodSeconds"],
|
|
||||||
"properties": {
|
|
||||||
"failureThreshold": {"type": "integer", "minimum": 1},
|
|
||||||
"periodSeconds": {"type": "integer", "minimum": 1},
|
|
||||||
"timeoutSeconds": {"type": "integer", "minimum": 1}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"httpProbes": {
|
|
||||||
"type": "object",
|
|
||||||
"additionalProperties": false,
|
|
||||||
"required": ["startup", "readiness", "liveness"],
|
|
||||||
"properties": {
|
|
||||||
"startup": {"$ref": "#/definitions/probeTiming"},
|
|
||||||
"readiness": {"$ref": "#/definitions/probeTiming"},
|
|
||||||
"liveness": {"$ref": "#/definitions/probeTiming"}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"processProbes": {
|
|
||||||
"type": "object",
|
|
||||||
"additionalProperties": false,
|
|
||||||
"required": ["startup", "liveness"],
|
|
||||||
"properties": {
|
|
||||||
"startup": {"$ref": "#/definitions/probeTiming"},
|
|
||||||
"liveness": {"$ref": "#/definitions/probeTiming"}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"resources": {
|
|
||||||
"type": "object",
|
|
||||||
"additionalProperties": false,
|
|
||||||
"required": ["requests", "limits"],
|
|
||||||
"properties": {
|
|
||||||
"requests": {"$ref": "#/definitions/resourcePair"},
|
|
||||||
"limits": {"$ref": "#/definitions/resourcePair"}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"resourcePair": {
|
|
||||||
"type": "object",
|
|
||||||
"additionalProperties": false,
|
|
||||||
"required": ["cpu", "memory"],
|
|
||||||
"properties": {
|
|
||||||
"cpu": {"type": ["string", "number"]},
|
|
||||||
"memory": {"type": "string", "minLength": 2}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"httpWorkload": {
|
|
||||||
"type": "object",
|
|
||||||
"additionalProperties": false,
|
|
||||||
"required": ["replicaCount", "service", "resources", "probes"],
|
|
||||||
"properties": {
|
|
||||||
"replicaCount": {"type": "integer", "minimum": 2, "maximum": 100},
|
|
||||||
"service": {
|
|
||||||
"type": "object",
|
|
||||||
"additionalProperties": false,
|
|
||||||
"required": ["type", "port"],
|
|
||||||
"properties": {
|
|
||||||
"type": {"enum": ["ClusterIP"]},
|
|
||||||
"port": {"type": "integer", "minimum": 1, "maximum": 65535}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"resources": {"$ref": "#/definitions/resources"},
|
|
||||||
"probes": {"$ref": "#/definitions/httpProbes"}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"ingressRoute": {
|
|
||||||
"type": "object",
|
|
||||||
"additionalProperties": false,
|
|
||||||
"required": ["host", "path", "tlsSecretName"],
|
|
||||||
"properties": {
|
|
||||||
"host": {"type": "string", "minLength": 1},
|
|
||||||
"path": {"type": "string", "pattern": "^/"},
|
|
||||||
"tlsSecretName": {"$ref": "#/definitions/kubernetesName"}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"databaseConfig": {
|
|
||||||
"type": "object",
|
|
||||||
"additionalProperties": false,
|
|
||||||
"required": ["poolSize", "maxOverflow", "poolRecycleSeconds", "poolTimeoutSeconds", "connectTimeoutSeconds"],
|
|
||||||
"properties": {
|
|
||||||
"poolSize": {"type": "integer", "minimum": 1, "maximum": 100},
|
|
||||||
"maxOverflow": {"type": "integer", "minimum": 0, "maximum": 100},
|
|
||||||
"poolRecycleSeconds": {"type": "integer", "minimum": 1},
|
|
||||||
"poolTimeoutSeconds": {"type": "number", "exclusiveMinimum": 0},
|
|
||||||
"connectTimeoutSeconds": {"type": "integer", "minimum": 1}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"ingestionConfig": {
|
|
||||||
"type": "object",
|
|
||||||
"additionalProperties": false,
|
|
||||||
"required": ["maxAttempts", "retryBackoffSeconds", "retryBackoffMaxSeconds", "workerLeaseSeconds", "recoveryIntervalSeconds", "batchMaxFiles", "batchMaxTotalBytes", "maxUploadBytes"],
|
|
||||||
"properties": {
|
|
||||||
"maxAttempts": {"type": "integer", "minimum": 1, "maximum": 10},
|
|
||||||
"retryBackoffSeconds": {"type": "integer", "minimum": 1},
|
|
||||||
"retryBackoffMaxSeconds": {"type": "integer", "minimum": 1},
|
|
||||||
"workerLeaseSeconds": {"type": "integer", "minimum": 30},
|
|
||||||
"recoveryIntervalSeconds": {"type": "integer", "minimum": 5},
|
|
||||||
"batchMaxFiles": {"type": "integer", "minimum": 1, "maximum": 100},
|
|
||||||
"batchMaxTotalBytes": {"type": "integer", "minimum": 1},
|
|
||||||
"maxUploadBytes": {"type": "integer", "minimum": 1}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"objectStorageConfig": {
|
|
||||||
"type": "object",
|
|
||||||
"additionalProperties": false,
|
|
||||||
"required": ["provider", "endpoint", "bucket", "keyPrefix", "timeoutSeconds"],
|
|
||||||
"properties": {
|
|
||||||
"provider": {"const": "bos"},
|
|
||||||
"endpoint": {"type": "string", "minLength": 1},
|
|
||||||
"bucket": {"type": "string", "minLength": 1},
|
|
||||||
"keyPrefix": {"type": "string", "minLength": 1},
|
|
||||||
"timeoutSeconds": {"type": "number", "exclusiveMinimum": 0}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"chunkingConfig": {
|
|
||||||
"type": "object",
|
|
||||||
"additionalProperties": false,
|
|
||||||
"required": ["maxTokens", "overlapTokens", "maxCharacters"],
|
|
||||||
"properties": {
|
|
||||||
"maxTokens": {"type": "integer", "minimum": 1},
|
|
||||||
"overlapTokens": {"type": "integer", "minimum": 0},
|
|
||||||
"maxCharacters": {"type": "integer", "minimum": 4}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"embeddingConfig": {
|
|
||||||
"type": "object",
|
|
||||||
"additionalProperties": false,
|
|
||||||
"required": ["provider", "model", "endpoint", "timeoutSeconds"],
|
|
||||||
"properties": {
|
|
||||||
"provider": {"const": "qianfan"},
|
|
||||||
"model": {"const": "bge-large-zh"},
|
|
||||||
"endpoint": {"type": "string", "minLength": 1},
|
|
||||||
"timeoutSeconds": {"type": "number", "exclusiveMinimum": 0}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"vectorStoreConfig": {
|
|
||||||
"type": "object",
|
|
||||||
"additionalProperties": false,
|
|
||||||
"required": ["provider", "uri", "user", "database", "table", "index", "batchSize", "timeoutSeconds"],
|
|
||||||
"properties": {
|
|
||||||
"provider": {"const": "oceanbase"},
|
|
||||||
"uri": {"type": "string", "minLength": 1},
|
|
||||||
"user": {"type": "string", "minLength": 1},
|
|
||||||
"database": {"type": "string", "minLength": 1},
|
|
||||||
"table": {"type": "string", "minLength": 1},
|
|
||||||
"index": {"type": "string", "minLength": 1},
|
|
||||||
"batchSize": {"type": "integer", "minimum": 1, "maximum": 1000},
|
|
||||||
"timeoutSeconds": {"type": "number", "exclusiveMinimum": 0}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"retrievalConfig": {
|
|
||||||
"type": "object",
|
|
||||||
"additionalProperties": false,
|
|
||||||
"required": ["indexProfileId", "denseTopN", "searchTopK"],
|
|
||||||
"properties": {
|
|
||||||
"indexProfileId": {"type": "string", "minLength": 1},
|
|
||||||
"denseTopN": {"type": "integer", "minimum": 1, "maximum": 64},
|
|
||||||
"searchTopK": {"type": "integer", "minimum": 1, "maximum": 64}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"rerankerConfig": {
|
|
||||||
"type": "object",
|
|
||||||
"additionalProperties": false,
|
|
||||||
"required": ["provider", "model", "endpoint", "timeoutSeconds"],
|
|
||||||
"properties": {
|
|
||||||
"provider": {"const": "qianfan"},
|
|
||||||
"model": {"const": "bce-reranker-base"},
|
|
||||||
"endpoint": {"type": "string", "minLength": 1},
|
|
||||||
"timeoutSeconds": {"type": "number", "exclusiveMinimum": 0}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"mcpConfig": {
|
|
||||||
"type": "object",
|
|
||||||
"additionalProperties": false,
|
|
||||||
"required": ["publicUrl", "issuerUrl", "clientId", "serviceIdentity", "requiredScope", "allowedHosts", "allowedOrigins", "maxRequestBodyBytes"],
|
|
||||||
"properties": {
|
|
||||||
"publicUrl": {"type": "string", "pattern": "^https://"},
|
|
||||||
"issuerUrl": {"type": "string", "pattern": "^https://"},
|
|
||||||
"clientId": {"type": "string", "minLength": 1},
|
|
||||||
"serviceIdentity": {"type": "string", "minLength": 1},
|
|
||||||
"requiredScope": {"const": "knowledge:search"},
|
|
||||||
"allowedHosts": {"type": "array", "minItems": 1, "items": {"type": "string", "minLength": 1}},
|
|
||||||
"allowedOrigins": {"type": "array", "minItems": 1, "items": {"type": "string", "pattern": "^https://"}},
|
|
||||||
"maxRequestBodyBytes": {"type": "integer", "minimum": 1024, "maximum": 4194304}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@ -1,218 +0,0 @@
|
|||||||
# DEPLOY-002 safe defaults. Real endpoints, immutable image coordinates and
|
|
||||||
# Secret names are supplied in an environment-specific values file.
|
|
||||||
nameOverride: ""
|
|
||||||
fullnameOverride: ""
|
|
||||||
|
|
||||||
existingSecret: knowledge-platform-runtime
|
|
||||||
imagePullSecrets: []
|
|
||||||
|
|
||||||
images:
|
|
||||||
api:
|
|
||||||
repository: knowledge-platform/enterprise-knowledge-api
|
|
||||||
tag: "0.1.0"
|
|
||||||
digest: ""
|
|
||||||
pullPolicy: IfNotPresent
|
|
||||||
mcp:
|
|
||||||
repository: knowledge-platform/enterprise-knowledge-mcp
|
|
||||||
tag: "0.1.0"
|
|
||||||
digest: ""
|
|
||||||
pullPolicy: IfNotPresent
|
|
||||||
worker:
|
|
||||||
repository: knowledge-platform/enterprise-knowledge-worker
|
|
||||||
tag: "0.1.0"
|
|
||||||
digest: ""
|
|
||||||
pullPolicy: IfNotPresent
|
|
||||||
|
|
||||||
serviceAccount:
|
|
||||||
create: true
|
|
||||||
name: ""
|
|
||||||
annotations: {}
|
|
||||||
|
|
||||||
podAnnotations: {}
|
|
||||||
podLabels: {}
|
|
||||||
|
|
||||||
api:
|
|
||||||
replicaCount: 2
|
|
||||||
service:
|
|
||||||
type: ClusterIP
|
|
||||||
port: 80
|
|
||||||
resources:
|
|
||||||
requests:
|
|
||||||
cpu: 250m
|
|
||||||
memory: 512Mi
|
|
||||||
limits:
|
|
||||||
cpu: "1"
|
|
||||||
memory: 1Gi
|
|
||||||
probes:
|
|
||||||
startup:
|
|
||||||
failureThreshold: 30
|
|
||||||
periodSeconds: 2
|
|
||||||
readiness:
|
|
||||||
periodSeconds: 10
|
|
||||||
timeoutSeconds: 2
|
|
||||||
failureThreshold: 3
|
|
||||||
liveness:
|
|
||||||
periodSeconds: 20
|
|
||||||
timeoutSeconds: 2
|
|
||||||
failureThreshold: 3
|
|
||||||
|
|
||||||
mcp:
|
|
||||||
replicaCount: 2
|
|
||||||
service:
|
|
||||||
type: ClusterIP
|
|
||||||
port: 80
|
|
||||||
resources:
|
|
||||||
requests:
|
|
||||||
cpu: 250m
|
|
||||||
memory: 512Mi
|
|
||||||
limits:
|
|
||||||
cpu: "1"
|
|
||||||
memory: 1Gi
|
|
||||||
probes:
|
|
||||||
startup:
|
|
||||||
failureThreshold: 30
|
|
||||||
periodSeconds: 2
|
|
||||||
readiness:
|
|
||||||
periodSeconds: 10
|
|
||||||
timeoutSeconds: 2
|
|
||||||
failureThreshold: 3
|
|
||||||
liveness:
|
|
||||||
periodSeconds: 20
|
|
||||||
timeoutSeconds: 2
|
|
||||||
failureThreshold: 3
|
|
||||||
|
|
||||||
worker:
|
|
||||||
replicaCount: 2
|
|
||||||
terminationGracePeriodSeconds: 60
|
|
||||||
resources:
|
|
||||||
requests:
|
|
||||||
cpu: 500m
|
|
||||||
memory: 1Gi
|
|
||||||
limits:
|
|
||||||
cpu: "2"
|
|
||||||
memory: 2Gi
|
|
||||||
probes:
|
|
||||||
startup:
|
|
||||||
failureThreshold: 30
|
|
||||||
periodSeconds: 2
|
|
||||||
liveness:
|
|
||||||
periodSeconds: 20
|
|
||||||
timeoutSeconds: 2
|
|
||||||
failureThreshold: 3
|
|
||||||
|
|
||||||
# Celery Beat is separated from scalable workers so only one recovery
|
|
||||||
# scheduler publishes due durable Jobs at a time.
|
|
||||||
scheduler:
|
|
||||||
enabled: true
|
|
||||||
resources:
|
|
||||||
requests:
|
|
||||||
cpu: 100m
|
|
||||||
memory: 256Mi
|
|
||||||
limits:
|
|
||||||
cpu: 500m
|
|
||||||
memory: 512Mi
|
|
||||||
probes:
|
|
||||||
startup:
|
|
||||||
failureThreshold: 30
|
|
||||||
periodSeconds: 2
|
|
||||||
liveness:
|
|
||||||
periodSeconds: 20
|
|
||||||
timeoutSeconds: 2
|
|
||||||
failureThreshold: 3
|
|
||||||
|
|
||||||
migration:
|
|
||||||
enabled: true
|
|
||||||
activeDeadlineSeconds: 600
|
|
||||||
backoffLimit: 0
|
|
||||||
ttlSecondsAfterFinished: 86400
|
|
||||||
resources:
|
|
||||||
requests:
|
|
||||||
cpu: 100m
|
|
||||||
memory: 256Mi
|
|
||||||
limits:
|
|
||||||
cpu: 500m
|
|
||||||
memory: 512Mi
|
|
||||||
|
|
||||||
ingress:
|
|
||||||
enabled: false
|
|
||||||
className: nginx
|
|
||||||
annotations: {}
|
|
||||||
api:
|
|
||||||
host: knowledge.example.invalid
|
|
||||||
path: /
|
|
||||||
tlsSecretName: knowledge-platform-api-tls
|
|
||||||
mcp:
|
|
||||||
host: mcp.example.invalid
|
|
||||||
path: /mcp
|
|
||||||
tlsSecretName: knowledge-platform-mcp-tls
|
|
||||||
|
|
||||||
# Only non-sensitive runtime settings belong here. The templates enumerate
|
|
||||||
# keys explicitly so credentials cannot be added to this ConfigMap by mistake.
|
|
||||||
config:
|
|
||||||
appName: enterprise-knowledge-platform
|
|
||||||
logLevel: INFO
|
|
||||||
database:
|
|
||||||
poolSize: 10
|
|
||||||
maxOverflow: 10
|
|
||||||
poolRecycleSeconds: 600
|
|
||||||
poolTimeoutSeconds: 30
|
|
||||||
connectTimeoutSeconds: 30
|
|
||||||
ingestion:
|
|
||||||
maxAttempts: 3
|
|
||||||
retryBackoffSeconds: 2
|
|
||||||
retryBackoffMaxSeconds: 60
|
|
||||||
workerLeaseSeconds: 300
|
|
||||||
recoveryIntervalSeconds: 30
|
|
||||||
batchMaxFiles: 20
|
|
||||||
batchMaxTotalBytes: 209715200
|
|
||||||
maxUploadBytes: 52428800
|
|
||||||
objectStorage:
|
|
||||||
provider: bos
|
|
||||||
endpoint: https://s3.example.invalid
|
|
||||||
bucket: knowledge-platform
|
|
||||||
keyPrefix: knowledge-platform
|
|
||||||
timeoutSeconds: 30
|
|
||||||
chunking:
|
|
||||||
maxTokens: 450
|
|
||||||
overlapTokens: 75
|
|
||||||
maxCharacters: 2000
|
|
||||||
embedding:
|
|
||||||
provider: qianfan
|
|
||||||
model: bge-large-zh
|
|
||||||
endpoint: https://qianfan.baidubce.com/v2/embeddings
|
|
||||||
timeoutSeconds: 30
|
|
||||||
vectorStore:
|
|
||||||
provider: oceanbase
|
|
||||||
uri: oceanbase.example.invalid:2881
|
|
||||||
user: knowledge-platform
|
|
||||||
database: knowledge_platform
|
|
||||||
table: knowledge_chunks_v1
|
|
||||||
index: vidx_knowledge_chunks_embedding
|
|
||||||
batchSize: 100
|
|
||||||
timeoutSeconds: 30
|
|
||||||
retrieval:
|
|
||||||
indexProfileId: demo-bge-large-zh-structure-v1-oceanbase-v1
|
|
||||||
denseTopN: 20
|
|
||||||
searchTopK: 5
|
|
||||||
reranker:
|
|
||||||
provider: qianfan
|
|
||||||
model: bce-reranker-base
|
|
||||||
endpoint: https://qianfan.baidubce.com/v2/rerank
|
|
||||||
timeoutSeconds: 30
|
|
||||||
mcp:
|
|
||||||
publicUrl: https://mcp.example.invalid/mcp
|
|
||||||
issuerUrl: https://mcp.example.invalid
|
|
||||||
clientId: workbuddy-enterprise
|
|
||||||
serviceIdentity: workbuddy-public-knowledge
|
|
||||||
requiredScope: knowledge:search
|
|
||||||
allowedHosts:
|
|
||||||
- mcp.example.invalid
|
|
||||||
allowedOrigins:
|
|
||||||
- https://mcp.example.invalid
|
|
||||||
maxRequestBodyBytes: 1048576
|
|
||||||
|
|
||||||
# Optional local dependency. Keep disabled for production and provide
|
|
||||||
# RABBITMQ_URL through existingSecret for an externally managed broker.
|
|
||||||
rabbitmq:
|
|
||||||
enabled: false
|
|
||||||
existingSecret: knowledge-platform-rabbitmq
|
|
||||||
@ -1,7 +1,6 @@
|
|||||||
-- ARCH-003 rebuildable OceanBase Vector index for bge-large-zh (1024 dimensions).
|
-- ARCH-003 rebuildable OceanBase Vector index for bge-large-zh (1024 dimensions).
|
||||||
-- Canonical metadata is co-located in OceanBase relational tables; this vector
|
-- PostgreSQL remains canonical; this table may be dropped and rebuilt from canonical chunks.
|
||||||
-- table remains a derived index and may be rebuilt from canonical chunks.
|
-- Requires OceanBase 4.6.0 or newer for explicit ANN scalar pre-filtering.
|
||||||
-- Requires OceanBase 4.3.5.4+; ARCH-006 verifies PRE-FILTER in the target test cluster.
|
|
||||||
|
|
||||||
CREATE TABLE IF NOT EXISTS knowledge_chunks_v1 (
|
CREATE TABLE IF NOT EXISTS knowledge_chunks_v1 (
|
||||||
index_record_id VARCHAR(36) NOT NULL,
|
index_record_id VARCHAR(36) NOT NULL,
|
||||||
|
|||||||
@ -1,12 +0,0 @@
|
|||||||
{
|
|
||||||
"mcpServers": {
|
|
||||||
"enterprise-knowledge": {
|
|
||||||
"type": "http",
|
|
||||||
"url": "${EKP_MCP_URL:-http://127.0.0.1:8766/mcp}",
|
|
||||||
"headers": {
|
|
||||||
"Authorization": "Bearer ${EKP_MCP_TOKEN}"
|
|
||||||
},
|
|
||||||
"description": "企业知识检索"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@ -11,22 +11,17 @@ Use OpenAPI as executable source of truth once implementation starts. Provider-s
|
|||||||
DEMO-009 公开以下最小接口。`docs/demo/DEMO_SCOPE.md` 中的 `/api/*` 逻辑路径按本契约统一
|
DEMO-009 公开以下最小接口。`docs/demo/DEMO_SCOPE.md` 中的 `/api/*` 逻辑路径按本契约统一
|
||||||
版本化为 `/api/v1/*`。
|
版本化为 `/api/v1/*`。
|
||||||
|
|
||||||
`DEMO-015` 已将单文件上传切换为持久化异步 Job;`DEMO-016` 已增加批量提交和聚合查询。
|
|
||||||
|
|
||||||
| Method | Path | 行为 |
|
| Method | Path | 行为 |
|
||||||
|---|---|---|
|
|---|---|---|
|
||||||
| `POST` | `/api/v1/documents` | multipart 单文件接收;持久化 Source、Document、Job 后返回 `202` |
|
| `POST` | `/api/v1/documents` | multipart 单文件上传并同步执行完整入库流水线 |
|
||||||
| `POST` | `/api/v1/ingestion-batches` | multipart `files` 多文件接收;返回批次与逐文件状态 |
|
|
||||||
| `GET` | `/api/v1/ingestion-batches/{batch_id}` | 返回批次汇总和逐文件最新 Job 状态 |
|
|
||||||
| `GET` | `/api/v1/documents` | 按创建时间倒序返回文档列表 |
|
| `GET` | `/api/v1/documents` | 按创建时间倒序返回文档列表 |
|
||||||
| `GET` | `/api/v1/documents/{document_id}` | 返回单个文档状态与安全错误信息 |
|
| `GET` | `/api/v1/documents/{document_id}` | 返回单个文档状态与安全错误信息 |
|
||||||
| `DELETE` | `/api/v1/documents/{document_id}` | 直接删除向量、Metadata、解析产物和原始文件 |
|
| `DELETE` | `/api/v1/documents/{document_id}` | 直接删除向量、Metadata、解析产物和原始文件 |
|
||||||
| `GET` | `/api/v1/jobs/{job_id}` | 返回阶段、状态、attempt、最大尝试次数和安全错误信息 |
|
|
||||||
| `POST` | `/api/v1/search` | Dense Top20、Reranker、TopK、来源补全 |
|
| `POST` | `/api/v1/search` | Dense Top20、Reranker、TopK、来源补全 |
|
||||||
|
|
||||||
上传字段名为 `file`,支持 PDF、DOCX、TXT 和 Markdown。成功返回 `202`、Document 字段及
|
上传字段名为 `file`,支持 PDF、DOCX、TXT 和 Markdown。成功返回 `201` 和 Document;
|
||||||
`job_id`、`job_status=queued`、`max_attempts`;响应不等待 Parse、Clean、Chunk、Embedding 或
|
Demo 同步流水线完成后状态为 `ready`。Document 响应不暴露 `storage_key`、Provider 配置或
|
||||||
Index。Document 响应不暴露 `storage_key`、Provider 配置或索引实现。
|
索引实现。
|
||||||
|
|
||||||
Search 请求:
|
Search 请求:
|
||||||
|
|
||||||
@ -57,87 +52,5 @@ Search 响应:
|
|||||||
```
|
```
|
||||||
|
|
||||||
公开错误语义:请求校验失败返回 `422`,不支持的上传类型返回 `415`,文件过大返回 `413`,
|
公开错误语义:请求校验失败返回 `422`,不支持的上传类型返回 `415`,文件过大返回 `413`,
|
||||||
文档、Job 或批次不存在返回 `404`,活动 Job 的文档删除返回 `409`,运行时 Provider 未配置返回
|
文档不存在返回 `404`,运行时 Provider 未配置返回 `503`。上游 Provider 或持久化异常只返回
|
||||||
`503`。上游 Provider 或持久化异常只返回经过收敛的 `5xx` 信息,不回显凭证、请求正文或
|
经过收敛的 `5xx` 信息,不回显凭证、请求正文或厂商响应。
|
||||||
厂商响应。
|
|
||||||
|
|
||||||
## Demo Admin 客户端路由
|
|
||||||
|
|
||||||
DEMO-010 提供 `/admin/upload`、`/admin/documents` 和 `/admin/search` 三个客户端路由,
|
|
||||||
`/admin` 默认展示上传页。它们是同源静态页面,不是新增业务 API,也不改变上述公开 REST
|
|
||||||
契约;所有业务操作仍调用 `/api/v1`。
|
|
||||||
|
|
||||||
## 当前公共知识库部署边界(PLAN-002)
|
|
||||||
|
|
||||||
当前阶段所有 Document 属于同一个公共知识可见域,不新增用户、知识库或 ACL 字段,也不把
|
|
||||||
这些完整未实现能力伪装为默认放行。Admin 写接口只允许部署在企业受控网络或统一接入网关后;
|
|
||||||
Ingress 是否进一步认证属于部署配置,但前端隐藏按钮或隐藏路由不构成安全控制。
|
|
||||||
|
|
||||||
集群重启、扩缩容和滚动部署不得改变本契约或要求浏览器重新注入 Provider 配置。数据库、BOS、
|
|
||||||
模型与队列凭据由服务端 Secret 提供,任何 API 响应都不得返回这些值。
|
|
||||||
|
|
||||||
## 异步入库契约(DEMO-015~016 已实现)
|
|
||||||
|
|
||||||
| Method | Path | 行为 |
|
|
||||||
|---|---|---|
|
|
||||||
| `POST` | `/api/v1/documents` | 已实现:单文件接收并返回 Document/Job 标识 |
|
|
||||||
| `POST` | `/api/v1/ingestion-batches` | multipart 多文件接收,返回批次及逐文件 Document/Job 标识 |
|
|
||||||
| `GET` | `/api/v1/ingestion-batches/{batch_id}` | 返回批次汇总和逐文件最新状态 |
|
|
||||||
| `GET` | `/api/v1/jobs/{job_id}` | 已实现:返回持久化 Job 状态 |
|
|
||||||
|
|
||||||
单文件响应示意:
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"id": "6e5ec370-5a45-4eb7-95d7-e84267ad8752",
|
|
||||||
"filename": "退款政策.md",
|
|
||||||
"media_type": "text/markdown",
|
|
||||||
"size_bytes": 1024,
|
|
||||||
"status": "uploaded",
|
|
||||||
"error_message": null,
|
|
||||||
"created_at": "2026-09-02T12:00:00Z",
|
|
||||||
"updated_at": "2026-09-02T12:00:00Z",
|
|
||||||
"job_id": "7adc4278-1157-4c3d-95a4-6926ad77a04e",
|
|
||||||
"job_status": "queued",
|
|
||||||
"max_attempts": 3
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
批量字段名为 `files`,成功返回 `202`。响应示意:
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"batch_id": "d21f8012-4c4e-4bb4-89af-e11c0f9f470e",
|
|
||||||
"status": "processing",
|
|
||||||
"total_files": 2,
|
|
||||||
"accepted_files": 1,
|
|
||||||
"rejected_files": 1,
|
|
||||||
"completed_files": 1,
|
|
||||||
"created_at": "2026-09-02T12:00:00Z",
|
|
||||||
"items": [
|
|
||||||
{
|
|
||||||
"item_index": 0,
|
|
||||||
"filename": "退款政策.md",
|
|
||||||
"acceptance_status": "accepted",
|
|
||||||
"document_id": "6e5ec370-5a45-4eb7-95d7-e84267ad8752",
|
|
||||||
"job_id": "7adc4278-1157-4c3d-95a4-6926ad77a04e",
|
|
||||||
"job_status": "queued",
|
|
||||||
"stage": null,
|
|
||||||
"attempt": 0,
|
|
||||||
"max_attempts": 3,
|
|
||||||
"error_message": null
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
Job 公开状态为 `queued | running | retrying | succeeded | failed | cancelled`,并通过 `stage`
|
|
||||||
表示 `parsing | cleaning | chunking | embedding | indexing`。错误字段只返回有界、安全摘要。
|
|
||||||
批次状态为 `processing | completed | partial_failure | failed`;逐文件接收状态为
|
|
||||||
`pending | accepted | rejected`。批次允许逐文件部分接收/失败,不以一个文件的处理结果回滚
|
|
||||||
其他文件。文件数量、单文件大小和请求总大小分别由 `BATCH_MAX_FILES`、`MAX_UPLOAD_BYTES`
|
|
||||||
和 `BATCH_MAX_TOTAL_BYTES` 限制。数量超限返回 `413`;类型、大小或内容不合规作为该文件的
|
|
||||||
`rejected` 安全结果返回,其他合规文件继续接收。
|
|
||||||
|
|
||||||
自动重试是服务端 Job 行为,不要求客户端再次上传,也不新增公开手动重试接口。每次重试前
|
|
||||||
清理派生数据但保留原始文件和稳定的 Document ID;达到次数上限或清理失败后进入 `failed`。
|
|
||||||
|
|||||||
@ -9,43 +9,3 @@ P0 business tools:
|
|||||||
- `list_knowledge_bases`
|
- `list_knowledge_bases`
|
||||||
|
|
||||||
Every request maps to a user/service identity and uses the same authorization/application services as REST. Do not expose OceanBase Vector, BOS, model names, parsers or other infrastructure choices.
|
Every request maps to a user/service identity and uses the same authorization/application services as REST. Do not expose OceanBase Vector, BOS, model names, parsers or other infrastructure choices.
|
||||||
|
|
||||||
## Demo priority slice(DEMO-012,已实现)
|
|
||||||
|
|
||||||
`DEMO-012` 已提升为当前最高实现优先级。首个可运行 Gateway 只注册
|
|
||||||
`search_knowledge`,其输入与 `/api/v1/search` 对齐:
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"query": "产品A退款周期是多少?",
|
|
||||||
"top_k": 5
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
输出保留业务来源字段:`document_id`、`filename`、`page`、`section`、`chunk_id`、`content`
|
|
||||||
和排序分数。Gateway 必须复用现有 Search Service 或正式 REST API,不得复制检索、来源补全
|
|
||||||
或排序逻辑。
|
|
||||||
|
|
||||||
Demo 验证也必须建立 user/service identity 映射边界,禁止使用共享超级管理员 Token。由于
|
|
||||||
Document ACL 尚未进入 Demo Scope,`DEMO-012` 只能证明身份传递、认证拒绝和业务服务复用,
|
|
||||||
不能声称完成 V1 ACL 授权闭环。错误响应和日志不得包含 Provider Endpoint、凭据、数据库表名
|
|
||||||
或模型名。
|
|
||||||
|
|
||||||
验收至少覆盖:MCP 工具发现、参数校验、成功检索、无结果、认证失败、安全错误收敛,以及
|
|
||||||
WorkBuddy Enterprise 的一次端到端调用。其余 P0 tools 继续留在完整 V1 `EKP-071`。
|
|
||||||
|
|
||||||
实现使用独立 Streamable HTTP Gateway,默认端点为 `http://127.0.0.1:8766/mcp`。请求必须
|
|
||||||
携带 `Authorization: Bearer <token>`;Token 只从运行环境读取,并映射为显式 service identity
|
|
||||||
和 `knowledge:search` scope。当前静态 service credential 是 Demo 兼容层,不替代 V1 OAuth、
|
|
||||||
ACL 或用户委托授权。
|
|
||||||
|
|
||||||
公开错误边界:缺失/无效 Token 为 HTTP `401`,scope 不足为 `403`;Tool 参数不合法返回 MCP
|
|
||||||
参数错误;检索运行失败只返回 `knowledge search failed`。Host/Origin allowlist 默认只允许
|
|
||||||
本机,远程部署必须显式注入公开域名。
|
|
||||||
|
|
||||||
## 当前公共知识库部署边界(PLAN-002)
|
|
||||||
|
|
||||||
集群阶段仍只暴露只读 `search_knowledge`,并为 WorkBuddy 配置稳定的 HTTPS MCP 地址。
|
|
||||||
Credential 由 Kubernetes Secret 在服务端运行时注入,客户端只保存管理员分发的最小
|
|
||||||
`knowledge:search` Token;Pod 重启或滚动升级不得要求重新生成配置。公共知识域不等于匿名
|
|
||||||
访问或超级管理员身份,未来引入非公共知识前必须升级为用户委托身份与 ACL-aware Search。
|
|
||||||
|
|||||||
@ -2,39 +2,22 @@
|
|||||||
|
|
||||||
- ADR-001: API First + Admin Console — Accepted
|
- ADR-001: API First + Admin Console — Accepted
|
||||||
- ADR-002: Modular monolith + workers + MCP gateway for V1 — Accepted
|
- ADR-002: Modular monolith + workers + MCP gateway for V1 — Accepted
|
||||||
- ADR-003: PostgreSQL + object storage are canonical — Superseded by ADR-012
|
- ADR-003: PostgreSQL + object storage are canonical — Accepted
|
||||||
- ADR-004: Mochow is initial retrieval backend but replaceable — Superseded by ADR-010
|
- ADR-004: Mochow is initial retrieval backend but replaceable — Superseded by ADR-010
|
||||||
- ADR-005: Provider abstraction for embedding/reranker/parser/LLM — Accepted
|
- ADR-005: Provider abstraction for embedding/reranker/parser/LLM — Accepted
|
||||||
- ADR-006: RAG + Knowledge Compilation — Accepted
|
- ADR-006: RAG + Knowledge Compilation — Accepted
|
||||||
- ADR-007: ACL before/during retrieval — Accepted
|
- ADR-007: ACL before/during retrieval — Accepted
|
||||||
- ADR-008: MCP hides infrastructure details — Accepted
|
- ADR-008: MCP hides infrastructure details — Accepted
|
||||||
- ADR-009: Baidu BOS is the primary object-storage provider; local storage remains for tests and local development — Accepted
|
- ADR-009: Baidu BOS is the primary object-storage provider; local storage remains for tests and local development — Accepted
|
||||||
- ADR-010: OceanBase Vector is the primary rebuildable retrieval index behind `VectorStore`; PostgreSQL remains canonical — Superseded by ADR-012
|
- ADR-010: OceanBase Vector is the primary rebuildable retrieval index behind `VectorStore`; PostgreSQL remains canonical — Accepted
|
||||||
- ADR-011: Qianfan `bge-large-zh` plus `bce-reranker-base` is the initial Demo model baseline — Accepted
|
- ADR-011: Qianfan `bge-large-zh` plus `bce-reranker-base` is the initial Demo model baseline — Accepted
|
||||||
- ADR-012: OceanBase MySQL relational tables + BOS are canonical; OceanBase vector tables remain rebuildable indexes — Accepted
|
|
||||||
|
|
||||||
## Architecture migration boundary
|
## Architecture migration boundary
|
||||||
|
|
||||||
ARCH-006 的只读环境核验确认,目标平台没有独立 PostgreSQL,OceanBase MySQL 模式同时承载
|
ADR-009~011 adopt the verified internal platform provider baseline without copying its
|
||||||
关系数据和向量数据。因此 ADR-012 取代 ADR-003/010 中的 PostgreSQL 部署假设:OceanBase
|
canonical-database design. ADR-003 remains binding: OceanBase Vector stores only rebuildable
|
||||||
关系表保存 canonical metadata,BOS 保存不可变源文件,OceanBase 专用向量表仍只保存可由
|
retrieval rows, while PostgreSQL and object storage retain canonical metadata and immutable
|
||||||
canonical Chunk 重建的检索行。共用数据库实例不改变两类数据的所有权边界,也不允许业务层
|
source data.
|
||||||
绕过 `VectorStore` 直接依赖 `pyobvector`。
|
|
||||||
|
|
||||||
目标 test 集群为 OceanBase 4.3.5.4 MySQL 模式。ARCH-006 已用只读 `EXPLAIN` 验证
|
|
||||||
`VECTOR_INDEX(... pre_filter)` 产生 `VECTOR INDEX ADAPTIVE SCAN (PRE-FILTER)`;版本下限与
|
|
||||||
运行检查据此调整为 4.3.5.4。生产升级和重新验收仍由运维流程控制。
|
|
||||||
|
|
||||||
## Public knowledge delivery boundary
|
|
||||||
|
|
||||||
PLAN-002 将当前交付阶段限定为企业受控网络内的单一公共知识可见域,并加入 Kubernetes/Helm
|
|
||||||
集群交付任务。这是部署范围和实施顺序调整,不是新的架构选型:继续采用 ADR-002 的模块化
|
|
||||||
单体 API、独立 Worker 和 MCP Gateway,以及架构既定的 Celery/RabbitMQ;OceanBase/BOS
|
|
||||||
边界继续遵循 ADR-012。
|
|
||||||
|
|
||||||
ADR-007 仍为 Accepted。当前阶段只是延后用户/组/角色和 Document ACL;开始接收任何非公共
|
|
||||||
知识前,必须先实现 Permission Resolver 和检索前/检索中 ACL 下推。公共知识库也不等于匿名
|
|
||||||
公网开放,Admin 写操作只允许经受控网络或统一接入网关访问,MCP 使用最小只读服务身份。
|
|
||||||
|
|
||||||
The migration retains the Demo's Dense Top20 → Reranker Top5 behavior and current lightweight
|
The migration retains the Demo's Dense Top20 → Reranker Top5 behavior and current lightweight
|
||||||
Parser scope. Top50 retrieval, distance thresholds, dual-query RRF, Marker/Pandoc and LLM-derived
|
Parser scope. Top50 retrieval, distance thresholds, dual-query RRF, Marker/Pandoc and LLM-derived
|
||||||
|
|||||||
@ -63,11 +63,10 @@ Wiki / FAQ / SOP / Summary 等派生知识
|
|||||||
┌────────────────────────┼────────────────────────┐
|
┌────────────────────────┼────────────────────────┐
|
||||||
│ │ │
|
│ │ │
|
||||||
▼ ▼ ▼
|
▼ ▼ ▼
|
||||||
┌────────────────────────┐ ┌──────────────┐
|
┌──────────────┐ ┌──────────────┐ ┌──────────────┐
|
||||||
│ OceanBase MySQL │ │ Object Store │
|
│ PostgreSQL │ │ Object Store │ │ OceanBase │
|
||||||
│ Canonical + Vector │ │ BOS │
|
│ Canonical DB │ │ BOS │ │ Vector Index │
|
||||||
│ (logical separation) │ └──────────────┘
|
└──────────────┘ └──────────────┘ └──────────────┘
|
||||||
└────────────────────────┘
|
|
||||||
▲ ▲
|
▲ ▲
|
||||||
│ │
|
│ │
|
||||||
┌───────┴────────────────────────┘
|
┌───────┴────────────────────────┘
|
||||||
@ -89,7 +88,7 @@ Wiki / FAQ / SOP / Summary 等派生知识
|
|||||||
|
|
||||||
1. **API First**:后台、MCP、其他业务系统均调用统一 API/Service。
|
1. **API First**:后台、MCP、其他业务系统均调用统一 API/Service。
|
||||||
2. **模块化单体优先**:V1 不拆大量微服务,API 服务 + 异步 Worker + MCP Gateway 足够。
|
2. **模块化单体优先**:V1 不拆大量微服务,API 服务 + 异步 Worker + MCP Gateway 足够。
|
||||||
3. **Canonical Data 逻辑独立**:OceanBase 关系表 + BOS 保存事实数据,同库向量表仅保存可重建索引。
|
3. **Canonical Data 独立**:PostgreSQL + 对象存储保存事实数据,Vector DB 仅保存可重建索引。
|
||||||
4. **Provider 可替换**:Embedding、Reranker、Vector DB、Parser、LLM 均通过 Adapter 抽象。
|
4. **Provider 可替换**:Embedding、Reranker、Vector DB、Parser、LLM 均通过 Adapter 抽象。
|
||||||
5. **MCP 不暴露底层实现**:客户端只看到 `search_knowledge` 等业务工具。
|
5. **MCP 不暴露底层实现**:客户端只看到 `search_knowledge` 等业务工具。
|
||||||
|
|
||||||
@ -100,12 +99,12 @@ Wiki / FAQ / SOP / Summary 等派生知识
|
|||||||
| 模块 | V1 当前选型 | 说明 |
|
| 模块 | V1 当前选型 | 说明 |
|
||||||
|---|---|---|
|
|---|---|---|
|
||||||
| Backend | Python + FastAPI | AI / 数据处理生态成熟,适合 MCP、异步任务和模型调用 |
|
| Backend | Python + FastAPI | AI / 数据处理生态成熟,适合 MCP、异步任务和模型调用 |
|
||||||
| Metadata DB | OceanBase MySQL | 保存用户、权限、文档、版本、任务、Compiled Knowledge 等 Canonical Data |
|
| Metadata DB | PostgreSQL | 保存用户、权限、文档、版本、任务、Compiled Knowledge 等 Canonical Data |
|
||||||
| Object Storage | Baidu BOS | 保存不可变原始文件、解析产物和中间文件;Local Adapter 仅用于测试/本地开发 |
|
| Object Storage | Baidu BOS | 保存不可变原始文件、解析产物和中间文件;Local Adapter 仅用于测试/本地开发 |
|
||||||
| Task Queue | Celery + RabbitMQ | 文件解析、Embedding、索引、Compilation 等异步任务 |
|
| Task Queue | Celery + RabbitMQ | 文件解析、Embedding、索引、Compilation 等异步任务 |
|
||||||
| PDF / OCR Parser | MinerU 优先 | 中文复杂 PDF、扫描件、表格、版面解析能力较强 |
|
| PDF / OCR Parser | MinerU 优先 | 中文复杂 PDF、扫描件、表格、版面解析能力较强 |
|
||||||
| Office Parser | Docling 补充 | DOCX/PPTX/XLSX/HTML 等格式统一解析 |
|
| Office Parser | Docling 补充 | DOCX/PPTX/XLSX/HTML 等格式统一解析 |
|
||||||
| Vector/Search DB | OceanBase Vector | 与关系表同库部署但逻辑隔离,必须通过 `VectorStore` 并可从 canonical Chunk 重建 |
|
| Vector/Search DB | OceanBase Vector | 当前主检索索引,必须通过 `VectorStore` 隔离并可从 PostgreSQL 重建 |
|
||||||
| Embedding | Qianfan bge-large-zh | 当前 1024 维基线,后续模型变更使用版本化 Shadow Index |
|
| Embedding | Qianfan bge-large-zh | 当前 1024 维基线,后续模型变更使用版本化 Shadow Index |
|
||||||
| Reranker | Qianfan bce-reranker-base | 当前 Dense Top20 → Reranker Top5 基线 |
|
| Reranker | Qianfan bce-reranker-base | 当前 Dense Top20 → Reranker Top5 基线 |
|
||||||
| LLM | Provider 抽象,云端优先 | 用于 Knowledge Compilation、分类、摘要等 |
|
| LLM | Provider 抽象,云端优先 | 用于 Knowledge Compilation、分类、摘要等 |
|
||||||
@ -174,46 +173,6 @@ Queued / Parsing / Cleaning / Chunking / Embedding / Indexing / Completed / Fail
|
|||||||
- 软删除;
|
- 软删除;
|
||||||
- 来源页码和章节追踪。
|
- 来源页码和章节追踪。
|
||||||
|
|
||||||
### 4.2.1 批量异步提交与失败恢复
|
|
||||||
|
|
||||||
API 只负责校验并持久化不可变源文件、Document 和 Ingestion Job,然后返回 `202`。批次是
|
|
||||||
提交与聚合查询边界,不是跨文件数据库事务;每个文件独立排队、执行和失败。
|
|
||||||
|
|
||||||
DEMO-016 以 canonical `ingestion_batches` / `ingestion_batch_items` 保存输入顺序和接收引用;
|
|
||||||
处理阶段、Attempt 与最终结果继续以 `ingestion_jobs` / `ingestion_attempts` 为事实源,批次表
|
|
||||||
不复制 Worker 状态。
|
|
||||||
|
|
||||||
```text
|
|
||||||
Batch Request
|
|
||||||
├─ Source A + Document A + Job A ─→ Worker
|
|
||||||
├─ Source B + Document B + Job B ─→ Worker
|
|
||||||
└─ Source C + Document C + Job C ─→ Worker
|
|
||||||
```
|
|
||||||
|
|
||||||
Worker 对可重试错误执行有界退避。重新执行前必须通过独立的派生数据清理服务按以下顺序
|
|
||||||
恢复干净状态:
|
|
||||||
|
|
||||||
```text
|
|
||||||
Vector/Sparse rows
|
|
||||||
→ ChunkIndexState
|
|
||||||
→ Chunks
|
|
||||||
→ Parsed artifact + parsed_storage_key
|
|
||||||
→ reset processing state
|
|
||||||
→ next attempt from immutable source
|
|
||||||
```
|
|
||||||
|
|
||||||
清理服务与 Document 删除语义分离:重试清理保留原始 BOS 对象、Document 和 Job/Attempt
|
|
||||||
历史。清理失败时 Job 终止,不能在未知状态继续执行。Job 通过数据库状态、唯一有效 Attempt
|
|
||||||
和幂等键避免重复投递产生并发执行;具体队列实现保持可替换,不进入业务服务接口。
|
|
||||||
|
|
||||||
公共知识库集群交付阶段由 Celery/RabbitMQ 作为首个生产执行器。API 只调用队列执行器抽象,
|
|
||||||
Celery task 只负责恢复应用上下文并调用 Ingestion Application Service;重试分类、Attempt 状态、
|
|
||||||
清理顺序和业务幂等仍由应用层持有。Broker 消息不是 Job 的 source of truth,Worker 重启或消息
|
|
||||||
重复投递必须以 OceanBase 中的持久化 Job/Attempt 状态收敛。
|
|
||||||
|
|
||||||
DEPLOY-001 将 API/Admin、MCP Gateway 与 Async Worker 构建为三个共享锁定 Python 运行时的
|
|
||||||
非 root 镜像 target;进程边界与上述模块边界一致,不新增服务或把 Provider 配置烘焙进镜像。
|
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
### 4.3 文档结构与 Chunk
|
### 4.3 文档结构与 Chunk
|
||||||
@ -367,7 +326,7 @@ profile_v2
|
|||||||
|
|
||||||
避免直接原地覆盖现有生产索引。
|
避免直接原地覆盖现有生产索引。
|
||||||
|
|
||||||
ARCH-005/006 将 Profile 生命周期以 `shadow / active / retired` 记录在 OceanBase 关系表;任一时刻至多
|
ARCH-005 将 Profile 生命周期以 `shadow / active / retired` 记录在 PostgreSQL;任一时刻至多
|
||||||
一个 Active Profile。重建从 canonical Chunk 生成确定性 `chunk_id + profile_id` 索引行,
|
一个 Active Profile。重建从 canonical Chunk 生成确定性 `chunk_id + profile_id` 索引行,
|
||||||
重复执行跳过已完整文档,激活前重新核对全部 ChunkIndexState,并在同一事务内锁定 Chunk
|
重复执行跳过已完整文档,激活前重新核对全部 ChunkIndexState,并在同一事务内锁定 Chunk
|
||||||
写入、退役旧 Profile、激活新 Profile。OceanBase 表本身仍不是 canonical source。
|
写入、退役旧 Profile、激活新 Profile。OceanBase 表本身仍不是 canonical source。
|
||||||
@ -405,9 +364,8 @@ VectorStoreProvider
|
|||||||
|
|
||||||
业务 Domain Model 不依赖 OceanBase/`pyobvector` SDK 数据结构。
|
业务 Domain Model 不依赖 OceanBase/`pyobvector` SDK 数据结构。
|
||||||
|
|
||||||
OceanBase Vector 表中的数据全部视为 **可重建索引**。同一数据库的关系表保存文档、Chunk、
|
OceanBase Vector 中的数据全部视为 **可重建索引**。PostgreSQL 保存文档、Chunk、Profile
|
||||||
Profile 和索引状态等 canonical 数据,BOS 保存不可变原始文件和派生产物。物理共库不允许
|
和索引状态等 canonical 数据,BOS 保存不可变原始文件和派生产物。
|
||||||
向量表反向成为业务事实来源。
|
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
@ -544,17 +502,6 @@ MinerU
|
|||||||
|
|
||||||
首要验证客户端:**WorkBuddy Enterprise**。
|
首要验证客户端:**WorkBuddy Enterprise**。
|
||||||
|
|
||||||
### 11.1 DEMO-012 实现边界
|
|
||||||
|
|
||||||
当前已落地独立 Streamable HTTP Gateway,默认只绑定 `127.0.0.1:8766/mcp`,只注册
|
|
||||||
`search_knowledge`。Gateway 与 REST 共享 `SearchService`,同步检索通过工作线程执行,避免
|
|
||||||
阻塞 MCP ASGI 事件循环。
|
|
||||||
|
|
||||||
Demo 使用运行时专用 Bearer Token 映射到 client/service identity 和最小
|
|
||||||
`knowledge:search` scope,并启用 Host/Origin allowlist。该边界用于 WorkBuddy 接入验证,
|
|
||||||
后续 `EKP-070` 仍需用正式 OAuth/企业身份系统和 Permission Resolver 替换静态 Token;
|
|
||||||
`DEMO-012` 不改变 ADR-007 的 ACL 要求。
|
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## 12. REST API
|
## 12. REST API
|
||||||
@ -655,11 +602,6 @@ ReindexJob
|
|||||||
EvaluationJob
|
EvaluationJob
|
||||||
```
|
```
|
||||||
|
|
||||||
当前增量实现先提供单个 `IngestionJob` 编排既有 Parse/Clean/Chunk/Embedding/Indexing 阶段,
|
|
||||||
保留阶段级状态和 Attempt 历史;以后拆分为独立阶段 Job 时不得改变公开批次/Job 查询契约。
|
|
||||||
Demo 默认最多 3 次总尝试,只有瞬时 Provider、网络、存储或数据库错误进入自动重试;输入
|
|
||||||
校验和确定性内容错误直接终止。
|
|
||||||
|
|
||||||
GPU 节点优先承担:
|
GPU 节点优先承担:
|
||||||
|
|
||||||
- MinerU / OCR;
|
- MinerU / OCR;
|
||||||
@ -672,7 +614,7 @@ LLM V1 可继续使用云端 Provider,不要求部署本地大模型。
|
|||||||
|
|
||||||
## 15. 数据存储原则
|
## 15. 数据存储原则
|
||||||
|
|
||||||
### OceanBase MySQL 关系表
|
### PostgreSQL
|
||||||
|
|
||||||
保存:
|
保存:
|
||||||
|
|
||||||
@ -721,13 +663,15 @@ Reverse Proxy / Ingress
|
|||||||
├─ Knowledge API
|
├─ Knowledge API
|
||||||
└─ MCP Gateway
|
└─ MCP Gateway
|
||||||
|
|
||||||
OceanBase MySQL(Canonical + Vector,逻辑隔离)
|
PostgreSQL
|
||||||
RabbitMQ
|
RabbitMQ
|
||||||
Baidu BOS
|
Baidu BOS
|
||||||
|
|
||||||
CPU Worker
|
CPU Worker
|
||||||
GPU Worker
|
GPU Worker
|
||||||
|
|
||||||
|
OceanBase Vector
|
||||||
|
|
||||||
Observability
|
Observability
|
||||||
```
|
```
|
||||||
|
|
||||||
@ -744,30 +688,6 @@ Observability
|
|||||||
|
|
||||||
未来根据负载再拆服务。
|
未来根据负载再拆服务。
|
||||||
|
|
||||||
### 16.1 当前公共知识库集群交付基线
|
|
||||||
|
|
||||||
PLAN-002 将当前交付范围收敛为企业受控网络内的单一公共知识可见域。该范围不重开 ADR-007:
|
|
||||||
所有现有文档共享同一可见性,因而当前检索不执行事后 ACL 过滤;接收任何非公共知识前,必须
|
|
||||||
先实现 Permission Resolver 和数据库/向量检索前置过滤。Admin 写接口不得直接暴露到匿名公网,
|
|
||||||
MCP 使用运行时 Secret 注入的最小 `knowledge:search` 服务身份。
|
|
||||||
|
|
||||||
集群中 API/Admin、MCP Gateway 和 Async Worker 分别部署并独立扩缩容,业务核心继续保持模块化
|
|
||||||
单体。当前 Parser 及云端 Embedding/Reranker 不需要 GPU Worker,因此它不进入最小交付,但角色
|
|
||||||
边界保留。OceanBase、BOS 和 AI Provider 使用外部服务;RabbitMQ 默认使用外部地址,可为开发/
|
|
||||||
演示显式启用 Chart 依赖。
|
|
||||||
|
|
||||||
Kubernetes/Helm 必须提供:生产镜像、`existingSecret`、串行 Migration Job、Ingress/TLS、
|
|
||||||
readiness/liveness、资源限制、非 root 与只读根文件系统、多副本和滚动更新。仓库的一键入口只
|
|
||||||
编排已有集群和外部服务,不创建云账号或把凭据写入 Values、镜像、命令行及日志。现有
|
|
||||||
`docker-compose.dev.yml` 包含已被 ADR-012 取代的 PostgreSQL/MinIO 假设,只能视为待清理的
|
|
||||||
本地开发资产,不能作为集群模板。详细依赖和验收见
|
|
||||||
[`PUBLIC_KNOWLEDGE_CLUSTER_PLAN.md`](../plans/PUBLIC_KNOWLEDGE_CLUSTER_PLAN.md)。
|
|
||||||
|
|
||||||
DEPLOY-002 的运行拓扑将 Celery Worker 与 Celery Beat 恢复调度器拆成独立 Pod:Worker 可横向
|
|
||||||
扩缩,Beat 固定单副本并使用 `Recreate`,避免每个 Worker 都发布周期恢复消息。二者仍调用同一
|
|
||||||
应用服务与持久化 Job/Attempt 状态,不形成新业务服务,也不改变 ADR-002。Migration 使用 Helm
|
|
||||||
`pre-install,pre-upgrade` Hook;失败会在业务 Deployment 更新前终止原子升级。
|
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## 17. V1 验收重点
|
## 17. V1 验收重点
|
||||||
@ -822,7 +742,7 @@ WorkBuddy 可以通过 MCP 完成知识搜索和 Source 获取。
|
|||||||
|
|
||||||
```text
|
```text
|
||||||
Backend Python + FastAPI
|
Backend Python + FastAPI
|
||||||
Metadata OceanBase MySQL relational tables
|
Metadata PostgreSQL
|
||||||
Object Storage Baidu BOS(Local Adapter 仅测试/本地开发)
|
Object Storage Baidu BOS(Local Adapter 仅测试/本地开发)
|
||||||
Queue RabbitMQ + Celery
|
Queue RabbitMQ + Celery
|
||||||
Parser MinerU 主力 + Docling 补充
|
Parser MinerU 主力 + Docling 补充
|
||||||
|
|||||||
@ -67,8 +67,8 @@ FastAPI
|
|||||||
辅助存储:
|
辅助存储:
|
||||||
|
|
||||||
```text
|
```text
|
||||||
OceanBase MySQL
|
SQLite / PostgreSQL
|
||||||
└── canonical document / chunk metadata
|
└── document / chunk metadata
|
||||||
|
|
||||||
Local Storage / BOS
|
Local Storage / BOS
|
||||||
└── original files
|
└── original files
|
||||||
@ -83,7 +83,7 @@ Demo 阶段优先简单实现,不引入不必要的基础设施。
|
|||||||
| 模块 | Demo 推荐选型 | 说明 |
|
| 模块 | Demo 推荐选型 | 说明 |
|
||||||
|---|---|---|
|
|---|---|---|
|
||||||
| Backend | FastAPI | 简单、开发效率高 |
|
| Backend | FastAPI | 简单、开发效率高 |
|
||||||
| Metadata DB | OceanBase MySQL | 目标环境使用 OceanBase;SQLite 仅用于离线测试 |
|
| Metadata DB | SQLite 或 PostgreSQL | 单机 Demo 可 SQLite;已有环境可直接 PostgreSQL |
|
||||||
| File Storage | 百度 BOS | Local Adapter 仅用于测试和本地开发,目标环境使用 BOS |
|
| File Storage | 百度 BOS | Local Adapter 仅用于测试和本地开发,目标环境使用 BOS |
|
||||||
| Parser | PyMuPDF + python-docx 或 MinerU | 优先快速跑通;复杂 PDF 后续增强 |
|
| Parser | PyMuPDF + python-docx 或 MinerU | 优先快速跑通;复杂 PDF 后续增强 |
|
||||||
| Cleaner | 自定义基础清洗 | 处理空行、空白、异常换行等 |
|
| Cleaner | 自定义基础清洗 | 处理空行、空白、异常换行等 |
|
||||||
@ -92,7 +92,7 @@ Demo 阶段优先简单实现,不引入不必要的基础设施。
|
|||||||
| Vector DB | OceanBase Vector | 当前默认,通过 `VectorStore` 隔离 SDK,不做多数据库运行时切换 |
|
| Vector DB | OceanBase Vector | 当前默认,通过 `VectorStore` 隔离 SDK,不做多数据库运行时切换 |
|
||||||
| Reranker | `bce-reranker-base` | 延续当前可调用的千帆 Reranker,Demo 中保留 |
|
| Reranker | `bce-reranker-base` | 延续当前可调用的千帆 Reranker,Demo 中保留 |
|
||||||
| Frontend | 简单 React/Vue 或 FastAPI HTML | 以最小可用为目标 |
|
| Frontend | 简单 React/Vue 或 FastAPI HTML | 以最小可用为目标 |
|
||||||
| Deployment | Kubernetes/Helm | PLAN-002 起面向企业受控网络交付;本地模式仅用于开发测试 |
|
| Deployment | 本地 / 单机 Docker | 暂不考虑 K8s |
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
@ -189,7 +189,7 @@ embedding
|
|||||||
|
|
||||||
Demo 阶段不实现 Qdrant/Milvus/pgvector,也不实现多 Vector Store 动态切换,但避免
|
Demo 阶段不实现 Qdrant/Milvus/pgvector,也不实现多 Vector Store 动态切换,但避免
|
||||||
OceanBase/pyobvector 调用散落在整个项目中。OceanBase Vector 行仍是可重建索引,不能替代
|
OceanBase/pyobvector 调用散落在整个项目中。OceanBase Vector 行仍是可重建索引,不能替代
|
||||||
同库关系表中的 canonical Chunk。
|
PostgreSQL canonical Chunk。
|
||||||
|
|
||||||
### 5.7 Dense Retrieval
|
### 5.7 Dense Retrieval
|
||||||
|
|
||||||
@ -305,7 +305,7 @@ Search Response 示例:
|
|||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## 8. Demo 后续增强
|
## 8. P1 可选增强
|
||||||
|
|
||||||
### DEMO-011 Hybrid Search
|
### DEMO-011 Hybrid Search
|
||||||
|
|
||||||
@ -323,26 +323,12 @@ Reranker
|
|||||||
|
|
||||||
### DEMO-012 MCP
|
### DEMO-012 MCP
|
||||||
|
|
||||||
**已完成。** 只实现一个最小 MCP Tool:`search_knowledge`,复用现有
|
只实现一个最小 MCP Tool:`search_knowledge`,内部调用 `/api/search`,用于验证 WorkBuddy 接入。
|
||||||
Search Service 或 `/api/v1/search`,用于验证标准 MCP、identity 边界和 WorkBuddy 接入。
|
|
||||||
不在本 Task 扩展其他 P0 tools 或声称完成尚未实施的 Document ACL。
|
|
||||||
|
|
||||||
### DEMO-013 OCR / MinerU
|
### DEMO-013 OCR / MinerU
|
||||||
|
|
||||||
增加扫描 PDF、图片、OCR 和复杂中文 PDF 处理能力。
|
增加扫描 PDF、图片、OCR 和复杂中文 PDF 处理能力。
|
||||||
|
|
||||||
### DEMO-015 异步入库 Job 与自动重试清理
|
|
||||||
|
|
||||||
**已完成。** 将现有同步入库流水线放入持久化 Ingestion Job。可重试失败自动执行有界重试;每次重试前
|
|
||||||
删除旧解析产物、Chunk、索引状态和向量数据,但保留不可变原始文件、Document 和 Attempt
|
|
||||||
历史。清理失败时终止,不在未知脏状态继续执行。通过执行器抽象接入 Celery/RabbitMQ,使
|
|
||||||
API 与 Worker 可独立部署,并以持久化状态处理 Worker 重启和重复消息。
|
|
||||||
|
|
||||||
### DEMO-016 批量异步上传
|
|
||||||
|
|
||||||
**已完成。** 在 DEMO-015 基础上支持一次提交多个文件、逐文件 Document/Job 状态、批次聚合查询和
|
|
||||||
Admin Console 状态展示。批次内文件相互隔离,不包含目录上传、压缩包解包或断点续传。
|
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## 9. 明确后置能力
|
## 9. 明确后置能力
|
||||||
@ -362,13 +348,12 @@ Admin Console 状态展示。批次内文件相互隔离,不包含目录上传
|
|||||||
- 多 Embedding Benchmark Platform;
|
- 多 Embedding Benchmark Platform;
|
||||||
- 多 Vector DB;
|
- 多 Vector DB;
|
||||||
- 动态 Provider;
|
- 动态 Provider;
|
||||||
|
- RabbitMQ / Celery;
|
||||||
|
- Kubernetes;
|
||||||
|
- 高可用;
|
||||||
- 灾备;
|
- 灾备;
|
||||||
- 完整 Observability。
|
- 完整 Observability。
|
||||||
|
|
||||||
用户/权限隔离属于完整 V1 后置能力,不因“公共知识库”而取消;本阶段所有已接收知识共享一个
|
|
||||||
可见域,且仅部署在企业受控网络或统一接入网关后。RabbitMQ/Celery 与最小 Kubernetes 可用性
|
|
||||||
已由 PLAN-002 纳入当前交付,跨区域容灾和完整平台级高可用仍不在范围内。
|
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## 10. Demo Task 清单
|
## 10. Demo Task 清单
|
||||||
@ -384,45 +369,17 @@ DEMO-007 Dense Retrieval
|
|||||||
DEMO-008 Reranker + Source Result
|
DEMO-008 Reranker + Source Result
|
||||||
DEMO-009 Document/Search REST API
|
DEMO-009 Document/Search REST API
|
||||||
DEMO-010 简单 Admin Console
|
DEMO-010 简单 Admin Console
|
||||||
DEMO-012 MCP search_knowledge(已提升优先级)
|
|
||||||
DEMO-014 地瓜机器人品牌化前端优化(用户新增)
|
|
||||||
DEMO-015 异步入库 Job + 自动重试前清理(用户新增)
|
|
||||||
DEMO-016 批量异步上传 + 状态展示(用户新增)
|
|
||||||
ARCH-001~006 Provider 架构迁移与真实验收
|
ARCH-001~006 Provider 架构迁移与真实验收
|
||||||
PLAN-002 公共知识库集群交付规划(已完成)
|
|
||||||
DEPLOY-001 API/Admin、MCP Gateway、Async Worker 生产镜像
|
|
||||||
DEPLOY-002 Kubernetes/Helm 一键部署
|
|
||||||
DEPLOY-003 可用性、滚动升级和远程 WorkBuddy 验收
|
|
||||||
```
|
```
|
||||||
|
|
||||||
实施状态(2026-09-02):`DEMO-001~DEMO-010`、用户新增的 `DEMO-014` 与
|
|
||||||
`ARCH-001~ARCH-006` 均已完成并通过当前 Scope 的闭环验收。`PLAN-001` 已完成现状审计和
|
|
||||||
任务拆分;`DEMO-012`、`DEMO-015~016` 已完成。`PLAN-002` 已将公共知识库集群交付纳入范围,
|
|
||||||
后续严格按 `DEPLOY-001~003` 顺序实施。`DEMO-011` 和 `DEMO-013` 仍为
|
|
||||||
可选项,不因本状态记录自动进入范围。
|
|
||||||
|
|
||||||
可选:
|
可选:
|
||||||
|
|
||||||
```text
|
```text
|
||||||
DEMO-011 Hybrid Search
|
DEMO-011 Hybrid Search
|
||||||
|
DEMO-012 MCP search_knowledge
|
||||||
DEMO-013 OCR / MinerU
|
DEMO-013 OCR / MinerU
|
||||||
```
|
```
|
||||||
|
|
||||||
已批准顺序:
|
|
||||||
|
|
||||||
```text
|
|
||||||
PLAN-002 Public Knowledge Cluster Plan
|
|
||||||
→ DEMO-015 Celery/RabbitMQ Async Job + Retry Cleanup
|
|
||||||
→ DEMO-016 Batch Async Upload + UI
|
|
||||||
→ DEPLOY-001 Production Images
|
|
||||||
→ DEPLOY-002 Kubernetes/Helm One-click Deploy
|
|
||||||
→ DEPLOY-003 Availability + Remote Acceptance
|
|
||||||
```
|
|
||||||
|
|
||||||
详细 Task 映射、依赖与验收边界见
|
|
||||||
[`docs/plans/ASYNC_INGESTION_MCP_PLAN.md`](../plans/ASYNC_INGESTION_MCP_PLAN.md) 和
|
|
||||||
[`docs/plans/PUBLIC_KNOWLEDGE_CLUSTER_PLAN.md`](../plans/PUBLIC_KNOWLEDGE_CLUSTER_PLAN.md)。
|
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## 11. Demo 测试数据
|
## 11. Demo 测试数据
|
||||||
@ -511,13 +468,10 @@ Demo 阶段不设严格生产指标。
|
|||||||
|
|
||||||
## 14. Codex 开发约束
|
## 14. Codex 开发约束
|
||||||
|
|
||||||
Codex 当前只能实现 `DEMO-001 ~ DEMO-010`、`DEMO-012`、`DEMO-014 ~ DEMO-016`、
|
Codex 当前只能实现 `DEMO-001 ~ DEMO-010` 和已批准的 `ARCH-001 ~ ARCH-006`。
|
||||||
`ARCH-001 ~ ARCH-006`、`PLAN-001 ~ PLAN-002` 和 `DEPLOY-001 ~ DEPLOY-003`。
|
ARCH 迁移任务不授权实现下列被排除的完整 V1 能力。
|
||||||
PLAN-002 只授权实现 Celery/RabbitMQ 异步执行器与最小 Kubernetes/Helm 集群交付,不授权
|
|
||||||
拆分额外微服务或扩大业务能力。
|
|
||||||
|
|
||||||
除非明确要求,否则禁止提前实现权限、审计、Knowledge Compilation、多数据库、多模型平台、
|
除非明确要求,否则禁止提前实现权限、审计、Knowledge Compilation、多数据库、多模型平台、微服务、Kubernetes 和企业级 Job Queue。
|
||||||
额外微服务、跨区域容灾和完整 Observability 平台。
|
|
||||||
|
|
||||||
当本文件与完整 PRD 的实现优先级冲突时:
|
当本文件与完整 PRD 的实现优先级冲突时:
|
||||||
|
|
||||||
@ -546,10 +500,10 @@ docs/demo/DEMO_SCOPE.md
|
|||||||
>
|
>
|
||||||
完整 V1 PRD
|
完整 V1 PRD
|
||||||
|
|
||||||
只允许实现本文件明确列出的 Demo、Architecture、Plan 与 Deploy Task,并且一次只执行一个
|
只允许实现 DEMO-001 ~ DEMO-010 以及本文件明确列出的 ARCH-001 ~ ARCH-006。
|
||||||
Task。当前严格顺序为 DEMO-016 → DEPLOY-001 → DEPLOY-002 → DEPLOY-003。
|
Provider 架构迁移任务按顺序完成后,再继续 DEMO-010。
|
||||||
|
|
||||||
当前公共知识库阶段的业务目标保持为:
|
当前 Demo 的目标只有:
|
||||||
|
|
||||||
Upload
|
Upload
|
||||||
→ Parse/Clean
|
→ Parse/Clean
|
||||||
@ -561,14 +515,6 @@ Upload
|
|||||||
→ Source Result
|
→ Source Result
|
||||||
→ Search UI
|
→ Search UI
|
||||||
|
|
||||||
并补齐以下交付能力:
|
|
||||||
|
|
||||||
Async Job / Retry Cleanup
|
|
||||||
→ Batch Async Upload
|
|
||||||
→ Production Images
|
|
||||||
→ Kubernetes/Helm One-click Deploy
|
|
||||||
→ Availability + Remote WorkBuddy Acceptance
|
|
||||||
|
|
||||||
不要提前实现:
|
不要提前实现:
|
||||||
|
|
||||||
- ACL
|
- ACL
|
||||||
@ -578,9 +524,9 @@ Async Job / Retry Cleanup
|
|||||||
- Knowledge Compilation
|
- Knowledge Compilation
|
||||||
- Multi Vector Store
|
- Multi Vector Store
|
||||||
- Embedding Benchmark Platform
|
- Embedding Benchmark Platform
|
||||||
- 额外微服务
|
- Celery/RabbitMQ
|
||||||
- 跨区域容灾
|
- Kubernetes
|
||||||
- 完整 Observability 平台
|
- 企业级高可用
|
||||||
|
|
||||||
先不要修改代码。
|
先不要修改代码。
|
||||||
|
|
||||||
@ -589,7 +535,7 @@ Async Job / Retry Cleanup
|
|||||||
1. 当前仓库状态;
|
1. 当前仓库状态;
|
||||||
2. 为完成 Demo 需要保留哪些现有结构;
|
2. 为完成 Demo 需要保留哪些现有结构;
|
||||||
3. 哪些完整 V1 设计可以暂时不实现;
|
3. 哪些完整 V1 设计可以暂时不实现;
|
||||||
4. 当前已批准 Task 的执行顺序;
|
4. DEMO-001 ~ DEMO-010 与 ARCH-001 ~ ARCH-006 的建议执行顺序;
|
||||||
5. 下一步应该实施的第一个 Task。
|
5. 下一步应该实施的第一个 Task。
|
||||||
```
|
```
|
||||||
|
|
||||||
@ -600,20 +546,10 @@ Async Job / Retry Cleanup
|
|||||||
```text
|
```text
|
||||||
Demo
|
Demo
|
||||||
↓
|
↓
|
||||||
MCP
|
|
||||||
↓
|
|
||||||
Async Job + Retry Cleanup
|
|
||||||
↓
|
|
||||||
Batch Async Upload
|
|
||||||
↓
|
|
||||||
Production Images
|
|
||||||
↓
|
|
||||||
Kubernetes/Helm One-click Deploy
|
|
||||||
↓
|
|
||||||
Availability + Remote WorkBuddy Acceptance
|
|
||||||
↓
|
|
||||||
Hybrid Search
|
Hybrid Search
|
||||||
↓
|
↓
|
||||||
|
MCP
|
||||||
|
↓
|
||||||
OCR / Complex Parsing
|
OCR / Complex Parsing
|
||||||
↓
|
↓
|
||||||
Document Version
|
Document Version
|
||||||
|
|||||||
@ -1,59 +0,0 @@
|
|||||||
# Demo Admin Console 使用说明
|
|
||||||
|
|
||||||
DEMO-010 在 FastAPI 服务中提供一个轻量、同源的管理客户端。页面只负责展示和交互,上传、
|
|
||||||
文档管理与检索仍全部通过 `/api/v1` 公开业务接口完成,不复制服务端业务逻辑。
|
|
||||||
|
|
||||||
## 启动与访问
|
|
||||||
|
|
||||||
完成运行时配置后启动 Worker 与 API:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
make run-worker
|
|
||||||
make run
|
|
||||||
```
|
|
||||||
|
|
||||||
页面地址:
|
|
||||||
|
|
||||||
| 页面 | 地址 | 对应业务接口 |
|
|
||||||
|---|---|---|
|
|
||||||
| 文档上传 | `/admin/upload` | `POST/GET /api/v1/ingestion-batches*` |
|
|
||||||
| 文档列表与详情 | `/admin/documents` | `GET/DELETE /api/v1/documents*` |
|
|
||||||
| Search Lab | `/admin/search` | `POST /api/v1/search` |
|
|
||||||
|
|
||||||
`/admin` 默认展示上传页。OpenAPI 文档仍位于 `/api/v1/docs`。
|
|
||||||
|
|
||||||
## 品牌视觉基线
|
|
||||||
|
|
||||||
DEMO-014 参考地瓜机器人公开官网的设计语言,使用品牌橙 `#FF3C00`、黑白高对比、中文
|
|
||||||
无衬线标题、紧凑数据字体、低圆角容器和宽松留白。贯穿页面标题的“计算信号轨”是唯一的
|
|
||||||
强调性图形,序号对应上传、文档、检索三步真实操作顺序。
|
|
||||||
|
|
||||||
品牌图形使用仓库内的代码原生 SVG/CSS 绘制,不复制或热链官网图片、字体和脚本;页面
|
|
||||||
运行时不依赖品牌官网。前端设计方法参考 GitHub `anthropics/skills` 的 Apache-2.0
|
|
||||||
`frontend-design` Skill,并以现有 Demo 信息架构、可访问性和业务正确性为优先约束。
|
|
||||||
|
|
||||||
## Demo 操作闭环
|
|
||||||
|
|
||||||
1. 在上传页选择一个或多个 PDF、DOCX、TXT 或 Markdown 文件并提交;
|
|
||||||
2. 接口返回 `202` 后,页面自动轮询批次,逐文件显示接收状态、处理阶段、尝试次数和安全错误;
|
|
||||||
3. 批次进入 `Completed`、`Partial failure` 或 `Failed` 后停止轮询,再在文档列表查看元数据;
|
|
||||||
4. 在 Search Lab 输入问题并查看 Top-K 的文件名、页码、章节、Chunk 内容、Dense Score 和
|
|
||||||
Rerank Score;
|
|
||||||
5. 如需清理 Demo 数据,在文档列表确认后直接删除。
|
|
||||||
|
|
||||||
## 范围与安全边界
|
|
||||||
|
|
||||||
- 页面静态文件位于 `apps/admin-web`,由同一个 FastAPI 进程提供,避免额外前端运行时;
|
|
||||||
- API 返回内容仅通过 DOM `textContent` 渲染,不作为 HTML 注入;
|
|
||||||
- 页面不保存或展示 Provider 凭据、内部 Endpoint、storage key 或 Index Profile;
|
|
||||||
- Console 只轮询当前批次的聚合接口,不读取 Provider/队列细节,也不提供人工重试按钮;
|
|
||||||
- 批量上传不包含目录上传、压缩包解包、断点续传或跨批次去重;
|
|
||||||
- 本任务不包含 ACL/RBAC、审计、Document Version、Compiled Knowledge、MCP、Hybrid/BM25
|
|
||||||
或完整 V1 Admin Console 导航;
|
|
||||||
- 删除是 Demo 的直接删除语义,确认后会清理 canonical Metadata、派生产物、源文件与向量行。
|
|
||||||
|
|
||||||
## 验证
|
|
||||||
|
|
||||||
静态路由和范围契约由 `tests/integration/test_admin_console.py` 覆盖;现有 REST 全链路测试覆盖
|
|
||||||
上传、列表、详情、搜索与删除。浏览器验收使用确定性本地 Provider,不读取真实企业文档,
|
|
||||||
并覆盖桌面与移动布局。
|
|
||||||
@ -1,97 +0,0 @@
|
|||||||
# 异步入库与失败恢复说明
|
|
||||||
|
|
||||||
## 范围
|
|
||||||
|
|
||||||
DEMO-015 将单文件上传从 API 请求线程迁移到独立 Celery Worker,DEMO-016 在其上增加逐文件
|
|
||||||
隔离的批量接收和状态聚合。OceanBase 中的
|
|
||||||
`ingestion_jobs` 和 `ingestion_attempts` 是执行状态事实源;RabbitMQ 消息只携带 Job ID,
|
|
||||||
不携带文件内容、Provider 凭据或业务状态。
|
|
||||||
|
|
||||||
批次只聚合接收与 Job 状态,不改变每个文件的独立执行和重试边界。
|
|
||||||
|
|
||||||
## 运行时配置
|
|
||||||
|
|
||||||
以下值必须通过环境变量或部署 Secret 注入,不得提交真实值:
|
|
||||||
|
|
||||||
```text
|
|
||||||
RABBITMQ_URL=amqp://<user>:<password>@<host>:5672/<vhost>
|
|
||||||
INGESTION_MAX_ATTEMPTS=3
|
|
||||||
INGESTION_RETRY_BACKOFF_SECONDS=2
|
|
||||||
INGESTION_RETRY_BACKOFF_MAX_SECONDS=60
|
|
||||||
INGESTION_WORKER_LEASE_SECONDS=300
|
|
||||||
INGESTION_RECOVERY_INTERVAL_SECONDS=30
|
|
||||||
BATCH_MAX_FILES=20
|
|
||||||
BATCH_MAX_TOTAL_BYTES=209715200
|
|
||||||
```
|
|
||||||
|
|
||||||
Worker 与 API 还需要相同的 OceanBase、BOS、Embedding、VectorStore 和 Index Profile 配置。
|
|
||||||
RabbitMQ 用户应只具备指定 vhost/queue 的最小权限。
|
|
||||||
|
|
||||||
## 启动顺序
|
|
||||||
|
|
||||||
```bash
|
|
||||||
uv run alembic upgrade head
|
|
||||||
make run-worker
|
|
||||||
make run
|
|
||||||
```
|
|
||||||
|
|
||||||
`make run-worker` 在当前 Demo 中同时启动 Worker 和恢复调度器。恢复调度器周期扫描:
|
|
||||||
|
|
||||||
- 尚未成功投递的 `queued` Job;
|
|
||||||
- 已到重试时间的 `retrying` Job;
|
|
||||||
- Worker 丢失后租约已过期的 `running` Job。
|
|
||||||
|
|
||||||
任务投递使用 late acknowledgement、worker-lost reject 和单任务 prefetch。重复消息会先锁定
|
|
||||||
Job 行;有效租约内不会创建第二个 Attempt。租约过期后,旧 Attempt 以安全错误关闭,再按重试
|
|
||||||
规则恢复。
|
|
||||||
|
|
||||||
## 重试与清理
|
|
||||||
|
|
||||||
确定性的内容错误(无法解析、清洗后为空、无 Chunk、派生数据身份不一致)直接进入 `failed`。
|
|
||||||
其余运行时/Provider 错误按指数退避重试,默认最多三次总 Attempt。
|
|
||||||
|
|
||||||
第二次及后续 Attempt 创建前严格执行:
|
|
||||||
|
|
||||||
```text
|
|
||||||
OceanBase Vector rows
|
|
||||||
→ ChunkIndexState
|
|
||||||
→ Chunk
|
|
||||||
→ parsed JSON object
|
|
||||||
→ reset Document processing state
|
|
||||||
```
|
|
||||||
|
|
||||||
不可变源文件、Document、Job 和已结束 Attempt 始终保留。任一清理步骤失败时不创建新 Attempt,
|
|
||||||
Job 直接以 `retry cleanup failed` 终止,禁止在未知脏状态继续执行。
|
|
||||||
|
|
||||||
## API 状态
|
|
||||||
|
|
||||||
`POST /api/v1/documents` 成功返回 `202` 和 `job_id`。查询:
|
|
||||||
|
|
||||||
```text
|
|
||||||
GET /api/v1/jobs/{job_id}
|
|
||||||
```
|
|
||||||
|
|
||||||
Job 状态为 `queued | running | retrying | succeeded | failed | cancelled`;阶段为
|
|
||||||
`parsing | cleaning | chunking | embedding | indexing`。公开错误只返回固定安全摘要,不保存或
|
|
||||||
返回 Broker/Provider 异常正文。活动 Job 的 Document 暂不允许删除,API 返回 `409`。
|
|
||||||
|
|
||||||
批量提交与查询:
|
|
||||||
|
|
||||||
```text
|
|
||||||
POST /api/v1/ingestion-batches # multipart 字段 files,可重复
|
|
||||||
GET /api/v1/ingestion-batches/{batch_id}
|
|
||||||
```
|
|
||||||
|
|
||||||
批次创建后逐文件返回 `accepted` 或 `rejected`;已接收文件继续通过上述 Job 状态异步推进。
|
|
||||||
批次聚合状态为 `processing | completed | partial_failure | failed`。请求文件数超过
|
|
||||||
`BATCH_MAX_FILES` 时整体返回 `413`;单文件或共享总字节预算超限只拒绝当前文件,不删除此前
|
|
||||||
已接收文件。默认总预算 200 MiB,且配置不得小于单文件 `MAX_UPLOAD_BYTES`。
|
|
||||||
|
|
||||||
## 故障检查
|
|
||||||
|
|
||||||
1. Job 长期为 `queued`:检查 RabbitMQ 连通性、恢复调度器以及 `ingestion` queue;
|
|
||||||
2. Job 长期为 `running`:检查 Worker,租约到期后恢复调度器会重新投递;
|
|
||||||
3. Job 为 `retrying`:等待 `next_run_at`,客户端不应重新上传同一文件;
|
|
||||||
4. Job 为 `failed` 且错误为 `retry cleanup failed`:停止人工重试,先核对向量、关系数据与 BOS
|
|
||||||
派生对象的一致性;
|
|
||||||
5. 不得通过日志打印 `RABBITMQ_URL`、数据库 URL、BOS/模型凭据或原始 Provider 响应。
|
|
||||||
@ -11,7 +11,7 @@ Adapter 继续用于单元测试和显式配置的本地开发。
|
|||||||
|
|
||||||
```text
|
```text
|
||||||
OBJECT_STORAGE_PROVIDER=bos
|
OBJECT_STORAGE_PROVIDER=bos
|
||||||
BOS_ENDPOINT=<已授权的 BOS Host 或 HTTP(S) Origin>
|
BOS_ENDPOINT=<已授权的 HTTP(S) BOS Origin>
|
||||||
BOS_ACCESS_KEY_ID=<运行时注入>
|
BOS_ACCESS_KEY_ID=<运行时注入>
|
||||||
BOS_SECRET_ACCESS_KEY=<运行时注入>
|
BOS_SECRET_ACCESS_KEY=<运行时注入>
|
||||||
BOS_BUCKET=<预创建的 Bucket>
|
BOS_BUCKET=<预创建的 Bucket>
|
||||||
@ -19,8 +19,7 @@ BOS_KEY_PREFIX=knowledge-platform
|
|||||||
BOS_TIMEOUT_SECONDS=30
|
BOS_TIMEOUT_SECONDS=30
|
||||||
```
|
```
|
||||||
|
|
||||||
- Endpoint 必须是无用户名、密码、Path、Query 和 Fragment 的 Host 或 HTTP(S) Origin;旧配置
|
- Endpoint 必须是无用户名、密码、Path、Query 和 Fragment 的显式 HTTP(S) Origin;
|
||||||
的纯 Host 会在 Provider 边界强制规范化为 HTTPS,禁止沿用 SDK 的默认 HTTP;
|
|
||||||
- Bucket 名必须符合 BOS 的小写字母、数字和连字符规则;
|
- Bucket 名必须符合 BOS 的小写字母、数字和连字符规则;
|
||||||
- Prefix 可为空;非空时使用安全的 POSIX 相对 key,不允许空段、`.`、`..` 或反斜杠;
|
- Prefix 可为空;非空时使用安全的 POSIX 相对 key,不允许空段、`.`、`..` 或反斜杠;
|
||||||
- 应用不会创建、删除或修改 Bucket 配置。
|
- 应用不会创建、删除或修改 Bucket 配置。
|
||||||
@ -53,8 +52,8 @@ SDK Response Body 或凭据。
|
|||||||
全局 List 权限。若生产策略要求来源不可物理删除,应在后续 Document Version/Retention
|
全局 List 权限。若生产策略要求来源不可物理删除,应在后续 Document Version/Retention
|
||||||
设计中单独处理;当前 Demo 的直接删除行为保持不变。
|
设计中单独处理;当前 Demo 的直接删除行为保持不变。
|
||||||
|
|
||||||
默认测试使用 Fake Client,不连接 BOS。真实连接、权限和临时对象清理已在 ARCH-006 的
|
默认测试使用 Fake Client,不连接 BOS。真实连接、权限和临时对象清理统一在 ARCH-006 的
|
||||||
受控 test 资源中验证;凭据只允许运行时注入,不得写入仓库、日志或测试快照。
|
受控测试资源中验证,并且只允许使用轮换后的凭据。
|
||||||
|
|
||||||
官方参考:
|
官方参考:
|
||||||
|
|
||||||
|
|||||||
@ -1,91 +0,0 @@
|
|||||||
# 生产容器镜像说明
|
|
||||||
|
|
||||||
## 范围
|
|
||||||
|
|
||||||
DEPLOY-001 使用同一个锁定运行时构建三个独立镜像目标:
|
|
||||||
|
|
||||||
| Target | 默认镜像名 | 入口 | 端口 | 镜像健康检查 |
|
|
||||||
|---|---|---|---|---|
|
|
||||||
| `api` | `enterprise-knowledge-api` | Uvicorn + API/Admin | `8000` | `GET /healthz` |
|
|
||||||
| `mcp` | `enterprise-knowledge-mcp` | `ekp-mcp` | `8766` | PID 1 存活 |
|
|
||||||
| `worker` | `enterprise-knowledge-worker` | `ekp-worker` | 无 | PID 1 存活 |
|
|
||||||
|
|
||||||
镜像没有内置数据库迁移、Helm、Ingress 或 Secret 对象;DEPLOY-002 已在独立 Chart 中提供这些
|
|
||||||
编排能力。API 镜像包含 Alembic 和 migrations,由一次性 Migration Job 覆盖命令后复用。详见
|
|
||||||
[`KUBERNETES_DEPLOYMENT.md`](KUBERNETES_DEPLOYMENT.md)。
|
|
||||||
|
|
||||||
## 可重复构建基线
|
|
||||||
|
|
||||||
- Python 基础镜像固定为 `3.12.14-slim-bookworm`,uv 固定为 `0.12.5`,两者同时锁定 manifest
|
|
||||||
digest;
|
|
||||||
- Python 依赖只使用提交的 `uv.lock`,以 `uv sync --frozen --no-dev --no-editable` 安装;
|
|
||||||
- 三个目标共享相同依赖层,不复制不同版本的业务运行时;
|
|
||||||
- 进程均以固定 UID/GID `10001:10001` 运行,不具备 root 权限;
|
|
||||||
- `.dockerignore` 排除 `.env`、Git、虚拟环境、缓存、本地存储、测试和开发产物;
|
|
||||||
- Bake 计划同时声明 `linux/amd64` 与 `linux/arm64`,不下载或打包平台专属模型。
|
|
||||||
|
|
||||||
基础镜像 tag/digest 与应用依赖均已固定;镜像发布系统仍应保存本项目镜像的构建 digest,并让
|
|
||||||
Helm 按 digest 部署。本项目镜像 digest 由实际 Registry 发布产生,不把某个环境的 Registry
|
|
||||||
地址写入仓库。
|
|
||||||
|
|
||||||
## 构建命令
|
|
||||||
|
|
||||||
先确认 Docker Buildx builder 支持两个目标平台:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
make images-print
|
|
||||||
```
|
|
||||||
|
|
||||||
构建全部目标:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
IMAGE_REGISTRY=registry.example.internal/knowledge \
|
|
||||||
IMAGE_TAG=<immutable-release-tag> \
|
|
||||||
make images
|
|
||||||
```
|
|
||||||
|
|
||||||
只有发布流水线才执行推送:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
IMAGE_REGISTRY=registry.example.internal/knowledge \
|
|
||||||
IMAGE_TAG=<immutable-release-tag> \
|
|
||||||
make images-push
|
|
||||||
```
|
|
||||||
|
|
||||||
不要把 Registry 密码放入变量或命令行;构建节点应提前通过其凭据助手登录。禁止使用
|
|
||||||
`latest` 作为集群部署版本。
|
|
||||||
|
|
||||||
## 运行时配置
|
|
||||||
|
|
||||||
镜像只内置以下非敏感进程默认值:`APP_ENV=production`、容器监听地址、静态页面绝对路径和
|
|
||||||
临时缓存目录。OceanBase、BOS、RabbitMQ、Embedding、Reranker 与 MCP Token 必须在容器启动
|
|
||||||
时由 Secret 注入。镜像构建参数、Layer、标签、健康检查和启动命令都不得包含这些值。
|
|
||||||
|
|
||||||
API 的 `/healthz` 只证明进程存活,不读取云端凭据或探测依赖。Kubernetes readiness、资源限制、
|
|
||||||
只读根文件系统所需临时卷及依赖就绪策略由 DEPLOY-002 Chart 配置。MCP endpoint 需要认证,Worker
|
|
||||||
没有 HTTP server,因此这两个镜像的 Docker 健康检查只验证主进程;业务可恢复性仍以持久化
|
|
||||||
Job/Attempt 和 DEPLOY-003 故障验收为准。
|
|
||||||
|
|
||||||
## 本地单目标检查
|
|
||||||
|
|
||||||
需要加载到本地 Docker 时,可直接构建当前平台的一个 target:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
docker build --target api -f deploy/docker/Dockerfile -t ekp-api:test .
|
|
||||||
docker build --target mcp -f deploy/docker/Dockerfile -t ekp-mcp:test .
|
|
||||||
docker build --target worker -f deploy/docker/Dockerfile -t ekp-worker:test .
|
|
||||||
```
|
|
||||||
|
|
||||||
启动时使用 `--env-file` 或编排系统的 Secret 引用,不在命令中展开敏感值。API/Admin smoke test
|
|
||||||
至少检查 `/healthz`、`/admin/upload` 和 `/api/v1/openapi.json`;MCP 与 Worker 的真实依赖启动
|
|
||||||
验收在具备 RabbitMQ 和目标 Provider 的受控环境执行。
|
|
||||||
|
|
||||||
## DEPLOY-001 构建验收
|
|
||||||
|
|
||||||
2026-09-02 已使用 Buildx 分别构建并加载三个 `linux/arm64` target 和三个 `linux/amd64`
|
|
||||||
target。API/Admin 以非 root 用户完成健康、静态页面和 OpenAPI smoke test;MCP/Worker 完成非
|
|
||||||
root 安装入口检查。构建历史与容器文件系统检查未发现 `.env`、Git、测试文件或敏感配置。
|
|
||||||
|
|
||||||
这次验收只生成本地 `deploy001-*` 测试 tag,没有登录或推送 Registry,因此不产生可供集群
|
|
||||||
引用的项目镜像 digest。DEPLOY-002 必须使用操作者指定的不可变 Registry tag/digest,不得把
|
|
||||||
本地测试 tag 当作发布版本。
|
|
||||||
@ -1,19 +1,3 @@
|
|||||||
# Definition of Done
|
# Definition of Done
|
||||||
|
|
||||||
A task is done only when scope is implemented, architecture invariants hold, format/lint/typecheck pass, relevant tests pass, migrations/contracts/docs are updated as needed, no secrets are committed, retrieval changes include RAG regression results, and known limitations are documented.
|
A task is done only when scope is implemented, architecture invariants hold, format/lint/typecheck pass, relevant tests pass, migrations/contracts/docs are updated as needed, no secrets are committed, retrieval changes include RAG regression results, and known limitations are documented.
|
||||||
|
|
||||||
For asynchronous ingestion or retry tasks, relevant tests must also prove durable Job/Attempt state,
|
|
||||||
bounded retry classification, idempotent duplicate delivery, cleanup-before-retry, preservation of the
|
|
||||||
immutable source, and terminal behavior when cleanup fails. Batch ingestion must cover per-file partial
|
|
||||||
failure and configured limits.
|
|
||||||
|
|
||||||
For MCP tasks, contract tests must cover tool discovery, identity propagation, authorization/error
|
|
||||||
boundaries, REST/application-service parity, infrastructure redaction, and the explicitly targeted MCP
|
|
||||||
client compatibility check.
|
|
||||||
|
|
||||||
For cluster delivery tasks, completion also requires reproducible production images, Secret-only
|
|
||||||
sensitive configuration, serialized migration, readiness/liveness probes, resource and pod-security
|
|
||||||
defaults, idempotent Helm install/upgrade, rolling-restart verification, durable worker recovery, stable
|
|
||||||
HTTPS Admin/MCP endpoints, and remote WorkBuddy acceptance. Manifests, rendered output, logs and images
|
|
||||||
must be checked for credential leakage. Public-knowledge delivery does not complete or waive V1
|
|
||||||
user/ACL tasks.
|
|
||||||
|
|||||||
@ -1,8 +1,8 @@
|
|||||||
# ARCH-005 Shadow Index 重建与切换
|
# ARCH-005 Shadow Index 重建与切换
|
||||||
|
|
||||||
ARCH-005 将正式运行默认值切换为 BOS、OceanBase Vector 和千帆 BCE Reranker;ARCH-006
|
ARCH-005 将正式运行默认值切换为 BOS、OceanBase Vector 和千帆 BCE Reranker。PostgreSQL
|
||||||
进一步确认 canonical 关系数据与向量索引同处 OceanBase MySQL 数据库。关系表保存 Document、
|
继续保存 canonical Document、Chunk、ChunkIndexState 和 Index Profile 生命周期;OceanBase
|
||||||
Chunk、ChunkIndexState 和 Index Profile;向量表只保存可以重建的检索行。旧 Mochow 运行配置、依赖和代码已移除,但本流程不会自动删除
|
只保存可以重建的检索行。旧 Mochow 运行配置、依赖和代码已移除,但本流程不会自动删除
|
||||||
旧外部索引,物理退役必须由受控运维流程另行确认。
|
旧外部索引,物理退役必须由受控运维流程另行确认。
|
||||||
|
|
||||||
## Profile 与迁移前提
|
## Profile 与迁移前提
|
||||||
@ -20,9 +20,9 @@ demo-bge-large-zh-structure-v1-oceanbase-v1
|
|||||||
|
|
||||||
1. 运行 `uv run alembic upgrade head`,创建 canonical `index_profiles` 表;
|
1. 运行 `uv run alembic upgrade head`,创建 canonical `index_profiles` 表;
|
||||||
2. 使用 `deploy/oceanbase/001_knowledge_chunks_v1.sql` 预创建目标表和 HNSW 索引;
|
2. 使用 `deploy/oceanbase/001_knowledge_chunks_v1.sql` 预创建目标表和 HNSW 索引;
|
||||||
3. 仅在运行时注入已授权的 OceanBase 与千帆凭据,不写入仓库或日志;
|
3. 注入 OceanBase 与千帆 Embedding 的轮换凭据,不使用配置清单中的历史 Secret;
|
||||||
4. 确认目标 OceanBase 为 4.3.5.4+,且应用账号只有目标数据库/表所需权限;
|
4. 确认目标 OceanBase 为 4.6.0+,且应用账号只有目标数据库/表所需权限;
|
||||||
5. 在重建期间避免修改 Chunk 策略;激活事务会阻止 canonical Chunk 并发写入穿过最终核对。
|
5. 在重建期间避免修改 Chunk 策略;激活事务会阻止 PostgreSQL Chunk 并发写入穿过最终核对。
|
||||||
|
|
||||||
## 顺序执行
|
## 顺序执行
|
||||||
|
|
||||||
@ -34,15 +34,15 @@ uv run ekp-index-rebuild verify
|
|||||||
uv run ekp-index-rebuild activate
|
uv run ekp-index-rebuild activate
|
||||||
```
|
```
|
||||||
|
|
||||||
- `build` 注册 Shadow Profile,从 OceanBase canonical Chunk 分文档生成 Embedding 并写入
|
- `build` 注册 Shadow Profile,从 PostgreSQL canonical Chunk 分文档生成 Embedding 并写入
|
||||||
OceanBase。命令中断后可以原样重试;已完整文档会跳过,部分目标 Profile 数据会先定向
|
OceanBase。命令中断后可以原样重试;已完整文档会跳过,部分目标 Profile 数据会先定向
|
||||||
清理再重建,不触碰 Active Profile;
|
清理再重建,不触碰 Active Profile;
|
||||||
- `verify` 要求目标策略下至少存在一个 canonical Chunk,并核对全部 Chunk 与匹配模型、
|
- `verify` 要求目标策略下至少存在一个 canonical Chunk,并核对全部 Chunk 与匹配模型、
|
||||||
维度的 ChunkIndexState。输出中的 `complete` 必须为 `true`;
|
维度的 ChunkIndexState。输出中的 `complete` 必须为 `true`;
|
||||||
- `activate` 在 OceanBase 事务内再次验证,锁定 Chunk 写入,将原 Active Profile 标为
|
- `activate` 在 PostgreSQL 事务内再次验证,锁定 Chunk 写入,将原 Active Profile 标为
|
||||||
`retired`,再将目标 Shadow Profile 标为 `active`。重复执行已成功的激活是幂等的。
|
`retired`,再将目标 Shadow Profile 标为 `active`。重复执行已成功的激活是幂等的。
|
||||||
|
|
||||||
激活后 API 从 OceanBase canonical 关系表解析 Active Profile,不再仅依赖进程环境中的 ID。新上传与查询
|
激活后 API 从 PostgreSQL 解析 Active Profile,不再仅依赖进程环境中的 ID。新上传与查询
|
||||||
统一使用 OceanBase Active Profile。部署仍应保留前一版本制品和旧索引回滚窗口;本 Task
|
统一使用 OceanBase Active Profile。部署仍应保留前一版本制品和旧索引回滚窗口;本 Task
|
||||||
不自动销毁任何旧外部数据。
|
不自动销毁任何旧外部数据。
|
||||||
|
|
||||||
@ -50,4 +50,4 @@ uv run ekp-index-rebuild activate
|
|||||||
|
|
||||||
离线测试覆盖多文档构建、重试跳过、完整性失败关闭、单 Active 约束、原子激活、迁移和
|
离线测试覆盖多文档构建、重试跳过、完整性失败关闭、单 Active 约束、原子激活、迁移和
|
||||||
CLI 输出脱敏。真实 BOS/OceanBase/Qianfan 连通性、DDL 权限、30~50 条脱敏 Golden Query、
|
CLI 输出脱敏。真实 BOS/OceanBase/Qianfan 连通性、DDL 权限、30~50 条脱敏 Golden Query、
|
||||||
召回质量和延迟属于 ARCH-006,且只能使用运行时授权凭据与受控测试资源。
|
召回质量和延迟属于 ARCH-006,且只能使用轮换后的凭据与受控测试资源。
|
||||||
|
|||||||
@ -1,143 +0,0 @@
|
|||||||
# Kubernetes / Helm 部署说明
|
|
||||||
|
|
||||||
## 1. 交付范围
|
|
||||||
|
|
||||||
DEPLOY-002 提供单一 Helm Chart,将以下运行角色部署到已有 Kubernetes 集群:
|
|
||||||
|
|
||||||
- API/Admin:默认两个副本,提供 `/healthz`、`/admin/*` 和 `/api/v1/*`;
|
|
||||||
- MCP Gateway:默认两个副本,提供经 Bearer Token 保护的 `/mcp`;
|
|
||||||
- Async Worker:默认两个副本,只消费 `ingestion` 队列;
|
|
||||||
- Recovery Scheduler:固定单副本,以 `Recreate` 策略运行 Celery Beat;
|
|
||||||
- Migration Job:每次安装/升级前串行执行 `alembic upgrade head`;
|
|
||||||
- 可选 RabbitMQ:只供开发/演示集群显式启用,生产默认使用外部 Broker。
|
|
||||||
|
|
||||||
OceanBase MySQL、BOS、Embedding/Reranker 服务继续使用集群外已有资源。Chart 不创建数据库、
|
|
||||||
BOS Bucket、域名、证书或云账号,也没有扩大公共知识库阶段的业务范围。
|
|
||||||
|
|
||||||
## 2. 前置条件
|
|
||||||
|
|
||||||
操作者需要提前准备:
|
|
||||||
|
|
||||||
1. 可用的 Kubernetes Namespace 和对应 `kubectl` 上下文;
|
|
||||||
2. Helm 3/4 与 Kubernetes 客户端;
|
|
||||||
3. 已推送到目标 Registry 的 API、MCP、Worker 不可变 tag 或 digest;
|
|
||||||
4. Namespace 内的运行时 Secret、TLS Secret,以及私有 Registry 所需的 ImagePullSecret;
|
|
||||||
5. 集群到 OceanBase、BOS、RabbitMQ 和模型服务的网络连通性。
|
|
||||||
|
|
||||||
Chart 默认引用 `knowledge-platform-runtime`,但不会创建它。该 Secret 至少包含以下键:
|
|
||||||
|
|
||||||
```text
|
|
||||||
DATABASE_URL
|
|
||||||
RABBITMQ_URL
|
|
||||||
BOS_ACCESS_KEY_ID
|
|
||||||
BOS_SECRET_ACCESS_KEY
|
|
||||||
BAIDU_QIANFAN_API_KEY
|
|
||||||
OCEANBASE_VECTOR_PASSWORD
|
|
||||||
MCP_AUTH_TOKEN
|
|
||||||
```
|
|
||||||
|
|
||||||
`RERANKER_API_KEY` 可单独提供;未提供时应用按既有契约复用
|
|
||||||
`BAIDU_QIANFAN_API_KEY`。`MCP_AUTH_TOKEN` 至少 32 个字符,只具备
|
|
||||||
`knowledge:search` 服务范围。不要使用 `kubectl --from-literal`、Shell 参数或 Helm
|
|
||||||
`--set` 传递以上值;应由受控 Secret 管理系统或运维流程预先写入 Namespace。
|
|
||||||
|
|
||||||
## 3. 环境 Values
|
|
||||||
|
|
||||||
复制 [`values.example.yaml`](../../deploy/helm/environments/values.example.yaml) 到仓库外的受控
|
|
||||||
路径,填写以下非敏感信息:
|
|
||||||
|
|
||||||
- 三个镜像的 Registry 路径与不可变 tag/digest;
|
|
||||||
- `existingSecret` 和 `imagePullSecrets` 的资源名称;
|
|
||||||
- OceanBase/BOS 的非敏感 Endpoint、数据库标识和 Bucket;
|
|
||||||
- Admin/API 与 MCP 的 Ingress Host、TLS Secret 名称;
|
|
||||||
- 与外部地址一致的 MCP Public URL、Issuer、Allowed Host/Origin;
|
|
||||||
- 副本、资源、超时和批量上传限制。
|
|
||||||
|
|
||||||
`values.schema.json` 拒绝未知顶层字段、`latest` tag、无效 digest、单副本 API/MCP、错误类型和
|
|
||||||
空的关键引用;模板还会校验 Chunk、检索和入库限制之间的大小关系。Values 只允许非敏感配置,
|
|
||||||
模板逐项生成 ConfigMap,不提供任意 `extraEnv` 入口,避免把凭据误写进渲染产物。
|
|
||||||
|
|
||||||
## 4. 安装与幂等升级
|
|
||||||
|
|
||||||
先做离线校验和清单检查:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
make deploy-lint VALUES=/secure/path/environment-values.yaml
|
|
||||||
make deploy-render VALUES=/secure/path/environment-values.yaml
|
|
||||||
```
|
|
||||||
|
|
||||||
安装或升级:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
NAMESPACE=knowledge-platform \
|
|
||||||
RELEASE=knowledge-platform \
|
|
||||||
make deploy VALUES=/secure/path/environment-values.yaml
|
|
||||||
```
|
|
||||||
|
|
||||||
`make deploy` 只接收 Values 文件路径。脚本会先验证 Chart、Namespace、运行时 Secret 及必需
|
|
||||||
键名、可选 RabbitMQ Secret 键名,以及启用 Ingress 时的两个 TLS Secret 引用,然后执行带失败
|
|
||||||
回滚和等待语义的 `helm upgrade --install`;脚本对 Helm 3 使用 `--atomic`,对 Helm 4 使用等价
|
|
||||||
的 `--rollback-on-failure`。它只验证 Secret 资源和键是否存在,不读取或输出 Secret 值。
|
|
||||||
|
|
||||||
Migration 是 `pre-install,pre-upgrade` Hook,固定名称配合
|
|
||||||
`before-hook-creation` 保证同一 Release 内不会并发执行。Helm 等待 Migration 成功后才更新业务
|
|
||||||
工作负载;失败时 Helm 保留上一版本。完成的 Job 最长保留一天供日志追查,并在下次升级前删除。
|
|
||||||
安装命令可以重复执行;数据库迁移必须保持 forward-only 与幂等。
|
|
||||||
|
|
||||||
## 5. Ingress、探针与 Pod 安全
|
|
||||||
|
|
||||||
生产环境启用 Ingress 后,Admin/API 与 MCP 使用独立稳定 HTTPS Host,也可由同一 Ingress
|
|
||||||
Controller 托管。证书只通过预建 TLS Secret 引用,不进入 Values。
|
|
||||||
|
|
||||||
- API 的 startup/readiness/liveness 调用无凭据 `/healthz`;
|
|
||||||
- MCP 的三类探针使用 TCP 端口,不发送或记录 Token;
|
|
||||||
- Worker/Scheduler 使用 PID 1 存活探针,业务恢复正确性由持久化 Job/Attempt 保证;
|
|
||||||
- 全部应用容器固定 UID/GID `10001`、禁止提权、丢弃 Linux capabilities、使用
|
|
||||||
`RuntimeDefault` seccomp 和只读根文件系统;
|
|
||||||
- `/tmp` 使用有大小上限的 `emptyDir`,ServiceAccount 不挂载 API Token;
|
|
||||||
- API/MCP 滚动更新使用 `maxUnavailable: 0`,未 Ready Pod 不进入 Service Endpoint。
|
|
||||||
|
|
||||||
`/healthz` 当前只表示 API 进程存活,不探测所有外部 Provider;真实依赖故障、滚动重启和
|
|
||||||
Worker 恢复行为由 DEPLOY-003 在目标集群验收。
|
|
||||||
|
|
||||||
## 6. 可选 Demo RabbitMQ
|
|
||||||
|
|
||||||
生产默认 `rabbitmq.enabled=false`,`RABBITMQ_URL` 指向外部受管 Broker。仅开发/演示时可开启
|
|
||||||
本地子 Chart;它使用单节点 StatefulSet 和 PVC,并要求预建 Secret 包含:
|
|
||||||
|
|
||||||
```text
|
|
||||||
username
|
|
||||||
password
|
|
||||||
erlang-cookie
|
|
||||||
```
|
|
||||||
|
|
||||||
同时必须让运行时 Secret 中的 `RABBITMQ_URL` 指向该 Service。子 Chart 使用固定
|
|
||||||
`rabbitmq:4.1.4-alpine` tag 与 multi-architecture manifest digest;它不具备生产 RabbitMQ 的
|
|
||||||
高可用、备份或升级保证,禁止作为生产默认值。卸载 Release 时 StatefulSet PVC 默认保留,
|
|
||||||
删除前必须由数据责任人确认。
|
|
||||||
|
|
||||||
## 7. 回滚、卸载与 Secret 轮换
|
|
||||||
|
|
||||||
查看历史并回滚时不要输出 Pod 环境变量:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
helm -n knowledge-platform history knowledge-platform
|
|
||||||
helm -n knowledge-platform rollback knowledge-platform <revision> --wait
|
|
||||||
```
|
|
||||||
|
|
||||||
更新外部 Secret 后,Secret 名称不变;用户和 WorkBuddy 配置无需重新注入。当前应通过受控
|
|
||||||
滚动重启让 Pod 读取新值,目标集群的无中断验证属于 DEPLOY-003。
|
|
||||||
|
|
||||||
卸载只删除本 Release 管理的 Kubernetes 对象:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
helm -n knowledge-platform uninstall knowledge-platform
|
|
||||||
```
|
|
||||||
|
|
||||||
外部 OceanBase、BOS、RabbitMQ、TLS/运行时 Secret 和可选 RabbitMQ PVC 不由卸载命令删除。
|
|
||||||
|
|
||||||
## 8. DEPLOY-002 验收边界
|
|
||||||
|
|
||||||
本任务完成 Chart lint、双路径渲染、严格 Kubernetes Schema、Values 失败关闭、Secret 泄漏扫描
|
|
||||||
以及本地集群两次安装/升级验证。目标企业集群的稳定 HTTPS、滚动重启、Worker 故障恢复、真实
|
|
||||||
Provider 和远程 WorkBuddy 验收统一属于后续 `DEPLOY-003`,不得提前宣称完成。
|
|
||||||
@ -1,67 +0,0 @@
|
|||||||
# ARCH-006 受控真实链路验收
|
|
||||||
|
|
||||||
本验收覆盖 OceanBase canonical relational metadata → BOS immutable source → 千帆
|
|
||||||
`bge-large-zh` → OceanBase Vector → 千帆 `bce-reranker-base` → canonical Source Result。数据集包含
|
|
||||||
32 条完全合成、脱敏 Query,不含真实客户、产品、政策或内部知识。
|
|
||||||
|
|
||||||
## 强制安全门禁
|
|
||||||
|
|
||||||
只有同时满足以下条件才允许执行:
|
|
||||||
|
|
||||||
1. 数据库、BOS 和千帆凭据已经所有者明确授权;配置清单中的现有值只允许本次 test
|
|
||||||
验收临时注入,禁止保存为 `.env`、写入日志或用于 fat/prod,完成后仍必须轮换;
|
|
||||||
2. OceanBase Database/Table 和 BOS Bucket/Prefix 已明确授权用于测试;
|
|
||||||
3. `APP_ENV=test`;
|
|
||||||
4. `BOS_KEY_PREFIX` 以 `live-acceptance/` 开头,避免写入共享业务前缀;
|
|
||||||
5. 显式设置 `EKP_RUN_LIVE_TESTS=1`、`EKP_CREDENTIALS_AUTHORIZED=1` 和
|
|
||||||
`EKP_LIVE_RESOURCE_ACK=1`。
|
|
||||||
|
|
||||||
任一条件不满足时,命令在创建 Provider 或外部连接前失败关闭,并且不输出 Endpoint、用户
|
|
||||||
名、SQL、响应正文、Chunk 内容或 Secret。仓库 `.env.example` 中三个开关固定为 `0`。
|
|
||||||
|
|
||||||
## 执行顺序
|
|
||||||
|
|
||||||
先完成 ARCH-005 切换:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
uv run alembic upgrade head
|
|
||||||
uv run ekp-index-rebuild build
|
|
||||||
uv run ekp-index-rebuild verify
|
|
||||||
uv run ekp-index-rebuild activate
|
|
||||||
```
|
|
||||||
|
|
||||||
确认 `verify` 和 `activate` 输出的 `complete` 均为 `true` 后,再运行:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
make live-acceptance
|
|
||||||
```
|
|
||||||
|
|
||||||
命令会使用随机运行前缀上传两份合成 Markdown,执行全部 32 条 Query,并在 `finally` 清理
|
|
||||||
本次运行的 OceanBase Vector 行、canonical Document/Chunk/State 以及 BOS 对象。清理失败会使验收
|
|
||||||
失败,且不会尝试删除不带本次随机文件名前缀的数据。
|
|
||||||
|
|
||||||
## 验收指标
|
|
||||||
|
|
||||||
默认门槛:
|
|
||||||
|
|
||||||
```text
|
|
||||||
Recall@1 >= 0.90
|
|
||||||
Recall@5 = 1.00
|
|
||||||
MRR >= 0.90
|
|
||||||
NDCG@5 >= 0.90
|
|
||||||
Search P95 <= 10000 ms
|
|
||||||
```
|
|
||||||
|
|
||||||
输出只包含 Query 数、Recall@1/3/5、MRR、NDCG@5、P50/P95 和 `passed`。如受控环境需要调整
|
|
||||||
门槛,只能通过 `EKP_LIVE_MIN_*` / `EKP_LIVE_MAX_P95_MS` 显式注入,并把实际门槛与结果记录
|
|
||||||
到受保护的验收日志;不要把 Endpoint、账号、凭据或原始响应提交到仓库。
|
|
||||||
|
|
||||||
## 验收结果(2026-09-02)
|
|
||||||
|
|
||||||
目标 test OceanBase 已完成 migration、专用向量表/HNSW、PRE-FILTER、BOS、Embedding、
|
|
||||||
Reranker 和 canonical Source Result 验收。32 条查询结果:Recall@1/3/5、MRR、NDCG@5 均为
|
|
||||||
`1.0`,P50 `3787.61 ms`,P95 `4759.21 ms`,`passed=true`。正式命令退出码为 `0`,数据库
|
|
||||||
复核 `live-` Document/Chunk/Vector 残留为 `0`。
|
|
||||||
|
|
||||||
首次正式运行出现过一次无细节瞬时失败;分阶段 Provider/单文档/32 查询诊断以及随后正式
|
|
||||||
运行均通过。后续稳定性任务应补充有界重试、限流指标与告警;这不阻塞本次受控验收。
|
|
||||||
@ -1,102 +0,0 @@
|
|||||||
# DEMO-012 MCP Gateway 使用说明
|
|
||||||
|
|
||||||
## 范围
|
|
||||||
|
|
||||||
DEMO-012 提供独立的 Enterprise Knowledge MCP Gateway,通过标准 Streamable HTTP 暴露一个
|
|
||||||
只读工具:`search_knowledge`。Gateway 直接复用 API 进程使用的 `SearchService` 组合,不复制
|
|
||||||
Dense Retrieval、Reranker 或 Source Enrichment 逻辑。
|
|
||||||
|
|
||||||
当前 Demo 只建立认证和身份传递边界:一个运行时注入的专用 Bearer Token 映射到一个
|
|
||||||
user/service identity,并且只有 `knowledge:search` scope。它不是共享超级管理员 Token,也不
|
|
||||||
代表 V1 OAuth、Document ACL 或完整 MCP P0 工具已经完成。
|
|
||||||
|
|
||||||
## 配置
|
|
||||||
|
|
||||||
必须通过运行环境提供至少 32 字符的随机 Token:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
export MCP_AUTH_TOKEN="<runtime-secret-with-at-least-32-characters>"
|
|
||||||
make run-mcp
|
|
||||||
```
|
|
||||||
|
|
||||||
默认地址为 `http://127.0.0.1:8766/mcp`。除现有数据库、BOS、Embedding、Vector 和 Reranker
|
|
||||||
配置外,Gateway 支持:
|
|
||||||
|
|
||||||
| 变量 | 默认值 | 说明 |
|
|
||||||
|---|---|---|
|
|
||||||
| `MCP_HOST` | `127.0.0.1` | 监听地址;本地默认不暴露到其他网卡 |
|
|
||||||
| `MCP_PORT` | `8766` | Gateway 端口 |
|
|
||||||
| `MCP_PATH` | `/mcp` | Streamable HTTP 单一端点 |
|
|
||||||
| `MCP_PUBLIC_URL` | `http://127.0.0.1:8766/mcp` | MCP Resource Server URL |
|
|
||||||
| `MCP_ISSUER_URL` | `http://127.0.0.1:8766` | Demo Token 的身份颁发方标识 |
|
|
||||||
| `MCP_CLIENT_ID` | `workbuddy-enterprise` | MCP Client 类型标识 |
|
|
||||||
| `MCP_SERVICE_IDENTITY` | `workbuddy-demo` | 传入业务边界的服务身份 |
|
|
||||||
| `MCP_REQUIRED_SCOPE` | `knowledge:search` | Tool 所需最小 scope |
|
|
||||||
| `MCP_ALLOWED_HOSTS` | 仅本机/test | DNS rebinding Host allowlist,JSON 数组 |
|
|
||||||
| `MCP_ALLOWED_ORIGINS` | 仅本机 | Origin allowlist,JSON 数组 |
|
|
||||||
|
|
||||||
远程部署必须使用 TLS 反向代理,并把公开域名加入 `MCP_ALLOWED_HOSTS`;有 `Origin` 的客户端还
|
|
||||||
必须显式配置 `MCP_ALLOWED_ORIGINS`。Token、真实域名和内部 Endpoint 不得提交 Git。
|
|
||||||
|
|
||||||
## WorkBuddy / CodeBuddy 配置
|
|
||||||
|
|
||||||
WorkBuddy Enterprise 官方客户端支持 `type: "http"`、远程 URL、自定义 Header 和环境变量
|
|
||||||
展开。可复制 `deploy/workbuddy-mcp.example.json` 作为项目配置,内容如下:
|
|
||||||
|
|
||||||
```jsonc
|
|
||||||
{
|
|
||||||
"mcpServers": {
|
|
||||||
"enterprise-knowledge": {
|
|
||||||
"type": "http",
|
|
||||||
"url": "${EKP_MCP_URL:-http://127.0.0.1:8766/mcp}",
|
|
||||||
"headers": {
|
|
||||||
"Authorization": "Bearer ${EKP_MCP_TOKEN}"
|
|
||||||
},
|
|
||||||
"description": "企业知识检索"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
只提交包含环境变量占位符的配置;`EKP_MCP_TOKEN` 必须由每个运行环境安全注入。连接后应只
|
|
||||||
发现 `mcp__enterprise_knowledge__search_knowledge` 一个工具。
|
|
||||||
|
|
||||||
## 安全和错误语义
|
|
||||||
|
|
||||||
- 缺失或错误 Token 返回 `401`,缺少 scope 返回 `403`;
|
|
||||||
- 未批准 Host 返回 `421`,未批准 Origin 返回 `403`;
|
|
||||||
- Query 长度和 `top_k` 在 MCP schema 层校验;
|
|
||||||
- 预期业务错误返回安全 Tool Error;未知 Provider/数据库错误收敛为
|
|
||||||
`knowledge search failed`;
|
|
||||||
- MCP 输出和 Gateway 日志不得包含 Token、Provider Endpoint、数据库表名、模型名或
|
|
||||||
Index Profile。
|
|
||||||
|
|
||||||
## 验证
|
|
||||||
|
|
||||||
```bash
|
|
||||||
uv run pytest tests/unit/test_mcp_auth.py tests/unit/test_mcp_search.py
|
|
||||||
uv run pytest tests/integration/test_mcp_server.py
|
|
||||||
```
|
|
||||||
|
|
||||||
集成测试使用官方 MCP Python Client 通过 Streamable HTTP 完成 initialize、tools/list 和
|
|
||||||
tools/call,并使用与 WorkBuddy 相同的 Bearer Header 配置验证身份传递、无结果、参数错误、
|
|
||||||
认证失败、Origin 防护和错误脱敏。
|
|
||||||
|
|
||||||
### WorkBuddy 现场验收记录
|
|
||||||
|
|
||||||
2026-09-02 使用 WorkBuddy Desktop `5.3.14` 自带的 CodeBuddy CLI `2.115.0` 完成兼容性
|
|
||||||
验收。客户端直接加载 `deploy/workbuddy-mcp.example.json`,健康检查为 `Connected`;工具
|
|
||||||
发现结果只有 `mcp__enterprise-knowledge__search_knowledge`,随后以字符串 `query` 和
|
|
||||||
`top_k=1` 完成 Tool 调用并读取结构化 `filename`、`section`、`content` 等来源字段。
|
|
||||||
|
|
||||||
现场验收使用 DEMO-012 的真实 Gateway、Bearer 验证、service identity 传递、输入 Schema 和
|
|
||||||
Tool 注册代码。为避免外部数据库与模型波动掩盖客户端兼容性,检索应用使用确定性验收返回;
|
|
||||||
真实 Provider 检索链路由 `ARCH-006` 的受控验收负责。非交互 CLI 需要批准 MCP Tool 调用,
|
|
||||||
本次只在单次会话中允许 `DeferExecuteTool`,没有永久启用跳过权限。测试结束后已删除临时 MCP
|
|
||||||
配置并停止本地 Gateway,未写入真实凭据。
|
|
||||||
|
|
||||||
参考:
|
|
||||||
|
|
||||||
- [MCP Python SDK](https://github.com/modelcontextprotocol/python-sdk)
|
|
||||||
- [MCP Streamable HTTP 规范](https://modelcontextprotocol.io/specification/2025-11-25/basic/transports)
|
|
||||||
- [WorkBuddy Enterprise MCP 使用文档](https://cloud.tencent.cn/document/product/1831/137039)
|
|
||||||
@ -1,13 +1,13 @@
|
|||||||
# ARCH-003 OceanBase Vector 配置与索引契约
|
# ARCH-003 OceanBase Vector 配置与索引契约
|
||||||
|
|
||||||
ARCH-003 在既有 `VectorStore` 契约后新增 OceanBase Vector Adapter。ARCH-006 确认 OceanBase
|
ARCH-003 在既有 `VectorStore` 契约后新增 OceanBase Vector Adapter。PostgreSQL 仍保存
|
||||||
MySQL 关系表保存 canonical Document、Chunk、Index Profile 和 Chunk Index State;同库向量表
|
canonical Document、Chunk、Index Profile 和 Chunk Index State;OceanBase 表只保存可由这些
|
||||||
只保存可由这些数据重建的检索行。ARCH-005 已完成运行时默认切换,并移除 Mochow 运行配置与依赖。
|
数据重建的检索行。ARCH-005 已完成运行时默认切换,并移除 Mochow 运行配置与依赖。
|
||||||
|
|
||||||
## 数据库前提
|
## 数据库前提
|
||||||
|
|
||||||
- OceanBase MySQL 租户版本至少为 4.3.5.4:ARCH-006 已在目标 test 集群用只读
|
- OceanBase MySQL 租户版本至少为 4.6.0:1024 维 Cosine HNSW 本身可由更早版本支持,
|
||||||
`EXPLAIN` 验证 `VECTOR_INDEX(... pre_filter)` 产生 PRE-FILTER 执行计划;
|
但本项目要求使用 `VECTOR_INDEX(... pre_filter)` 显式保证 Profile 在 ANN 前过滤;
|
||||||
- 目标数据库和最小权限账号由受控部署流程预创建;
|
- 目标数据库和最小权限账号由受控部署流程预创建;
|
||||||
- Vector 内存、资源规格和 HNSW 参数由 DBA 在受控环境确认;
|
- Vector 内存、资源规格和 HNSW 参数由 DBA 在受控环境确认;
|
||||||
- 应用启动和请求路径不会创建、修改或删除表结构。
|
- 应用启动和请求路径不会创建、修改或删除表结构。
|
||||||
@ -62,8 +62,8 @@ OCEANBASE_VECTOR_TIMEOUT_SECONDS=30
|
|||||||
- 返回 UUID、Profile、字段数、距离范围和排序不满足契约时失败关闭,不生成 Source Result。
|
- 返回 UUID、Profile、字段数、距离范围和排序不满足契约时失败关闭,不生成 Source Result。
|
||||||
|
|
||||||
默认测试使用 Fake Client,不连接 OceanBase。真实版本、DDL、权限、Cosine HNSW 召回、过滤
|
默认测试使用 Fake Client,不连接 OceanBase。真实版本、DDL、权限、Cosine HNSW 召回、过滤
|
||||||
行为与 30~50 条脱敏 Golden Query 统一在 ARCH-006 的受控资源中验证。凭据只能在运行时
|
行为与 30~50 条脱敏 Golden Query 统一在 ARCH-006 的受控资源中验证,并且只允许使用轮换
|
||||||
临时注入,禁止进入仓库、日志或测试快照。
|
后的凭据。
|
||||||
|
|
||||||
官方参考:
|
官方参考:
|
||||||
|
|
||||||
|
|||||||
@ -42,7 +42,7 @@ Adapter 只向业务层返回原候选索引和 Reranker 分数,不暴露千
|
|||||||
|
|
||||||
Reranker 对 provider-neutral Dense Candidate 的 `content` 评分;选出 Top5 后,
|
Reranker 对 provider-neutral Dense Candidate 的 `content` 评分;选出 Top5 后,
|
||||||
`SearchService` 使用
|
`SearchService` 使用
|
||||||
`chunk_id` 一次性回查 OceanBase canonical `chunks` 关系表,并由 canonical row 生成:
|
`chunk_id` 一次性回查 PostgreSQL canonical `chunks` 表,并由 canonical row 生成:
|
||||||
|
|
||||||
```text
|
```text
|
||||||
document_id
|
document_id
|
||||||
|
|||||||
@ -2,21 +2,19 @@
|
|||||||
|
|
||||||
## 范围
|
## 范围
|
||||||
|
|
||||||
DEMO-009 将入库和检索服务公开为版本化 REST API,DEMO-015 将单文件入库切换为持久化异步
|
DEMO-009 将已经完成的同步入库和检索服务公开为版本化 REST API。API 层只负责输入校验、
|
||||||
Job,DEMO-016 增加批量接收与聚合查询。API 层只负责输入校验、
|
|
||||||
错误映射和响应序列化,不重复实现 Parser、Embedding、VectorStore 或 Reranker 业务逻辑。
|
错误映射和响应序列化,不重复实现 Parser、Embedding、VectorStore 或 Reranker 业务逻辑。
|
||||||
|
|
||||||
当前不包含 ACL/RBAC、软删除、目录/压缩包上传、断点续传或 Hybrid Search。
|
本任务不包含 Admin Console、ACL/RBAC、异步任务、软删除或 Hybrid Search。
|
||||||
|
|
||||||
## 启动前提
|
## 启动前提
|
||||||
|
|
||||||
1. 按 `docs/development/BOS_SETUP.md` 配置 BOS;
|
1. 按 `docs/development/BOS_SETUP.md` 配置 BOS;
|
||||||
2. 按 `docs/development/OCEANBASE_VECTOR_SETUP.md` 配置 OceanBase Vector 和千帆 Embedding;
|
2. 按 `docs/development/OCEANBASE_VECTOR_SETUP.md` 配置 OceanBase Vector 和千帆 Embedding;
|
||||||
3. 按 `docs/development/RERANKER_SETUP.md` 配置 Reranker;
|
3. 按 `docs/development/RERANKER_SETUP.md` 配置 Reranker;
|
||||||
4. 按 `docs/development/ASYNC_INGESTION_SETUP.md` 配置 RabbitMQ 并启动 Worker;
|
4. 执行 `uv run alembic upgrade head`;
|
||||||
5. 执行 `uv run alembic upgrade head`;
|
5. 新旧索引迁移按 `docs/development/INDEX_REBUILD.md` 构建、验证并激活 Profile;
|
||||||
6. 新旧索引迁移按 `docs/development/INDEX_REBUILD.md` 构建、验证并激活 Profile;
|
6. 执行 `make run`。
|
||||||
7. 执行 `make run`。
|
|
||||||
|
|
||||||
`/healthz` 和 OpenAPI 可以在没有云端凭证时启动和访问。业务依赖在首次访问 P0 业务接口时
|
`/healthz` 和 OpenAPI 可以在没有云端凭证时启动和访问。业务依赖在首次访问 P0 业务接口时
|
||||||
延迟创建;配置不完整时返回 `503`,不会将凭证名以外的配置值暴露给客户端。
|
延迟创建;配置不完整时返回 `503`,不会将凭证名以外的配置值暴露给客户端。
|
||||||
@ -27,29 +25,13 @@ Swagger UI:`/api/v1/docs`
|
|||||||
|
|
||||||
## 示例
|
## 示例
|
||||||
|
|
||||||
上传并异步排队:
|
上传并同步入库:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
curl -F 'file=@./policy.md;type=text/markdown' \
|
curl -F 'file=@./policy.md;type=text/markdown' \
|
||||||
http://127.0.0.1:8000/api/v1/documents
|
http://127.0.0.1:8000/api/v1/documents
|
||||||
```
|
```
|
||||||
|
|
||||||
使用响应中的 `job_id` 查询进度:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
curl http://127.0.0.1:8000/api/v1/jobs/<job-id>
|
|
||||||
```
|
|
||||||
|
|
||||||
批量上传并查询聚合进度:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
curl -F 'files=@./policy.md;type=text/markdown' \
|
|
||||||
-F 'files=@./manual.pdf;type=application/pdf' \
|
|
||||||
http://127.0.0.1:8000/api/v1/ingestion-batches
|
|
||||||
|
|
||||||
curl http://127.0.0.1:8000/api/v1/ingestion-batches/<batch-id>
|
|
||||||
```
|
|
||||||
|
|
||||||
检索:
|
检索:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
@ -58,19 +40,17 @@ curl -H 'Content-Type: application/json' \
|
|||||||
http://127.0.0.1:8000/api/v1/search
|
http://127.0.0.1:8000/api/v1/search
|
||||||
```
|
```
|
||||||
|
|
||||||
上传只等待不可变源文件、Document 与 Job 持久化,成功返回 `202`。Worker 后台执行 Parse、
|
Demo 上传请求会一直等待 Parse、Clean、Chunk、Embedding 和 OceanBase Upsert 完成。大文
|
||||||
Clean、Chunk、Embedding 和 OceanBase Upsert;客户端通过 Job 状态判断何时可检索。
|
件或云端调用较慢时,请求耗时会相应增加;异步任务属于 Demo 后置能力。
|
||||||
|
|
||||||
## 删除一致性
|
## 删除一致性
|
||||||
|
|
||||||
删除按以下顺序执行:
|
删除按以下顺序执行:
|
||||||
|
|
||||||
1. 删除所有已记录 Index Profile 以及当前 Profile 的 OceanBase 行;
|
1. 删除所有已记录 Index Profile 以及当前 Profile 的 OceanBase 行;
|
||||||
2. 删除 OceanBase canonical 关系表中的 ChunkIndexState、Chunk 和 Document;
|
2. 删除 PostgreSQL 中的 ChunkIndexState、Chunk 和 Document;
|
||||||
3. 删除解析 JSON 与不可变原始文件。
|
3. 删除解析 JSON 与不可变原始文件。
|
||||||
|
|
||||||
`queued`、`running` 或 `retrying` Job 对应的 Document 返回 `409`,避免删除与 Worker 并发。
|
|
||||||
|
|
||||||
向量清理失败时保留 canonical Metadata 和原始文件。Metadata 提交后若文件清理失败,接口
|
向量清理失败时保留 canonical Metadata 和原始文件。Metadata 提交后若文件清理失败,接口
|
||||||
返回明确的 `500`,可能留下不可访问的孤立文件;这是 Demo 直接删除的已知边界,正式 V1
|
返回明确的 `500`,可能留下不可访问的孤立文件;这是 Demo 直接删除的已知边界,正式 V1
|
||||||
应通过可重试清理任务和审计记录强化。
|
应通过可重试清理任务和审计记录强化。
|
||||||
|
|||||||
@ -11,11 +11,6 @@ Provide deterministic fakes:
|
|||||||
|
|
||||||
External integrations use separate integration suites/markers.
|
External integrations use separate integration suites/markers.
|
||||||
|
|
||||||
Async ingestion tests run with an in-process recording dispatcher and deterministic providers. They
|
|
||||||
must cover persisted Job/Attempt history, safe error summaries, bounded retry, cleanup-before-retry,
|
|
||||||
active-lease duplicate delivery, expired-lease recovery, immutable-source preservation and terminal
|
|
||||||
cleanup failure without requiring a live RabbitMQ broker.
|
|
||||||
|
|
||||||
E2E covers upload -> parse -> chunk -> embed -> index -> retrieve -> source trace, and source -> compilation draft -> review -> publish -> retrieval.
|
E2E covers upload -> parse -> chunk -> embed -> index -> retrieve -> source trace, and source -> compilation draft -> review -> publish -> retrieval.
|
||||||
|
|
||||||
Any parsing/chunking/embedding/fusion/retrieval/reranking/query-routing change must run the Golden Dataset evaluation.
|
Any parsing/chunking/embedding/fusion/retrieval/reranking/query-routing change must run the Golden Dataset evaluation.
|
||||||
|
|||||||
@ -6,20 +6,4 @@ Benchmark the full pipeline: Embedding × Retrieval Strategy × Fusion × Rerank
|
|||||||
|
|
||||||
Initial embedding candidates: bge-large-zh, Qwen3-Embedding-0.6B, Qwen3-Embedding-4B, optional 8B.
|
Initial embedding candidates: bge-large-zh, Qwen3-Embedding-0.6B, Qwen3-Embedding-4B, optional 8B.
|
||||||
|
|
||||||
Use synthetic/public/anonymized fixtures only. ARCH-006 expands the checked-in dataset to 32
|
Use synthetic/public/anonymized fixtures only.
|
||||||
fully synthetic queries across refund, technical support, lodging, and reimbursement-material
|
|
||||||
sources. Offline Fake Provider regression must remain deterministic; target-provider acceptance
|
|
||||||
uses the same records through `make live-acceptance` and reports aggregate metrics only.
|
|
||||||
|
|
||||||
Current live gates:
|
|
||||||
|
|
||||||
```text
|
|
||||||
Recall@1 >= 0.90
|
|
||||||
Recall@5 = 1.00
|
|
||||||
MRR >= 0.90
|
|
||||||
NDCG@5 >= 0.90
|
|
||||||
Latency P95 <= 10000 ms
|
|
||||||
```
|
|
||||||
|
|
||||||
Threshold overrides are allowed only as explicit test-run environment inputs and must be recorded
|
|
||||||
with the run result. See `docs/development/LIVE_ACCEPTANCE.md` for safety and cleanup requirements.
|
|
||||||
|
|||||||
@ -2,13 +2,13 @@
|
|||||||
|
|
||||||
## 目标基线
|
## 目标基线
|
||||||
|
|
||||||
本计划依据 2026-09-02 对最新内部配置与 test 环境的只读核验调整 EKP Provider 和数据库
|
本计划依据 2026-09-01 核对的内部平台配置现状调整 EKP Provider 选型,同时保留本仓库
|
||||||
选型,同时保留 canonical-data 和 API-first 原则。
|
已经 Accepted 的 canonical-data 和 API-first 原则。
|
||||||
|
|
||||||
```text
|
```text
|
||||||
OceanBase MySQL canonical Metadata / Chunk / Index State
|
PostgreSQL canonical Metadata / Chunk / Index State
|
||||||
Baidu BOS immutable source and derived artifacts
|
Baidu BOS immutable source and derived artifacts
|
||||||
OceanBase Vector co-located rebuildable Dense retrieval index
|
OceanBase Vector rebuildable Dense retrieval index
|
||||||
Qianfan bge-large-zh 1024-dimensional Embedding
|
Qianfan bge-large-zh 1024-dimensional Embedding
|
||||||
Qianfan BCE Reranker bce-reranker-base
|
Qianfan BCE Reranker bce-reranker-base
|
||||||
```
|
```
|
||||||
@ -19,7 +19,7 @@ FakeRerankerProvider。
|
|||||||
|
|
||||||
## 明确保留的行为
|
## 明确保留的行为
|
||||||
|
|
||||||
- OceanBase 关系表保持 canonical database,专用向量表只保存可重建索引;
|
- PostgreSQL 保持 canonical database,OceanBase 只保存可重建索引;
|
||||||
- 原始文件不可变;
|
- 原始文件不可变;
|
||||||
- Dense Top20 → Reranker Top5;
|
- Dense Top20 → Reranker Top5;
|
||||||
- 当前 PDF/DOCX/TXT/Markdown Parser 和 `structure_v1` Chunk;
|
- 当前 PDF/DOCX/TXT/Markdown Parser 和 `structure_v1` Chunk;
|
||||||
@ -29,7 +29,7 @@ FakeRerankerProvider。
|
|||||||
|
|
||||||
## 明确不随迁移引入
|
## 明确不随迁移引入
|
||||||
|
|
||||||
- 将 OceanBase 向量表当作 canonical source;
|
- OceanBase 取代 PostgreSQL canonical database;
|
||||||
- Top50、距离阈值、标准答案捷径、双查询和 RRF;
|
- Top50、距离阈值、标准答案捷径、双查询和 RRF;
|
||||||
- Marker/Pandoc、LLM 摘要/问题/关键词索引;
|
- Marker/Pandoc、LLM 摘要/问题/关键词索引;
|
||||||
- RBAC/ACL、异步队列、MCP、Hybrid/BM25 或 Admin Console 扩展;
|
- RBAC/ACL、异步队列、MCP、Hybrid/BM25 或 Admin Console 扩展;
|
||||||
@ -64,22 +64,19 @@ Evaluation;保持 Top5 API 行为。
|
|||||||
|
|
||||||
### ARCH-005 Runtime Cutover
|
### ARCH-005 Runtime Cutover
|
||||||
|
|
||||||
将 API 运行时切换到 BOS、OceanBase Vector 和 BCE Reranker;提供从 canonical
|
将 API 运行时切换到 BOS、OceanBase Vector 和 BCE Reranker;提供从 PostgreSQL canonical
|
||||||
Chunk 构建新 shadow index 的可重试命令,验证后激活新 Profile,再清理 Mochow 运行配置。
|
Chunk 构建新 shadow index 的可重试命令,验证后激活新 Profile,再清理 Mochow 运行配置。
|
||||||
|
|
||||||
依赖:ARCH-004。
|
依赖:ARCH-004。
|
||||||
|
|
||||||
### ARCH-006 真实验收
|
### ARCH-006 真实验收
|
||||||
|
|
||||||
修正先前错误的 PostgreSQL 前提:将 canonical SQLAlchemy/Alembic 迁移到 OceanBase MySQL,
|
在受控测试资源中执行 PostgreSQL → BOS → Qianfan Embedding → OceanBase Vector → BCE
|
||||||
同库保留逻辑隔离的可重建向量表;在受控 test 资源中执行 OceanBase canonical → BOS →
|
Reranker → canonical Source Result 全链路,并完成 30~50 条脱敏 Golden Query 验收。
|
||||||
Qianfan Embedding → OceanBase Vector → BCE Reranker → canonical Source Result 全链路,并完成
|
|
||||||
30~50 条脱敏 Golden Query 验收。
|
|
||||||
|
|
||||||
依赖:ARCH-005、已授权的 test 运行凭据、数据库/表和 BOS 专用前缀。
|
依赖:ARCH-005、轮换后的运行凭据、已授权的测试数据库/表和 BOS 前缀。
|
||||||
|
|
||||||
## 安全边界
|
## 安全边界
|
||||||
|
|
||||||
内部配置清单中的现有凭据按已暴露处理。经所有者明确要求,本次只允许把 test 凭据临时
|
内部配置清单中的现有凭据按已暴露处理,只能使用轮换后的运行时 Secret。禁止读取凭据后
|
||||||
注入 ARCH-006 进程;不得生成 `.env`、测试快照或调试日志,不得连接 fat/prod,验收后仍应
|
生成 `.env`、测试快照或调试日志;真实集成测试必须使用专用测试资源并清理临时对象。
|
||||||
轮换。真实集成测试必须使用专用表/BOS 前缀并清理本次临时对象。
|
|
||||||
|
|||||||
@ -1,123 +0,0 @@
|
|||||||
# MCP 与异步批量入库优先级方案
|
|
||||||
|
|
||||||
**Task:** PLAN-001
|
|
||||||
**Status:** Accepted planning baseline
|
|
||||||
**Date:** 2026-09-02
|
|
||||||
|
|
||||||
## 1. 现状判断
|
|
||||||
|
|
||||||
| 能力 | 当前实现 | 结论 |
|
|
||||||
|---|---|---|
|
|
||||||
| 单文件上传 | `POST /api/v1/documents` 持久化 Source/Document/Job 后返回 `202`,Worker 后台处理 | DEMO-015 已实现 |
|
|
||||||
| 批量上传 | 批次 API、逐文件 Job 状态和 Admin Console 自动轮询 | DEMO-016 已实现 |
|
|
||||||
| 失败补偿 | 阶段内补偿 + 重试前清理全部派生数据,保留不可变源和执行历史 | DEMO-015 已实现 |
|
|
||||||
| 自动重试 | OceanBase Job/Attempt + Celery/RabbitMQ + 有界退避 + Worker 租约恢复 | DEMO-015 已实现 |
|
|
||||||
| 重试前清理 | 重试前删除向量、索引状态、Chunk、解析产物,保留源文件与历史 | DEMO-015 已实现 |
|
|
||||||
| MCP | 独立 Gateway、认证边界、`search_knowledge` 与 WorkBuddy 协议级验收 | DEMO-012 已实现 |
|
|
||||||
|
|
||||||
PLAN-001 的异步执行缺口已由 DEMO-015 补齐,批量提交与状态 UI 已由 DEMO-016 补齐。
|
|
||||||
|
|
||||||
## 2. Task 归并结论
|
|
||||||
|
|
||||||
### DEMO-012:MCP `search_knowledge`(提升为最高优先级)
|
|
||||||
|
|
||||||
保留既有 Task ID,不新建重复任务。范围限定为:
|
|
||||||
|
|
||||||
- 提供可运行的标准 MCP Gateway;
|
|
||||||
- 只暴露 `search_knowledge`;
|
|
||||||
- 复用现有 Search Service 或正式 `/api/v1/search`,不复制检索逻辑;
|
|
||||||
- 建立 WorkBuddy 所需的 user/service identity 边界,不使用共享超级管理员凭据;
|
|
||||||
- 完成协议契约、错误收敛、凭据脱敏和 WorkBuddy 兼容性验证。
|
|
||||||
|
|
||||||
该 Task 是 `EKP-070`、`EKP-071`、`EKP-072` 的最小垂直切片,不能据此把三个完整 V1 Task 标记为完成。
|
|
||||||
|
|
||||||
### DEMO-015:异步入库 Job 与自动重试清理(新增)
|
|
||||||
|
|
||||||
从 `EKP-017` 和 `EKP-023` 抽取 Demo 所需的最小基础能力,并复用 DEMO-003~006 的处理服务:
|
|
||||||
|
|
||||||
- 新增持久化 Ingestion Job/Attempt,记录阶段、状态、尝试次数、安全错误和时间戳;
|
|
||||||
- 源文件成功写入 BOS 且 Document 已持久化后立即排队,API 不等待解析和索引;
|
|
||||||
- 同一 Document 同时最多一个有效执行,任务投递和消费具备幂等保护;
|
|
||||||
- 对可重试的瞬时错误自动执行有界重试和退避;Demo 默认最多 3 次总尝试,允许通过非敏感配置调整;
|
|
||||||
- 格式不支持、文件为空、无可提取内容等确定性错误直接失败,不进行无意义重试;
|
|
||||||
- 每次重新执行前,按顺序清理该 Document 的向量行、`ChunkIndexState`、Chunk 和解析产物,再从不可变原始文件重新处理;
|
|
||||||
- 原始文件、Document 标识和审计所需的 Job/Attempt 历史不得被重试清理删除;
|
|
||||||
- 任一清理步骤失败时禁止启动下一次尝试,并记录为终止失败,避免在未知脏状态上重复执行。
|
|
||||||
- 通过执行器抽象接入 Celery/RabbitMQ,使 API 和 Worker 可独立部署;业务应用服务不得直接
|
|
||||||
依赖 Celery API,确保执行器仍可替换;
|
|
||||||
- Worker 重启、消息重复投递和确认超时必须保持持久化 Job 状态一致,不能产生双重有效
|
|
||||||
Attempt 或重复派生数据。
|
|
||||||
|
|
||||||
该 Task 只完成公共知识库入库所需的 Job/Retry 与队列基线,不扩展到完整审计系统、
|
|
||||||
Compilation、OCR、跨区域高可用或通用企业级 Job 平台。
|
|
||||||
|
|
||||||
### DEMO-016:批量异步上传与状态展示(已完成)
|
|
||||||
|
|
||||||
建立在 DEMO-015 之上,不与 Job 基础合并:
|
|
||||||
|
|
||||||
- 新增批量上传 API,一次接收多个 Demo 支持类型文件;
|
|
||||||
- 每个文件独立生成 Document 和 Ingestion Job,单个文件失败不回滚已成功接收的其他文件;
|
|
||||||
- 返回批次 ID、逐文件 Document ID、Job ID 和接收状态,不等待后台处理完成;
|
|
||||||
- 提供批次/Job 查询能力,并在 Admin Console 展示逐文件阶段、重试次数和最终结果;
|
|
||||||
- 文件数量、单文件大小和请求总大小均受可配置限制;
|
|
||||||
- 不在本 Task 引入目录上传、压缩包解包、断点续传或跨批次去重。
|
|
||||||
|
|
||||||
该 Task 对应 `EKP-021` 的批量异步扩展及 `EKP-080`/`EKP-081` 的最小状态 UI 切片,不能据此把完整 V1 Task 标记为完成。
|
|
||||||
|
|
||||||
## 3. 实施顺序与依赖
|
|
||||||
|
|
||||||
```text
|
|
||||||
DEMO-009 + ARCH-006 → DEMO-012(已完成)
|
|
||||||
|
|
||||||
PLAN-002(已完成)
|
|
||||||
└──→ DEMO-003 → DEMO-004 → DEMO-005 → DEMO-006
|
|
||||||
└──→ DEMO-015 Async Job + Celery/RabbitMQ + Retry Cleanup
|
|
||||||
└──→ DEMO-016 Batch Async Upload + UI
|
|
||||||
└──→ DEPLOY-001 → DEPLOY-002 → DEPLOY-003
|
|
||||||
```
|
|
||||||
|
|
||||||
推荐严格按以下顺序执行并逐项闭环:
|
|
||||||
|
|
||||||
1. `PLAN-002`(已完成)
|
|
||||||
2. `DEMO-015`(已完成)
|
|
||||||
3. `DEMO-016`(已完成)
|
|
||||||
4. `DEPLOY-001`
|
|
||||||
5. `DEPLOY-002`
|
|
||||||
6. `DEPLOY-003`
|
|
||||||
|
|
||||||
MCP 与异步入库没有代码级前置依赖,因此提高 MCP 优先级不会阻塞后续批量上传。`DEMO-016` 必须等待 `DEMO-015`,否则会把同步流水线简单并发放大,缺少可靠的状态、隔离和恢复边界。
|
|
||||||
|
|
||||||
## 4. 验收边界
|
|
||||||
|
|
||||||
### DEMO-012
|
|
||||||
|
|
||||||
- MCP Inspector/标准客户端能发现并调用 `search_knowledge`;
|
|
||||||
- WorkBuddy 使用已映射的 user/service identity 完成一次端到端检索;
|
|
||||||
- REST 与 MCP 对同一查询复用同一业务逻辑并返回一致的来源字段;
|
|
||||||
- 响应与日志不泄露 OceanBase、BOS、模型名、Endpoint 或凭据。
|
|
||||||
|
|
||||||
### DEMO-015
|
|
||||||
|
|
||||||
- 上传接口在源文件和 Job 持久化后返回 `202`,后台完成现有全流水线;
|
|
||||||
- 注入一次可重试故障后可观察到自动重试,并最终成功;
|
|
||||||
- 第二次尝试开始前,旧解析产物、Chunk、索引状态和向量均已清理,原始源文件仍存在;
|
|
||||||
- 永久错误不重试,达到次数上限后稳定进入 `failed`;
|
|
||||||
- 并发或重复投递不会产生双重有效执行和重复派生数据。
|
|
||||||
- RabbitMQ 消息确认超时或 Worker 重启后,Job 可恢复到终态且不会重复生成派生数据。
|
|
||||||
|
|
||||||
### DEMO-016
|
|
||||||
|
|
||||||
- 一次提交多个文件后立即得到逐文件 Document/Job 标识;
|
|
||||||
- 批次内成功、处理中、重试中和失败状态互不影响且可查询;
|
|
||||||
- Admin Console 无需刷新即可轮询展示逐文件进度和最终状态;
|
|
||||||
- 批次边界、大小限制、部分失败和重试清理均有集成测试。
|
|
||||||
|
|
||||||
## 5. 非目标与 ADR 影响
|
|
||||||
|
|
||||||
- PLAN-002 已把 Celery/RabbitMQ 纳入 `DEMO-015`,并以独立 `DEPLOY-*` Task 纳入最小
|
|
||||||
Kubernetes、多副本和滚动升级能力;跨区域容灾、自动云资源编排和完整 Observability
|
|
||||||
仍为非目标。
|
|
||||||
- 不改变 OceanBase/BOS canonical、向量索引可重建、原始文档不可变、Provider 抽象和 MCP 隐藏基础设施等 Accepted ADR。
|
|
||||||
- 公共知识库只是阶段性单一可见域;ADR-007 保持 Accepted,用户/ACL 实现后置且不得用全局
|
|
||||||
Top-K 后过滤替代。
|
|
||||||
- 本次任务拆分和交付边界不需要新增或重开 Accepted ADR。
|
|
||||||
@ -1,15 +1,15 @@
|
|||||||
# 项目进度
|
# 项目进度
|
||||||
|
|
||||||
当前阶段:生产容器镜像闭环完成,准备实施 Kubernetes/Helm 一键部署
|
当前阶段:Provider 架构迁移
|
||||||
|
|
||||||
已完成:DEMO-001~DEMO-010、DEMO-012、DEMO-014~DEMO-016、ARCH-001~ARCH-006、PLAN-001~PLAN-002、DEPLOY-001
|
已完成:DEMO-001~DEMO-009、ARCH-001、ARCH-002、ARCH-003、ARCH-004、ARCH-005
|
||||||
进行中:无
|
进行中:无
|
||||||
阻塞:无
|
阻塞:无
|
||||||
|
|
||||||
下一任务:`DEPLOY-002`,完成并合并后执行 `DEPLOY-003`。
|
下一任务:ARCH-006
|
||||||
|
|
||||||
Provider 架构迁移按 `ARCH-001 → ARCH-002 → ARCH-003 → ARCH-004 → ARCH-005 → ARCH-006`
|
Provider 架构迁移按 `ARCH-001 → ARCH-002 → ARCH-003 → ARCH-004 → ARCH-005 → ARCH-006`
|
||||||
执行并已完成。DEMO-010 在迁移闭环验收后实施,未扩大为完整 V1 Admin Console。
|
执行;每个 Task 验证后独立推送并合并 `main`。全部迁移完成后再继续 DEMO-010。
|
||||||
|
|
||||||
## 任务记录
|
## 任务记录
|
||||||
|
|
||||||
@ -195,232 +195,3 @@ Provider 架构迁移按 `ARCH-001 → ARCH-002 → ARCH-003 → ARCH-004 → AR
|
|||||||
外部索引,保留受控回滚和运维退役边界。
|
外部索引,保留受控回滚和运维退役边界。
|
||||||
- 新增重建/重试/完整性失败/单 Active/激活/迁移/CLI 脱敏测试及中文操作文档。真实资源
|
- 新增重建/重试/完整性失败/单 Active/激活/迁移/CLI 脱敏测试及中文操作文档。真实资源
|
||||||
构建、BOS/OceanBase/Qianfan 连通性和 30~50 条脱敏查询验收仍属于 ARCH-006。
|
构建、BOS/OceanBase/Qianfan 连通性和 30~50 条脱敏查询验收仍属于 ARCH-006。
|
||||||
|
|
||||||
### ARCH-006(已完成)
|
|
||||||
|
|
||||||
- Golden Dataset 已扩展为 32 条完全合成 Query,覆盖退款、技术支持、住宿标准和报销材料;
|
|
||||||
Dense、Reranker、canonical Source Result 的离线回归全部通过。
|
|
||||||
- 新增 `ekp-live-acceptance` / `make live-acceptance`,实际覆盖上传、解析、Chunk、Embedding、
|
|
||||||
OceanBase Dense Top20、BCE Reranker Top5、canonical 来源核对及本次运行数据清理。
|
|
||||||
- live 命令强制要求 `APP_ENV=test`、专用 `live-acceptance/` BOS Prefix、凭据授权确认、资源
|
|
||||||
授权确认和显式运行开关;当前环境下已验证会在创建 Provider 前失败关闭且不泄露细节。
|
|
||||||
- 根据最新配置清单和 test 环境只读探测,确认没有独立 PostgreSQL;目标为 OceanBase
|
|
||||||
4.3.5.4 MySQL 模式,同一数据库承载 canonical 关系数据和逻辑隔离的向量表。ADR-012 已
|
|
||||||
取代先前 PostgreSQL 部署假设,业务事实来源与可重建索引的逻辑边界保持不变。
|
|
||||||
- 已在目标 test 集群只读验证数据库连通、版本与兼容模式,并通过 `EXPLAIN` 确认
|
|
||||||
`VECTOR_INDEX(... pre_filter)` 生成 `VECTOR INDEX ADAPTIVE SCAN (PRE-FILTER)`;未读取
|
|
||||||
业务正文或输出任何连接信息/凭据。
|
|
||||||
- canonical SQLAlchemy 默认已切换为 `mysql+pymysql`,数据库 URL 使用 `SecretStr`,新增
|
|
||||||
有界连接池/超时配置;新增 forward-only migration,以生成列唯一索引保证 OceanBase 与
|
|
||||||
SQLite 下至多一个 Active Profile;PostgreSQL 驱动已移除。
|
|
||||||
- 配置清单中的 test 凭据仅按用户明确授权临时注入本次验收,不保存为 `.env`、不提交、
|
|
||||||
不连接 fat/prod;完成后仍记录为待轮换风险。
|
|
||||||
- 已在 test OceanBase 执行全部 forward-only migration,并对本项目表完成列、类型、默认值、
|
|
||||||
主键的 scoped schema drift 核对;OceanBase 扩展语法导致标准 MySQL 反射器不能可靠比较的
|
|
||||||
二级索引/唯一约束/外键,由 SQLite 全量 drift、真实唯一约束事务和 live 级联删除补充验证。创建
|
|
||||||
`knowledge_chunks_v1` 12 列专用向量表和 Cosine HNSW,确认 PRE-FILTER 执行计划。迁移前
|
|
||||||
已确认所有目标表均不存在,没有覆盖原有 31 张业务表;事务内双 Active 写入实测被唯一
|
|
||||||
索引拒绝并完整回滚。
|
|
||||||
- 已完成 OceanBase canonical → BOS → `bge-large-zh` → OceanBase Vector →
|
|
||||||
`bce-reranker-base` → canonical Source Result 正式验收:32 条查询的 Recall@1/3/5、MRR、
|
|
||||||
NDCG@5 均为 `1.0`,P50 `3787.61 ms`、P95 `4759.21 ms`,通过 P95 `10000 ms` 门槛。
|
|
||||||
- 正式命令退出码为 `0`;两份合成文档、canonical Chunk/State、向量行与 BOS 对象均走随机
|
|
||||||
前缀清理,数据库复核 `live-` Document/Chunk/Vector 残留均为 `0`。首次 live 运行曾出现
|
|
||||||
一次瞬时失败,分阶段诊断及随后完整 32 查询均成功,仍需在后续稳定性任务关注 Provider
|
|
||||||
重试、限流和告警。
|
|
||||||
|
|
||||||
### DEMO-010(已完成)
|
|
||||||
|
|
||||||
- 新增同源轻量 Admin Console,提供 `/admin/upload`、`/admin/documents` 和
|
|
||||||
`/admin/search` 三个直接可访问页面;FastAPI 只负责静态文件路由,页面业务操作全部复用
|
|
||||||
`/api/v1`,没有复制上传、文档管理或检索逻辑。
|
|
||||||
- 上传页支持 PDF、DOCX、TXT、Markdown 文件选择、处理状态和结果摘要;文档页展示文件名、
|
|
||||||
类型、上传时间、状态,支持详情与确认删除;Search Lab 显式展示 Rank、Filename、Page、
|
|
||||||
Section、Chunk Content、Dense Score 和 Rerank Score。
|
|
||||||
- 前端使用安全 DOM 文本节点渲染 API 数据,不注入 HTML,不展示 storage key、Index Profile、
|
|
||||||
Provider Endpoint 或凭据;只实现 Demo 规定的三个页面,没有引入 ACL/RBAC、任务、审计、
|
|
||||||
Compiled Knowledge、MCP、Hybrid/BM25 或额外前端构建工具链。
|
|
||||||
- 新增静态路由、资产、范围和 API First 契约测试;浏览器使用本地确定性 Provider 完成
|
|
||||||
`README.md` 上传 → Ready → 列表/详情 → Top5 检索,并验证桌面和 390px 移动布局无横向
|
|
||||||
溢出、无前端警告或错误。删除的 UI 确认和服务端清理语义由现有 REST 全链路测试覆盖。
|
|
||||||
- 完整验证结果:Ruff Format/Lint、Mypy、JavaScript 语法、Git diff、依赖锁、Alembic Head/
|
|
||||||
离线升级均通过;Unit + Integration `150 passed`,RAG Evaluation `5 passed`。本任务没有
|
|
||||||
修改检索算法或 Provider,不重复调用真实资源验收;ARCH-006 的 32 条真实查询结果继续
|
|
||||||
作为底层链路基线。
|
|
||||||
- 已知非阻塞风险保持不变:配置清单中的历史 test 凭据仍应轮换;真实 Provider 瞬时失败的
|
|
||||||
重试、限流与告警留待后续明确 Task。Demo Console 本身没有新增凭据或生产 Endpoint。
|
|
||||||
|
|
||||||
### DEMO-014(已完成)
|
|
||||||
|
|
||||||
- 将 DEMO-010 通用绿色界面升级为地瓜机器人品牌化 Admin Console,使用唯一品牌强调色
|
|
||||||
`#FF3C00`、黑白高对比、无衬线中文排版、低圆角工业组件和与三步操作对应的计算信号轨;
|
|
||||||
没有增加页面、接口或业务能力。
|
|
||||||
- 品牌标记由内联 SVG 与 CSS 原生绘制,不下载、复制或热链官网资产。页面没有新增外部
|
|
||||||
Runtime、字体请求或第三方脚本,继续保持原生 HTML/CSS/JS 和同源 `/api/v1` 调用。
|
|
||||||
- GitHub 调研确认官方 `openai/skills` 当前没有通用前端设计 Skill;选择并安装了 Apache-2.0
|
|
||||||
的 `anthropics/skills/skills/frontend-design`。该 Skill 只影响视觉策划和自检方法,不进入
|
|
||||||
项目运行依赖或仓库产物。
|
|
||||||
- 浏览器使用本地确定性 Provider 再次完成 README 上传 → Ready → 文档列表 → Top5 检索;
|
|
||||||
1440px 桌面端和移动端均无根页面横向溢出、无前端警告或错误。复验中发现隐藏表头将移动
|
|
||||||
页面撑宽至 503px,已改为 `aria-label`,最终根宽度与 463px 视口一致,表格只在容器内滚动。
|
|
||||||
- 完整验证结果:Ruff Format/Lint、Mypy、JavaScript 语法与 Git diff 通过;Unit、Integration
|
|
||||||
和 RAG Evaluation 合计 `155 passed`;Alembic Head 与离线升级通过。本任务不修改 API、
|
|
||||||
检索算法、Provider 或数据库,不重复调用真实资源验收。
|
|
||||||
|
|
||||||
### PLAN-001(已完成)
|
|
||||||
|
|
||||||
- 审计确认当前 `/api/v1/documents` 仅接受一个文件,并在请求线程同步执行解析、Chunk、
|
|
||||||
Embedding 和索引;API 与 Admin Console 均没有批量提交、Job 查询或后台状态轮询。
|
|
||||||
- 当前上传、解析和向量索引具备阶段内失败补偿,影子索引重建具备目标 Profile 的重试安全
|
|
||||||
清理,但正常上传流水线没有持久化 Job/Attempt、自动调度退避或统一的重试前派生数据清理,
|
|
||||||
因而不能视为已实现自动重试。
|
|
||||||
- 保留既有 `DEMO-012` 并提升为最高优先级;新增 `DEMO-015` 承担 Async Job、自动重试和
|
|
||||||
重试前清理,新增 `DEMO-016` 承担批量异步上传和逐文件状态 UI。严格执行顺序为
|
|
||||||
`DEMO-012 → DEMO-015 → DEMO-016`。
|
|
||||||
- 已更新 PRD、Architecture、API/MCP Contract、Execution Plan、Task List、Demo Scope 与 DoD;
|
|
||||||
本任务只形成规划和待实现契约,没有修改业务代码、数据库 Migration 或 Accepted ADR。
|
|
||||||
- 验证结果:Ruff Format/Lint、Mypy、Git diff 与 Alembic Head/离线升级通过;Unit + Integration
|
|
||||||
`150 passed`,RAG Evaluation `5 passed`。首次受限网络运行因无法解析 PyPI 失败,获准联网
|
|
||||||
获取已锁定的构建依赖后全套复验通过。
|
|
||||||
- 下一任务:`DEMO-012`。详细依赖、范围和验收标准见
|
|
||||||
`docs/plans/ASYNC_INGESTION_MCP_PLAN.md`。
|
|
||||||
|
|
||||||
### DEMO-012(已完成)
|
|
||||||
|
|
||||||
- 新增独立 `ekp-mcp` Streamable HTTP Gateway 和 `make run-mcp` 入口,使用官方 MCP Python
|
|
||||||
SDK v2;默认只监听 `127.0.0.1:8766/mcp`,且只注册 `search_knowledge` 一个业务 Tool。
|
|
||||||
- Gateway 直接复用现有 `SearchService`,没有复制 Dense Retrieval、Reranker 或 Source
|
|
||||||
Enrichment;结构化输出与 REST 保持相同来源字段,不返回 Index Profile 或 Provider 信息。
|
|
||||||
- 新增运行时专用 Bearer Token 验证,将凭据映射为显式 client/service identity 和最小
|
|
||||||
`knowledge:search` scope;Token 至少 32 字符并通过 `SecretStr` 管理,不允许写入仓库。
|
|
||||||
当前身份层是 Demo 边界,不宣称完成 V1 OAuth、Document ACL 或用户委托授权。
|
|
||||||
- 启用 Host/Origin allowlist 和请求体限制;缺失/错误认证、scope、参数和检索异常均按安全
|
|
||||||
边界收敛,响应和日志不回显 Token、内部 Endpoint、数据库、模型或索引实现。
|
|
||||||
- 新增官方 MCP Client 的协议级测试,覆盖 initialize、tools/list、tools/call、唯一 Tool、
|
|
||||||
identity 传递、REST 字段一致性、无结果、参数拒绝、认证失败、Origin 防护和错误脱敏;
|
|
||||||
WorkBuddy 的 HTTP + Bearer + 环境变量配置记录于 `docs/development/MCP_SETUP.md`。
|
|
||||||
- 验证结果:Ruff Format/Lint、Mypy、Git diff、Alembic Head/离线升级全部通过;Unit +
|
|
||||||
Integration `165 passed`,RAG Evaluation `5 passed`。本任务未改变检索算法或数据库结构,
|
|
||||||
不重复消耗真实 Embedding/OceanBase/Reranker 资源。
|
|
||||||
- 2026-09-02 补充完成 WorkBuddy 现场验收:本机 WorkBuddy Desktop `5.3.14` 自带的
|
|
||||||
CodeBuddy CLI `2.115.0` 通过仓库交付的 `deploy/workbuddy-mcp.example.json` 建立
|
|
||||||
Streamable HTTP + Bearer 连接,健康检查为 `Connected`,仅发现
|
|
||||||
`mcp__enterprise-knowledge__search_knowledge`,并以 `query` + `top_k=1` 完成真实 Tool
|
|
||||||
调用、读取结构化来源字段。验收使用真实 Gateway、认证、身份传递、Schema 和 Tool
|
|
||||||
注册代码,检索应用返回确定性数据,以隔离 WorkBuddy 兼容性与外部 Provider 波动。
|
|
||||||
- 非交互客户端调用使用仅对当前会话放行 `DeferExecuteTool` 的权限设置,没有持久化放宽权限;
|
|
||||||
临时客户端配置已删除、Gateway 已停止,仓库和用户配置中均未留下测试 Token。至此
|
|
||||||
`DEMO-012` 约定的 WorkBuddy 客户端验收项闭环;公网 HTTPS 部署验证属于部署环境验收,
|
|
||||||
不阻塞本地 Demo Task 完成。
|
|
||||||
- 下一任务:`DEMO-015`。
|
|
||||||
|
|
||||||
### PLAN-002(已完成)
|
|
||||||
|
|
||||||
- 将当前交付目标明确为企业受控网络内的单一公共知识库:本阶段所有知识共享一个可见域,
|
|
||||||
暂缓用户、角色、知识库空间和 Document ACL,但不废弃 ADR-007,也不把未实现项标记完成。
|
|
||||||
- 保持 Accepted ADR 的模块化单体边界,确定 API/Admin、MCP Gateway 和 Async Worker 独立
|
|
||||||
部署;OceanBase/BOS/AI Provider 使用外部服务,当前云端模型链路不启用 GPU Worker。
|
|
||||||
- 将 Celery/RabbitMQ 正式并入 `DEMO-015`,并新增 `DEPLOY-001~003`,严格顺序为
|
|
||||||
`DEMO-015 → DEMO-016 → DEPLOY-001 → DEPLOY-002 → DEPLOY-003`。
|
|
||||||
- 定义 `make deploy VALUES=...` 的一键部署边界:操作者预先提供集群访问与 Secret,仓库负责
|
|
||||||
生产镜像、串行 Migration、Helm 安装/升级、Ingress/TLS、探针、资源与 Pod 安全配置;不得
|
|
||||||
将凭据写入 Git、镜像、命令行或日志。
|
|
||||||
- 明确现有 `docker-compose.dev.yml` 的 PostgreSQL/MinIO 假设已被 ADR-012 取代,不作为集群
|
|
||||||
交付模板;新增 `docs/plans/PUBLIC_KNOWLEDGE_CLUSTER_PLAN.md` 作为任务范围和验收基线。
|
|
||||||
- 本任务仅修改规划、产品、架构、API/MCP 契约、Demo Scope 和 DoD 文档,没有修改业务代码、
|
|
||||||
数据库 Migration 或 Accepted ADR 状态。
|
|
||||||
- 验证结果:Markdown 相对链接、敏感值模式和 Git diff 检查通过;Ruff Format/Lint、Mypy、
|
|
||||||
Alembic Head/离线升级通过;Unit + Integration `165 passed`,RAG Evaluation `5 passed`。
|
|
||||||
- 下一任务:`DEMO-015`。
|
|
||||||
|
|
||||||
### DEMO-015(已完成)
|
|
||||||
|
|
||||||
- 新增 OceanBase canonical `ingestion_jobs` / `ingestion_attempts` 与 forward-only migration;
|
|
||||||
Document 唯一 Job、Attempt 编号上限和单一运行中 Attempt 均由数据库约束保护。
|
|
||||||
- `POST /api/v1/documents` 改为持久化不可变 Source、Document、Job 后返回 `202`,新增
|
|
||||||
`GET /api/v1/jobs/{job_id}`;活动 Job 的 Document 删除返回 `409`,避免与 Worker 竞态。
|
|
||||||
- 新增队列无关的 Dispatcher 与 Job Runner,首个适配器使用 Celery/RabbitMQ;消息只包含 Job
|
|
||||||
ID,启用 late ack、worker-lost reject、单任务 prefetch 与周期性持久化 Job 恢复扫描。
|
|
||||||
- 瞬时错误按配置执行指数退避和最多三次总 Attempt;Worker 租约有效时重复投递不创建新
|
|
||||||
Attempt,租约过期后关闭旧 Attempt 并恢复。错误记录为固定安全摘要,不保存 Provider 正文。
|
|
||||||
- 第二次及后续 Attempt 前依次清理向量、ChunkIndexState、Chunk 与 parsed JSON,再从不可变
|
|
||||||
源文件重建;清理失败直接终止且不创建新 Attempt。Job 创建失败会补偿 Document 与源对象。
|
|
||||||
- Admin Console 仅将上传提示调整为“已接收并进入队列”;批量选择、Job 轮询和逐文件状态 UI
|
|
||||||
没有提前实现,继续属于 `DEMO-016`。
|
|
||||||
- 验证结果:Ruff Format/Lint、Mypy、JavaScript 语法、依赖锁、Alembic Head/离线 MySQL SQL
|
|
||||||
与 SQLite schema drift 均通过;Unit + Integration `179 passed`,RAG Evaluation `5 passed`;
|
|
||||||
Worker 入口确认注册 `ekp.ingestion.run` 与 `ekp.ingestion.recover`。
|
|
||||||
- 环境限制:当前工作站没有 Docker/RabbitMQ,也没有持久化注入目标 OceanBase URL,因此未做
|
|
||||||
真实 Broker 断连/确认超时和目标库 migration;这些真实环境项必须在 DEPLOY-003 集群验收
|
|
||||||
重跑。本任务以 Celery 配置测试、SQLite 持久化/唯一约束和故障注入集成测试闭环代码语义。
|
|
||||||
- 下一任务:`DEMO-016`。
|
|
||||||
|
|
||||||
### DEMO-016(已完成)
|
|
||||||
|
|
||||||
- 新增 canonical `ingestion_batches` / `ingestion_batch_items` 及 forward-only migration,保存
|
|
||||||
输入顺序、逐文件接收结果与 Document/Job 引用;异步阶段和最终结果仍以 Job 为事实源,
|
|
||||||
批次不是跨文件事务。
|
|
||||||
- 新增 `POST /api/v1/ingestion-batches` 与 `GET /api/v1/ingestion-batches/{batch_id}`;批量
|
|
||||||
请求字段为 `files`,成功返回 `202`、聚合状态和逐文件接收/Job 状态。类型、大小或内容错误
|
|
||||||
只拒绝当前文件,错误映射为固定安全摘要,其他文件继续接收。
|
|
||||||
- 新增 `BATCH_MAX_FILES` 与 `BATCH_MAX_TOTAL_BYTES`,复用 `MAX_UPLOAD_BYTES` 约束单文件;
|
|
||||||
上传流共享总字节预算,不把整批文件读入内存。数量超限在创建批次前返回 `413`。
|
|
||||||
- Admin Console 支持多选提交,显示当前批次的接收数量、完成数量、逐文件阶段、Attempt 与安全
|
|
||||||
错误,并每 1.5 秒自动轮询;进入 `completed`、`partial_failure` 或 `failed` 后停止轮询。
|
|
||||||
DOM 数据继续只通过文本节点渲染。
|
|
||||||
- 集成测试覆盖顺序稳定、部分拒绝不回滚、数量/单文件/批次总大小、全成功/部分失败聚合、
|
|
||||||
未知批次、OpenAPI 和前端契约;迁移 drift 增加两张新表。Ruff Format/Lint、Mypy、
|
|
||||||
JavaScript 语法、依赖锁、Alembic Head/离线升级均通过;Unit + Integration `183 passed`,
|
|
||||||
RAG Evaluation `5 passed`。
|
|
||||||
- 浏览器验收确认桌面布局和 390px 窄屏布局正常、无根页面横向溢出、无控制台警告或错误;
|
|
||||||
本地页面不注入云端凭据,批次结果行为由确定性 API 集成测试验证。
|
|
||||||
- 未扩展目录上传、压缩包解包、断点续传、手动重试、去重、用户或 ACL;自动重试和清理继续
|
|
||||||
复用 DEMO-015,Accepted ADR 未变更。
|
|
||||||
- 下一任务:`DEPLOY-001`。
|
|
||||||
|
|
||||||
### DEPLOY-001(已完成)
|
|
||||||
|
|
||||||
- 新增共享生产 Dockerfile 的 `api`、`mcp`、`worker` 三个 target,使用精确 Python/uv 版本、
|
|
||||||
frozen production dependencies、固定非 root UID/GID、exec-form 启动命令和逐 target 健康检查。
|
|
||||||
- 新增 amd64/arm64 Buildx Bake 计划、镜像构建/渲染/推送 Make 入口和 `.dockerignore`;构建上下文
|
|
||||||
排除凭据、本地状态、缓存、测试与开发产物,镜像不包含环境 Endpoint 或 Secret。
|
|
||||||
- Admin 静态目录新增类型化运行时覆盖,使非 editable 安装后的 API 镜像仍能提供同源页面;
|
|
||||||
源码运行继续使用既有默认目录。
|
|
||||||
- 新增不依赖 Docker daemon 的镜像契约测试与中文运行文档;基础 Python/uv 镜像同时固定精确
|
|
||||||
tag 与 manifest digest,Registry 地址和本项目发布 digest 继续由部署环境提供。
|
|
||||||
- 使用 Docker Buildx `0.36.1` + Colima 分别实际构建并加载全部三个 `linux/arm64` 和三个
|
|
||||||
`linux/amd64` target;最终 digest 固定版本再次构建命中相同业务 layer。没有执行 Registry
|
|
||||||
push,也没有在构建参数、history 或镜像文件系统中发现凭据。
|
|
||||||
- API 镜像以 `10001:10001` 启动并进入 healthy,`/healthz`、`/admin/upload`、OpenAPI 及
|
|
||||||
DEMO-016 批量路径 smoke test 通过;MCP/Worker 镜像以同一非 root 用户完成安装入口导入,
|
|
||||||
默认 CMD 与 HEALTHCHECK 元数据核对通过。真实 Provider/Broker 启动仍属于 DEPLOY-003。
|
|
||||||
- Ruff Format/Lint、Mypy、Docker Bake 渲染、JavaScript/依赖锁/Alembic 与完整测试门禁通过;
|
|
||||||
Unit + Integration `187 passed`,RAG Evaluation `5 passed`。下一任务:`DEPLOY-002`。
|
|
||||||
|
|
||||||
### DEPLOY-002(已完成)
|
|
||||||
|
|
||||||
- 新增 `deploy/helm/knowledge-platform` Helm Chart,部署 API/Admin、MCP Gateway、Async Worker、
|
|
||||||
单实例 Recovery Scheduler、串行 Migration Job、Service、Ingress/TLS 和可选 Demo RabbitMQ;
|
|
||||||
API/MCP 默认各两个副本,Worker 可独立扩缩,Beat 使用 `Recreate` 避免重复调度。
|
|
||||||
- 新增严格 `values.schema.json` 与仓库外环境 Values 示例,拒绝未知顶层字段、`latest` tag、
|
|
||||||
非法 digest、API/MCP 单副本和错误类型;跨字段模板校验继续约束批量大小、重试、Chunk 和
|
|
||||||
Top-K 关系。应用和可选 RabbitMQ 镜像均支持 digest,Demo RabbitMQ 已固定 manifest digest。
|
|
||||||
- 敏感配置只通过预建 `existingSecret` 注入;ConfigMap 仅渲染显式白名单内的非敏感字段,
|
|
||||||
没有任意 `extraEnv` 或 Secret 对象。`scripts/deploy.sh` 在部署前校验 Values、Namespace、运行时
|
|
||||||
Secret/必需键、可选 RabbitMQ Secret/键和启用 Ingress 时的 TLS Secret 引用,不读取 Secret
|
|
||||||
值;Helm 3/4 分别使用等价的原子失败回滚参数。
|
|
||||||
- Migration 使用 `pre-install,pre-upgrade` Hook、固定名称和 `before-hook-creation`,安装/升级在
|
|
||||||
Migration 成功前不更新业务 Pod。应用容器固定非 root UID/GID、只读根文件系统、丢弃全部
|
|
||||||
capabilities、禁用 ServiceAccount Token,并为临时目录、RabbitMQ 日志和 Beat 状态提供
|
|
||||||
`emptyDir`;API/MCP/Worker/Scheduler 均配置启动和存活/就绪探针及资源 requests/limits。
|
|
||||||
- 本机使用 K3s `v1.35.0+k3s1`、Helm `4.2.4` 和 DEPLOY-001 ARM64 镜像完成真实安装;Migration
|
|
||||||
成功,API/MCP 为 `2/2`,Worker、Scheduler 与可选 RabbitMQ 全部 Ready。首次安装暴露的
|
|
||||||
RabbitMQ 只读日志目录和 Beat 调度文件权限问题已修复,随后相同 Values 连续升级至 Revision
|
|
||||||
3~5 均成功;API `/healthz` 和 Admin 上传页 smoke test 通过。
|
|
||||||
- 默认、环境示例和 RabbitMQ 三种 Helm lint 均通过;含 Ingress/TLS/RabbitMQ 的 14 个最终
|
|
||||||
资源通过 kubeconform strict 校验。仓库、Helm manifest、Pod 日志三路 Secret 值扫描均为 `0`;
|
|
||||||
Ruff Format/Lint、Mypy、Alembic、Bash 语法与完整门禁通过,Unit + Integration `195 passed`,
|
|
||||||
RAG Evaluation `5 passed`。
|
|
||||||
- 目标企业集群的稳定 HTTPS、滚动重启、Worker 故障恢复、真实外部依赖监测与远程 WorkBuddy
|
|
||||||
仍属于 `DEPLOY-003`;本任务没有把这些项提前标记完成,也没有改变 Accepted ADR 或公共知识库
|
|
||||||
范围。下一任务:`DEPLOY-003`。
|
|
||||||
|
|||||||
@ -1,123 +0,0 @@
|
|||||||
# 公共知识库集群交付计划
|
|
||||||
|
|
||||||
**Task:** PLAN-002
|
|
||||||
**Status:** Accepted planning baseline
|
|
||||||
**Date:** 2026-09-02
|
|
||||||
|
|
||||||
## 1. 阶段目标与边界
|
|
||||||
|
|
||||||
当前阶段只交付一个面向企业内部受控网络的公共知识库,先保证服务持续可用、可重复部署,
|
|
||||||
并让使用者打开 Admin Console 或连接 WorkBuddy 后即可使用,无需感知后端进程启停。
|
|
||||||
|
|
||||||
“公共”表示本阶段所有已接收知识处于同一个可见域,不代表面向公网匿名开放,也不代表
|
|
||||||
ADR-007 被废弃。用户、用户组、角色、知识库多空间、Document ACL、SSO、OAuth 用户委托等
|
|
||||||
完整 V1 能力继续后置;一旦需要接收非公共知识,必须先实现 Permission Resolver 和检索前/
|
|
||||||
检索中 ACL 下推,禁止先做全局 Top-K 再过滤。
|
|
||||||
|
|
||||||
Admin 写操作只部署在企业受控网络或统一接入网关后。MCP 继续使用运行时注入、只具备
|
|
||||||
`knowledge:search` scope 的服务凭据;静态 Demo credential 不得演变为共享超级管理员 Token。
|
|
||||||
|
|
||||||
## 2. 目标部署拓扑
|
|
||||||
|
|
||||||
```text
|
|
||||||
Ingress / TLS
|
|
||||||
│
|
|
||||||
┌──────────────┴──────────────┐
|
|
||||||
│ │
|
|
||||||
Admin Console + Knowledge API MCP Gateway
|
|
||||||
replicas replicas
|
|
||||||
│ │
|
|
||||||
└──────── Application Services ────────┐
|
|
||||||
│
|
|
||||||
┌─────────────────────────────┼─────────────┐
|
|
||||||
│ │ │
|
|
||||||
OceanBase MySQL BOS RabbitMQ
|
|
||||||
canonical + vector │
|
|
||||||
▼
|
|
||||||
Async Worker(s)
|
|
||||||
```
|
|
||||||
|
|
||||||
沿用 ADR-002:业务核心仍为模块化单体,不拆分额外微服务;API、MCP Gateway 和异步 Worker
|
|
||||||
分别作为可独立扩缩容的进程/容器交付。当前 Parser 与云端 Embedding/Reranker 不需要本地 GPU,
|
|
||||||
因此 GPU Worker 不进入本轮最小部署,未来引入本地 OCR/模型时再按既有架构角色启用。
|
|
||||||
|
|
||||||
OceanBase、BOS 和模型服务使用集群外已有服务。RabbitMQ 默认使用外部地址;Helm Chart 可以
|
|
||||||
提供显式开关安装开发/演示依赖,但生产值不得把队列数据生命周期绑定到应用 Pod。
|
|
||||||
|
|
||||||
## 3. “一键部署”的定义
|
|
||||||
|
|
||||||
在操作者已具备 Kubernetes 访问权限、目标 Namespace 和 Secret 的前提下,仓库提供单一入口:
|
|
||||||
|
|
||||||
```text
|
|
||||||
make deploy VALUES=<environment-values.yaml>
|
|
||||||
```
|
|
||||||
|
|
||||||
该入口负责:
|
|
||||||
|
|
||||||
- 校验非敏感配置和 `existingSecret` 引用,不把凭据写入 Git、镜像、命令行或日志;
|
|
||||||
- 运行一次性、可追踪且串行的数据库 Migration Job;
|
|
||||||
- 安装或升级 API/Admin、MCP Gateway、Async Worker、Service 和可选 RabbitMQ 依赖;
|
|
||||||
- 配置 Ingress/TLS、资源 requests/limits、非 root、只读根文件系统和最小 ServiceAccount;
|
|
||||||
- 配置 readiness/liveness probe、副本数和滚动升级策略;
|
|
||||||
- 支持重复执行和幂等升级,并提供对应卸载/回滚说明。
|
|
||||||
|
|
||||||
本定义不承诺自动创建 OceanBase、BOS、域名证书或云厂商账号,也不允许在仓库内保存这些
|
|
||||||
资源的敏感凭据。
|
|
||||||
|
|
||||||
## 4. Task 顺序与依赖
|
|
||||||
|
|
||||||
```text
|
|
||||||
PLAN-002
|
|
||||||
└──→ DEMO-015 Celery/RabbitMQ Async Job + Retry Cleanup
|
|
||||||
└──→ DEMO-016 Batch Async Upload + Status UI
|
|
||||||
└──→ DEPLOY-001 Production Images
|
|
||||||
└──→ DEPLOY-002 Kubernetes/Helm One-click Deploy
|
|
||||||
└──→ DEPLOY-003 Availability + Remote Acceptance
|
|
||||||
```
|
|
||||||
|
|
||||||
### DEMO-015
|
|
||||||
|
|
||||||
实现持久化 Job/Attempt、重试前清理、幂等消费和有界自动重试,并把执行器正式接入
|
|
||||||
Celery/RabbitMQ。队列调用保持在抽象边界后,不允许业务服务直接依赖 Celery API。
|
|
||||||
|
|
||||||
### DEMO-016
|
|
||||||
|
|
||||||
**已完成。** 基于 DEMO-015 提供逐文件隔离的批量异步提交、批次/Job 查询和 Admin Console
|
|
||||||
状态展示;下一交付门槛为 DEPLOY-001 的生产镜像。
|
|
||||||
|
|
||||||
### DEPLOY-001
|
|
||||||
|
|
||||||
**已完成。** 交付 API/Admin、MCP Gateway、Async Worker 的生产容器镜像与启动约定;镜像使用固定依赖、
|
|
||||||
非 root 用户、健康检查和多架构可构建基线,不内置凭据。
|
|
||||||
|
|
||||||
### DEPLOY-002
|
|
||||||
|
|
||||||
**已完成。** 交付 Kubernetes/Helm Chart、严格环境 Values Schema、`existingSecret`、串行
|
|
||||||
Migration Hook、Ingress/TLS、Pod 安全、资源限制、探针和 `make deploy` 入口;Worker 与单实例
|
|
||||||
恢复调度器已分离。现有 `docker-compose.dev.yml` 仍仅作为旧的本地开发文件,因其
|
|
||||||
PostgreSQL/MinIO 假设与 ADR-012 不一致,不得作为集群交付模板。下一交付门槛为 DEPLOY-003
|
|
||||||
的目标集群可用性与远程验收。
|
|
||||||
|
|
||||||
### DEPLOY-003
|
|
||||||
|
|
||||||
在目标集群完成安装、幂等升级、滚动重启、Worker 故障恢复和远程 WorkBuddy 验收;形成运行
|
|
||||||
手册、备份/恢复责任边界和最小日志/指标检查。灾备演练与完整 Observability 平台仍属后置项。
|
|
||||||
|
|
||||||
## 5. 最小验收标准
|
|
||||||
|
|
||||||
- 新 Namespace 可由单一命令完成安装,重复执行不会破坏已有数据;
|
|
||||||
- Migration 在应用升级前串行成功,失败时新版本不继续接管流量;
|
|
||||||
- Admin/API 与 MCP 使用稳定 HTTPS 地址,重启或滚动升级不要求最终用户重新注入配置;
|
|
||||||
- API/MCP 支持至少两个副本,readiness 未通过的 Pod 不接收流量;
|
|
||||||
- Worker 重启后持久化 Job 可继续恢复,重复消息不产生双重有效执行或重复派生数据;
|
|
||||||
- 远程 WorkBuddy 仅发现批准的 `search_knowledge` Tool,并完成一次真实检索;
|
|
||||||
- 清单、镜像、日志和验收产物不包含凭据,敏感值均来自 Secret;
|
|
||||||
- 关键请求、Job 和外部依赖失败具备可定位的结构化日志和最小指标。
|
|
||||||
|
|
||||||
## 6. 明确后置能力
|
|
||||||
|
|
||||||
- 用户、用户组、角色、多租户、知识库空间隔离和 Document ACL;
|
|
||||||
- LDAP/SSO、OAuth 用户委托、公共互联网匿名访问;
|
|
||||||
- GPU Worker、本地模型、OCR/MinerU;
|
|
||||||
- 跨区域容灾、自动云资源编排和完整 Observability 平台;
|
|
||||||
- 未被既有 Demo/V1 明确批准的业务能力。
|
|
||||||
@ -7,7 +7,7 @@
|
|||||||
- [x] ARCH-003 OceanBase VectorStore adapter and index schema
|
- [x] ARCH-003 OceanBase VectorStore adapter and index schema
|
||||||
- [x] ARCH-004 Qianfan `bce-reranker-base` alignment
|
- [x] ARCH-004 Qianfan `bce-reranker-base` alignment
|
||||||
- [x] ARCH-005 Runtime cutover, shadow rebuild workflow and legacy configuration cleanup
|
- [x] ARCH-005 Runtime cutover, shadow rebuild workflow and legacy configuration cleanup
|
||||||
- [x] ARCH-006 OceanBase canonical cutover, controlled live integration and RAG acceptance
|
- [ ] ARCH-006 Controlled live integration and RAG acceptance
|
||||||
|
|
||||||
Dependencies:
|
Dependencies:
|
||||||
|
|
||||||
@ -23,30 +23,6 @@ ARCH-001
|
|||||||
Each architecture task uses an independent branch and is merged into `main` only after its own
|
Each architecture task uses an independent branch and is merged into `main` only after its own
|
||||||
validation. Provider credentials and internal endpoints are runtime inputs and never task artifacts.
|
validation. Provider credentials and internal endpoints are runtime inputs and never task artifacts.
|
||||||
|
|
||||||
## Post-Demo priority plan
|
|
||||||
|
|
||||||
- [x] PLAN-001 MCP 与异步批量入库现状审计、任务拆分和契约基线
|
|
||||||
- [x] DEMO-012 MCP `search_knowledge` Gateway + identity + WorkBuddy verification
|
|
||||||
- [x] PLAN-002 公共知识库集群交付范围、任务拆分和一键部署基线
|
|
||||||
- [x] DEMO-015 Async ingestion Job + Celery/RabbitMQ + bounded retry + pre-retry cleanup
|
|
||||||
- [x] DEMO-016 Batch async upload API + per-file Job status + Admin Console
|
|
||||||
- [x] DEPLOY-001 Production images for API/Admin, MCP Gateway and Async Worker
|
|
||||||
- [x] DEPLOY-002 Kubernetes/Helm one-click deployment + Secret/Migration/Ingress baseline
|
|
||||||
- [ ] DEPLOY-003 Availability, rolling upgrade and remote WorkBuddy acceptance
|
|
||||||
|
|
||||||
Dependencies and acceptance boundaries are defined in
|
|
||||||
[`ASYNC_INGESTION_MCP_PLAN.md`](ASYNC_INGESTION_MCP_PLAN.md) and
|
|
||||||
[`PUBLIC_KNOWLEDGE_CLUSTER_PLAN.md`](PUBLIC_KNOWLEDGE_CLUSTER_PLAN.md). These Demo/Deploy tasks are
|
|
||||||
vertical slices of the V1 backlog below; completing a slice does not automatically complete its
|
|
||||||
corresponding `EKP-*` task. The current public-knowledge phase defers, but does not complete or
|
|
||||||
supersede, the unchecked user/ACL tasks.
|
|
||||||
|
|
||||||
```text
|
|
||||||
DEMO-009 + ARCH-006 → DEMO-012
|
|
||||||
PLAN-002
|
|
||||||
└→ DEMO-003~006 → DEMO-015 → DEMO-016 → DEPLOY-001 → DEPLOY-002 → DEPLOY-003
|
|
||||||
```
|
|
||||||
|
|
||||||
## Phase 0
|
## Phase 0
|
||||||
- [ ] EKP-001 Repository bootstrap
|
- [ ] EKP-001 Repository bootstrap
|
||||||
- [ ] EKP-002 Backend Python toolchain
|
- [ ] EKP-002 Backend Python toolchain
|
||||||
@ -58,7 +34,7 @@ PLAN-002
|
|||||||
|
|
||||||
## Canonical domain / ingestion
|
## Canonical domain / ingestion
|
||||||
- [ ] EKP-010 Core domain package
|
- [ ] EKP-010 Core domain package
|
||||||
- [ ] EKP-011 OceanBase MySQL schema and migrations
|
- [ ] EKP-011 PostgreSQL schema and migrations
|
||||||
- [ ] EKP-012 User/group/role model
|
- [ ] EKP-012 User/group/role model
|
||||||
- [ ] EKP-013 Knowledge Base model
|
- [ ] EKP-013 Knowledge Base model
|
||||||
- [ ] EKP-014 Folder model
|
- [ ] EKP-014 Folder model
|
||||||
|
|||||||
@ -9,26 +9,3 @@
|
|||||||
7. Admin Console
|
7. Admin Console
|
||||||
8. Knowledge Compilation
|
8. Knowledge Compilation
|
||||||
9. Evaluation, security and release hardening
|
9. Evaluation, security and release hardening
|
||||||
|
|
||||||
## 当前增量实施顺序(2026-09-02)
|
|
||||||
|
|
||||||
`DEMO-012` 已完成 MCP `search_knowledge` 与本地 WorkBuddy 验收。根据 PLAN-002,当前阶段按
|
|
||||||
“单一公共知识可见域、企业受控网络、持续可用和一键集群部署”推进,不提前实现用户/权限隔离:
|
|
||||||
|
|
||||||
1. `PLAN-002`:确定公共知识库交付边界、目标拓扑、Task 拆分与验收门槛;
|
|
||||||
2. `DEMO-015`(已完成):持久化异步 Job、Celery/RabbitMQ 执行器、自动重试和重试前清理;
|
|
||||||
3. `DEMO-016`(已完成):批量异步上传、批次/Job 查询和 Admin Console 状态展示;
|
|
||||||
4. `DEPLOY-001`(已完成):API/Admin、MCP Gateway、Async Worker 生产镜像;
|
|
||||||
5. `DEPLOY-002`(已完成):Kubernetes/Helm、Secret、Migration、Ingress/TLS 和一键部署入口;
|
|
||||||
6. `DEPLOY-003`:多副本可用性、滚动升级、Worker 恢复和远程 WorkBuddy 验收。
|
|
||||||
|
|
||||||
严格依赖为:
|
|
||||||
|
|
||||||
```text
|
|
||||||
PLAN-002 → DEMO-015 → DEMO-016 → DEPLOY-001 → DEPLOY-002 → DEPLOY-003
|
|
||||||
```
|
|
||||||
|
|
||||||
`DEMO-016` 不得在缺少持久化状态、幂等消费和恢复边界时直接并发调用同步流水线;集群交付也
|
|
||||||
不得继续使用 `docker-compose.dev.yml` 中已被 ADR-012 取代的 PostgreSQL/MinIO 假设。详细
|
|
||||||
范围和验收标准见 [`ASYNC_INGESTION_MCP_PLAN.md`](ASYNC_INGESTION_MCP_PLAN.md) 与
|
|
||||||
[`PUBLIC_KNOWLEDGE_CLUSTER_PLAN.md`](PUBLIC_KNOWLEDGE_CLUSTER_PLAN.md)。
|
|
||||||
|
|||||||
@ -10,21 +10,6 @@
|
|||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## 当前阶段交付覆盖(PLAN-002)
|
|
||||||
|
|
||||||
为优先达成服务可用和一键集群部署,当前阶段只交付企业受控网络内的单一公共知识库:所有
|
|
||||||
已接收知识处于同一个可见域,用户打开 Admin Console 或连接 WorkBuddy 即可使用,无需感知
|
|
||||||
后端进程启停。用户、用户组、角色、知识库多空间、Document ACL、LDAP/SSO 和 OAuth 用户
|
|
||||||
委托仍是本 PRD 的后续 V1 要求,本阶段不实现,也不得标记为完成。
|
|
||||||
|
|
||||||
该阶段覆盖不会取代安全原则:公共知识库不等于匿名公网开放;Admin 写操作必须位于受控网络
|
|
||||||
或统一接入网关后,MCP 使用运行时注入的最小 `knowledge:search` 服务凭据。一旦平台需要接收
|
|
||||||
非公共知识,必须先完成 Permission Resolver 与检索前/检索中 ACL 下推,禁止全局 Top-K 后置
|
|
||||||
过滤。实施顺序和集群验收边界见
|
|
||||||
[`PUBLIC_KNOWLEDGE_CLUSTER_PLAN.md`](../plans/PUBLIC_KNOWLEDGE_CLUSTER_PLAN.md)。
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
# 1. 项目背景
|
# 1. 项目背景
|
||||||
|
|
||||||
企业内部存在大量分散于 PDF、Word、PPT、Excel、Markdown、图片、扫描件及其他文件中的知识资产。
|
企业内部存在大量分散于 PDF、Word、PPT、Excel、Markdown、图片、扫描件及其他文件中的知识资产。
|
||||||
@ -462,19 +447,6 @@ V1 必须支持:
|
|||||||
|
|
||||||
Document processing 必须异步执行。
|
Document processing 必须异步执行。
|
||||||
|
|
||||||
源文件与 Document/Job 成功持久化后返回 `202 Accepted`。批量上传中的每个文件拥有独立的
|
|
||||||
Document ID 和 Ingestion Job;单个文件处理失败不得阻塞或回滚批次内其他已接收文件。
|
|
||||||
|
|
||||||
当前 Demo 已实现多文件 `202` 接收、批次聚合查询和逐文件 Job 状态;目录上传、压缩包解包、
|
|
||||||
断点续传以及完整 V1 的 Hash/Version/ACL 能力仍按各自 Task 后置。
|
|
||||||
|
|
||||||
公共知识库集群交付使用分别可扩缩的 API/Admin、MCP Gateway 和 Async Worker 生产镜像;
|
|
||||||
镜像只包含锁定代码与非敏感默认值,环境凭据必须在部署时注入。
|
|
||||||
|
|
||||||
集群编排由 Helm Chart 统一管理:API/MCP 默认多副本,Worker 与单实例恢复调度器分离;敏感值
|
|
||||||
只引用预建 Kubernetes Secret,数据库 Migration 在安装/升级前串行完成。Ingress/TLS、探针、
|
|
||||||
资源限制和 Pod 安全默认值属于部署契约,不改变 REST/MCP 业务接口。
|
|
||||||
|
|
||||||
**REQ-INGEST-004**
|
**REQ-INGEST-004**
|
||||||
|
|
||||||
展示处理阶段:
|
展示处理阶段:
|
||||||
@ -496,18 +468,10 @@ Failed
|
|||||||
失败必须展示:
|
失败必须展示:
|
||||||
|
|
||||||
- Failed Stage;
|
- Failed Stage;
|
||||||
- Attempt Count;
|
|
||||||
- 是否仍会自动重试;
|
|
||||||
- Error Code;
|
- Error Code;
|
||||||
- Error Message;
|
- Error Message;
|
||||||
- Retry。
|
- Retry。
|
||||||
|
|
||||||
**REQ-INGEST-006**
|
|
||||||
|
|
||||||
可重试的瞬时执行失败必须自动进行有界重试。每次重新执行前必须清理该 Document 的旧解析
|
|
||||||
产物、Chunk、Chunk Index State 和向量/稀疏索引数据,再从不可变原始文件开始处理。清理不得
|
|
||||||
删除原始文件、Document 标识或 Job/Attempt 历史;清理失败时不得继续重试。
|
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
# 11. 文件版本和去重
|
# 11. 文件版本和去重
|
||||||
@ -1850,9 +1814,6 @@ Cancelled
|
|||||||
Retry
|
Retry
|
||||||
```
|
```
|
||||||
|
|
||||||
Retry 必须记录 Attempt 历史并区分可重试的瞬时错误与确定性永久错误;重试执行前遵循
|
|
||||||
`REQ-INGEST-006` 的派生数据清理规则。
|
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
# 57. 删除语义
|
# 57. 删除语义
|
||||||
@ -2414,7 +2375,7 @@ API First
|
|||||||
```
|
```
|
||||||
|
|
||||||
运行时切换约束:索引 Provider 或 Profile 变更必须以新 Profile 构建 Shadow Index;Profile
|
运行时切换约束:索引 Provider 或 Profile 变更必须以新 Profile 构建 Shadow Index;Profile
|
||||||
生命周期和激活状态保存在 OceanBase canonical 关系表。只有 canonical Chunk 与 ChunkIndexState 完整核对
|
生命周期和激活状态保存在 PostgreSQL。只有 canonical Chunk 与 ChunkIndexState 完整核对
|
||||||
通过后才能原子激活,禁止复用旧 Profile ID 或原地覆盖当前 Active Index。
|
通过后才能原子激活,禁止复用旧 Profile ID 或原地覆盖当前 Active Index。
|
||||||
|
|
||||||
---
|
---
|
||||||
@ -2436,7 +2397,7 @@ API First + Admin Console
|
|||||||
**DEC-002**
|
**DEC-002**
|
||||||
|
|
||||||
OceanBase Vector 作为当前生产主检索索引,同时必须保留 Vector Store 替换能力。
|
OceanBase Vector 作为当前生产主检索索引,同时必须保留 Vector Store 替换能力。
|
||||||
OceanBase 关系表保存 canonical 数据,同库 Vector Store 表只保存可重建索引。
|
PostgreSQL 继续保存 canonical 数据,Vector Store 只保存可重建索引。
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
@ -2492,7 +2453,7 @@ PRD 评审通过后进入《总体技术架构设计》。
|
|||||||
|
|
||||||
1. Backend 技术栈;
|
1. Backend 技术栈;
|
||||||
2. Admin Console 技术栈;
|
2. Admin Console 技术栈;
|
||||||
3. OceanBase MySQL canonical 数据模型;
|
3. PostgreSQL 数据模型;
|
||||||
4. Object Storage;
|
4. Object Storage;
|
||||||
5. Task Queue;
|
5. Task Queue;
|
||||||
6. MinerU vs Docling;
|
6. MinerU vs Docling;
|
||||||
@ -2578,8 +2539,8 @@ Cloud LLM
|
|||||||
# 77. 调研依据备注
|
# 77. 调研依据备注
|
||||||
|
|
||||||
内部最新配置基线确认现有知识平台主链使用千帆 `bge-large-zh`、`bce-reranker-base`、
|
内部最新配置基线确认现有知识平台主链使用千帆 `bge-large-zh`、`bce-reranker-base`、
|
||||||
BOS 与 OceanBase MySQL 模式;OceanBase 同时承载 canonical 关系表和逻辑隔离的可重建
|
BOS 与 OceanBase Vector。该信息只用于确定 Provider 选型,不把连接地址、账号、密钥或
|
||||||
向量表。连接地址、账号和密钥仍只能作为运行时 Secret,不进入产品文档或提交记录。
|
历史环境加载方式写入产品文档和代码。
|
||||||
|
|
||||||
OceanBase Vector 的生产接入必须验证向量检索、标量 Filter、索引参数和版本兼容性;
|
OceanBase Vector 的生产接入必须验证向量检索、标量 Filter、索引参数和版本兼容性;
|
||||||
BM25/Hybrid 在相应任务验收前仍按产品能力要求保留,不在本轮 Provider 迁移中提前扩展。
|
BM25/Hybrid 在相应任务验收前仍按产品能力要求保留,不在本轮 Provider 迁移中提前扩展。
|
||||||
|
|||||||
@ -1,32 +1,2 @@
|
|||||||
{"id":"refund-001","query":"产品A退款周期","expected_documents":["sample-product-a-policy.md"],"expected_sections":["退款"],"tags":["产品","短查询"]}
|
{"id":"sample-001","query":"员工出差住宿标准是多少?","expected_documents":["sample-travel-policy.md"],"expected_sections":["住宿标准"],"tags":["制度","自然语言"]}
|
||||||
{"id":"refund-002","query":"示例产品 A 的退款需要几个自然日?","expected_documents":["sample-product-a-policy.md"],"expected_sections":["退款"],"tags":["产品","自然语言"]}
|
{"id":"sample-002","query":"产品A退款周期","expected_documents":["sample-product-a-policy.md"],"expected_sections":["退款"],"tags":["产品","短查询"]}
|
||||||
{"id":"refund-003","query":"申请产品A退款后多久完成?","expected_documents":["sample-product-a-policy.md"],"expected_sections":["退款"],"tags":["产品","同义表达"]}
|
|
||||||
{"id":"refund-004","query":"退款规则里写的处理周期是什么?","expected_documents":["sample-product-a-policy.md"],"expected_sections":["退款"],"tags":["产品","规则"]}
|
|
||||||
{"id":"refund-005","query":"产品 A 提交退款申请后的等待时间","expected_documents":["sample-product-a-policy.md"],"expected_sections":["退款"],"tags":["产品","长查询"]}
|
|
||||||
{"id":"refund-006","query":"退款是否按自然日计算?","expected_documents":["sample-product-a-policy.md"],"expected_sections":["退款"],"tags":["产品","精确术语"]}
|
|
||||||
{"id":"refund-007","query":"请查找产品A有关退款时限的说明","expected_documents":["sample-product-a-policy.md"],"expected_sections":["退款"],"tags":["产品","指令式"]}
|
|
||||||
{"id":"refund-008","query":"示例服务政策中的退款周期是多少天?","expected_documents":["sample-product-a-policy.md"],"expected_sections":["退款"],"tags":["产品","文档定位"]}
|
|
||||||
{"id":"support-001","query":"产品A遇到问题如何联系技术支持?","expected_documents":["sample-product-a-policy.md"],"expected_sections":["技术支持"],"tags":["支持","自然语言"]}
|
|
||||||
{"id":"support-002","query":"测试用户在哪里提交支持问题?","expected_documents":["sample-product-a-policy.md"],"expected_sections":["技术支持"],"tags":["支持","入口"]}
|
|
||||||
{"id":"support-003","query":"技术支持入口是否提供真实联系方式?","expected_documents":["sample-product-a-policy.md"],"expected_sections":["技术支持"],"tags":["支持","边界"]}
|
|
||||||
{"id":"support-004","query":"我想提交产品问题,支持说明是什么?","expected_documents":["sample-product-a-policy.md"],"expected_sections":["技术支持"],"tags":["支持","同义表达"]}
|
|
||||||
{"id":"support-005","query":"测试用户的虚构支持入口","expected_documents":["sample-product-a-policy.md"],"expected_sections":["技术支持"],"tags":["支持","短查询"]}
|
|
||||||
{"id":"support-006","query":"支持入口能否用于测试用户反馈问题?","expected_documents":["sample-product-a-policy.md"],"expected_sections":["技术支持"],"tags":["支持","多条件"]}
|
|
||||||
{"id":"support-007","query":"请定位服务政策里关于提交问题的段落","expected_documents":["sample-product-a-policy.md"],"expected_sections":["技术支持"],"tags":["支持","文档定位"]}
|
|
||||||
{"id":"support-008","query":"产品技术支持部分有没有联系电话?","expected_documents":["sample-product-a-policy.md"],"expected_sections":["技术支持"],"tags":["支持","否定事实"]}
|
|
||||||
{"id":"lodging-001","query":"员工出差住宿标准是多少?","expected_documents":["sample-travel-policy.md"],"expected_sections":["住宿标准"],"tags":["制度","自然语言"]}
|
|
||||||
{"id":"lodging-002","query":"出差时应如何选择住宿?","expected_documents":["sample-travel-policy.md"],"expected_sections":["住宿标准"],"tags":["制度","行为"]}
|
|
||||||
{"id":"lodging-003","query":"住宿地点需要满足哪些安全和交通条件?","expected_documents":["sample-travel-policy.md"],"expected_sections":["住宿标准"],"tags":["制度","多条件"]}
|
|
||||||
{"id":"lodging-004","query":"出差住宿是否必须在批准预算内?","expected_documents":["sample-travel-policy.md"],"expected_sections":["住宿标准"],"tags":["制度","预算"]}
|
|
||||||
{"id":"lodging-005","query":"批准预算范围内的安全住宿","expected_documents":["sample-travel-policy.md"],"expected_sections":["住宿标准"],"tags":["制度","短查询"]}
|
|
||||||
{"id":"lodging-006","query":"员工出差选酒店时要考虑什么?","expected_documents":["sample-travel-policy.md"],"expected_sections":["住宿标准"],"tags":["制度","同义表达"]}
|
|
||||||
{"id":"lodging-007","query":"请找出关于安全且交通便利住宿的说明","expected_documents":["sample-travel-policy.md"],"expected_sections":["住宿标准"],"tags":["制度","文档定位"]}
|
|
||||||
{"id":"lodging-008","query":"出差住宿示例标准能否作为实际报销依据?","expected_documents":["sample-travel-policy.md"],"expected_sections":["住宿标准"],"tags":["制度","边界"]}
|
|
||||||
{"id":"expense-001","query":"出差报销需要提交哪些材料?","expected_documents":["sample-travel-policy.md"],"expected_sections":["报销材料"],"tags":["报销","自然语言"]}
|
|
||||||
{"id":"expense-002","query":"报销时是否需要行程单?","expected_documents":["sample-travel-policy.md"],"expected_sections":["报销材料"],"tags":["报销","材料"]}
|
|
||||||
{"id":"expense-003","query":"合规票据是不是报销必需材料?","expected_documents":["sample-travel-policy.md"],"expected_sections":["报销材料"],"tags":["报销","票据"]}
|
|
||||||
{"id":"expense-004","query":"提交报销要附上什么凭证?","expected_documents":["sample-travel-policy.md"],"expected_sections":["报销材料"],"tags":["报销","同义表达"]}
|
|
||||||
{"id":"expense-005","query":"提交报销的示例行程单和合规票据","expected_documents":["sample-travel-policy.md"],"expected_sections":["报销材料"],"tags":["报销","短查询"]}
|
|
||||||
{"id":"expense-006","query":"行程单和票据应在什么时候提交?","expected_documents":["sample-travel-policy.md"],"expected_sections":["报销材料"],"tags":["报销","多条件"]}
|
|
||||||
{"id":"expense-007","query":"请定位管理制度中有关合规票据的段落","expected_documents":["sample-travel-policy.md"],"expected_sections":["报销材料"],"tags":["报销","文档定位"]}
|
|
||||||
{"id":"expense-008","query":"报销材料段落是否只用于自动化测试?","expected_documents":["sample-travel-policy.md"],"expected_sections":["报销材料"],"tags":["报销","边界"]}
|
|
||||||
|
|||||||
@ -1,27 +1,6 @@
|
|||||||
# Database migrations
|
# Database migrations
|
||||||
|
|
||||||
Migrations are forward-only and operate on OceanBase MySQL canonical metadata. Run
|
Migrations are forward-only and operate on PostgreSQL canonical metadata. Run
|
||||||
`make migration-check` to render the full upgrade path without changing a
|
`make migration-check` to render the full upgrade path without changing a
|
||||||
database. Integration tests also apply the migration to a temporary database
|
database. Integration tests also apply the migration to a temporary database
|
||||||
and compare it with SQLAlchemy metadata.
|
and compare it with SQLAlchemy metadata.
|
||||||
|
|
||||||
The target OceanBase database is shared. Alembic autogenerate and drift checks
|
|
||||||
are intentionally scoped to tables present in EKP `Base.metadata`; unrelated
|
|
||||||
application tables and the separately provisioned vector-index table are never
|
|
||||||
reflected or treated as migration targets.
|
|
||||||
|
|
||||||
OceanBase adds `BLOCK_SIZE ... LOCAL` syntax that SQLAlchemy's stock MySQL
|
|
||||||
reflector does not recognize for secondary indexes, unique constraints, and
|
|
||||||
foreign keys. Real OceanBase drift checks therefore compare columns, types,
|
|
||||||
defaults, and primary keys while excluding those three falsely reported object
|
|
||||||
types. Full constraint comparison still runs against SQLite in integration
|
|
||||||
tests; target-cluster checks separately verify HNSW/PRE-FILTER and the
|
|
||||||
single-active generated unique index.
|
|
||||||
|
|
||||||
DEMO-015 adds `ingestion_jobs` and `ingestion_attempts`. The former is the canonical execution state;
|
|
||||||
the latter preserves numbered history and uses a generated nullable active marker plus a unique index
|
|
||||||
to reject two running Attempts for the same Job on both OceanBase MySQL and SQLite.
|
|
||||||
|
|
||||||
DEMO-016 adds `ingestion_batches` and `ingestion_batch_items`. They retain input order and per-file
|
|
||||||
acceptance references while Job rows remain the processing state source of truth; a batch is an aggregate,
|
|
||||||
not a cross-file transaction.
|
|
||||||
|
|||||||
@ -1,7 +1,6 @@
|
|||||||
"""Alembic environment for canonical document metadata migrations."""
|
"""Alembic environment for canonical document metadata migrations."""
|
||||||
|
|
||||||
from logging.config import fileConfig
|
from logging.config import fileConfig
|
||||||
from typing import Any
|
|
||||||
|
|
||||||
from alembic import context
|
from alembic import context
|
||||||
from sqlalchemy import engine_from_config, pool
|
from sqlalchemy import engine_from_config, pool
|
||||||
@ -14,47 +13,12 @@ if config.config_file_name is not None:
|
|||||||
fileConfig(config.config_file_name)
|
fileConfig(config.config_file_name)
|
||||||
|
|
||||||
target_metadata = Base.metadata
|
target_metadata = Base.metadata
|
||||||
TARGET_TABLE_NAMES = frozenset(target_metadata.tables)
|
|
||||||
|
|
||||||
|
|
||||||
def _include_name(
|
|
||||||
name: str | None,
|
|
||||||
type_: str,
|
|
||||||
parent_names: dict[str, str | None],
|
|
||||||
) -> bool:
|
|
||||||
"""Keep Alembic reflection inside EKP-owned canonical tables.
|
|
||||||
|
|
||||||
The target OceanBase database is shared with another application and
|
|
||||||
contains provider-specific VECTOR columns. Excluding unrelated tables
|
|
||||||
before reflection prevents schema checks from inspecting or proposing
|
|
||||||
changes to objects outside this repository's ownership boundary.
|
|
||||||
"""
|
|
||||||
del parent_names
|
|
||||||
return type_ != "table" or name in TARGET_TABLE_NAMES
|
|
||||||
|
|
||||||
|
|
||||||
def _include_oceanbase_object(
|
|
||||||
object_: Any,
|
|
||||||
name: str | None,
|
|
||||||
type_: str,
|
|
||||||
reflected: bool,
|
|
||||||
compare_to: Any,
|
|
||||||
) -> bool:
|
|
||||||
"""Skip constraints the stock MySQL reflector cannot parse from OceanBase.
|
|
||||||
|
|
||||||
OceanBase appends `BLOCK_SIZE ... LOCAL` to secondary constraints. SQLAlchemy
|
|
||||||
consequently reports model constraints as missing even when they exist.
|
|
||||||
Columns, types, defaults and primary keys remain in the real drift check;
|
|
||||||
constraints are covered by migration tests and targeted live invariants.
|
|
||||||
"""
|
|
||||||
del object_, name, reflected, compare_to
|
|
||||||
return type_ not in {"index", "unique_constraint", "foreign_key_constraint"}
|
|
||||||
|
|
||||||
|
|
||||||
def _database_url() -> str:
|
def _database_url() -> str:
|
||||||
"""Allow tests to override the URL without changing runtime settings."""
|
"""Allow tests to override the URL without changing runtime settings."""
|
||||||
override = config.attributes.get("database_url")
|
override = config.attributes.get("database_url")
|
||||||
return override if isinstance(override, str) else get_settings().database_url.get_secret_value()
|
return override if isinstance(override, str) else get_settings().database_url
|
||||||
|
|
||||||
|
|
||||||
def run_migrations_offline() -> None:
|
def run_migrations_offline() -> None:
|
||||||
@ -65,7 +29,6 @@ def run_migrations_offline() -> None:
|
|||||||
literal_binds=True,
|
literal_binds=True,
|
||||||
dialect_opts={"paramstyle": "named"},
|
dialect_opts={"paramstyle": "named"},
|
||||||
compare_type=True,
|
compare_type=True,
|
||||||
include_name=_include_name,
|
|
||||||
)
|
)
|
||||||
|
|
||||||
with context.begin_transaction():
|
with context.begin_transaction():
|
||||||
@ -85,10 +48,6 @@ def run_migrations_online() -> None:
|
|||||||
connection=connection,
|
connection=connection,
|
||||||
target_metadata=target_metadata,
|
target_metadata=target_metadata,
|
||||||
compare_type=True,
|
compare_type=True,
|
||||||
include_name=_include_name,
|
|
||||||
include_object=(
|
|
||||||
_include_oceanbase_object if connection.dialect.name == "mysql" else None
|
|
||||||
),
|
|
||||||
)
|
)
|
||||||
|
|
||||||
with context.begin_transaction():
|
with context.begin_transaction():
|
||||||
|
|||||||
@ -1,41 +0,0 @@
|
|||||||
"""Make single-active profile enforcement portable to OceanBase MySQL.
|
|
||||||
|
|
||||||
Revision ID: 20260902_0006
|
|
||||||
Revises: 20260902_0005
|
|
||||||
Create Date: 2026-09-02
|
|
||||||
"""
|
|
||||||
|
|
||||||
from collections.abc import Sequence
|
|
||||||
|
|
||||||
import sqlalchemy as sa
|
|
||||||
from alembic import op
|
|
||||||
|
|
||||||
revision: str = "20260902_0006"
|
|
||||||
down_revision: str | None = "20260902_0005"
|
|
||||||
branch_labels: str | Sequence[str] | None = None
|
|
||||||
depends_on: str | Sequence[str] | None = None
|
|
||||||
|
|
||||||
|
|
||||||
def upgrade() -> None:
|
|
||||||
"""Replace dialect-specific partial uniqueness with a generated marker."""
|
|
||||||
op.drop_index("uq_index_profiles_single_active", table_name="index_profiles")
|
|
||||||
op.add_column(
|
|
||||||
"index_profiles",
|
|
||||||
sa.Column(
|
|
||||||
"active_marker",
|
|
||||||
sa.String(length=6),
|
|
||||||
sa.Computed("CASE WHEN status = 'active' THEN 'active' ELSE NULL END"),
|
|
||||||
nullable=True,
|
|
||||||
),
|
|
||||||
)
|
|
||||||
op.create_index(
|
|
||||||
"uq_index_profiles_single_active",
|
|
||||||
"index_profiles",
|
|
||||||
["active_marker"],
|
|
||||||
unique=True,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def downgrade() -> None:
|
|
||||||
"""Reject downgrade because project migrations are forward-only."""
|
|
||||||
raise NotImplementedError("Database migrations are forward-only")
|
|
||||||
@ -1,174 +0,0 @@
|
|||||||
"""Create durable ingestion jobs and attempt history.
|
|
||||||
|
|
||||||
Revision ID: 20260902_0007
|
|
||||||
Revises: 20260902_0006
|
|
||||||
Create Date: 2026-09-02
|
|
||||||
"""
|
|
||||||
|
|
||||||
from collections.abc import Sequence
|
|
||||||
|
|
||||||
import sqlalchemy as sa
|
|
||||||
from alembic import op
|
|
||||||
|
|
||||||
revision: str = "20260902_0007"
|
|
||||||
down_revision: str | None = "20260902_0006"
|
|
||||||
branch_labels: str | Sequence[str] | None = None
|
|
||||||
depends_on: str | Sequence[str] | None = None
|
|
||||||
|
|
||||||
|
|
||||||
def upgrade() -> None:
|
|
||||||
"""Persist queue-independent Job state and bounded execution attempts."""
|
|
||||||
op.create_table(
|
|
||||||
"ingestion_jobs",
|
|
||||||
sa.Column("id", sa.Uuid(), nullable=False),
|
|
||||||
sa.Column("document_id", sa.Uuid(), nullable=False),
|
|
||||||
sa.Column(
|
|
||||||
"status",
|
|
||||||
sa.Enum(
|
|
||||||
"queued",
|
|
||||||
"running",
|
|
||||||
"retrying",
|
|
||||||
"succeeded",
|
|
||||||
"failed",
|
|
||||||
"cancelled",
|
|
||||||
name="ingestion_job_status",
|
|
||||||
native_enum=False,
|
|
||||||
create_constraint=False,
|
|
||||||
),
|
|
||||||
nullable=False,
|
|
||||||
),
|
|
||||||
sa.Column(
|
|
||||||
"stage",
|
|
||||||
sa.Enum(
|
|
||||||
"parsing",
|
|
||||||
"cleaning",
|
|
||||||
"chunking",
|
|
||||||
"embedding",
|
|
||||||
"indexing",
|
|
||||||
name="ingestion_stage",
|
|
||||||
native_enum=False,
|
|
||||||
create_constraint=False,
|
|
||||||
),
|
|
||||||
nullable=True,
|
|
||||||
),
|
|
||||||
sa.Column("attempt_count", sa.Integer(), nullable=False),
|
|
||||||
sa.Column("max_attempts", sa.Integer(), nullable=False),
|
|
||||||
sa.Column("error_message", sa.Text(), nullable=True),
|
|
||||||
sa.Column("next_run_at", sa.DateTime(timezone=True), nullable=True),
|
|
||||||
sa.Column("lease_expires_at", sa.DateTime(timezone=True), nullable=True),
|
|
||||||
sa.Column(
|
|
||||||
"created_at",
|
|
||||||
sa.DateTime(timezone=True),
|
|
||||||
server_default=sa.text("CURRENT_TIMESTAMP"),
|
|
||||||
nullable=False,
|
|
||||||
),
|
|
||||||
sa.Column(
|
|
||||||
"updated_at",
|
|
||||||
sa.DateTime(timezone=True),
|
|
||||||
server_default=sa.text("CURRENT_TIMESTAMP"),
|
|
||||||
nullable=False,
|
|
||||||
),
|
|
||||||
sa.Column("completed_at", sa.DateTime(timezone=True), nullable=True),
|
|
||||||
sa.CheckConstraint("attempt_count >= 0", name="attempt_count_non_negative"),
|
|
||||||
sa.CheckConstraint("max_attempts > 0", name="max_attempts_positive"),
|
|
||||||
sa.CheckConstraint("attempt_count <= max_attempts", name="attempt_count_bounded"),
|
|
||||||
sa.CheckConstraint(
|
|
||||||
"status IN ('queued', 'running', 'retrying', 'succeeded', 'failed', 'cancelled')",
|
|
||||||
name="status_valid",
|
|
||||||
),
|
|
||||||
sa.CheckConstraint(
|
|
||||||
"stage IS NULL OR stage IN "
|
|
||||||
"('parsing', 'cleaning', 'chunking', 'embedding', 'indexing')",
|
|
||||||
name="stage_valid",
|
|
||||||
),
|
|
||||||
sa.ForeignKeyConstraint(
|
|
||||||
["document_id"],
|
|
||||||
["documents.id"],
|
|
||||||
name="fk_ingestion_jobs_document_id_documents",
|
|
||||||
ondelete="CASCADE",
|
|
||||||
),
|
|
||||||
sa.PrimaryKeyConstraint("id", name="pk_ingestion_jobs"),
|
|
||||||
sa.UniqueConstraint("document_id", name="uq_ingestion_jobs_document_id"),
|
|
||||||
)
|
|
||||||
op.create_index(
|
|
||||||
"ix_ingestion_jobs_status_next_run",
|
|
||||||
"ingestion_jobs",
|
|
||||||
["status", "next_run_at"],
|
|
||||||
unique=False,
|
|
||||||
)
|
|
||||||
op.create_table(
|
|
||||||
"ingestion_attempts",
|
|
||||||
sa.Column("id", sa.Uuid(), nullable=False),
|
|
||||||
sa.Column("job_id", sa.Uuid(), nullable=False),
|
|
||||||
sa.Column("attempt_number", sa.Integer(), nullable=False),
|
|
||||||
sa.Column(
|
|
||||||
"status",
|
|
||||||
sa.Enum(
|
|
||||||
"running",
|
|
||||||
"succeeded",
|
|
||||||
"failed",
|
|
||||||
name="ingestion_attempt_status",
|
|
||||||
native_enum=False,
|
|
||||||
create_constraint=False,
|
|
||||||
),
|
|
||||||
nullable=False,
|
|
||||||
),
|
|
||||||
sa.Column(
|
|
||||||
"stage",
|
|
||||||
sa.Enum(
|
|
||||||
"parsing",
|
|
||||||
"cleaning",
|
|
||||||
"chunking",
|
|
||||||
"embedding",
|
|
||||||
"indexing",
|
|
||||||
name="ingestion_attempt_stage",
|
|
||||||
native_enum=False,
|
|
||||||
create_constraint=False,
|
|
||||||
),
|
|
||||||
nullable=True,
|
|
||||||
),
|
|
||||||
sa.Column("retryable", sa.Boolean(), nullable=True),
|
|
||||||
sa.Column("error_message", sa.Text(), nullable=True),
|
|
||||||
sa.Column(
|
|
||||||
"active_marker",
|
|
||||||
sa.Integer(),
|
|
||||||
sa.Computed("CASE WHEN status = 'running' THEN 1 ELSE NULL END"),
|
|
||||||
nullable=True,
|
|
||||||
),
|
|
||||||
sa.Column(
|
|
||||||
"started_at",
|
|
||||||
sa.DateTime(timezone=True),
|
|
||||||
server_default=sa.text("CURRENT_TIMESTAMP"),
|
|
||||||
nullable=False,
|
|
||||||
),
|
|
||||||
sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True),
|
|
||||||
sa.CheckConstraint("attempt_number > 0", name="attempt_number_positive"),
|
|
||||||
sa.CheckConstraint(
|
|
||||||
"status IN ('running', 'succeeded', 'failed')",
|
|
||||||
name="status_valid",
|
|
||||||
),
|
|
||||||
sa.CheckConstraint(
|
|
||||||
"stage IS NULL OR stage IN "
|
|
||||||
"('parsing', 'cleaning', 'chunking', 'embedding', 'indexing')",
|
|
||||||
name="stage_valid",
|
|
||||||
),
|
|
||||||
sa.ForeignKeyConstraint(
|
|
||||||
["job_id"],
|
|
||||||
["ingestion_jobs.id"],
|
|
||||||
name="fk_ingestion_attempts_job_id_ingestion_jobs",
|
|
||||||
ondelete="CASCADE",
|
|
||||||
),
|
|
||||||
sa.PrimaryKeyConstraint("id", name="pk_ingestion_attempts"),
|
|
||||||
sa.UniqueConstraint("job_id", "attempt_number", name="uq_ingestion_attempts_job_number"),
|
|
||||||
)
|
|
||||||
op.create_index(
|
|
||||||
"uq_ingestion_attempts_active",
|
|
||||||
"ingestion_attempts",
|
|
||||||
["job_id", "active_marker"],
|
|
||||||
unique=True,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def downgrade() -> None:
|
|
||||||
"""Reject downgrade because project migrations are forward-only."""
|
|
||||||
raise NotImplementedError("Database migrations are forward-only")
|
|
||||||
@ -1,109 +0,0 @@
|
|||||||
"""Create ingestion batch aggregation metadata.
|
|
||||||
|
|
||||||
Revision ID: 20260902_0008
|
|
||||||
Revises: 20260902_0007
|
|
||||||
Create Date: 2026-09-02
|
|
||||||
"""
|
|
||||||
|
|
||||||
from collections.abc import Sequence
|
|
||||||
|
|
||||||
import sqlalchemy as sa
|
|
||||||
from alembic import op
|
|
||||||
|
|
||||||
revision: str = "20260902_0008"
|
|
||||||
down_revision: str | None = "20260902_0007"
|
|
||||||
branch_labels: str | Sequence[str] | None = None
|
|
||||||
depends_on: str | Sequence[str] | None = None
|
|
||||||
|
|
||||||
|
|
||||||
def upgrade() -> None:
|
|
||||||
"""Persist ordered per-file acceptance without making a batch transactional."""
|
|
||||||
op.create_table(
|
|
||||||
"ingestion_batches",
|
|
||||||
sa.Column("id", sa.Uuid(), nullable=False),
|
|
||||||
sa.Column("total_files", sa.Integer(), nullable=False),
|
|
||||||
sa.Column(
|
|
||||||
"created_at",
|
|
||||||
sa.DateTime(timezone=True),
|
|
||||||
server_default=sa.text("CURRENT_TIMESTAMP"),
|
|
||||||
nullable=False,
|
|
||||||
),
|
|
||||||
sa.CheckConstraint("total_files > 0", name="total_files_positive"),
|
|
||||||
sa.PrimaryKeyConstraint("id", name="pk_ingestion_batches"),
|
|
||||||
)
|
|
||||||
op.create_table(
|
|
||||||
"ingestion_batch_items",
|
|
||||||
sa.Column("id", sa.Uuid(), nullable=False),
|
|
||||||
sa.Column("batch_id", sa.Uuid(), nullable=False),
|
|
||||||
sa.Column("item_index", sa.Integer(), nullable=False),
|
|
||||||
sa.Column("filename", sa.String(length=255), nullable=False),
|
|
||||||
sa.Column(
|
|
||||||
"acceptance_status",
|
|
||||||
sa.Enum(
|
|
||||||
"pending",
|
|
||||||
"accepted",
|
|
||||||
"rejected",
|
|
||||||
name="batch_item_acceptance_status",
|
|
||||||
native_enum=False,
|
|
||||||
create_constraint=False,
|
|
||||||
),
|
|
||||||
nullable=False,
|
|
||||||
),
|
|
||||||
sa.Column("document_id", sa.Uuid(), nullable=True),
|
|
||||||
sa.Column("job_id", sa.Uuid(), nullable=True),
|
|
||||||
sa.Column("error_message", sa.Text(), nullable=True),
|
|
||||||
sa.Column(
|
|
||||||
"created_at",
|
|
||||||
sa.DateTime(timezone=True),
|
|
||||||
server_default=sa.text("CURRENT_TIMESTAMP"),
|
|
||||||
nullable=False,
|
|
||||||
),
|
|
||||||
sa.Column(
|
|
||||||
"updated_at",
|
|
||||||
sa.DateTime(timezone=True),
|
|
||||||
server_default=sa.text("CURRENT_TIMESTAMP"),
|
|
||||||
nullable=False,
|
|
||||||
),
|
|
||||||
sa.CheckConstraint("item_index >= 0", name="item_index_non_negative"),
|
|
||||||
sa.CheckConstraint("length(filename) > 0", name="filename_not_empty"),
|
|
||||||
sa.CheckConstraint(
|
|
||||||
"acceptance_status IN ('pending', 'accepted', 'rejected')",
|
|
||||||
name="acceptance_status_valid",
|
|
||||||
),
|
|
||||||
sa.ForeignKeyConstraint(
|
|
||||||
["batch_id"],
|
|
||||||
["ingestion_batches.id"],
|
|
||||||
name="fk_ingestion_batch_items_batch_id_ingestion_batches",
|
|
||||||
ondelete="CASCADE",
|
|
||||||
),
|
|
||||||
sa.ForeignKeyConstraint(
|
|
||||||
["document_id"],
|
|
||||||
["documents.id"],
|
|
||||||
name="fk_ingestion_batch_items_document_id_documents",
|
|
||||||
ondelete="SET NULL",
|
|
||||||
),
|
|
||||||
sa.ForeignKeyConstraint(
|
|
||||||
["job_id"],
|
|
||||||
["ingestion_jobs.id"],
|
|
||||||
name="fk_ingestion_batch_items_job_id_ingestion_jobs",
|
|
||||||
ondelete="SET NULL",
|
|
||||||
),
|
|
||||||
sa.PrimaryKeyConstraint("id", name="pk_ingestion_batch_items"),
|
|
||||||
)
|
|
||||||
op.create_index(
|
|
||||||
"uq_ingestion_batch_items_order",
|
|
||||||
"ingestion_batch_items",
|
|
||||||
["batch_id", "item_index"],
|
|
||||||
unique=True,
|
|
||||||
)
|
|
||||||
op.create_index(
|
|
||||||
"uq_ingestion_batch_items_job_id",
|
|
||||||
"ingestion_batch_items",
|
|
||||||
["job_id"],
|
|
||||||
unique=True,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def downgrade() -> None:
|
|
||||||
"""Reject downgrade because project migrations are forward-only."""
|
|
||||||
raise NotImplementedError("Database migrations are forward-only")
|
|
||||||
@ -4,9 +4,8 @@ from functools import lru_cache
|
|||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Literal
|
from typing import Literal
|
||||||
|
|
||||||
from pydantic import AnyHttpUrl, Field, SecretStr, model_validator
|
from pydantic import Field, SecretStr, model_validator
|
||||||
from pydantic_settings import BaseSettings, SettingsConfigDict
|
from pydantic_settings import BaseSettings, SettingsConfigDict
|
||||||
from sqlalchemy.engine import make_url
|
|
||||||
|
|
||||||
|
|
||||||
class Settings(BaseSettings):
|
class Settings(BaseSettings):
|
||||||
@ -25,23 +24,7 @@ class Settings(BaseSettings):
|
|||||||
api_host: str = "0.0.0.0"
|
api_host: str = "0.0.0.0"
|
||||||
api_port: int = Field(default=8000, ge=1, le=65535)
|
api_port: int = Field(default=8000, ge=1, le=65535)
|
||||||
api_prefix: str = "/api/v1"
|
api_prefix: str = "/api/v1"
|
||||||
admin_web_root: Path | None = None
|
database_url: str = "postgresql+psycopg://localhost:5432/ekp"
|
||||||
# SQLAlchemy URLs commonly embed a password, so configuration rendering
|
|
||||||
# must mask this value and unwrap it only at the database driver boundary.
|
|
||||||
database_url: SecretStr = SecretStr("mysql+pymysql://localhost:2881/ekp")
|
|
||||||
database_pool_size: int = Field(default=10, ge=1, le=100)
|
|
||||||
database_max_overflow: int = Field(default=10, ge=0, le=100)
|
|
||||||
database_pool_recycle_seconds: int = Field(default=600, ge=1)
|
|
||||||
database_pool_timeout_seconds: float = Field(default=30.0, gt=0)
|
|
||||||
database_connect_timeout_seconds: int = Field(default=30, ge=1)
|
|
||||||
rabbitmq_url: SecretStr | None = None
|
|
||||||
ingestion_max_attempts: int = Field(default=3, ge=1, le=10)
|
|
||||||
ingestion_retry_backoff_seconds: int = Field(default=2, ge=1, le=3600)
|
|
||||||
ingestion_retry_backoff_max_seconds: int = Field(default=60, ge=1, le=86400)
|
|
||||||
ingestion_worker_lease_seconds: int = Field(default=300, ge=30, le=86400)
|
|
||||||
ingestion_recovery_interval_seconds: int = Field(default=30, ge=5, le=3600)
|
|
||||||
batch_max_files: int = Field(default=20, ge=1, le=100)
|
|
||||||
batch_max_total_bytes: int = Field(default=200 * 1024 * 1024, gt=0)
|
|
||||||
object_storage_provider: Literal["local", "bos"] = "bos"
|
object_storage_provider: Literal["local", "bos"] = "bos"
|
||||||
object_storage_path: Path = Path("storage")
|
object_storage_path: Path = Path("storage")
|
||||||
bos_endpoint: str | None = None
|
bos_endpoint: str | None = None
|
||||||
@ -76,22 +59,6 @@ class Settings(BaseSettings):
|
|||||||
reranker_api_key: SecretStr | None = None
|
reranker_api_key: SecretStr | None = None
|
||||||
reranker_timeout_seconds: float = Field(default=30.0, gt=0)
|
reranker_timeout_seconds: float = Field(default=30.0, gt=0)
|
||||||
search_top_k: int = Field(default=5, ge=1, le=64)
|
search_top_k: int = Field(default=5, ge=1, le=64)
|
||||||
mcp_host: str = "127.0.0.1"
|
|
||||||
mcp_port: int = Field(default=8766, ge=1, le=65535)
|
|
||||||
mcp_path: str = "/mcp"
|
|
||||||
mcp_public_url: AnyHttpUrl = AnyHttpUrl("http://127.0.0.1:8766/mcp")
|
|
||||||
mcp_issuer_url: AnyHttpUrl = AnyHttpUrl("http://127.0.0.1:8766")
|
|
||||||
mcp_auth_token: SecretStr | None = None
|
|
||||||
mcp_client_id: str = Field(default="workbuddy-enterprise", min_length=1, max_length=200)
|
|
||||||
mcp_service_identity: str = Field(default="workbuddy-demo", min_length=1, max_length=200)
|
|
||||||
mcp_required_scope: str = Field(default="knowledge:search", min_length=1, max_length=200)
|
|
||||||
mcp_allowed_hosts: list[str] = ["127.0.0.1:*", "localhost:*", "[::1]:*", "testserver"]
|
|
||||||
mcp_allowed_origins: list[str] = [
|
|
||||||
"http://127.0.0.1:*",
|
|
||||||
"http://localhost:*",
|
|
||||||
"http://[::1]:*",
|
|
||||||
]
|
|
||||||
mcp_max_request_body_bytes: int = Field(default=1024 * 1024, ge=1024, le=4 * 1024 * 1024)
|
|
||||||
|
|
||||||
@model_validator(mode="after")
|
@model_validator(mode="after")
|
||||||
def validate_chunk_window(self) -> "Settings":
|
def validate_chunk_window(self) -> "Settings":
|
||||||
@ -100,29 +67,6 @@ class Settings(BaseSettings):
|
|||||||
raise ValueError("chunk_overlap_tokens must be less than chunk_max_tokens")
|
raise ValueError("chunk_overlap_tokens must be less than chunk_max_tokens")
|
||||||
if self.search_top_k > self.dense_top_n:
|
if self.search_top_k > self.dense_top_n:
|
||||||
raise ValueError("search_top_k must not exceed dense_top_n")
|
raise ValueError("search_top_k must not exceed dense_top_n")
|
||||||
if self.ingestion_retry_backoff_max_seconds < self.ingestion_retry_backoff_seconds:
|
|
||||||
raise ValueError(
|
|
||||||
"ingestion_retry_backoff_max_seconds must be at least the initial backoff"
|
|
||||||
)
|
|
||||||
longest_provider_timeout = max(
|
|
||||||
self.bos_timeout_seconds,
|
|
||||||
self.embedding_timeout_seconds,
|
|
||||||
self.oceanbase_vector_timeout_seconds,
|
|
||||||
)
|
|
||||||
if self.ingestion_worker_lease_seconds <= longest_provider_timeout:
|
|
||||||
raise ValueError("ingestion_worker_lease_seconds must exceed provider timeouts")
|
|
||||||
if self.batch_max_total_bytes < self.max_upload_bytes:
|
|
||||||
raise ValueError("batch_max_total_bytes must be at least max_upload_bytes")
|
|
||||||
if not self.mcp_path.startswith("/") or self.mcp_path.endswith("/"):
|
|
||||||
raise ValueError("mcp_path must start with '/' and must not end with '/'")
|
|
||||||
if not self.mcp_allowed_hosts:
|
|
||||||
raise ValueError("mcp_allowed_hosts must not be empty")
|
|
||||||
database_url = make_url(self.database_url.get_secret_value())
|
|
||||||
if database_url.get_backend_name() == "sqlite":
|
|
||||||
if self.app_env != "test":
|
|
||||||
raise ValueError("SQLite database URLs are restricted to APP_ENV=test")
|
|
||||||
elif database_url.drivername != "mysql+pymysql":
|
|
||||||
raise ValueError("canonical database must use OceanBase through mysql+pymysql")
|
|
||||||
return self
|
return self
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@ -1,35 +1,14 @@
|
|||||||
"""SQLAlchemy engine and session construction for canonical metadata."""
|
"""SQLAlchemy engine and session construction for canonical metadata."""
|
||||||
|
|
||||||
from sqlalchemy import Engine, create_engine
|
from sqlalchemy import Engine, create_engine
|
||||||
from sqlalchemy.engine import make_url
|
|
||||||
from sqlalchemy.orm import Session, sessionmaker
|
from sqlalchemy.orm import Session, sessionmaker
|
||||||
|
|
||||||
from ekp_core.config import Settings
|
from ekp_core.config import Settings
|
||||||
|
|
||||||
|
|
||||||
def create_database_engine(settings: Settings) -> Engine:
|
def create_database_engine(settings: Settings) -> Engine:
|
||||||
"""Create the canonical engine without opening a connection eagerly.
|
"""Create the metadata engine without opening a connection eagerly."""
|
||||||
|
return create_engine(settings.database_url, pool_pre_ping=True)
|
||||||
OceanBase uses the synchronous PyMySQL driver in this Demo. Pool and
|
|
||||||
socket timeouts are explicit so an outage cannot hold API workers
|
|
||||||
indefinitely. SQLite remains available only for isolated tests.
|
|
||||||
"""
|
|
||||||
database_url = settings.database_url.get_secret_value()
|
|
||||||
url = make_url(database_url)
|
|
||||||
options: dict[str, object] = {"pool_pre_ping": True}
|
|
||||||
if url.get_backend_name() == "mysql":
|
|
||||||
options.update(
|
|
||||||
pool_size=settings.database_pool_size,
|
|
||||||
max_overflow=settings.database_max_overflow,
|
|
||||||
pool_recycle=settings.database_pool_recycle_seconds,
|
|
||||||
pool_timeout=settings.database_pool_timeout_seconds,
|
|
||||||
connect_args={
|
|
||||||
"connect_timeout": settings.database_connect_timeout_seconds,
|
|
||||||
"read_timeout": settings.database_connect_timeout_seconds,
|
|
||||||
"write_timeout": settings.database_connect_timeout_seconds,
|
|
||||||
},
|
|
||||||
)
|
|
||||||
return create_engine(database_url, **options)
|
|
||||||
|
|
||||||
|
|
||||||
def create_session_factory(engine: Engine) -> sessionmaker[Session]:
|
def create_session_factory(engine: Engine) -> sessionmaker[Session]:
|
||||||
|
|||||||
@ -1,6 +1,5 @@
|
|||||||
"""Application services for building and persisting document chunks."""
|
"""Application services for building and persisting document chunks."""
|
||||||
|
|
||||||
from collections.abc import Callable
|
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from hashlib import sha256
|
from hashlib import sha256
|
||||||
from uuid import UUID, uuid5
|
from uuid import UUID, uuid5
|
||||||
@ -12,7 +11,7 @@ from ekp_core.chunking import ChunkDraft, ChunkingStrategy
|
|||||||
from ekp_core.document_parsing import DocumentIngestionService, DocumentParsingService
|
from ekp_core.document_parsing import DocumentIngestionService, DocumentParsingService
|
||||||
from ekp_core.document_upload import DocumentUploadService
|
from ekp_core.document_upload import DocumentUploadService
|
||||||
from ekp_core.parsed_document_artifact import deserialize_parsed_document
|
from ekp_core.parsed_document_artifact import deserialize_parsed_document
|
||||||
from ekp_domain import Chunk, Document, DocumentStatus, IngestionStage, ParsedDocument
|
from ekp_domain import Chunk, Document, DocumentStatus, ParsedDocument
|
||||||
from ekp_object_storage import BinarySource, ObjectStorage
|
from ekp_object_storage import BinarySource, ObjectStorage
|
||||||
|
|
||||||
MAX_CHUNKING_ERROR_LENGTH = 2000
|
MAX_CHUNKING_ERROR_LENGTH = 2000
|
||||||
@ -55,16 +54,10 @@ class DocumentChunkingService:
|
|||||||
self._storage = storage
|
self._storage = storage
|
||||||
self._strategy = strategy
|
self._strategy = strategy
|
||||||
|
|
||||||
def process(
|
def process(self, document_id: UUID) -> tuple[Chunk, ...]:
|
||||||
self,
|
"""Run DEMO-005 while reserving READY for the later index stage."""
|
||||||
document_id: UUID,
|
|
||||||
stage_callback: Callable[[IngestionStage], None] | None = None,
|
|
||||||
) -> tuple[Chunk, ...]:
|
|
||||||
"""Build chunks while optionally reporting the durable Job stage."""
|
|
||||||
filename, parsed_key = self._start_chunking(document_id)
|
filename, parsed_key = self._start_chunking(document_id)
|
||||||
try:
|
try:
|
||||||
if stage_callback is not None:
|
|
||||||
stage_callback(IngestionStage.CHUNKING)
|
|
||||||
with self._storage.open(parsed_key) as source:
|
with self._storage.open(parsed_key) as source:
|
||||||
parsed = deserialize_parsed_document(source.read())
|
parsed = deserialize_parsed_document(source.read())
|
||||||
if parsed.document_id != document_id or parsed.filename != filename:
|
if parsed.document_id != document_id or parsed.filename != filename:
|
||||||
|
|||||||
@ -1,6 +1,5 @@
|
|||||||
"""Embedding and vector-index orchestration for canonical document chunks."""
|
"""Embedding and vector-index orchestration for canonical document chunks."""
|
||||||
|
|
||||||
from collections.abc import Callable
|
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from uuid import UUID, uuid5
|
from uuid import UUID, uuid5
|
||||||
|
|
||||||
@ -8,14 +7,7 @@ from sqlalchemy import select
|
|||||||
from sqlalchemy.orm import Session, sessionmaker
|
from sqlalchemy.orm import Session, sessionmaker
|
||||||
|
|
||||||
from ekp_core.document_chunking import DocumentPipelineResult, DocumentPipelineService
|
from ekp_core.document_chunking import DocumentPipelineResult, DocumentPipelineService
|
||||||
from ekp_domain import (
|
from ekp_domain import Chunk, ChunkIndexState, Document, DocumentStatus, IndexProfile
|
||||||
Chunk,
|
|
||||||
ChunkIndexState,
|
|
||||||
Document,
|
|
||||||
DocumentStatus,
|
|
||||||
IndexProfile,
|
|
||||||
IngestionStage,
|
|
||||||
)
|
|
||||||
from ekp_embedding import EmbeddingProvider
|
from ekp_embedding import EmbeddingProvider
|
||||||
from ekp_object_storage import BinarySource
|
from ekp_object_storage import BinarySource
|
||||||
from ekp_vector_store import VectorRecord, VectorStore
|
from ekp_vector_store import VectorRecord, VectorStore
|
||||||
@ -75,16 +67,12 @@ class DocumentIndexingService:
|
|||||||
self._vector_store = vector_store
|
self._vector_store = vector_store
|
||||||
self._index_profile = index_profile
|
self._index_profile = index_profile
|
||||||
|
|
||||||
def process(
|
def process(self, document_id: UUID) -> DocumentIndexingResult:
|
||||||
self,
|
"""Index all chunks for one profile, compensating any partial provider write."""
|
||||||
document_id: UUID,
|
|
||||||
stage_callback: Callable[[IngestionStage], None] | None = None,
|
|
||||||
) -> DocumentIndexingResult:
|
|
||||||
"""Embed/index chunks while optionally reporting durable Job stages."""
|
|
||||||
chunks = self._start_indexing(document_id)
|
chunks = self._start_indexing(document_id)
|
||||||
upsert_attempted = False
|
upsert_attempted = False
|
||||||
try:
|
try:
|
||||||
vectors = self._embed_chunks(chunks, stage_callback)
|
vectors = self._embed_chunks(chunks)
|
||||||
records = tuple(
|
records = tuple(
|
||||||
VectorRecord(
|
VectorRecord(
|
||||||
index_record_id=uuid5(
|
index_record_id=uuid5(
|
||||||
@ -105,8 +93,6 @@ class DocumentIndexingService:
|
|||||||
)
|
)
|
||||||
for chunk, vector in zip(chunks, vectors, strict=True)
|
for chunk, vector in zip(chunks, vectors, strict=True)
|
||||||
)
|
)
|
||||||
if stage_callback is not None:
|
|
||||||
stage_callback(IngestionStage.INDEXING)
|
|
||||||
upsert_attempted = True
|
upsert_attempted = True
|
||||||
self._vector_store.upsert(records)
|
self._vector_store.upsert(records)
|
||||||
self._record_success(document_id, chunks)
|
self._record_success(document_id, chunks)
|
||||||
@ -166,19 +152,13 @@ class DocumentIndexingService:
|
|||||||
session.commit()
|
session.commit()
|
||||||
return chunks
|
return chunks
|
||||||
|
|
||||||
def _embed_chunks(
|
def _embed_chunks(self, chunks: tuple[Chunk, ...]) -> tuple[tuple[float, ...], ...]:
|
||||||
self,
|
"""Respect provider batch limits and validate every returned vector."""
|
||||||
chunks: tuple[Chunk, ...],
|
|
||||||
stage_callback: Callable[[IngestionStage], None] | None,
|
|
||||||
) -> tuple[tuple[float, ...], ...]:
|
|
||||||
"""Embed bounded batches and renew an optional Job lease before each call."""
|
|
||||||
vectors: list[tuple[float, ...]] = []
|
vectors: list[tuple[float, ...]] = []
|
||||||
batch_size = self._embedding_provider.max_batch_size
|
batch_size = self._embedding_provider.max_batch_size
|
||||||
if batch_size <= 0:
|
if batch_size <= 0:
|
||||||
raise EmbeddingResultError("embedding provider has an invalid batch size")
|
raise EmbeddingResultError("embedding provider has an invalid batch size")
|
||||||
for start in range(0, len(chunks), batch_size):
|
for start in range(0, len(chunks), batch_size):
|
||||||
if stage_callback is not None:
|
|
||||||
stage_callback(IngestionStage.EMBEDDING)
|
|
||||||
batch = chunks[start : start + batch_size]
|
batch = chunks[start : start + batch_size]
|
||||||
embedded = self._embedding_provider.embed([chunk.content for chunk in batch])
|
embedded = self._embedding_provider.embed([chunk.content for chunk in batch])
|
||||||
if len(embedded) != len(batch):
|
if len(embedded) != len(batch):
|
||||||
|
|||||||
@ -5,7 +5,7 @@ from uuid import UUID
|
|||||||
from sqlalchemy import delete, select
|
from sqlalchemy import delete, select
|
||||||
from sqlalchemy.orm import Session, sessionmaker
|
from sqlalchemy.orm import Session, sessionmaker
|
||||||
|
|
||||||
from ekp_domain import Chunk, ChunkIndexState, Document, IngestionJob, IngestionJobStatus
|
from ekp_domain import Chunk, ChunkIndexState, Document
|
||||||
from ekp_object_storage import ObjectStorage
|
from ekp_object_storage import ObjectStorage
|
||||||
from ekp_vector_store import VectorStore
|
from ekp_vector_store import VectorStore
|
||||||
|
|
||||||
@ -18,10 +18,6 @@ class DocumentArtifactCleanupError(RuntimeError):
|
|||||||
"""Raised when deleted metadata leaves an external storage artifact behind."""
|
"""Raised when deleted metadata leaves an external storage artifact behind."""
|
||||||
|
|
||||||
|
|
||||||
class DocumentIngestionActiveError(RuntimeError):
|
|
||||||
"""Raised when direct deletion would race an active asynchronous Job."""
|
|
||||||
|
|
||||||
|
|
||||||
class DocumentManagementService:
|
class DocumentManagementService:
|
||||||
"""Read canonical documents and coordinate the Demo's direct deletion.
|
"""Read canonical documents and coordinate the Demo's direct deletion.
|
||||||
|
|
||||||
@ -67,22 +63,6 @@ class DocumentManagementService:
|
|||||||
document = session.get(Document, document_id)
|
document = session.get(Document, document_id)
|
||||||
if document is None:
|
if document is None:
|
||||||
raise DocumentNotFoundError(str(document_id))
|
raise DocumentNotFoundError(str(document_id))
|
||||||
active_job_id = session.scalar(
|
|
||||||
select(IngestionJob.id)
|
|
||||||
.where(
|
|
||||||
IngestionJob.document_id == document_id,
|
|
||||||
IngestionJob.status.in_(
|
|
||||||
(
|
|
||||||
IngestionJobStatus.QUEUED,
|
|
||||||
IngestionJobStatus.RUNNING,
|
|
||||||
IngestionJobStatus.RETRYING,
|
|
||||||
)
|
|
||||||
),
|
|
||||||
)
|
|
||||||
.limit(1)
|
|
||||||
)
|
|
||||||
if active_job_id is not None:
|
|
||||||
raise DocumentIngestionActiveError(str(document_id))
|
|
||||||
|
|
||||||
source_keys = tuple(
|
source_keys = tuple(
|
||||||
key
|
key
|
||||||
|
|||||||
@ -1,6 +1,5 @@
|
|||||||
"""Application services for parsing and persisting derived document content."""
|
"""Application services for parsing and persisting derived document content."""
|
||||||
|
|
||||||
from collections.abc import Callable
|
|
||||||
from io import BytesIO
|
from io import BytesIO
|
||||||
from uuid import UUID
|
from uuid import UUID
|
||||||
|
|
||||||
@ -9,7 +8,7 @@ from sqlalchemy.orm import Session, sessionmaker
|
|||||||
from ekp_core.document_cleaner import clean_parsed_document
|
from ekp_core.document_cleaner import clean_parsed_document
|
||||||
from ekp_core.document_upload import DocumentUploadService
|
from ekp_core.document_upload import DocumentUploadService
|
||||||
from ekp_core.parsed_document_artifact import serialize_parsed_document
|
from ekp_core.parsed_document_artifact import serialize_parsed_document
|
||||||
from ekp_domain import Document, DocumentStatus, IngestionStage, ParsedDocument
|
from ekp_domain import Document, DocumentStatus, ParsedDocument
|
||||||
from ekp_object_storage import BinarySource, ObjectStorage
|
from ekp_object_storage import BinarySource, ObjectStorage
|
||||||
from ekp_parser import DocumentParseError, Parser
|
from ekp_parser import DocumentParseError, Parser
|
||||||
|
|
||||||
@ -49,23 +48,15 @@ class DocumentParsingService:
|
|||||||
self._storage = storage
|
self._storage = storage
|
||||||
self._parser = parser
|
self._parser = parser
|
||||||
|
|
||||||
def process(
|
def process(self, document_id: UUID) -> ParsedDocument:
|
||||||
self,
|
"""Run DEMO-004 synchronously and persist success or failure state."""
|
||||||
document_id: UUID,
|
|
||||||
stage_callback: Callable[[IngestionStage], None] | None = None,
|
|
||||||
) -> ParsedDocument:
|
|
||||||
"""Parse and clean while optionally reporting durable Job stage boundaries."""
|
|
||||||
filename, source_key = self._start_processing(document_id)
|
filename, source_key = self._start_processing(document_id)
|
||||||
parsed_key = f"parsed/{document_id}/document.json"
|
parsed_key = f"parsed/{document_id}/document.json"
|
||||||
parsed_object_stored = False
|
parsed_object_stored = False
|
||||||
|
|
||||||
try:
|
try:
|
||||||
if stage_callback is not None:
|
|
||||||
stage_callback(IngestionStage.PARSING)
|
|
||||||
with self._storage.open(source_key) as source:
|
with self._storage.open(source_key) as source:
|
||||||
parsed = self._parser.parse(document_id, filename, source)
|
parsed = self._parser.parse(document_id, filename, source)
|
||||||
if stage_callback is not None:
|
|
||||||
stage_callback(IngestionStage.CLEANING)
|
|
||||||
cleaned = clean_parsed_document(parsed)
|
cleaned = clean_parsed_document(parsed)
|
||||||
if not cleaned.blocks:
|
if not cleaned.blocks:
|
||||||
raise EmptyParsedDocumentError(
|
raise EmptyParsedDocumentError(
|
||||||
|
|||||||
@ -1,9 +1,8 @@
|
|||||||
"""Validated, transactional source-document upload application service."""
|
"""Validated, transactional source-document upload application service."""
|
||||||
|
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from uuid import UUID, uuid4
|
from uuid import uuid4
|
||||||
|
|
||||||
from sqlalchemy import delete
|
|
||||||
from sqlalchemy.orm import Session, sessionmaker
|
from sqlalchemy.orm import Session, sessionmaker
|
||||||
|
|
||||||
from ekp_domain import Document
|
from ekp_domain import Document
|
||||||
@ -133,26 +132,6 @@ class DocumentUploadService:
|
|||||||
|
|
||||||
return document
|
return document
|
||||||
|
|
||||||
def rollback_acceptance(self, document_id: UUID) -> None:
|
|
||||||
"""Compensate a just-uploaded source when durable Job creation fails.
|
|
||||||
|
|
||||||
This narrow method is only valid before parsing starts; normal document
|
|
||||||
deletion remains coordinated by ``DocumentManagementService``.
|
|
||||||
"""
|
|
||||||
with self._session_factory() as session:
|
|
||||||
document = session.get(Document, document_id)
|
|
||||||
if document is None:
|
|
||||||
return
|
|
||||||
storage_key = document.storage_key
|
|
||||||
session.execute(delete(Document).where(Document.id == document.id))
|
|
||||||
session.commit()
|
|
||||||
try:
|
|
||||||
self._storage.delete(storage_key)
|
|
||||||
except Exception as cleanup_error:
|
|
||||||
raise UploadRollbackError(
|
|
||||||
f"job creation failed and source cleanup failed for {storage_key}"
|
|
||||||
) from cleanup_error
|
|
||||||
|
|
||||||
|
|
||||||
def _validate_filename(filename: str) -> str:
|
def _validate_filename(filename: str) -> str:
|
||||||
"""Accept a plain basename that can be traced without path ambiguity."""
|
"""Accept a plain basename that can be traced without path ambiguity."""
|
||||||
|
|||||||
@ -4,7 +4,7 @@ from dataclasses import dataclass
|
|||||||
from datetime import UTC, datetime
|
from datetime import UTC, datetime
|
||||||
from uuid import UUID
|
from uuid import UUID
|
||||||
|
|
||||||
from sqlalchemy import delete, select
|
from sqlalchemy import delete, select, text
|
||||||
from sqlalchemy.orm import Session, sessionmaker
|
from sqlalchemy.orm import Session, sessionmaker
|
||||||
|
|
||||||
from ekp_core.document_indexing import DocumentIndexingService
|
from ekp_core.document_indexing import DocumentIndexingService
|
||||||
@ -49,7 +49,7 @@ class IndexBuildSummary:
|
|||||||
|
|
||||||
|
|
||||||
class IndexRebuildService:
|
class IndexRebuildService:
|
||||||
"""Build a new profile from canonical chunks without mutating an active index."""
|
"""Build a new profile from PostgreSQL chunks without mutating an active index."""
|
||||||
|
|
||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
@ -255,10 +255,10 @@ class IndexRebuildService:
|
|||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _lock_canonical_chunks(session: Session) -> None:
|
def _lock_canonical_chunks(session: Session) -> None:
|
||||||
"""Prevent canonical chunk writes between verification and activation."""
|
"""Prevent PostgreSQL chunk writes between final verification and activation."""
|
||||||
bind = session.get_bind()
|
bind = session.get_bind()
|
||||||
if bind.dialect.name == "mysql":
|
if bind.dialect.name == "postgresql":
|
||||||
session.execute(select(Chunk.chunk_id).with_for_update()).all()
|
session.execute(text("LOCK TABLE chunks IN SHARE MODE"))
|
||||||
|
|
||||||
def _state_matches_profile(self, state: ChunkIndexState) -> bool:
|
def _state_matches_profile(self, state: ChunkIndexState) -> bool:
|
||||||
return (
|
return (
|
||||||
|
|||||||
@ -1,270 +0,0 @@
|
|||||||
"""Batch submission and aggregate status over independent ingestion Jobs."""
|
|
||||||
|
|
||||||
from dataclasses import dataclass
|
|
||||||
from enum import StrEnum
|
|
||||||
from typing import Protocol
|
|
||||||
from uuid import UUID
|
|
||||||
|
|
||||||
from sqlalchemy import select
|
|
||||||
from sqlalchemy.orm import Session, sessionmaker
|
|
||||||
|
|
||||||
from ekp_core.document_upload import (
|
|
||||||
DocumentTooLargeError,
|
|
||||||
DocumentUploadError,
|
|
||||||
EmptyDocumentError,
|
|
||||||
MediaTypeMismatchError,
|
|
||||||
UnsupportedDocumentTypeError,
|
|
||||||
)
|
|
||||||
from ekp_core.ingestion_jobs import AsyncIngestionSubmissionService
|
|
||||||
from ekp_domain import (
|
|
||||||
BatchItemAcceptanceStatus,
|
|
||||||
IngestionBatch,
|
|
||||||
IngestionBatchItem,
|
|
||||||
IngestionJob,
|
|
||||||
IngestionJobStatus,
|
|
||||||
)
|
|
||||||
from ekp_object_storage import BinarySource
|
|
||||||
|
|
||||||
SAFE_UNSUPPORTED_ERROR = "unsupported document type or media type"
|
|
||||||
SAFE_FILE_SIZE_ERROR = "document exceeds the configured upload limit"
|
|
||||||
SAFE_BATCH_SIZE_ERROR = "batch exceeds the configured total upload limit"
|
|
||||||
SAFE_EMPTY_ERROR = "document must contain content"
|
|
||||||
SAFE_ACCEPTANCE_ERROR = "document could not be accepted"
|
|
||||||
|
|
||||||
|
|
||||||
class BatchNotFoundError(LookupError):
|
|
||||||
"""Raised when a requested ingestion batch does not exist."""
|
|
||||||
|
|
||||||
|
|
||||||
class BatchFileCountLimitError(ValueError):
|
|
||||||
"""Raised before persistence when the multipart file count is invalid."""
|
|
||||||
|
|
||||||
|
|
||||||
class BatchAggregateStatus(StrEnum):
|
|
||||||
"""Computed batch state derived from item acceptance and durable Jobs."""
|
|
||||||
|
|
||||||
PROCESSING = "processing"
|
|
||||||
COMPLETED = "completed"
|
|
||||||
PARTIAL_FAILURE = "partial_failure"
|
|
||||||
FAILED = "failed"
|
|
||||||
|
|
||||||
|
|
||||||
class BatchTotalSizeLimitError(ValueError):
|
|
||||||
"""Raised while streaming the file that crosses the shared batch budget."""
|
|
||||||
|
|
||||||
|
|
||||||
class _ReadableSource(Protocol):
|
|
||||||
def read(self, size: int = -1, /) -> bytes:
|
|
||||||
"""Read bytes from an uploaded file."""
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True, slots=True)
|
|
||||||
class BatchSource:
|
|
||||||
"""Provider-neutral input for one ordered multipart file."""
|
|
||||||
|
|
||||||
filename: str
|
|
||||||
media_type: str | None
|
|
||||||
source: _ReadableSource
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True, slots=True)
|
|
||||||
class BatchItemView:
|
|
||||||
"""Detached batch item plus its current optional Job state."""
|
|
||||||
|
|
||||||
item: IngestionBatchItem
|
|
||||||
job: IngestionJob | None
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True, slots=True)
|
|
||||||
class BatchView:
|
|
||||||
"""Detached aggregate used by REST without exposing persistence sessions."""
|
|
||||||
|
|
||||||
batch: IngestionBatch
|
|
||||||
items: tuple[BatchItemView, ...]
|
|
||||||
|
|
||||||
@property
|
|
||||||
def status(self) -> BatchAggregateStatus:
|
|
||||||
"""Aggregate terminal results without copying Job state into batch rows."""
|
|
||||||
if any(
|
|
||||||
view.item.acceptance_status == BatchItemAcceptanceStatus.PENDING
|
|
||||||
or (
|
|
||||||
view.item.acceptance_status == BatchItemAcceptanceStatus.ACCEPTED
|
|
||||||
and (
|
|
||||||
view.job is None
|
|
||||||
or view.job.status
|
|
||||||
in {
|
|
||||||
IngestionJobStatus.QUEUED,
|
|
||||||
IngestionJobStatus.RUNNING,
|
|
||||||
IngestionJobStatus.RETRYING,
|
|
||||||
}
|
|
||||||
)
|
|
||||||
)
|
|
||||||
for view in self.items
|
|
||||||
):
|
|
||||||
return BatchAggregateStatus.PROCESSING
|
|
||||||
succeeded = sum(
|
|
||||||
view.job is not None and view.job.status == IngestionJobStatus.SUCCEEDED
|
|
||||||
for view in self.items
|
|
||||||
)
|
|
||||||
failed = len(self.items) - succeeded
|
|
||||||
if failed == 0:
|
|
||||||
return BatchAggregateStatus.COMPLETED
|
|
||||||
if succeeded == 0:
|
|
||||||
return BatchAggregateStatus.FAILED
|
|
||||||
return BatchAggregateStatus.PARTIAL_FAILURE
|
|
||||||
|
|
||||||
|
|
||||||
class _BatchByteBudget:
|
|
||||||
"""Count bytes across independently streamed files without buffering them."""
|
|
||||||
|
|
||||||
def __init__(self, limit_bytes: int) -> None:
|
|
||||||
self._limit_bytes = limit_bytes
|
|
||||||
self._consumed_bytes = 0
|
|
||||||
|
|
||||||
def wrap(self, source: _ReadableSource) -> BinarySource:
|
|
||||||
return _BatchLimitedSource(source, self)
|
|
||||||
|
|
||||||
def consume(self, size_bytes: int) -> None:
|
|
||||||
self._consumed_bytes += size_bytes
|
|
||||||
if self._consumed_bytes > self._limit_bytes:
|
|
||||||
raise BatchTotalSizeLimitError(SAFE_BATCH_SIZE_ERROR)
|
|
||||||
|
|
||||||
|
|
||||||
class _BatchLimitedSource:
|
|
||||||
"""Apply one shared request budget through the ObjectStorage stream contract."""
|
|
||||||
|
|
||||||
def __init__(self, source: _ReadableSource, budget: _BatchByteBudget) -> None:
|
|
||||||
self._source = source
|
|
||||||
self._budget = budget
|
|
||||||
|
|
||||||
def read(self, size: int = -1, /) -> bytes:
|
|
||||||
chunk = self._source.read(size)
|
|
||||||
self._budget.consume(len(chunk))
|
|
||||||
return chunk
|
|
||||||
|
|
||||||
|
|
||||||
class IngestionBatchService:
|
|
||||||
"""Accept files independently and persist ordered aggregate status."""
|
|
||||||
|
|
||||||
def __init__(
|
|
||||||
self,
|
|
||||||
session_factory: sessionmaker[Session],
|
|
||||||
submissions: AsyncIngestionSubmissionService,
|
|
||||||
*,
|
|
||||||
max_files: int,
|
|
||||||
max_total_bytes: int,
|
|
||||||
) -> None:
|
|
||||||
if max_files <= 0 or max_total_bytes <= 0:
|
|
||||||
raise ValueError("batch limits must be positive")
|
|
||||||
self._session_factory = session_factory
|
|
||||||
self._submissions = submissions
|
|
||||||
self._max_files = max_files
|
|
||||||
self._max_total_bytes = max_total_bytes
|
|
||||||
|
|
||||||
def submit(self, sources: tuple[BatchSource, ...]) -> BatchView:
|
|
||||||
"""Create a batch and independently accept or reject every ordered file."""
|
|
||||||
if not sources or len(sources) > self._max_files:
|
|
||||||
raise BatchFileCountLimitError(
|
|
||||||
f"batch must contain between 1 and {self._max_files} files"
|
|
||||||
)
|
|
||||||
batch_id, item_ids = self._create_pending_batch(sources)
|
|
||||||
budget = _BatchByteBudget(self._max_total_bytes)
|
|
||||||
for item_id, source in zip(item_ids, sources, strict=True):
|
|
||||||
try:
|
|
||||||
submission = self._submissions.submit(
|
|
||||||
source.filename,
|
|
||||||
source.media_type,
|
|
||||||
budget.wrap(source.source),
|
|
||||||
)
|
|
||||||
except Exception as error:
|
|
||||||
self._record_rejected(item_id, _safe_acceptance_error(error))
|
|
||||||
continue
|
|
||||||
self._record_accepted(
|
|
||||||
item_id,
|
|
||||||
document_id=submission.document.id,
|
|
||||||
job_id=submission.job.id,
|
|
||||||
)
|
|
||||||
return self.get(batch_id)
|
|
||||||
|
|
||||||
def get(self, batch_id: UUID) -> BatchView:
|
|
||||||
"""Return ordered items and fresh Job state for polling clients."""
|
|
||||||
with self._session_factory() as session:
|
|
||||||
batch = session.get(IngestionBatch, batch_id)
|
|
||||||
if batch is None:
|
|
||||||
raise BatchNotFoundError(str(batch_id))
|
|
||||||
items = tuple(
|
|
||||||
session.scalars(
|
|
||||||
select(IngestionBatchItem)
|
|
||||||
.where(IngestionBatchItem.batch_id == batch_id)
|
|
||||||
.order_by(IngestionBatchItem.item_index)
|
|
||||||
).all()
|
|
||||||
)
|
|
||||||
views = tuple(
|
|
||||||
BatchItemView(
|
|
||||||
item=item,
|
|
||||||
job=session.get(IngestionJob, item.job_id) if item.job_id is not None else None,
|
|
||||||
)
|
|
||||||
for item in items
|
|
||||||
)
|
|
||||||
return BatchView(batch=batch, items=views)
|
|
||||||
|
|
||||||
def _create_pending_batch(
|
|
||||||
self, sources: tuple[BatchSource, ...]
|
|
||||||
) -> tuple[UUID, tuple[UUID, ...]]:
|
|
||||||
"""Persist every input position before starting external storage writes."""
|
|
||||||
with self._session_factory() as session:
|
|
||||||
batch = IngestionBatch(total_files=len(sources))
|
|
||||||
session.add(batch)
|
|
||||||
session.flush()
|
|
||||||
items = tuple(
|
|
||||||
IngestionBatchItem(
|
|
||||||
batch_id=batch.id,
|
|
||||||
item_index=index,
|
|
||||||
filename=_display_filename(source.filename),
|
|
||||||
)
|
|
||||||
for index, source in enumerate(sources)
|
|
||||||
)
|
|
||||||
session.add_all(items)
|
|
||||||
session.commit()
|
|
||||||
return batch.id, tuple(item.id for item in items)
|
|
||||||
|
|
||||||
def _record_accepted(self, item_id: UUID, *, document_id: UUID, job_id: UUID) -> None:
|
|
||||||
with self._session_factory() as session:
|
|
||||||
item = session.get(IngestionBatchItem, item_id)
|
|
||||||
if item is None:
|
|
||||||
raise BatchNotFoundError(str(item_id))
|
|
||||||
item.acceptance_status = BatchItemAcceptanceStatus.ACCEPTED
|
|
||||||
item.document_id = document_id
|
|
||||||
item.job_id = job_id
|
|
||||||
item.error_message = None
|
|
||||||
session.commit()
|
|
||||||
|
|
||||||
def _record_rejected(self, item_id: UUID, error_message: str) -> None:
|
|
||||||
with self._session_factory() as session:
|
|
||||||
item = session.get(IngestionBatchItem, item_id)
|
|
||||||
if item is None:
|
|
||||||
raise BatchNotFoundError(str(item_id))
|
|
||||||
item.acceptance_status = BatchItemAcceptanceStatus.REJECTED
|
|
||||||
item.error_message = error_message
|
|
||||||
session.commit()
|
|
||||||
|
|
||||||
|
|
||||||
def _display_filename(filename: str) -> str:
|
|
||||||
"""Store a bounded safe label even when the upload filename is invalid."""
|
|
||||||
display = filename.replace("\x00", "<EFBFBD>").strip()
|
|
||||||
return display[:255] or "<unnamed>"
|
|
||||||
|
|
||||||
|
|
||||||
def _safe_acceptance_error(error: Exception) -> str:
|
|
||||||
"""Map per-file failures without persisting exception text or credentials."""
|
|
||||||
if isinstance(error, (UnsupportedDocumentTypeError, MediaTypeMismatchError)):
|
|
||||||
return SAFE_UNSUPPORTED_ERROR
|
|
||||||
if isinstance(error, DocumentTooLargeError):
|
|
||||||
return SAFE_FILE_SIZE_ERROR
|
|
||||||
if isinstance(error, BatchTotalSizeLimitError):
|
|
||||||
return SAFE_BATCH_SIZE_ERROR
|
|
||||||
if isinstance(error, EmptyDocumentError):
|
|
||||||
return SAFE_EMPTY_ERROR
|
|
||||||
if isinstance(error, DocumentUploadError):
|
|
||||||
return SAFE_ACCEPTANCE_ERROR
|
|
||||||
return SAFE_ACCEPTANCE_ERROR
|
|
||||||
@ -1,588 +0,0 @@
|
|||||||
"""Durable asynchronous ingestion orchestration and retry cleanup."""
|
|
||||||
|
|
||||||
from dataclasses import dataclass
|
|
||||||
from datetime import UTC, datetime, timedelta
|
|
||||||
from enum import StrEnum
|
|
||||||
from math import ceil
|
|
||||||
from typing import Protocol
|
|
||||||
from uuid import UUID
|
|
||||||
|
|
||||||
from sqlalchemy import delete, select
|
|
||||||
from sqlalchemy.orm import Session, sessionmaker
|
|
||||||
|
|
||||||
from ekp_core.document_chunking import (
|
|
||||||
DocumentAlreadyChunkedError,
|
|
||||||
DocumentChunkingService,
|
|
||||||
DocumentNotParsedError,
|
|
||||||
EmptyChunkSetError,
|
|
||||||
ParsedDocumentMismatchError,
|
|
||||||
)
|
|
||||||
from ekp_core.document_chunking import (
|
|
||||||
DocumentNotFoundError as ChunkingDocumentNotFoundError,
|
|
||||||
)
|
|
||||||
from ekp_core.document_indexing import (
|
|
||||||
DocumentAlreadyIndexedError,
|
|
||||||
DocumentHasNoChunksError,
|
|
||||||
DocumentIndexingService,
|
|
||||||
)
|
|
||||||
from ekp_core.document_indexing import (
|
|
||||||
DocumentNotFoundError as IndexingDocumentNotFoundError,
|
|
||||||
)
|
|
||||||
from ekp_core.document_parsing import (
|
|
||||||
DocumentAlreadyParsedError,
|
|
||||||
DocumentParsingService,
|
|
||||||
EmptyParsedDocumentError,
|
|
||||||
)
|
|
||||||
from ekp_core.document_parsing import (
|
|
||||||
DocumentNotFoundError as ParsingDocumentNotFoundError,
|
|
||||||
)
|
|
||||||
from ekp_core.document_upload import DocumentUploadService
|
|
||||||
from ekp_core.parsed_document_artifact import ParsedDocumentArtifactError
|
|
||||||
from ekp_domain import (
|
|
||||||
Chunk,
|
|
||||||
ChunkIndexState,
|
|
||||||
Document,
|
|
||||||
DocumentStatus,
|
|
||||||
IngestionAttempt,
|
|
||||||
IngestionAttemptStatus,
|
|
||||||
IngestionJob,
|
|
||||||
IngestionJobStatus,
|
|
||||||
IngestionStage,
|
|
||||||
)
|
|
||||||
from ekp_object_storage import BinarySource, ObjectStorage
|
|
||||||
from ekp_parser import DocumentParseError
|
|
||||||
from ekp_vector_store import VectorStore
|
|
||||||
|
|
||||||
SAFE_TRANSIENT_ERROR = "temporary ingestion failure"
|
|
||||||
SAFE_PERMANENT_ERROR = "document content could not be processed"
|
|
||||||
SAFE_CLEANUP_ERROR = "retry cleanup failed"
|
|
||||||
SAFE_LEASE_ERROR = "worker execution lease expired"
|
|
||||||
|
|
||||||
|
|
||||||
class IngestionJobNotFoundError(LookupError):
|
|
||||||
"""Raised when a requested ingestion Job does not exist."""
|
|
||||||
|
|
||||||
|
|
||||||
class IngestionCleanupError(RuntimeError):
|
|
||||||
"""Raised when derived data cannot be restored to a known clean state."""
|
|
||||||
|
|
||||||
|
|
||||||
class JobDispatcher(Protocol):
|
|
||||||
"""Replaceable message-delivery boundary used by submission services."""
|
|
||||||
|
|
||||||
def enqueue(self, job_id: UUID) -> None:
|
|
||||||
"""Request asynchronous execution of one already-persisted Job."""
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True, slots=True)
|
|
||||||
class IngestionSubmission:
|
|
||||||
"""Stable identifiers returned after source and Job persistence."""
|
|
||||||
|
|
||||||
document: Document
|
|
||||||
job: IngestionJob
|
|
||||||
|
|
||||||
|
|
||||||
class JobRunAction(StrEnum):
|
|
||||||
"""Instruction returned to a queue adapter after one delivery."""
|
|
||||||
|
|
||||||
COMPLETE = "complete"
|
|
||||||
RETRY = "retry"
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True, slots=True)
|
|
||||||
class JobRunResult:
|
|
||||||
"""Queue-neutral result for acknowledging or rescheduling a delivery."""
|
|
||||||
|
|
||||||
action: JobRunAction
|
|
||||||
status: IngestionJobStatus
|
|
||||||
retry_delay_seconds: int | None = None
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True, slots=True)
|
|
||||||
class _Claim:
|
|
||||||
"""Internal durable claim outcome used before external provider work."""
|
|
||||||
|
|
||||||
run: bool
|
|
||||||
cleanup_required: bool = False
|
|
||||||
result: JobRunResult | None = None
|
|
||||||
|
|
||||||
|
|
||||||
class IngestionJobService:
|
|
||||||
"""Create and query Job state independently from the broker implementation."""
|
|
||||||
|
|
||||||
def __init__(self, session_factory: sessionmaker[Session], max_attempts: int) -> None:
|
|
||||||
if max_attempts <= 0:
|
|
||||||
raise ValueError("max_attempts must be positive")
|
|
||||||
self._session_factory = session_factory
|
|
||||||
self._max_attempts = max_attempts
|
|
||||||
|
|
||||||
def create(self, document_id: UUID) -> IngestionJob:
|
|
||||||
"""Create exactly one queued ingestion Job for a canonical Document."""
|
|
||||||
with self._session_factory() as session:
|
|
||||||
job = IngestionJob(document_id=document_id, max_attempts=self._max_attempts)
|
|
||||||
session.add(job)
|
|
||||||
session.commit()
|
|
||||||
return job
|
|
||||||
|
|
||||||
def get(self, job_id: UUID) -> IngestionJob:
|
|
||||||
"""Return one detached Job or a stable not-found error."""
|
|
||||||
with self._session_factory() as session:
|
|
||||||
job = session.get(IngestionJob, job_id)
|
|
||||||
if job is None:
|
|
||||||
raise IngestionJobNotFoundError(str(job_id))
|
|
||||||
return job
|
|
||||||
|
|
||||||
def list_dispatchable(self, limit: int = 100) -> tuple[UUID, ...]:
|
|
||||||
"""Find due queued/retrying Jobs and expired running leases for recovery."""
|
|
||||||
if not 1 <= limit <= 1000:
|
|
||||||
raise ValueError("dispatch limit must be between 1 and 1000")
|
|
||||||
now = datetime.now(UTC)
|
|
||||||
with self._session_factory() as session:
|
|
||||||
jobs = session.scalars(
|
|
||||||
select(IngestionJob)
|
|
||||||
.where(
|
|
||||||
(IngestionJob.status == IngestionJobStatus.QUEUED)
|
|
||||||
| (
|
|
||||||
(IngestionJob.status == IngestionJobStatus.RETRYING)
|
|
||||||
& ((IngestionJob.next_run_at.is_(None)) | (IngestionJob.next_run_at <= now))
|
|
||||||
)
|
|
||||||
| (
|
|
||||||
(IngestionJob.status == IngestionJobStatus.RUNNING)
|
|
||||||
& (
|
|
||||||
(IngestionJob.lease_expires_at.is_(None))
|
|
||||||
| (IngestionJob.lease_expires_at <= now)
|
|
||||||
)
|
|
||||||
)
|
|
||||||
)
|
|
||||||
.order_by(IngestionJob.created_at, IngestionJob.id)
|
|
||||||
.limit(limit)
|
|
||||||
).all()
|
|
||||||
return tuple(job.id for job in jobs)
|
|
||||||
|
|
||||||
|
|
||||||
class AsyncIngestionSubmissionService:
|
|
||||||
"""Persist immutable source and Job before best-effort broker delivery."""
|
|
||||||
|
|
||||||
def __init__(
|
|
||||||
self,
|
|
||||||
upload_service: DocumentUploadService,
|
|
||||||
jobs: IngestionJobService,
|
|
||||||
dispatcher: JobDispatcher,
|
|
||||||
) -> None:
|
|
||||||
self._upload_service = upload_service
|
|
||||||
self._jobs = jobs
|
|
||||||
self._dispatcher = dispatcher
|
|
||||||
|
|
||||||
def submit(
|
|
||||||
self,
|
|
||||||
filename: str,
|
|
||||||
media_type: str | None,
|
|
||||||
source: BinarySource,
|
|
||||||
) -> IngestionSubmission:
|
|
||||||
"""Accept one source and return without executing expensive stages inline.
|
|
||||||
|
|
||||||
Broker delivery is deliberately best effort after the canonical commit.
|
|
||||||
A periodic dispatcher recovers queued Jobs when the broker was temporarily
|
|
||||||
unavailable, so the request never treats a message as the source of truth.
|
|
||||||
"""
|
|
||||||
document = self._upload_service.upload(filename, media_type, source)
|
|
||||||
try:
|
|
||||||
job = self._jobs.create(document.id)
|
|
||||||
except Exception:
|
|
||||||
self._upload_service.rollback_acceptance(document.id)
|
|
||||||
raise
|
|
||||||
try:
|
|
||||||
self._dispatcher.enqueue(job.id)
|
|
||||||
except Exception:
|
|
||||||
# Keeping the Job queued is intentional. The recovery dispatcher can
|
|
||||||
# safely redeliver it without asking the client to upload again.
|
|
||||||
pass
|
|
||||||
return IngestionSubmission(document=document, job=job)
|
|
||||||
|
|
||||||
|
|
||||||
class DerivedDocumentCleanupService:
|
|
||||||
"""Remove only rebuildable derivatives before a retry begins."""
|
|
||||||
|
|
||||||
def __init__(
|
|
||||||
self,
|
|
||||||
session_factory: sessionmaker[Session],
|
|
||||||
storage: ObjectStorage,
|
|
||||||
vector_store: VectorStore,
|
|
||||||
active_index_profile_id: str,
|
|
||||||
) -> None:
|
|
||||||
self._session_factory = session_factory
|
|
||||||
self._storage = storage
|
|
||||||
self._vector_store = vector_store
|
|
||||||
self._active_index_profile_id = active_index_profile_id
|
|
||||||
|
|
||||||
def cleanup(self, document_id: UUID) -> None:
|
|
||||||
"""Clean vector, relational and parsed artifacts while preserving source/Job."""
|
|
||||||
try:
|
|
||||||
with self._session_factory() as session:
|
|
||||||
document = session.get(Document, document_id)
|
|
||||||
if document is None:
|
|
||||||
raise IngestionCleanupError("document metadata is missing")
|
|
||||||
parsed_key = document.parsed_storage_key
|
|
||||||
profile_ids = set(
|
|
||||||
session.scalars(
|
|
||||||
select(ChunkIndexState.index_profile_id)
|
|
||||||
.join(Chunk, Chunk.chunk_id == ChunkIndexState.chunk_id)
|
|
||||||
.where(Chunk.document_id == document_id)
|
|
||||||
.distinct()
|
|
||||||
).all()
|
|
||||||
)
|
|
||||||
profile_ids.add(self._active_index_profile_id)
|
|
||||||
|
|
||||||
# External vector rows are deleted before their canonical evidence.
|
|
||||||
for profile_id in sorted(profile_ids):
|
|
||||||
self._vector_store.delete_document(document_id, profile_id)
|
|
||||||
|
|
||||||
with self._session_factory() as session:
|
|
||||||
chunk_ids = select(Chunk.chunk_id).where(Chunk.document_id == document_id)
|
|
||||||
session.execute(
|
|
||||||
delete(ChunkIndexState).where(ChunkIndexState.chunk_id.in_(chunk_ids))
|
|
||||||
)
|
|
||||||
session.execute(delete(Chunk).where(Chunk.document_id == document_id))
|
|
||||||
session.commit()
|
|
||||||
|
|
||||||
if parsed_key is not None and self._storage.exists(parsed_key):
|
|
||||||
self._storage.delete(parsed_key)
|
|
||||||
|
|
||||||
with self._session_factory() as session:
|
|
||||||
document = session.get(Document, document_id)
|
|
||||||
if document is None:
|
|
||||||
raise IngestionCleanupError("document metadata is missing")
|
|
||||||
document.parsed_storage_key = None
|
|
||||||
document.status = DocumentStatus.UPLOADED
|
|
||||||
document.error_message = None
|
|
||||||
session.commit()
|
|
||||||
except IngestionCleanupError:
|
|
||||||
raise
|
|
||||||
except Exception as error:
|
|
||||||
raise IngestionCleanupError(SAFE_CLEANUP_ERROR) from error
|
|
||||||
|
|
||||||
|
|
||||||
class IngestionJobRunner:
|
|
||||||
"""Execute one durable attempt and return queue-neutral retry instructions."""
|
|
||||||
|
|
||||||
def __init__(
|
|
||||||
self,
|
|
||||||
session_factory: sessionmaker[Session],
|
|
||||||
parsing: DocumentParsingService,
|
|
||||||
chunking: DocumentChunkingService,
|
|
||||||
indexing: DocumentIndexingService,
|
|
||||||
cleanup: DerivedDocumentCleanupService,
|
|
||||||
*,
|
|
||||||
retry_backoff_seconds: int,
|
|
||||||
retry_backoff_max_seconds: int,
|
|
||||||
lease_seconds: int,
|
|
||||||
) -> None:
|
|
||||||
if retry_backoff_seconds <= 0 or retry_backoff_max_seconds <= 0 or lease_seconds <= 0:
|
|
||||||
raise ValueError("retry and lease durations must be positive")
|
|
||||||
self._session_factory = session_factory
|
|
||||||
self._parsing = parsing
|
|
||||||
self._chunking = chunking
|
|
||||||
self._indexing = indexing
|
|
||||||
self._cleanup = cleanup
|
|
||||||
self._retry_backoff_seconds = retry_backoff_seconds
|
|
||||||
self._retry_backoff_max_seconds = retry_backoff_max_seconds
|
|
||||||
self._lease_seconds = lease_seconds
|
|
||||||
|
|
||||||
def run(self, job_id: UUID) -> JobRunResult:
|
|
||||||
"""Claim, optionally clean, execute and durably finalize one delivery."""
|
|
||||||
claim = self._claim(job_id)
|
|
||||||
if not claim.run:
|
|
||||||
assert claim.result is not None
|
|
||||||
return claim.result
|
|
||||||
|
|
||||||
if claim.cleanup_required:
|
|
||||||
try:
|
|
||||||
document_id = self._job_document_id(job_id)
|
|
||||||
self._cleanup.cleanup(document_id)
|
|
||||||
except Exception:
|
|
||||||
self._finish_cleanup_failure(job_id)
|
|
||||||
return JobRunResult(JobRunAction.COMPLETE, IngestionJobStatus.FAILED)
|
|
||||||
|
|
||||||
attempt_number, document_id = self._start_attempt(job_id)
|
|
||||||
|
|
||||||
def stage_callback(stage: IngestionStage) -> None:
|
|
||||||
"""Bind this attempt while pipeline stages remain Job-agnostic."""
|
|
||||||
self._record_stage(job_id, attempt_number, stage)
|
|
||||||
|
|
||||||
try:
|
|
||||||
self._parsing.process(document_id, stage_callback=stage_callback)
|
|
||||||
self._chunking.process(document_id, stage_callback=stage_callback)
|
|
||||||
self._indexing.process(document_id, stage_callback=stage_callback)
|
|
||||||
except Exception as error:
|
|
||||||
return self._finish_failure(job_id, attempt_number, document_id, error)
|
|
||||||
|
|
||||||
self._finish_success(job_id, attempt_number, document_id)
|
|
||||||
return JobRunResult(JobRunAction.COMPLETE, IngestionJobStatus.SUCCEEDED)
|
|
||||||
|
|
||||||
def _claim(self, job_id: UUID) -> _Claim:
|
|
||||||
"""Serialize deliveries and recover only after an execution lease expires."""
|
|
||||||
now = datetime.now(UTC)
|
|
||||||
with self._session_factory() as session:
|
|
||||||
job = session.scalar(
|
|
||||||
select(IngestionJob).where(IngestionJob.id == job_id).with_for_update()
|
|
||||||
)
|
|
||||||
if job is None:
|
|
||||||
raise IngestionJobNotFoundError(str(job_id))
|
|
||||||
if job.status in {
|
|
||||||
IngestionJobStatus.SUCCEEDED,
|
|
||||||
IngestionJobStatus.FAILED,
|
|
||||||
IngestionJobStatus.CANCELLED,
|
|
||||||
}:
|
|
||||||
return _Claim(
|
|
||||||
run=False,
|
|
||||||
result=JobRunResult(JobRunAction.COMPLETE, job.status),
|
|
||||||
)
|
|
||||||
if job.status == IngestionJobStatus.RUNNING:
|
|
||||||
lease_expiry = _as_utc(job.lease_expires_at)
|
|
||||||
if lease_expiry is not None and lease_expiry > now:
|
|
||||||
delay = max(1, ceil((lease_expiry - now).total_seconds()))
|
|
||||||
return _Claim(
|
|
||||||
run=False,
|
|
||||||
result=JobRunResult(
|
|
||||||
JobRunAction.RETRY,
|
|
||||||
IngestionJobStatus.RUNNING,
|
|
||||||
delay,
|
|
||||||
),
|
|
||||||
)
|
|
||||||
self._expire_active_attempt(session, job, now)
|
|
||||||
if job.attempt_count >= job.max_attempts:
|
|
||||||
job.status = IngestionJobStatus.FAILED
|
|
||||||
job.lease_expires_at = None
|
|
||||||
job.completed_at = now
|
|
||||||
document = session.get(Document, job.document_id)
|
|
||||||
if document is not None:
|
|
||||||
document.status = DocumentStatus.FAILED
|
|
||||||
document.error_message = SAFE_LEASE_ERROR
|
|
||||||
session.commit()
|
|
||||||
return _Claim(
|
|
||||||
run=False,
|
|
||||||
result=JobRunResult(JobRunAction.COMPLETE, job.status),
|
|
||||||
)
|
|
||||||
job.status = IngestionJobStatus.RETRYING
|
|
||||||
next_run = _as_utc(job.next_run_at)
|
|
||||||
if (
|
|
||||||
job.status == IngestionJobStatus.RETRYING
|
|
||||||
and next_run is not None
|
|
||||||
and next_run > now
|
|
||||||
):
|
|
||||||
delay = max(1, ceil((next_run - now).total_seconds()))
|
|
||||||
return _Claim(
|
|
||||||
run=False,
|
|
||||||
result=JobRunResult(
|
|
||||||
JobRunAction.RETRY,
|
|
||||||
IngestionJobStatus.RETRYING,
|
|
||||||
delay,
|
|
||||||
),
|
|
||||||
)
|
|
||||||
|
|
||||||
cleanup_required = job.attempt_count > 0
|
|
||||||
job.status = IngestionJobStatus.RUNNING
|
|
||||||
job.lease_expires_at = now + timedelta(seconds=self._lease_seconds)
|
|
||||||
job.next_run_at = None
|
|
||||||
session.commit()
|
|
||||||
return _Claim(run=True, cleanup_required=cleanup_required)
|
|
||||||
|
|
||||||
def _expire_active_attempt(self, session: Session, job: IngestionJob, now: datetime) -> None:
|
|
||||||
"""Close an abandoned Attempt before a replacement can be created."""
|
|
||||||
attempt = session.scalar(
|
|
||||||
select(IngestionAttempt)
|
|
||||||
.where(
|
|
||||||
IngestionAttempt.job_id == job.id,
|
|
||||||
IngestionAttempt.status == IngestionAttemptStatus.RUNNING,
|
|
||||||
)
|
|
||||||
.with_for_update()
|
|
||||||
)
|
|
||||||
if attempt is not None:
|
|
||||||
attempt.status = IngestionAttemptStatus.FAILED
|
|
||||||
attempt.retryable = True
|
|
||||||
attempt.error_message = SAFE_LEASE_ERROR
|
|
||||||
attempt.finished_at = now
|
|
||||||
job.error_message = SAFE_LEASE_ERROR
|
|
||||||
|
|
||||||
def _job_document_id(self, job_id: UUID) -> UUID:
|
|
||||||
with self._session_factory() as session:
|
|
||||||
job = session.get(IngestionJob, job_id)
|
|
||||||
if job is None:
|
|
||||||
raise IngestionJobNotFoundError(str(job_id))
|
|
||||||
return job.document_id
|
|
||||||
|
|
||||||
def _start_attempt(self, job_id: UUID) -> tuple[int, UUID]:
|
|
||||||
"""Create the next numbered Attempt only after retry cleanup succeeds."""
|
|
||||||
now = datetime.now(UTC)
|
|
||||||
with self._session_factory() as session:
|
|
||||||
job = session.scalar(
|
|
||||||
select(IngestionJob).where(IngestionJob.id == job_id).with_for_update()
|
|
||||||
)
|
|
||||||
if job is None:
|
|
||||||
raise IngestionJobNotFoundError(str(job_id))
|
|
||||||
attempt_number = job.attempt_count + 1
|
|
||||||
job.attempt_count = attempt_number
|
|
||||||
job.stage = IngestionStage.PARSING
|
|
||||||
job.error_message = None
|
|
||||||
job.lease_expires_at = now + timedelta(seconds=self._lease_seconds)
|
|
||||||
session.add(
|
|
||||||
IngestionAttempt(
|
|
||||||
job_id=job.id,
|
|
||||||
attempt_number=attempt_number,
|
|
||||||
stage=IngestionStage.PARSING,
|
|
||||||
)
|
|
||||||
)
|
|
||||||
session.commit()
|
|
||||||
return attempt_number, job.document_id
|
|
||||||
|
|
||||||
def _record_stage(self, job_id: UUID, attempt_number: int, stage: IngestionStage) -> None:
|
|
||||||
"""Persist progress and renew the worker lease at every stage boundary."""
|
|
||||||
with self._session_factory() as session:
|
|
||||||
job = session.get(IngestionJob, job_id)
|
|
||||||
attempt = session.scalar(
|
|
||||||
select(IngestionAttempt).where(
|
|
||||||
IngestionAttempt.job_id == job_id,
|
|
||||||
IngestionAttempt.attempt_number == attempt_number,
|
|
||||||
)
|
|
||||||
)
|
|
||||||
if job is None or attempt is None:
|
|
||||||
raise IngestionJobNotFoundError(str(job_id))
|
|
||||||
job.stage = stage
|
|
||||||
job.lease_expires_at = datetime.now(UTC) + timedelta(seconds=self._lease_seconds)
|
|
||||||
attempt.stage = stage
|
|
||||||
session.commit()
|
|
||||||
|
|
||||||
def _finish_success(self, job_id: UUID, attempt_number: int, document_id: UUID) -> None:
|
|
||||||
"""Atomically close Job/Attempt after the document becomes ready."""
|
|
||||||
now = datetime.now(UTC)
|
|
||||||
with self._session_factory() as session:
|
|
||||||
job, attempt = self._load_job_attempt(session, job_id, attempt_number)
|
|
||||||
document = session.get(Document, document_id)
|
|
||||||
if document is None:
|
|
||||||
raise IngestionJobNotFoundError(str(job_id))
|
|
||||||
attempt.status = IngestionAttemptStatus.SUCCEEDED
|
|
||||||
attempt.retryable = False
|
|
||||||
attempt.finished_at = now
|
|
||||||
job.status = IngestionJobStatus.SUCCEEDED
|
|
||||||
job.error_message = None
|
|
||||||
job.lease_expires_at = None
|
|
||||||
job.completed_at = now
|
|
||||||
document.status = DocumentStatus.READY
|
|
||||||
document.error_message = None
|
|
||||||
session.commit()
|
|
||||||
|
|
||||||
def _finish_failure(
|
|
||||||
self,
|
|
||||||
job_id: UUID,
|
|
||||||
attempt_number: int,
|
|
||||||
document_id: UUID,
|
|
||||||
error: Exception,
|
|
||||||
) -> JobRunResult:
|
|
||||||
"""Classify safely, bound retry count, and schedule exponential backoff."""
|
|
||||||
retryable = _is_retryable(error)
|
|
||||||
safe_error = SAFE_TRANSIENT_ERROR if retryable else SAFE_PERMANENT_ERROR
|
|
||||||
now = datetime.now(UTC)
|
|
||||||
with self._session_factory() as session:
|
|
||||||
job, attempt = self._load_job_attempt(session, job_id, attempt_number)
|
|
||||||
document = session.get(Document, document_id)
|
|
||||||
attempt.status = IngestionAttemptStatus.FAILED
|
|
||||||
attempt.retryable = retryable
|
|
||||||
attempt.error_message = safe_error
|
|
||||||
attempt.finished_at = now
|
|
||||||
job.error_message = safe_error
|
|
||||||
job.lease_expires_at = None
|
|
||||||
if retryable and job.attempt_count < job.max_attempts:
|
|
||||||
delay = min(
|
|
||||||
self._retry_backoff_seconds * (2 ** (job.attempt_count - 1)),
|
|
||||||
self._retry_backoff_max_seconds,
|
|
||||||
)
|
|
||||||
job.status = IngestionJobStatus.RETRYING
|
|
||||||
job.next_run_at = now + timedelta(seconds=delay)
|
|
||||||
if document is not None:
|
|
||||||
document.status = DocumentStatus.PROCESSING
|
|
||||||
document.error_message = safe_error
|
|
||||||
session.commit()
|
|
||||||
return JobRunResult(JobRunAction.RETRY, job.status, delay)
|
|
||||||
|
|
||||||
job.status = IngestionJobStatus.FAILED
|
|
||||||
job.next_run_at = None
|
|
||||||
job.completed_at = now
|
|
||||||
if document is not None:
|
|
||||||
document.status = DocumentStatus.FAILED
|
|
||||||
document.error_message = safe_error
|
|
||||||
session.commit()
|
|
||||||
return JobRunResult(JobRunAction.COMPLETE, job.status)
|
|
||||||
|
|
||||||
def _finish_cleanup_failure(self, job_id: UUID) -> None:
|
|
||||||
"""Fail closed without creating a new Attempt on unknown dirty state."""
|
|
||||||
now = datetime.now(UTC)
|
|
||||||
with self._session_factory() as session:
|
|
||||||
job = session.get(IngestionJob, job_id)
|
|
||||||
if job is None:
|
|
||||||
raise IngestionJobNotFoundError(str(job_id))
|
|
||||||
document = session.get(Document, job.document_id)
|
|
||||||
job.status = IngestionJobStatus.FAILED
|
|
||||||
job.error_message = SAFE_CLEANUP_ERROR
|
|
||||||
job.next_run_at = None
|
|
||||||
job.lease_expires_at = None
|
|
||||||
job.completed_at = now
|
|
||||||
if document is not None:
|
|
||||||
document.status = DocumentStatus.FAILED
|
|
||||||
document.error_message = SAFE_CLEANUP_ERROR
|
|
||||||
session.commit()
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def _load_job_attempt(
|
|
||||||
session: Session, job_id: UUID, attempt_number: int
|
|
||||||
) -> tuple[IngestionJob, IngestionAttempt]:
|
|
||||||
job = session.get(IngestionJob, job_id)
|
|
||||||
attempt = session.scalar(
|
|
||||||
select(IngestionAttempt).where(
|
|
||||||
IngestionAttempt.job_id == job_id,
|
|
||||||
IngestionAttempt.attempt_number == attempt_number,
|
|
||||||
)
|
|
||||||
)
|
|
||||||
if job is None or attempt is None:
|
|
||||||
raise IngestionJobNotFoundError(str(job_id))
|
|
||||||
return job, attempt
|
|
||||||
|
|
||||||
|
|
||||||
def dispatch_pending_jobs(jobs: IngestionJobService, dispatcher: JobDispatcher) -> int:
|
|
||||||
"""Best-effort redelivery loop used by Celery beat after broker/API failures."""
|
|
||||||
dispatched = 0
|
|
||||||
for job_id in jobs.list_dispatchable():
|
|
||||||
try:
|
|
||||||
dispatcher.enqueue(job_id)
|
|
||||||
except Exception:
|
|
||||||
continue
|
|
||||||
dispatched += 1
|
|
||||||
return dispatched
|
|
||||||
|
|
||||||
|
|
||||||
def _is_retryable(error: Exception) -> bool:
|
|
||||||
"""Treat known content/state violations as permanent and providers as transient."""
|
|
||||||
permanent_errors = (
|
|
||||||
DocumentParseError,
|
|
||||||
EmptyParsedDocumentError,
|
|
||||||
EmptyChunkSetError,
|
|
||||||
ParsedDocumentMismatchError,
|
|
||||||
ParsedDocumentArtifactError,
|
|
||||||
DocumentAlreadyParsedError,
|
|
||||||
DocumentAlreadyChunkedError,
|
|
||||||
DocumentAlreadyIndexedError,
|
|
||||||
DocumentNotParsedError,
|
|
||||||
DocumentHasNoChunksError,
|
|
||||||
ParsingDocumentNotFoundError,
|
|
||||||
ChunkingDocumentNotFoundError,
|
|
||||||
IndexingDocumentNotFoundError,
|
|
||||||
)
|
|
||||||
return not isinstance(error, permanent_errors)
|
|
||||||
|
|
||||||
|
|
||||||
def _as_utc(value: datetime | None) -> datetime | None:
|
|
||||||
"""Normalize SQLite's naive DateTime values to the UTC runtime contract."""
|
|
||||||
if value is None:
|
|
||||||
return None
|
|
||||||
if value.tzinfo is None:
|
|
||||||
return value.replace(tzinfo=UTC)
|
|
||||||
return value.astimezone(UTC)
|
|
||||||
@ -59,7 +59,7 @@ class SearchResult:
|
|||||||
|
|
||||||
|
|
||||||
class SearchService:
|
class SearchService:
|
||||||
"""Compose Dense Top-N, reranking, and canonical source enrichment."""
|
"""Compose Dense Top-N, reranking, and PostgreSQL source enrichment."""
|
||||||
|
|
||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
@ -158,7 +158,7 @@ class SearchService:
|
|||||||
self,
|
self,
|
||||||
selected: tuple[tuple[DenseSearchCandidate, float], ...],
|
selected: tuple[tuple[DenseSearchCandidate, float], ...],
|
||||||
) -> tuple[SourceResult, ...]:
|
) -> tuple[SourceResult, ...]:
|
||||||
"""Rehydrate final content and provenance from canonical relational rows."""
|
"""Rehydrate final content and provenance from canonical PostgreSQL rows."""
|
||||||
chunk_ids = [candidate.chunk_id for candidate, _ in selected]
|
chunk_ids = [candidate.chunk_id for candidate, _ in selected]
|
||||||
with self._session_factory() as session:
|
with self._session_factory() as session:
|
||||||
chunks = {
|
chunks = {
|
||||||
|
|||||||
@ -8,36 +8,16 @@ from ekp_domain.index_profile import (
|
|||||||
IndexProfileStatus,
|
IndexProfileStatus,
|
||||||
StoredIndexProfile,
|
StoredIndexProfile,
|
||||||
)
|
)
|
||||||
from ekp_domain.ingestion_batch import (
|
|
||||||
BatchItemAcceptanceStatus,
|
|
||||||
IngestionBatch,
|
|
||||||
IngestionBatchItem,
|
|
||||||
)
|
|
||||||
from ekp_domain.ingestion_job import (
|
|
||||||
IngestionAttempt,
|
|
||||||
IngestionAttemptStatus,
|
|
||||||
IngestionJob,
|
|
||||||
IngestionJobStatus,
|
|
||||||
IngestionStage,
|
|
||||||
)
|
|
||||||
from ekp_domain.parsed_document import ParsedBlock, ParsedBlockType, ParsedDocument
|
from ekp_domain.parsed_document import ParsedBlock, ParsedBlockType, ParsedDocument
|
||||||
|
|
||||||
__all__ = [
|
__all__ = [
|
||||||
"Base",
|
"Base",
|
||||||
"BatchItemAcceptanceStatus",
|
|
||||||
"Chunk",
|
"Chunk",
|
||||||
"ChunkIndexState",
|
"ChunkIndexState",
|
||||||
"Document",
|
"Document",
|
||||||
"DocumentStatus",
|
"DocumentStatus",
|
||||||
"IndexProfile",
|
"IndexProfile",
|
||||||
"IndexProfileStatus",
|
"IndexProfileStatus",
|
||||||
"IngestionAttempt",
|
|
||||||
"IngestionAttemptStatus",
|
|
||||||
"IngestionJob",
|
|
||||||
"IngestionJobStatus",
|
|
||||||
"IngestionStage",
|
|
||||||
"IngestionBatch",
|
|
||||||
"IngestionBatchItem",
|
|
||||||
"ParsedBlock",
|
"ParsedBlock",
|
||||||
"ParsedBlockType",
|
"ParsedBlockType",
|
||||||
"ParsedDocument",
|
"ParsedDocument",
|
||||||
|
|||||||
@ -6,7 +6,7 @@ from datetime import datetime
|
|||||||
from enum import StrEnum
|
from enum import StrEnum
|
||||||
from uuid import UUID
|
from uuid import UUID
|
||||||
|
|
||||||
from sqlalchemy import CheckConstraint, Computed, DateTime, ForeignKey, Index, Integer, String, func
|
from sqlalchemy import CheckConstraint, DateTime, ForeignKey, Index, Integer, String, func, text
|
||||||
from sqlalchemy import Enum as SqlEnum
|
from sqlalchemy import Enum as SqlEnum
|
||||||
from sqlalchemy.orm import Mapped, mapped_column
|
from sqlalchemy.orm import Mapped, mapped_column
|
||||||
|
|
||||||
@ -60,10 +60,14 @@ class StoredIndexProfile(Base):
|
|||||||
"status IN ('shadow', 'active', 'retired')",
|
"status IN ('shadow', 'active', 'retired')",
|
||||||
name="profile_status_valid",
|
name="profile_status_valid",
|
||||||
),
|
),
|
||||||
# A generated nullable marker provides portable partial-uniqueness:
|
# Both supported test/runtime dialects enforce at most one active profile.
|
||||||
# every active row maps to the same value, while NULL shadow/retired
|
Index(
|
||||||
# markers may repeat under OceanBase MySQL and SQLite.
|
"uq_index_profiles_single_active",
|
||||||
Index("uq_index_profiles_single_active", "active_marker", unique=True),
|
"status",
|
||||||
|
unique=True,
|
||||||
|
postgresql_where=text("status = 'active'"),
|
||||||
|
sqlite_where=text("status = 'active'"),
|
||||||
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
profile_id: Mapped[str] = mapped_column(String(128), primary_key=True)
|
profile_id: Mapped[str] = mapped_column(String(128), primary_key=True)
|
||||||
@ -82,11 +86,6 @@ class StoredIndexProfile(Base):
|
|||||||
nullable=False,
|
nullable=False,
|
||||||
default=IndexProfileStatus.SHADOW,
|
default=IndexProfileStatus.SHADOW,
|
||||||
)
|
)
|
||||||
active_marker: Mapped[str | None] = mapped_column(
|
|
||||||
String(6),
|
|
||||||
Computed("CASE WHEN status = 'active' THEN 'active' ELSE NULL END"),
|
|
||||||
nullable=True,
|
|
||||||
)
|
|
||||||
created_at: Mapped[datetime] = mapped_column(
|
created_at: Mapped[datetime] = mapped_column(
|
||||||
DateTime(timezone=True),
|
DateTime(timezone=True),
|
||||||
nullable=False,
|
nullable=False,
|
||||||
|
|||||||
@ -1,96 +0,0 @@
|
|||||||
"""Canonical batch submission metadata for per-file ingestion tracking."""
|
|
||||||
|
|
||||||
from datetime import datetime
|
|
||||||
from enum import StrEnum
|
|
||||||
from uuid import UUID, uuid4
|
|
||||||
|
|
||||||
from sqlalchemy import (
|
|
||||||
CheckConstraint,
|
|
||||||
DateTime,
|
|
||||||
ForeignKey,
|
|
||||||
Index,
|
|
||||||
Integer,
|
|
||||||
String,
|
|
||||||
Text,
|
|
||||||
Uuid,
|
|
||||||
func,
|
|
||||||
)
|
|
||||||
from sqlalchemy import Enum as SqlEnum
|
|
||||||
from sqlalchemy.orm import Mapped, mapped_column
|
|
||||||
|
|
||||||
from ekp_domain.document import Base
|
|
||||||
|
|
||||||
|
|
||||||
class BatchItemAcceptanceStatus(StrEnum):
|
|
||||||
"""Source acceptance state, independent from its asynchronous Job state."""
|
|
||||||
|
|
||||||
PENDING = "pending"
|
|
||||||
ACCEPTED = "accepted"
|
|
||||||
REJECTED = "rejected"
|
|
||||||
|
|
||||||
|
|
||||||
class IngestionBatch(Base):
|
|
||||||
"""Stable aggregation identifier for one bounded multipart submission."""
|
|
||||||
|
|
||||||
__tablename__ = "ingestion_batches"
|
|
||||||
__table_args__ = (CheckConstraint("total_files > 0", name="total_files_positive"),)
|
|
||||||
|
|
||||||
id: Mapped[UUID] = mapped_column(Uuid(as_uuid=True), primary_key=True, default=uuid4)
|
|
||||||
total_files: Mapped[int] = mapped_column(Integer, nullable=False)
|
|
||||||
created_at: Mapped[datetime] = mapped_column(
|
|
||||||
DateTime(timezone=True), nullable=False, server_default=func.now()
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
class IngestionBatchItem(Base):
|
|
||||||
"""One ordered input and its optional accepted Document/Job references."""
|
|
||||||
|
|
||||||
__tablename__ = "ingestion_batch_items"
|
|
||||||
__table_args__ = (
|
|
||||||
CheckConstraint("item_index >= 0", name="item_index_non_negative"),
|
|
||||||
CheckConstraint("length(filename) > 0", name="filename_not_empty"),
|
|
||||||
CheckConstraint(
|
|
||||||
"acceptance_status IN ('pending', 'accepted', 'rejected')",
|
|
||||||
name="acceptance_status_valid",
|
|
||||||
),
|
|
||||||
Index("uq_ingestion_batch_items_order", "batch_id", "item_index", unique=True),
|
|
||||||
Index("uq_ingestion_batch_items_job_id", "job_id", unique=True),
|
|
||||||
)
|
|
||||||
|
|
||||||
id: Mapped[UUID] = mapped_column(Uuid(as_uuid=True), primary_key=True, default=uuid4)
|
|
||||||
batch_id: Mapped[UUID] = mapped_column(
|
|
||||||
Uuid(as_uuid=True),
|
|
||||||
ForeignKey("ingestion_batches.id", ondelete="CASCADE"),
|
|
||||||
nullable=False,
|
|
||||||
)
|
|
||||||
item_index: Mapped[int] = mapped_column(Integer, nullable=False)
|
|
||||||
filename: Mapped[str] = mapped_column(String(255), nullable=False)
|
|
||||||
acceptance_status: Mapped[BatchItemAcceptanceStatus] = mapped_column(
|
|
||||||
SqlEnum(
|
|
||||||
BatchItemAcceptanceStatus,
|
|
||||||
name="batch_item_acceptance_status",
|
|
||||||
native_enum=False,
|
|
||||||
create_constraint=False,
|
|
||||||
validate_strings=True,
|
|
||||||
values_callable=lambda values: [item.value for item in values],
|
|
||||||
),
|
|
||||||
nullable=False,
|
|
||||||
default=BatchItemAcceptanceStatus.PENDING,
|
|
||||||
)
|
|
||||||
document_id: Mapped[UUID | None] = mapped_column(
|
|
||||||
Uuid(as_uuid=True),
|
|
||||||
ForeignKey("documents.id", ondelete="SET NULL"),
|
|
||||||
nullable=True,
|
|
||||||
)
|
|
||||||
job_id: Mapped[UUID | None] = mapped_column(
|
|
||||||
Uuid(as_uuid=True),
|
|
||||||
ForeignKey("ingestion_jobs.id", ondelete="SET NULL"),
|
|
||||||
nullable=True,
|
|
||||||
)
|
|
||||||
error_message: Mapped[str | None] = mapped_column(Text, nullable=True)
|
|
||||||
created_at: Mapped[datetime] = mapped_column(
|
|
||||||
DateTime(timezone=True), nullable=False, server_default=func.now()
|
|
||||||
)
|
|
||||||
updated_at: Mapped[datetime] = mapped_column(
|
|
||||||
DateTime(timezone=True), nullable=False, server_default=func.now(), onupdate=func.now()
|
|
||||||
)
|
|
||||||
@ -1,166 +0,0 @@
|
|||||||
"""Durable ingestion Job and Attempt state for asynchronous processing."""
|
|
||||||
|
|
||||||
from datetime import datetime
|
|
||||||
from enum import StrEnum
|
|
||||||
from uuid import UUID, uuid4
|
|
||||||
|
|
||||||
from sqlalchemy import (
|
|
||||||
Boolean,
|
|
||||||
CheckConstraint,
|
|
||||||
Computed,
|
|
||||||
DateTime,
|
|
||||||
ForeignKey,
|
|
||||||
Index,
|
|
||||||
Integer,
|
|
||||||
Text,
|
|
||||||
UniqueConstraint,
|
|
||||||
Uuid,
|
|
||||||
func,
|
|
||||||
)
|
|
||||||
from sqlalchemy import Enum as SqlEnum
|
|
||||||
from sqlalchemy.orm import Mapped, mapped_column
|
|
||||||
|
|
||||||
from ekp_domain.document import Base
|
|
||||||
|
|
||||||
|
|
||||||
class IngestionJobStatus(StrEnum):
|
|
||||||
"""Public lifecycle states for one document ingestion Job."""
|
|
||||||
|
|
||||||
QUEUED = "queued"
|
|
||||||
RUNNING = "running"
|
|
||||||
RETRYING = "retrying"
|
|
||||||
SUCCEEDED = "succeeded"
|
|
||||||
FAILED = "failed"
|
|
||||||
CANCELLED = "cancelled"
|
|
||||||
|
|
||||||
|
|
||||||
class IngestionStage(StrEnum):
|
|
||||||
"""Observable stages within one ingestion attempt."""
|
|
||||||
|
|
||||||
PARSING = "parsing"
|
|
||||||
CLEANING = "cleaning"
|
|
||||||
CHUNKING = "chunking"
|
|
||||||
EMBEDDING = "embedding"
|
|
||||||
INDEXING = "indexing"
|
|
||||||
|
|
||||||
|
|
||||||
class IngestionAttemptStatus(StrEnum):
|
|
||||||
"""Terminal or active state of one bounded execution attempt."""
|
|
||||||
|
|
||||||
RUNNING = "running"
|
|
||||||
SUCCEEDED = "succeeded"
|
|
||||||
FAILED = "failed"
|
|
||||||
|
|
||||||
|
|
||||||
def _enum_column(enum_type: type[StrEnum], name: str) -> SqlEnum:
|
|
||||||
"""Build consistent portable string enums for OceanBase and SQLite tests."""
|
|
||||||
return SqlEnum(
|
|
||||||
enum_type,
|
|
||||||
name=name,
|
|
||||||
native_enum=False,
|
|
||||||
create_constraint=False,
|
|
||||||
validate_strings=True,
|
|
||||||
values_callable=lambda values: [item.value for item in values],
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
class IngestionJob(Base):
|
|
||||||
"""Canonical orchestration state; broker messages are only delivery hints."""
|
|
||||||
|
|
||||||
__tablename__ = "ingestion_jobs"
|
|
||||||
__table_args__ = (
|
|
||||||
CheckConstraint("attempt_count >= 0", name="attempt_count_non_negative"),
|
|
||||||
CheckConstraint("max_attempts > 0", name="max_attempts_positive"),
|
|
||||||
CheckConstraint("attempt_count <= max_attempts", name="attempt_count_bounded"),
|
|
||||||
CheckConstraint(
|
|
||||||
"status IN ('queued', 'running', 'retrying', 'succeeded', 'failed', 'cancelled')",
|
|
||||||
name="status_valid",
|
|
||||||
),
|
|
||||||
CheckConstraint(
|
|
||||||
"stage IS NULL OR stage IN "
|
|
||||||
"('parsing', 'cleaning', 'chunking', 'embedding', 'indexing')",
|
|
||||||
name="stage_valid",
|
|
||||||
),
|
|
||||||
UniqueConstraint("document_id", name="uq_ingestion_jobs_document_id"),
|
|
||||||
Index("ix_ingestion_jobs_status_next_run", "status", "next_run_at"),
|
|
||||||
)
|
|
||||||
|
|
||||||
id: Mapped[UUID] = mapped_column(Uuid(as_uuid=True), primary_key=True, default=uuid4)
|
|
||||||
document_id: Mapped[UUID] = mapped_column(
|
|
||||||
Uuid(as_uuid=True),
|
|
||||||
ForeignKey("documents.id", ondelete="CASCADE"),
|
|
||||||
nullable=False,
|
|
||||||
)
|
|
||||||
status: Mapped[IngestionJobStatus] = mapped_column(
|
|
||||||
_enum_column(IngestionJobStatus, "ingestion_job_status"),
|
|
||||||
nullable=False,
|
|
||||||
default=IngestionJobStatus.QUEUED,
|
|
||||||
)
|
|
||||||
stage: Mapped[IngestionStage | None] = mapped_column(
|
|
||||||
_enum_column(IngestionStage, "ingestion_stage"),
|
|
||||||
nullable=True,
|
|
||||||
)
|
|
||||||
attempt_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
|
||||||
max_attempts: Mapped[int] = mapped_column(Integer, nullable=False)
|
|
||||||
error_message: Mapped[str | None] = mapped_column(Text, nullable=True)
|
|
||||||
next_run_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
|
||||||
lease_expires_at: Mapped[datetime | None] = mapped_column(
|
|
||||||
DateTime(timezone=True), nullable=True
|
|
||||||
)
|
|
||||||
created_at: Mapped[datetime] = mapped_column(
|
|
||||||
DateTime(timezone=True), nullable=False, server_default=func.now()
|
|
||||||
)
|
|
||||||
updated_at: Mapped[datetime] = mapped_column(
|
|
||||||
DateTime(timezone=True), nullable=False, server_default=func.now(), onupdate=func.now()
|
|
||||||
)
|
|
||||||
completed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
|
||||||
|
|
||||||
|
|
||||||
class IngestionAttempt(Base):
|
|
||||||
"""Immutable-numbered execution history with at most one active row per Job."""
|
|
||||||
|
|
||||||
__tablename__ = "ingestion_attempts"
|
|
||||||
__table_args__ = (
|
|
||||||
CheckConstraint("attempt_number > 0", name="attempt_number_positive"),
|
|
||||||
CheckConstraint(
|
|
||||||
"status IN ('running', 'succeeded', 'failed')",
|
|
||||||
name="status_valid",
|
|
||||||
),
|
|
||||||
CheckConstraint(
|
|
||||||
"stage IS NULL OR stage IN "
|
|
||||||
"('parsing', 'cleaning', 'chunking', 'embedding', 'indexing')",
|
|
||||||
name="stage_valid",
|
|
||||||
),
|
|
||||||
UniqueConstraint("job_id", "attempt_number", name="uq_ingestion_attempts_job_number"),
|
|
||||||
# NULL values may repeat, while every running attempt maps to 1. This
|
|
||||||
# provides portable partial uniqueness on OceanBase MySQL and SQLite.
|
|
||||||
Index("uq_ingestion_attempts_active", "job_id", "active_marker", unique=True),
|
|
||||||
)
|
|
||||||
|
|
||||||
id: Mapped[UUID] = mapped_column(Uuid(as_uuid=True), primary_key=True, default=uuid4)
|
|
||||||
job_id: Mapped[UUID] = mapped_column(
|
|
||||||
Uuid(as_uuid=True),
|
|
||||||
ForeignKey("ingestion_jobs.id", ondelete="CASCADE"),
|
|
||||||
nullable=False,
|
|
||||||
)
|
|
||||||
attempt_number: Mapped[int] = mapped_column(Integer, nullable=False)
|
|
||||||
status: Mapped[IngestionAttemptStatus] = mapped_column(
|
|
||||||
_enum_column(IngestionAttemptStatus, "ingestion_attempt_status"),
|
|
||||||
nullable=False,
|
|
||||||
default=IngestionAttemptStatus.RUNNING,
|
|
||||||
)
|
|
||||||
stage: Mapped[IngestionStage | None] = mapped_column(
|
|
||||||
_enum_column(IngestionStage, "ingestion_attempt_stage"),
|
|
||||||
nullable=True,
|
|
||||||
)
|
|
||||||
retryable: Mapped[bool | None] = mapped_column(Boolean, nullable=True)
|
|
||||||
error_message: Mapped[str | None] = mapped_column(Text, nullable=True)
|
|
||||||
active_marker: Mapped[int | None] = mapped_column(
|
|
||||||
Integer,
|
|
||||||
Computed("CASE WHEN status = 'running' THEN 1 ELSE NULL END"),
|
|
||||||
nullable=True,
|
|
||||||
)
|
|
||||||
started_at: Mapped[datetime] = mapped_column(
|
|
||||||
DateTime(timezone=True), nullable=False, server_default=func.now()
|
|
||||||
)
|
|
||||||
finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
|
||||||
@ -147,7 +147,7 @@ def create_bos_object_storage(
|
|||||||
"""Build the official BOS client from runtime-injected, validated settings."""
|
"""Build the official BOS client from runtime-injected, validated settings."""
|
||||||
if not access_key_id or not secret_access_key:
|
if not access_key_id or not secret_access_key:
|
||||||
raise BosConfigurationError("BOS access key ID and secret access key are required")
|
raise BosConfigurationError("BOS access key ID and secret access key are required")
|
||||||
normalized_endpoint = _normalize_endpoint(endpoint)
|
_validate_endpoint(endpoint)
|
||||||
if not BOS_BUCKET_NAME.fullmatch(bucket_name):
|
if not BOS_BUCKET_NAME.fullmatch(bucket_name):
|
||||||
raise BosConfigurationError("invalid BOS bucket name")
|
raise BosConfigurationError("invalid BOS bucket name")
|
||||||
if timeout_seconds <= 0:
|
if timeout_seconds <= 0:
|
||||||
@ -164,7 +164,7 @@ def create_bos_object_storage(
|
|||||||
|
|
||||||
config = BceClientConfiguration(
|
config = BceClientConfiguration(
|
||||||
credentials=BceCredentials(access_key_id, secret_access_key),
|
credentials=BceCredentials(access_key_id, secret_access_key),
|
||||||
endpoint=normalized_endpoint,
|
endpoint=endpoint,
|
||||||
connection_timeout_in_mills=int(timeout_seconds * 1000),
|
connection_timeout_in_mills=int(timeout_seconds * 1000),
|
||||||
)
|
)
|
||||||
client = BosClient(config)
|
client = BosClient(config)
|
||||||
@ -192,10 +192,9 @@ def _status_code(error: Exception) -> int | None:
|
|||||||
return status_code if isinstance(status_code, int) else None
|
return status_code if isinstance(status_code, int) else None
|
||||||
|
|
||||||
|
|
||||||
def _normalize_endpoint(endpoint: str) -> str:
|
def _validate_endpoint(endpoint: str) -> None:
|
||||||
"""Validate a BOS origin and force host-only legacy settings onto HTTPS."""
|
"""Require an explicit HTTP(S) origin without embedded credentials or paths."""
|
||||||
candidate = endpoint if "://" in endpoint else f"https://{endpoint}"
|
parsed = urlsplit(endpoint)
|
||||||
parsed = urlsplit(candidate)
|
|
||||||
if (
|
if (
|
||||||
parsed.scheme not in {"http", "https"}
|
parsed.scheme not in {"http", "https"}
|
||||||
or not parsed.hostname
|
or not parsed.hostname
|
||||||
@ -206,4 +205,3 @@ def _normalize_endpoint(endpoint: str) -> str:
|
|||||||
or parsed.fragment
|
or parsed.fragment
|
||||||
):
|
):
|
||||||
raise BosConfigurationError("invalid BOS endpoint")
|
raise BosConfigurationError("invalid BOS endpoint")
|
||||||
return candidate.rstrip("/")
|
|
||||||
|
|||||||
@ -336,16 +336,16 @@ def _validate_uri(uri: str) -> None:
|
|||||||
|
|
||||||
|
|
||||||
def _validate_oceanbase_version(version_nums: object) -> None:
|
def _validate_oceanbase_version(version_nums: object) -> None:
|
||||||
"""Require the tested OceanBase 4.3.5 patch level for filtered ANN."""
|
"""Require the first release with an explicit vector pre-filter hint."""
|
||||||
if (
|
if (
|
||||||
not isinstance(version_nums, (list, tuple))
|
not isinstance(version_nums, (list, tuple))
|
||||||
or len(version_nums) != 4
|
or len(version_nums) != 4
|
||||||
or any(isinstance(part, bool) or not isinstance(part, int) for part in version_nums)
|
or any(isinstance(part, bool) or not isinstance(part, int) for part in version_nums)
|
||||||
):
|
):
|
||||||
raise OceanBaseVectorStoreError("OceanBase version validation failed")
|
raise OceanBaseVectorStoreError("OceanBase version validation failed")
|
||||||
if tuple(version_nums) < (4, 3, 5, 4):
|
if tuple(version_nums) < (4, 6, 0, 0):
|
||||||
raise OceanBaseConfigurationError(
|
raise OceanBaseConfigurationError(
|
||||||
"OceanBase 4.3.5.4 or newer is required for filtered vector search"
|
"OceanBase 4.6.0 or newer is required for vector pre-filtering"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@ -11,10 +11,8 @@ requires-python = ">=3.12,<3.14"
|
|||||||
dependencies = [
|
dependencies = [
|
||||||
"alembic>=1.18,<2",
|
"alembic>=1.18,<2",
|
||||||
"bce-python-sdk>=0.9.72,<0.10",
|
"bce-python-sdk>=0.9.72,<0.10",
|
||||||
"celery>=5.5,<6",
|
|
||||||
"fastapi>=0.115,<1",
|
"fastapi>=0.115,<1",
|
||||||
"mcp>=2,<3",
|
"psycopg[binary]>=3.2,<4",
|
||||||
"pymysql>=1.1,<2",
|
|
||||||
"pymupdf>=1.26,<2",
|
"pymupdf>=1.26,<2",
|
||||||
"pydantic-settings>=2.7,<3",
|
"pydantic-settings>=2.7,<3",
|
||||||
"python-multipart>=0.0.20,<1",
|
"python-multipart>=0.0.20,<1",
|
||||||
@ -26,9 +24,6 @@ dependencies = [
|
|||||||
|
|
||||||
[project.scripts]
|
[project.scripts]
|
||||||
ekp-index-rebuild = "ekp_api.index_rebuild:main"
|
ekp-index-rebuild = "ekp_api.index_rebuild:main"
|
||||||
ekp-live-acceptance = "ekp_api.live_acceptance:main"
|
|
||||||
ekp-mcp = "ekp_mcp.main:main"
|
|
||||||
ekp-worker = "ekp_api.worker:main"
|
|
||||||
|
|
||||||
[dependency-groups]
|
[dependency-groups]
|
||||||
dev = [
|
dev = [
|
||||||
@ -42,7 +37,6 @@ dev = [
|
|||||||
[tool.setuptools.packages.find]
|
[tool.setuptools.packages.find]
|
||||||
where = [
|
where = [
|
||||||
"apps/api/src",
|
"apps/api/src",
|
||||||
"apps/mcp-server/src",
|
|
||||||
"packages/core/src",
|
"packages/core/src",
|
||||||
"packages/domain/src",
|
"packages/domain/src",
|
||||||
"packages/providers/object_storage/src",
|
"packages/providers/object_storage/src",
|
||||||
@ -62,7 +56,6 @@ select = ["ASYNC", "B", "E4", "E7", "E9", "F", "I", "UP"]
|
|||||||
[tool.ruff.lint.isort]
|
[tool.ruff.lint.isort]
|
||||||
known-first-party = [
|
known-first-party = [
|
||||||
"ekp_api",
|
"ekp_api",
|
||||||
"ekp_mcp",
|
|
||||||
"ekp_core",
|
"ekp_core",
|
||||||
"ekp_domain",
|
"ekp_domain",
|
||||||
"ekp_embedding",
|
"ekp_embedding",
|
||||||
@ -77,7 +70,6 @@ python_version = "3.12"
|
|||||||
strict = true
|
strict = true
|
||||||
mypy_path = [
|
mypy_path = [
|
||||||
"apps/api/src",
|
"apps/api/src",
|
||||||
"apps/mcp-server/src",
|
|
||||||
"packages/core/src",
|
"packages/core/src",
|
||||||
"packages/domain/src",
|
"packages/domain/src",
|
||||||
"packages/providers/object_storage/src",
|
"packages/providers/object_storage/src",
|
||||||
|
|||||||
@ -1,130 +0,0 @@
|
|||||||
#!/usr/bin/env bash
|
|
||||||
set -euo pipefail
|
|
||||||
|
|
||||||
# DEPLOY-002 accepts a values file path only. Secret contents remain in
|
|
||||||
# Kubernetes and are never expanded into this process or Helm arguments.
|
|
||||||
values_file="${1:-}"
|
|
||||||
chart_dir="${CHART:-deploy/helm/knowledge-platform}"
|
|
||||||
release_name="${RELEASE:-knowledge-platform}"
|
|
||||||
namespace="${NAMESPACE:-knowledge-platform}"
|
|
||||||
deploy_timeout="${DEPLOY_TIMEOUT:-10m}"
|
|
||||||
|
|
||||||
if [[ -z "${values_file}" || ! -f "${values_file}" ]]; then
|
|
||||||
echo "VALUES must point to an existing environment values file" >&2
|
|
||||||
exit 2
|
|
||||||
fi
|
|
||||||
|
|
||||||
for tool in helm kubectl; do
|
|
||||||
if ! command -v "${tool}" >/dev/null 2>&1; then
|
|
||||||
echo "required deployment tool is missing: ${tool}" >&2
|
|
||||||
exit 2
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
|
|
||||||
# Helm 4 renamed --atomic to --rollback-on-failure. Select the equivalent flag
|
|
||||||
# by major version while preserving compatibility with supported Helm 3.
|
|
||||||
helm_version="$(helm version --template '{{.Version}}')"
|
|
||||||
if [[ "${helm_version}" =~ ^v?([0-9]+) ]] && (( BASH_REMATCH[1] >= 4 )); then
|
|
||||||
rollback_flag=(--rollback-on-failure)
|
|
||||||
else
|
|
||||||
rollback_flag=(--atomic)
|
|
||||||
fi
|
|
||||||
|
|
||||||
helm lint "${chart_dir}" --values "${values_file}"
|
|
||||||
|
|
||||||
# The test-hook ConfigMap contains resource names only. Its fixed, controlled
|
|
||||||
# shape lets awk extract names without adding a YAML parser or reading Secrets.
|
|
||||||
preflight_yaml="$({
|
|
||||||
helm template "${release_name}" "${chart_dir}" \
|
|
||||||
--namespace "${namespace}" \
|
|
||||||
--values "${values_file}" \
|
|
||||||
--show-only templates/tests/preflight-references.yaml
|
|
||||||
})"
|
|
||||||
runtime_secret="$({
|
|
||||||
printf '%s\n' "${preflight_yaml}" | awk '$1 == "runtimeSecret:" {gsub(/\"/, "", $2); print $2}'
|
|
||||||
})"
|
|
||||||
ingress_enabled="$({
|
|
||||||
printf '%s\n' "${preflight_yaml}" | awk '$1 == "ingressEnabled:" {gsub(/\"/, "", $2); print $2}'
|
|
||||||
})"
|
|
||||||
rabbitmq_enabled="$({
|
|
||||||
printf '%s\n' "${preflight_yaml}" | awk '$1 == "rabbitmqEnabled:" {gsub(/\"/, "", $2); print $2}'
|
|
||||||
})"
|
|
||||||
|
|
||||||
if [[ -z "${runtime_secret}" || -z "${ingress_enabled}" || -z "${rabbitmq_enabled}" ]]; then
|
|
||||||
echo "failed to resolve Helm Secret references" >&2
|
|
||||||
exit 2
|
|
||||||
fi
|
|
||||||
|
|
||||||
secret_has_key() {
|
|
||||||
local secret_name="$1"
|
|
||||||
local key_name="$2"
|
|
||||||
local present
|
|
||||||
present="$({
|
|
||||||
kubectl -n "${namespace}" get secret "${secret_name}" \
|
|
||||||
-o go-template="{{with .data}}{{if index . \"${key_name}\"}}present{{end}}{{end}}"
|
|
||||||
})"
|
|
||||||
[[ "${present}" == "present" ]]
|
|
||||||
}
|
|
||||||
|
|
||||||
if ! kubectl get namespace "${namespace}" >/dev/null; then
|
|
||||||
echo "target namespace does not exist: ${namespace}" >&2
|
|
||||||
exit 2
|
|
||||||
fi
|
|
||||||
if ! kubectl -n "${namespace}" get secret "${runtime_secret}" >/dev/null; then
|
|
||||||
echo "referenced runtime Secret does not exist: ${runtime_secret}" >&2
|
|
||||||
exit 2
|
|
||||||
fi
|
|
||||||
required_runtime_keys=(
|
|
||||||
DATABASE_URL
|
|
||||||
RABBITMQ_URL
|
|
||||||
BOS_ACCESS_KEY_ID
|
|
||||||
BOS_SECRET_ACCESS_KEY
|
|
||||||
BAIDU_QIANFAN_API_KEY
|
|
||||||
OCEANBASE_VECTOR_PASSWORD
|
|
||||||
MCP_AUTH_TOKEN
|
|
||||||
)
|
|
||||||
for key in "${required_runtime_keys[@]}"; do
|
|
||||||
if ! secret_has_key "${runtime_secret}" "${key}"; then
|
|
||||||
echo "referenced runtime Secret is missing required key: ${key}" >&2
|
|
||||||
exit 2
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
|
|
||||||
if [[ "${ingress_enabled}" == "true" ]]; then
|
|
||||||
for key in apiTlsSecret mcpTlsSecret; do
|
|
||||||
tls_secret="$({
|
|
||||||
printf '%s\n' "${preflight_yaml}" | awk -v field="${key}:" '$1 == field {gsub(/\"/, "", $2); print $2}'
|
|
||||||
})"
|
|
||||||
if ! kubectl -n "${namespace}" get secret "${tls_secret}" >/dev/null; then
|
|
||||||
echo "referenced TLS Secret does not exist: ${tls_secret}" >&2
|
|
||||||
exit 2
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [[ "${rabbitmq_enabled}" == "true" ]]; then
|
|
||||||
rabbitmq_secret="$({
|
|
||||||
printf '%s\n' "${preflight_yaml}" | awk '$1 == "rabbitmqSecret:" {gsub(/\"/, "", $2); print $2}'
|
|
||||||
})"
|
|
||||||
if ! kubectl -n "${namespace}" get secret "${rabbitmq_secret}" >/dev/null; then
|
|
||||||
echo "referenced RabbitMQ Secret does not exist: ${rabbitmq_secret}" >&2
|
|
||||||
exit 2
|
|
||||||
fi
|
|
||||||
for key in username password erlang-cookie; do
|
|
||||||
if ! secret_has_key "${rabbitmq_secret}" "${key}"; then
|
|
||||||
echo "referenced RabbitMQ Secret is missing required key: ${key}" >&2
|
|
||||||
exit 2
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
fi
|
|
||||||
|
|
||||||
# The version-specific rollback flag keeps the previous release active when a
|
|
||||||
# pre-upgrade migration or rollout fails. Helm serializes operations for this
|
|
||||||
# release and waits for hook/workload readiness before returning success.
|
|
||||||
helm upgrade --install "${release_name}" "${chart_dir}" \
|
|
||||||
--namespace "${namespace}" \
|
|
||||||
--values "${values_file}" \
|
|
||||||
"${rollback_flag[@]}" \
|
|
||||||
--wait \
|
|
||||||
--timeout "${deploy_timeout}" \
|
|
||||||
--history-max 10
|
|
||||||
@ -24,22 +24,18 @@ class GoldenKeywordEmbeddingProvider:
|
|||||||
"""Small semantic fixture that keeps retrieval evaluation offline and stable."""
|
"""Small semantic fixture that keeps retrieval evaluation offline and stable."""
|
||||||
|
|
||||||
model_name = "golden-keyword-v1"
|
model_name = "golden-keyword-v1"
|
||||||
dimensions = 4
|
dimensions = 2
|
||||||
max_batch_size = 16
|
max_batch_size = 16
|
||||||
|
|
||||||
def embed(self, texts: Sequence[str]) -> tuple[tuple[float, ...], ...]:
|
def embed(self, texts: Sequence[str]) -> tuple[tuple[float, ...], ...]:
|
||||||
vectors: list[tuple[float, ...]] = []
|
vectors: list[tuple[float, ...]] = []
|
||||||
for text in texts:
|
for text in texts:
|
||||||
if "报销" in text or "行程单" in text or "票据" in text:
|
if "住宿" in text or "出差" in text:
|
||||||
vectors.append((0.0, 0.0, 0.0, 1.0))
|
vectors.append((1.0, 0.0))
|
||||||
elif "住宿" in text or "出差" in text or "酒店" in text:
|
|
||||||
vectors.append((1.0, 0.0, 0.0, 0.0))
|
|
||||||
elif "支持" in text or "问题" in text or "联系方式" in text:
|
|
||||||
vectors.append((0.0, 0.0, 1.0, 0.0))
|
|
||||||
elif "退款" in text or "产品A" in text or "产品 A" in text:
|
elif "退款" in text or "产品A" in text or "产品 A" in text:
|
||||||
vectors.append((0.0, 1.0, 0.0, 0.0))
|
vectors.append((0.0, 1.0))
|
||||||
else:
|
else:
|
||||||
vectors.append((0.1, 0.1, 0.1, 0.1))
|
vectors.append((0.1, 0.1))
|
||||||
return tuple(vectors)
|
return tuple(vectors)
|
||||||
|
|
||||||
|
|
||||||
@ -58,7 +54,7 @@ def test_dense_top_twenty_recalls_each_golden_source(tmp_path: Path) -> None:
|
|||||||
)
|
)
|
||||||
embedding = GoldenKeywordEmbeddingProvider()
|
embedding = GoldenKeywordEmbeddingProvider()
|
||||||
vector_store = FakeVectorStore()
|
vector_store = FakeVectorStore()
|
||||||
profile = IndexProfile("golden-dense-v1", "structure_v1", embedding.model_name, 4)
|
profile = IndexProfile("golden-dense-v1", "structure_v1", embedding.model_name, 2)
|
||||||
indexing = DocumentIndexingService(session_factory, embedding, vector_store, profile)
|
indexing = DocumentIndexingService(session_factory, embedding, vector_store, profile)
|
||||||
pipeline = DocumentIndexingPipelineService(
|
pipeline = DocumentIndexingPipelineService(
|
||||||
DocumentPipelineService(upload, parsing, chunking),
|
DocumentPipelineService(upload, parsing, chunking),
|
||||||
|
|||||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
x
Reference in New Issue
Block a user