Compare commits

..

No commits in common. "codex/deploy-003-cluster-acceptance" and "main" have entirely different histories.

35 changed files with 15 additions and 1177 deletions

View File

@ -1,4 +1,4 @@
.PHONY: help install run run-mcp run-worker images images-print images-push deploy deploy-lint deploy-render cluster-acceptance mcp-remote-acceptance format format-check lint typecheck test test-unit test-integration migration-check eval live-acceptance up down
.PHONY: help install run run-mcp run-worker images images-print images-push deploy deploy-lint deploy-render format format-check lint typecheck test test-unit test-integration migration-check eval live-acceptance up down
UV_CACHE_DIR ?= /tmp/ekp-uv-cache
export UV_CACHE_DIR
@ -15,8 +15,6 @@ help:
@echo " deploy Install or upgrade Kubernetes with VALUES=<file>"
@echo " deploy-lint Validate Helm values and templates"
@echo " deploy-render Render non-secret Kubernetes manifests"
@echo " cluster-acceptance Validate the explicitly selected target cluster"
@echo " mcp-remote-acceptance Verify remote MCP protocol and real search"
@echo " format Format code"
@echo " format-check Check formatting without modifying files"
@echo " lint Run linters"
@ -52,10 +50,6 @@ deploy-lint:
deploy-render:
@test -n "$(VALUES)" || (echo "VALUES must point to an environment values file" >&2; exit 2)
helm template "$${RELEASE:-knowledge-platform}" deploy/helm/knowledge-platform --namespace "$${NAMESPACE:-knowledge-platform}" --values "$(VALUES)"
cluster-acceptance:
./scripts/cluster_acceptance.sh "$(VALUES)"
mcp-remote-acceptance:
uv run python scripts/mcp_remote_acceptance.py
format:
uv run ruff format .
format-check:

View File

@ -69,11 +69,6 @@ The Streamable HTTP endpoint defaults to `http://127.0.0.1:8766/mcp` and exposes
only `search_knowledge`. See `docs/development/MCP_SETUP.md` for WorkBuddy configuration,
identity boundaries, remote allowlists, and protocol verification.
Kubernetes installation, target-cluster acceptance, rolling recovery, metrics and backup
responsibility are documented in `docs/development/KUBERNETES_DEPLOYMENT.md` and
`docs/development/CLUSTER_OPERATIONS.md`. Cluster acceptance always requires an explicitly named
Kubernetes context; credentials remain in Kubernetes Secret or the client process environment.
The health endpoint is available at `http://localhost:8000/healthz`, and the
versioned OpenAPI document is available at
`http://localhost:8000/api/v1/openapi.json`.

View File

@ -24,9 +24,6 @@ def create_celery_app(settings: Settings) -> Celery:
task_reject_on_worker_lost=True,
worker_prefetch_multiplier=1,
broker_connection_retry_on_startup=True,
# Preserve the shared JSON handler instead of letting Celery replace it
# with process-specific plain-text formatting.
worker_hijack_root_logger=False,
timezone="UTC",
enable_utc=True,
task_routes={

View File

@ -1,26 +1,17 @@
"""FastAPI application entry point."""
import logging
import re
from collections.abc import Awaitable, Callable
from pathlib import Path
from time import monotonic
from uuid import uuid4
from fastapi import FastAPI, Request, Response
from fastapi.responses import FileResponse, PlainTextResponse
from fastapi import FastAPI
from fastapi.responses import FileResponse
from fastapi.staticfiles import StaticFiles
from pydantic import BaseModel
from ekp_api.metrics import HttpMetrics
from ekp_api.providers import ApiServices
from ekp_api.routes import router
from ekp_core.config import Settings, get_settings
from ekp_core.observability import bind_request_id, configure_json_logging, reset_request_id
ADMIN_WEB_ROOT = Path(__file__).resolve().parents[3] / "admin-web"
REQUEST_ID_PATTERN = re.compile(r"^[A-Za-z0-9._:-]{1,64}$")
logger = logging.getLogger(__name__)
class HealthResponse(BaseModel):
@ -37,7 +28,6 @@ def create_app(
) -> FastAPI:
"""Create the API application with explicit, testable settings."""
resolved_settings = settings or get_settings()
configure_json_logging(resolved_settings.log_level)
application = FastAPI(
title=resolved_settings.app_name,
version="0.1.0",
@ -46,7 +36,6 @@ def create_app(
)
application.state.settings = resolved_settings
application.state.api_services = api_services
application.state.http_metrics = HttpMetrics()
application.include_router(router, prefix=resolved_settings.api_prefix)
admin_web_root = (resolved_settings.admin_web_root or ADMIN_WEB_ROOT).resolve()
@ -58,48 +47,6 @@ def create_app(
name="admin-assets",
)
@application.middleware("http")
async def observe_http(
request: Request,
call_next: Callable[[Request], Awaitable[Response]],
) -> Response:
"""Correlate requests and emit bounded metrics without body/query data."""
incoming_request_id = request.headers.get("x-request-id", "")
request_id = (
incoming_request_id
if REQUEST_ID_PATTERN.fullmatch(incoming_request_id)
else uuid4().hex
)
token = bind_request_id(request_id)
started_at = monotonic()
status_code = 500
try:
response = await call_next(request)
status_code = response.status_code
response.headers["x-request-id"] = request_id
return response
finally:
duration_seconds = monotonic() - started_at
route = request.scope.get("route")
route_path = getattr(route, "path", "unmatched")
application.state.http_metrics.observe(
request.method,
route_path,
status_code,
duration_seconds,
)
logger.info(
"HTTP request completed",
extra={
"event": "http_request_completed",
"method": request.method,
"path": route_path,
"status_code": status_code,
"duration_ms": round(duration_seconds * 1000, 3),
},
)
reset_request_id(token)
@application.get("/admin", include_in_schema=False)
@application.get("/admin/upload", include_in_schema=False)
@application.get("/admin/documents", include_in_schema=False)
@ -116,14 +63,6 @@ def create_app(
environment=resolved_settings.app_env,
)
@application.get("/metrics", include_in_schema=False)
async def metrics() -> PlainTextResponse:
"""Expose aggregate process/HTTP metrics without provider or user data."""
return PlainTextResponse(
application.state.http_metrics.render(),
media_type="text/plain; version=0.0.4",
)
return application

View File

@ -1,59 +0,0 @@
"""Small dependency-free Prometheus endpoint for cluster smoke monitoring."""
from collections import defaultdict
from threading import Lock
from time import monotonic
class HttpMetrics:
"""Keep bounded per-process HTTP counters using route templates as labels."""
def __init__(self) -> None:
self._started_at = monotonic()
self._lock = Lock()
self._counts: defaultdict[tuple[str, str, int], int] = defaultdict(int)
self._durations: defaultdict[tuple[str, str], tuple[int, float]] = defaultdict(
lambda: (0, 0.0)
)
def observe(self, method: str, path: str, status_code: int, duration_seconds: float) -> None:
"""Record one request without retaining query strings or user content."""
with self._lock:
self._counts[(method, path, status_code)] += 1
count, total = self._durations[(method, path)]
self._durations[(method, path)] = (count + 1, total + duration_seconds)
def render(self) -> str:
"""Return Prometheus text exposition with stable low-cardinality labels."""
with self._lock:
counts = tuple(sorted(self._counts.items()))
durations = tuple(sorted(self._durations.items()))
lines = [
"# HELP ekp_process_uptime_seconds Process uptime in seconds.",
"# TYPE ekp_process_uptime_seconds gauge",
f"ekp_process_uptime_seconds {monotonic() - self._started_at:.6f}",
"# HELP ekp_http_requests_total HTTP requests handled by this API process.",
"# TYPE ekp_http_requests_total counter",
]
for (method, path, status_code), value in counts:
labels = _labels(method=method, path=path, status=str(status_code))
lines.append(f"ekp_http_requests_total{{{labels}}} {value}")
lines.extend(
(
"# HELP ekp_http_request_duration_seconds HTTP request duration.",
"# TYPE ekp_http_request_duration_seconds summary",
)
)
for (method, path), (count, total) in durations:
labels = _labels(method=method, path=path)
lines.append(f"ekp_http_request_duration_seconds_count{{{labels}}} {count}")
lines.append(f"ekp_http_request_duration_seconds_sum{{{labels}}} {total:.6f}")
return "\n".join(lines) + "\n"
def _labels(**values: str) -> str:
"""Escape the limited label set according to Prometheus text rules."""
return ",".join(
f'{name}="{value.replace(chr(92), chr(92) * 2).replace(chr(10), chr(92) + "n").replace(chr(34), chr(92) + chr(34))}"'
for name, value in values.items()
)

View File

@ -1,6 +1,5 @@
"""Versioned document and search routes for the retrieval Demo."""
import logging
from typing import Annotated
from uuid import UUID
@ -31,7 +30,6 @@ from ekp_core.ingestion_batches import BatchFileCountLimitError, BatchNotFoundEr
from ekp_core.ingestion_jobs import IngestionJobNotFoundError
router = APIRouter()
logger = logging.getLogger(__name__)
def get_api_services(request: Request) -> ApiServices:
@ -42,10 +40,6 @@ def get_api_services(request: Request) -> ApiServices:
try:
services = create_api_services(request.app.state.settings)
except ProviderConfigurationError as error:
logger.error(
"API provider composition failed",
extra={"event": "api_provider_configuration_failed"},
)
raise HTTPException(
status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
detail="API providers are not fully configured",
@ -87,22 +81,10 @@ def upload_document(
) from error
except Exception as error:
# Provider and persistence exceptions are intentionally not reflected to clients.
logger.error(
"Document acceptance failed",
extra={"event": "document_acceptance_failed"},
)
raise HTTPException(
status_code=status.HTTP_502_BAD_GATEWAY,
detail="document acceptance failed",
) from error
logger.info(
"Document accepted for ingestion",
extra={
"event": "document_accepted",
"document_id": str(submission.document.id),
"job_id": str(submission.job.id),
},
)
return IngestionAcceptedResponse.from_submission(submission)
@ -149,18 +131,10 @@ def upload_ingestion_batch(
detail="batch file count exceeds the configured limit",
) from error
except Exception as error:
logger.error(
"Batch acceptance failed",
extra={"event": "ingestion_batch_acceptance_failed"},
)
raise HTTPException(
status_code=status.HTTP_502_BAD_GATEWAY,
detail="batch could not be accepted",
) from error
logger.info(
"Ingestion batch accepted",
extra={"event": "ingestion_batch_accepted", "batch_id": str(view.batch.id)},
)
return IngestionBatchResponse.from_view(view)
@ -220,22 +194,11 @@ def delete_document(document_id: UUID, services: Services) -> Response:
detail="document ingestion is still active",
) from error
except DocumentArtifactCleanupError as error:
logger.error(
"Document artifact cleanup failed",
extra={
"event": "document_artifact_cleanup_failed",
"document_id": str(document_id),
},
)
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail="document metadata was deleted but artifact cleanup failed",
) from error
except Exception as error:
logger.error(
"Document deletion failed",
extra={"event": "document_deletion_failed", "document_id": str(document_id)},
)
raise HTTPException(
status_code=status.HTTP_502_BAD_GATEWAY,
detail="document deletion failed",
@ -254,10 +217,6 @@ def search_knowledge(payload: SearchRequest, services: Services) -> SearchRespon
detail="search query must contain non-whitespace text",
) from error
except Exception as error:
logger.error(
"Knowledge search provider chain failed",
extra={"event": "knowledge_search_failed"},
)
raise HTTPException(
status_code=status.HTTP_502_BAD_GATEWAY,
detail="search providers failed",

View File

@ -1,6 +1,5 @@
"""Celery worker entry point for durable ingestion Jobs."""
import logging
from functools import lru_cache
from typing import Any
from uuid import UUID
@ -13,12 +12,9 @@ from ekp_api.celery_queue import (
from ekp_api.providers import WorkerServices, create_worker_services
from ekp_core.config import get_settings
from ekp_core.ingestion_jobs import JobRunAction, dispatch_pending_jobs
from ekp_core.observability import configure_json_logging
_settings = get_settings()
configure_json_logging(_settings.log_level)
celery_app = create_celery_app(_settings)
logger = logging.getLogger(__name__)
@lru_cache
@ -38,20 +34,7 @@ def run_ingestion(self: Any, job_id: str) -> None:
Celery delivery retries are unbounded because valid-lease deferrals are not
business Attempts; the durable runner independently enforces max_attempts.
"""
logger.info(
"Ingestion delivery received",
extra={"event": "ingestion_delivery_received", "job_id": job_id},
)
result = _worker_services().runner.run(UUID(job_id))
logger.info(
"Ingestion delivery completed",
extra={
"event": "ingestion_delivery_completed",
"job_id": job_id,
"job_status": result.status.value,
"retry_delay_seconds": result.retry_delay_seconds,
},
)
if result.action == JobRunAction.RETRY:
raise self.retry(countdown=result.retry_delay_seconds or 1, max_retries=None)
@ -60,15 +43,7 @@ def run_ingestion(self: Any, job_id: str) -> None:
def recover_ingestion_jobs() -> int:
"""Redeliver due durable Jobs after API/broker or worker interruptions."""
services = _worker_services()
dispatch_count = dispatch_pending_jobs(services.jobs, services.dispatcher)
logger.info(
"Durable ingestion recovery scan completed",
extra={
"event": "ingestion_recovery_scan_completed",
"dispatch_count": dispatch_count,
},
)
return dispatch_count
return dispatch_pending_jobs(services.jobs, services.dispatcher)
def main() -> None:

View File

@ -3,14 +3,12 @@
from mcp.server.transport_security import TransportSecuritySettings
from ekp_core.config import get_settings
from ekp_core.observability import configure_json_logging
from ekp_mcp.server import create_mcp_server
def main() -> None:
"""Run the authenticated Streamable HTTP server from environment settings."""
settings = get_settings()
configure_json_logging(settings.log_level)
server = create_mcp_server(settings)
server.run(
transport="streamable-http",

View File

@ -100,10 +100,7 @@ def create_mcp_server(
# Do not reflect provider, database, endpoint, or credential details
# into MCP content or logs. Operations can correlate this generic
# event with upstream sanitized service telemetry.
logger.error(
"MCP knowledge search failed",
extra={"event": "mcp_knowledge_search_failed"},
)
logger.error("MCP knowledge search failed")
raise ToolError("knowledge search failed") from None
return server

View File

@ -17,16 +17,6 @@ images:
tag: "0.1.0"
digest: ""
podDisruptionBudget:
enabled: true
apiMinAvailable: 1
mcpMinAvailable: 1
observability:
metrics:
enabled: true
path: /metrics
ingress:
enabled: true
className: nginx

View File

@ -64,14 +64,6 @@ app.kubernetes.io/instance: {{ .Release.Name }}
{{- if gt (float64 .Values.config.retrieval.searchTopK) (float64 .Values.config.retrieval.denseTopN) -}}
{{- fail "config.retrieval.searchTopK must be <= denseTopN" -}}
{{- end -}}
{{- if .Values.podDisruptionBudget.enabled -}}
{{- if ge (int .Values.podDisruptionBudget.apiMinAvailable) (int .Values.api.replicaCount) -}}
{{- fail "podDisruptionBudget.apiMinAvailable must be < api.replicaCount" -}}
{{- end -}}
{{- if ge (int .Values.podDisruptionBudget.mcpMinAvailable) (int .Values.mcp.replicaCount) -}}
{{- fail "podDisruptionBudget.mcpMinAvailable must be < mcp.replicaCount" -}}
{{- end -}}
{{- end -}}
{{- end -}}
{{- define "knowledge-platform.podSecurityContext" -}}

View File

@ -28,11 +28,6 @@ spec:
annotations:
checksum/runtime-config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }}
checksum/runtime-secret-reference: {{ .Values.existingSecret | sha256sum }}
{{- if .Values.observability.metrics.enabled }}
prometheus.io/scrape: "true"
prometheus.io/port: "8000"
prometheus.io/path: {{ .Values.observability.metrics.path | quote }}
{{- end }}
{{- with .Values.podAnnotations }}
{{- toYaml . | nindent 8 }}
{{- end }}

View File

@ -1,31 +0,0 @@
{{- if .Values.podDisruptionBudget.enabled }}
# DEPLOY-003 protects both user-facing paths during voluntary disruptions. PDBs
# do not cover involuntary failures, which remain handled by replica recovery.
apiVersion: policy/v1
kind: PodDisruptionBudget
metadata:
name: {{ include "knowledge-platform.fullname" . }}-api
labels:
{{- include "knowledge-platform.labels" . | nindent 4 }}
app.kubernetes.io/component: api
spec:
minAvailable: {{ .Values.podDisruptionBudget.apiMinAvailable }}
selector:
matchLabels:
{{- include "knowledge-platform.selectorLabels" . | nindent 6 }}
app.kubernetes.io/component: api
---
apiVersion: policy/v1
kind: PodDisruptionBudget
metadata:
name: {{ include "knowledge-platform.fullname" . }}-mcp
labels:
{{- include "knowledge-platform.labels" . | nindent 4 }}
app.kubernetes.io/component: mcp
spec:
minAvailable: {{ .Values.podDisruptionBudget.mcpMinAvailable }}
selector:
matchLabels:
{{- include "knowledge-platform.selectorLabels" . | nindent 6 }}
app.kubernetes.io/component: mcp
{{- end }}

View File

@ -11,13 +11,6 @@ metadata:
data:
runtimeSecret: {{ .Values.existingSecret | quote }}
ingressEnabled: {{ .Values.ingress.enabled | quote }}
apiHost: {{ .Values.ingress.api.host | quote }}
mcpHost: {{ .Values.ingress.mcp.host | quote }}
apiDeployment: {{ printf "%s-api" (include "knowledge-platform.fullname" .) | quote }}
mcpDeployment: {{ printf "%s-mcp" (include "knowledge-platform.fullname" .) | quote }}
workerDeployment: {{ printf "%s-worker" (include "knowledge-platform.fullname" .) | quote }}
schedulerDeployment: {{ printf "%s-scheduler" (include "knowledge-platform.fullname" .) | quote }}
metricsPath: {{ .Values.observability.metrics.path | quote }}
apiTlsSecret: {{ .Values.ingress.api.tlsSecretName | quote }}
mcpTlsSecret: {{ .Values.ingress.mcp.tlsSecretName | quote }}
rabbitmqEnabled: {{ .Values.rabbitmq.enabled | quote }}

View File

@ -7,8 +7,6 @@
"existingSecret",
"images",
"serviceAccount",
"podDisruptionBudget",
"observability",
"api",
"mcp",
"worker",
@ -53,32 +51,6 @@
},
"podAnnotations": {"$ref": "#/definitions/stringMap"},
"podLabels": {"$ref": "#/definitions/stringMap"},
"podDisruptionBudget": {
"type": "object",
"additionalProperties": false,
"required": ["enabled", "apiMinAvailable", "mcpMinAvailable"],
"properties": {
"enabled": {"type": "boolean"},
"apiMinAvailable": {"type": "integer", "minimum": 1},
"mcpMinAvailable": {"type": "integer", "minimum": 1}
}
},
"observability": {
"type": "object",
"additionalProperties": false,
"required": ["metrics"],
"properties": {
"metrics": {
"type": "object",
"additionalProperties": false,
"required": ["enabled", "path"],
"properties": {
"enabled": {"type": "boolean"},
"path": {"const": "/metrics"}
}
}
}
},
"api": {"$ref": "#/definitions/httpWorkload"},
"mcp": {"$ref": "#/definitions/httpWorkload"},
"worker": {

View File

@ -31,18 +31,6 @@ serviceAccount:
podAnnotations: {}
podLabels: {}
# DEPLOY-003 keeps one API and MCP replica available during voluntary node
# maintenance. This complements, rather than replaces, RollingUpdate settings.
podDisruptionBudget:
enabled: true
apiMinAvailable: 1
mcpMinAvailable: 1
observability:
metrics:
enabled: true
path: /metrics
api:
replicaCount: 2
service:

View File

@ -76,12 +76,6 @@ Ingress 是否进一步认证属于部署配置,但前端隐藏按钮或隐藏
集群重启、扩缩容和滚动部署不得改变本契约或要求浏览器重新注入 Provider 配置。数据库、BOS、
模型与队列凭据由服务端 Secret 提供,任何 API 响应都不得返回这些值。
DEPLOY-003 增加两个不进入 `/api/v1` OpenAPI 的运维端点:`GET /healthz` 只表示 API 进程可用;
`GET /metrics` 使用 Prometheus text format 返回进程存活时间、按 HTTP method/路由模板/status
聚合的请求量和按 method/路由模板聚合的耗时。两者不得包含 Query、文件名、正文、身份、
Provider Endpoint 或凭据。响应附带安全 `X-Request-ID`,客户端提供的值只有匹配
`[A-Za-z0-9._:-]{1,64}` 时才会沿用。
## 异步入库契约(DEMO-015~016 已实现)
| Method | Path | 行为 |

View File

@ -49,7 +49,3 @@ ACL 或用户委托授权。
Credential 由 Kubernetes Secret 在服务端运行时注入,客户端只保存管理员分发的最小
`knowledge:search` Token;Pod 重启或滚动升级不得要求重新生成配置。公共知识域不等于匿名
访问或超级管理员身份,未来引入非公共知识前必须升级为用户委托身份与 ACL-aware Search。
DEPLOY-003 的远程验收必须从最终 HTTPS 地址只发现 `search_knowledge` 并完成一次真实、有来源
结果的调用。Gateway 失败日志只记录固定事件名,不写请求 Query、异常正文、Provider Endpoint
或 Token;具体请求量与 Pod 可用性由集群入口/Kubernetes 指标补充,当前不新增 MCP 业务 Tool。

View File

@ -768,11 +768,6 @@ DEPLOY-002 的运行拓扑将 Celery Worker 与 Celery Beat 恢复调度器拆
应用服务与持久化 Job/Attempt 状态,不形成新业务服务,也不改变 ADR-002。Migration 使用 Helm
`pre-install,pre-upgrade` Hook;失败会在业务 Deployment 更新前终止原子升级。
DEPLOY-003 在上述拓扑内为 API/MCP 增加 PodDisruptionBudget,并保持双副本和零不可用滚动
策略;Worker 继续依赖 OceanBase 的持久化 Job/Attempt、执行租约和独立恢复调度器收敛故障。
应用进程统一输出字段白名单约束的 JSON 事件,API 暴露低基数 `/metrics` 供集群抓取。该最小
可观测性不引入新的业务服务或完整 Observability 平台。
---
## 17. V1 验收重点

View File

@ -1,147 +0,0 @@
# 公共知识库集群运行与验收手册
## 1. 范围与安全边界
本手册覆盖 DEPLOY-003 的目标集群安装、幂等升级、滚动恢复、Worker 故障恢复、最小日志/指标、
备份责任和远程 MCP/WorkBuddy 验收。当前仍是企业受控网络内的单一公共知识库,不增加用户、
租户或 Document ACL,也不提供完整 Observability 平台和跨区域灾备。
所有敏感值必须由 Secret 管理系统写入 Kubernetes Secret。Values、Git、镜像、命令行参数、
验收记录和日志中不得出现数据库 URL、RabbitMQ URL、云密钥、Provider Key 或 MCP Token。
## 2. 交付责任边界
| 资源 | 平台仓库负责 | 环境/运维负责 |
|---|---|---|
| 应用镜像 | 可重复构建的 API、MCP、Worker target | 目标 Registry、不可变 tag/digest、镜像扫描和保留策略 |
| Kubernetes | Helm Chart、Migration、探针、PDB、资源和 Pod 安全默认值 | 集群、Namespace、Ingress Controller、配额和网络策略 |
| 敏感配置 | Secret 引用和必需键名预检 | Secret 创建、轮换、审计和访问控制 |
| HTTPS | Ingress Host/TLS Secret 引用 | DNS、证书签发/续期与 TLS Secret |
| 数据 | canonical/可重建边界和恢复顺序 | OceanBase、BOS、RabbitMQ 的备份、容量、HA 和恢复演练 |
| 监控 | JSON 事件、API `/metrics`、验收检查 | 日志/指标采集、留存、查询、告警和 On-call |
## 3. 部署前检查
操作者需要准备仓库外的环境 Values、Namespace、运行时 Secret、TLS Secret、ImagePullSecret,
并确认集群可访问 OceanBase、BOS、RabbitMQ、Embedding 和 Reranker。先显式选择并核对 context:
```bash
kubectl config current-context
kubectl auth can-i get deployments -n knowledge-platform
kubectl auth can-i patch deployments -n knowledge-platform
```
镜像必须使用 Registry 中已存在的不可变 tag 或 digest,禁止 `latest`。部署前执行:
```bash
make deploy-lint VALUES=/secure/path/environment-values.yaml
make deploy-render VALUES=/secure/path/environment-values.yaml >/tmp/ekp-rendered.yaml
```
渲染文件只能包含非敏感 ConfigMap 和 Secret 引用;完成检查后应删除临时文件。
## 4. 安装、幂等升级与只读验收
首次安装或普通升级使用同一入口:
```bash
NAMESPACE=knowledge-platform RELEASE=knowledge-platform \
make deploy VALUES=/secure/path/environment-values.yaml
```
只读验收不会触发安装、重启或故障注入,但强制 context 与操作者声明完全一致:
```bash
ACCEPTANCE_KUBE_CONTEXT="$(kubectl config current-context)" \
NAMESPACE=knowledge-platform RELEASE=knowledge-platform \
make cluster-acceptance VALUES=/secure/path/environment-values.yaml
```
## 5. 完整可用性验收
完整模式会连续执行两次相同 Helm 入口、滚动重启 API/MCP,并在一个合成入库 Job 运行时强制
删除一个明确选中的 Worker Pod。只能在已批准的验收 Namespace 执行:
```bash
context="$(kubectl config current-context)"
ACCEPTANCE_MODE=full \
ACCEPTANCE_KUBE_CONTEXT="${context}" \
ACCEPTANCE_CONFIRM="DEPLOY-003:${context}:knowledge-platform:knowledge-platform" \
NAMESPACE=knowledge-platform RELEASE=knowledge-platform \
make cluster-acceptance VALUES=/secure/path/environment-values.yaml
```
脚本验证以下内容:
- 相同 Values 的安装/升级可以重复成功,Migration Hook 先于工作负载更新完成;
- API/MCP 至少两个副本全部 Ready/Available,PDB 存在;
- Admin、健康检查、指标和 MCP 均通过稳定 HTTPS Host 到达;
- API/MCP 滚动期间连续健康探测无中断;
- Worker 被非优雅中断后,同一 canonical Job 经租约到期和恢复调度器完成,不要求重新上传;
- 合成文档验收完成后调用公开删除 API 清理,脚本不访问 Secret 值。
若合成文档处理过快,脚本会失败而不是虚报 Worker 中断已验证。可在受控验收环境调整 Provider
延迟或使用批准的非敏感验收文件重跑,不得加入测试后门。
## 6. 远程 MCP 与 WorkBuddy
先用官方 MCP Python Client 验证远程协议、唯一 Tool 和一次真实检索。Token 通过静默输入进入
当前进程环境,不写入命令参数或配置文件:
```bash
export EKP_MCP_URL=https://mcp.example.internal/mcp
export EKP_MCP_ACCEPTANCE_QUERY='已批准的公共知识验收问题'
read -rsp 'MCP token: ' EKP_MCP_TOKEN && export EKP_MCP_TOKEN && echo
make mcp-remote-acceptance
unset EKP_MCP_TOKEN
```
随后在与最终用户相同网络路径的 WorkBuddy 客户端加载
[`workbuddy-mcp.example.json`](../../deploy/workbuddy-mcp.example.json),只通过客户端安全环境注入
`EKP_MCP_URL` 和 `EKP_MCP_TOKEN`。验收必须确认:连接状态正常、只发现
`search_knowledge`、真实问题返回至少一个带来源字段的结果。记录客户端版本、时间、网络区域、
Tool 名和成功/失败即可,禁止复制 Token、正文或内部 Provider 信息。
## 7. 日志、指标与最小告警
应用事件为单行 JSON,允许字段包括 `event`、`request_id`、`job_id`、`attempt_number`、`stage`、
`job_status` 和耗时;异常正文不会被格式化器写出。常用检查:
```bash
kubectl -n knowledge-platform logs deployment/knowledge-platform-api --since=15m
kubectl -n knowledge-platform logs deployment/knowledge-platform-worker --since=15m
curl --fail https://knowledge.example.internal/metrics
kubectl -n knowledge-platform get deployments,pods,poddisruptionbudgets
```
环境监控至少采集并告警:API/MCP Ready 副本不足、Pod 重启持续增加、HTTP 5xx、P95 请求耗时、
Worker 无 Ready 副本、`ingestion_attempt_failed`/`ingestion_retry_cleanup_failed`、恢复扫描持续
失败、Migration Job 失败,以及 OceanBase/BOS/RabbitMQ/模型服务在应用日志中的安全失败事件。
阈值由真实流量基线确定,不在本任务中固化完整监控平台。
## 8. 故障处置
API/MCP 不可用时先检查 Deployment/Endpoint/Ingress/TLS,再按单个 Deployment 滚动重启;禁止
直接打印 Pod 环境变量。Job 卡住时使用 Admin/API 的 Job ID 查询状态,再按相同 Job ID 搜索
Worker JSON 事件。处于有效租约的 Job 不应手工重复提交;租约到期后恢复调度器会重投递。
更新同名 Secret 不会自动改变已运行进程。由运维完成 Secret 轮换后,依次滚动重启 API、MCP、
Worker 和 Scheduler,并复跑只读验收。稳定 Host 和客户端 Token 未轮换时,Admin/WorkBuddy
无需重新配置;MCP Token 轮换时必须通过客户端 Secret 分发流程更新。
## 9. 备份与恢复顺序
OceanBase 关系表和 BOS 不可变源文件共同构成事实数据,必须由资源所有者执行一致性备份策略。
OceanBase 向量表是可重建索引,不得代替 canonical 备份。RabbitMQ 消息只提供执行提示,
canonical Job/Attempt 才是恢复依据,但生产 Broker 仍应按其 SLA 做持久化和 HA。
恢复顺序为:恢复/验证 OceanBase canonical 表 → 恢复并核对 BOS 对象 → 部署相同或兼容应用版本
并执行 forward-only Migration → 恢复 RabbitMQ 连通 → 启动 Worker/Scheduler → 按 active Index
Profile 重建缺失向量 → 验证来源追踪和真实检索。Helm 回滚只能回退兼容的应用清单,不能反向
撤销数据库 Migration。完整灾备演练仍是后置任务。
## 10. 验收记录
每次目标集群验收按 [`DEPLOY_003_ACCEPTANCE.md`](DEPLOY_003_ACCEPTANCE.md) 记录非敏感结果。
只有完整自动化验收、真实 MCP 协议调用和远程 WorkBuddy 三部分全部通过,DEPLOY-003 才能
标记完成并合并到 `main`。

View File

@ -1,52 +0,0 @@
# DEPLOY-003 目标集群验收记录
**状态:** 等待目标环境
**Task:** DEPLOY-003
**记录规则:** 只记录非敏感标识和聚合结果,不记录 Secret、业务正文或内部 Provider 响应。
## 环境信息
- Kubernetes context:待提供
- Namespace / Release:待提供
- API HTTPS Host:待提供(只记录公开受控 Host)
- MCP HTTPS Host:待提供(只记录公开受控 Host)
- 镜像 tag/digest:待提供
- 验收时间与操作者:待提供
## 自动化验收
- [ ] 同一 Values 连续两次 `upgrade --install` 成功
- [ ] Migration Hook 成功且失败保护已核对
- [ ] API/MCP 各至少两个 Ready/Available 副本
- [ ] API/MCP PodDisruptionBudget 生效
- [ ] Admin、健康检查、`/metrics` 和 MCP 使用有效 HTTPS
- [ ] API/MCP 滚动重启期间连续健康探测无失败
- [ ] Worker 非优雅中断后同一 Job 自动恢复成功
- [ ] 验收合成文档已清理
- [ ] Manifest、镜像与验收日志凭据扫描为 0
执行结果与命令退出码:待执行。
## 远程 MCP / WorkBuddy
- [ ] 官方 MCP Client 只发现 `search_knowledge`
- [ ] 官方 MCP Client 完成一次真实、有来源的检索
- [ ] 最终用户网络路径上的 WorkBuddy 连接成功
- [ ] WorkBuddy 只发现批准 Tool 并完成真实检索
客户端版本、网络区域和非敏感结果:待执行。
## 运行保障
- [ ] 日志采集可按 `request_id` / `job_id` 定位
- [ ] 最小指标已被环境监控系统抓取
- [ ] Ready 副本、5xx、耗时、Pod 重启和 Job 失败告警责任人已确认
- [ ] OceanBase/BOS/RabbitMQ 备份责任人、周期和恢复顺序已确认
## 结论
当前仓库实现已提供 PDB、结构化日志、最小指标、完整验收脚本和运行手册;本机没有目标
Kubernetes context、Registry、DNS/TLS 或远程 WorkBuddy 环境参数,因此所有目标环境复选项
保持未完成,DEPLOY-003 不得提前闭环。

View File

@ -96,17 +96,10 @@ Controller 托管。证书只通过预建 TLS Secret 引用,不进入 Values
`RuntimeDefault` seccomp 和只读根文件系统;
- `/tmp` 使用有大小上限的 `emptyDir`,ServiceAccount 不挂载 API Token;
- API/MCP 滚动更新使用 `maxUnavailable: 0`,未 Ready Pod 不进入 Service Endpoint。
- API/MCP 默认各有一个 `minAvailable: 1` 的 PodDisruptionBudget,避免节点维护同时驱逐全部
用户入口;不可用故障仍由 Deployment 副本恢复处理。
- API Pod 默认以标准注解暴露 `/metrics`,只包含进程存活时间、按路由模板聚合的请求量和
请求耗时,不包含 Query、文件名、正文、Endpoint 或凭据。
`/healthz` 当前只表示 API 进程存活,不探测所有外部 Provider;真实依赖故障、滚动重启和
Worker 恢复行为由 DEPLOY-003 在目标集群验收。
运行、滚动更新、Worker 故障恢复、备份责任与远程 MCP/WorkBuddy 验收流程见
[`CLUSTER_OPERATIONS.md`](CLUSTER_OPERATIONS.md)。
## 6. 可选 Demo RabbitMQ
生产默认 `rabbitmq.enabled=false`,`RABBITMQ_URL` 指向外部受管 Broker。仅开发/演示时可开启

View File

@ -1,12 +1,12 @@
# 项目进度
当前阶段:DEPLOY-003 集群可用性与远程验收进行中
当前阶段:生产容器镜像闭环完成,准备实施 Kubernetes/Helm 一键部署
已完成:DEMO-001~DEMO-010、DEMO-012、DEMO-014~DEMO-016、ARCH-001~ARCH-006、PLAN-001~PLAN-002、DEPLOY-001~DEPLOY-002
进行中:DEPLOY-003
阻塞:目标 Kubernetes context、Registry 镜像坐标、DNS/TLS 与远程 WorkBuddy 环境参数未提供
已完成:DEMO-001~DEMO-010、DEMO-012、DEMO-014~DEMO-016、ARCH-001~ARCH-006、PLAN-001~PLAN-002、DEPLOY-001
进行中:无
阻塞:无
下一门槛:在明确目标集群执行 DEPLOY-003 完整验收;通过前不开始后续任务。
下一任务:`DEPLOY-002`,完成并合并后执行 `DEPLOY-003`。
Provider 架构迁移按 `ARCH-001 → ARCH-002 → ARCH-003 → ARCH-004 → ARCH-005 → ARCH-006`
执行并已完成。DEMO-010 在迁移闭环验收后实施,未扩大为完整 V1 Admin Console。
@ -424,25 +424,3 @@ Provider 架构迁移按 `ARCH-001 → ARCH-002 → ARCH-003 → ARCH-004 → AR
- 目标企业集群的稳定 HTTPS、滚动重启、Worker 故障恢复、真实外部依赖监测与远程 WorkBuddy
仍属于 `DEPLOY-003`;本任务没有把这些项提前标记完成,也没有改变 Accepted ADR 或公共知识库
范围。下一任务:`DEPLOY-003`。
### DEPLOY-003(进行中)
- 新增 API/MCP PodDisruptionBudget,继续使用双副本和 `maxUnavailable: 0`,覆盖节点维护与滚动
更新的最小可用性;没有引入新微服务或扩大公共知识库范围。
- 新增字段白名单 JSON 日志、请求相关 ID、API 低基数 `/metrics`,并为上传、批次、检索、MCP、
Worker Attempt/Stage/Retry/Cleanup/Recovery 补充安全事件;异常正文、Query、文件名、Provider
Endpoint 和凭据不进入结构化字段。
- 新增显式 context 核对和 `ACCEPTANCE_CONFIRM` 二次确认的集群验收入口;完整模式验证相同 Values
两次升级、HTTPS、PDB、连续滚动探测,并在合成 Job 运行时非优雅删除单个 Worker Pod,等待
canonical 租约与恢复调度器收敛后清理验收文档。
- 新增官方 MCP Client 远程验收脚本;Token 只从进程环境读取,输出只包含 Tool/结果存在性,
不包含 Token 或检索正文。远程 WorkBuddy 仍需在最终用户网络路径现场确认。
- 新增中文集群运行手册和验收记录,明确 OceanBase canonical/BOS、可重建向量、RabbitMQ、TLS、
Registry、日志指标与备份恢复的责任边界。完整 Observability 平台和灾备演练保持后置。
- 仓库侧验证已通过 Ruff Format/Lint、Mypy、Bash 语法、三种 Helm lint、13 个渲染资源的
kubeconform strict、Alembic Head/离线 SQL、Unit + Integration `200 passed` 和 RAG Evaluation
`5 passed`;本地运行 API 的 `/healthz`、`/metrics` 与 JSON 事件烟测通过。
- 新生产镜像的首次本地重建被 Docker Hub 匿名授权连接重置阻塞,两次重试结果一致,没有改动
固定 digest 或镜像来源规避;当前机器同时没有目标 Kubernetes context、目标 Registry 坐标、
DNS/TLS 或远程 WorkBuddy 参数。因此现场复选项保持未完成;当前任务分支仅保存并推送仓库侧
阶段性实现,在现场验收闭环前不合并 `main`。

View File

@ -100,10 +100,8 @@ PostgreSQL/MinIO 假设与 ADR-012 不一致,不得作为集群交付模板。
### DEPLOY-003
**进行中。** 仓库侧已实现 PDB、结构化日志、最小 HTTP 指标、显式 context/二次确认保护的
验收脚本,并形成运行手册和备份/恢复责任边界。仍须在目标集群完成安装、幂等升级、滚动重启、
Worker 故障恢复、日志/指标接入和远程 WorkBuddy 验收;在这些现场项完成前不得闭环。灾备演练
与完整 Observability 平台仍属后置项。
在目标集群完成安装、幂等升级、滚动重启、Worker 故障恢复和远程 WorkBuddy 验收;形成运行
手册、备份/恢复责任边界和最小日志/指标检查。灾备演练与完整 Observability 平台仍属后置项。
## 5. 最小验收标准

View File

@ -32,9 +32,7 @@ validation. Provider credentials and internal endpoints are runtime inputs and n
- [x] DEMO-016 Batch async upload API + per-file Job status + Admin Console
- [x] DEPLOY-001 Production images for API/Admin, MCP Gateway and Async Worker
- [x] DEPLOY-002 Kubernetes/Helm one-click deployment + Secret/Migration/Ingress baseline
- [ ] DEPLOY-003 Availability, rolling upgrade and remote WorkBuddy acceptance(进行中:仓库侧
PDB、结构化日志、最小指标、验收脚本与运行手册已实现;等待目标集群/Registry/DNS/TLS 和
远程 WorkBuddy 实测,不得提前勾选)
- [ ] DEPLOY-003 Availability, rolling upgrade and remote WorkBuddy acceptance
Dependencies and acceptance boundaries are defined in
[`ASYNC_INGESTION_MCP_PLAN.md`](ASYNC_INGESTION_MCP_PLAN.md) and

View File

@ -20,8 +20,7 @@
3. `DEMO-016`(已完成):批量异步上传、批次/Job 查询和 Admin Console 状态展示;
4. `DEPLOY-001`(已完成):API/Admin、MCP Gateway、Async Worker 生产镜像;
5. `DEPLOY-002`(已完成):Kubernetes/Helm、Secret、Migration、Ingress/TLS 和一键部署入口;
6. `DEPLOY-003`(进行中):仓库侧可用性保护、结构化日志、最小指标和验收入口已实现;仍需在
明确目标集群完成幂等升级、滚动升级、Worker 恢复和远程 WorkBuddy 现场验收。
6. `DEPLOY-003`:多副本可用性、滚动升级、Worker 恢复和远程 WorkBuddy 验收。
严格依赖为:

View File

@ -23,10 +23,6 @@
过滤。实施顺序和集群验收边界见
[`PUBLIC_KNOWLEDGE_CLUSTER_PLAN.md`](../plans/PUBLIC_KNOWLEDGE_CLUSTER_PLAN.md)。
公共知识库交付还要求 API/Admin 与 MCP 各至少两个副本、滚动升级期间稳定 HTTPS 地址可用、
Worker 中断后以 canonical Job/Attempt 恢复,并提供不含业务正文或凭据的结构化事件日志与最小
HTTP 指标。完整监控平台、跨区域灾备和自动云资源编排不随本阶段扩入 V1 Scope。
---
# 1. 项目背景

View File

@ -1,6 +1,5 @@
"""Durable asynchronous ingestion orchestration and retry cleanup."""
import logging
from dataclasses import dataclass
from datetime import UTC, datetime, timedelta
from enum import StrEnum
@ -58,7 +57,6 @@ SAFE_TRANSIENT_ERROR = "temporary ingestion failure"
SAFE_PERMANENT_ERROR = "document content could not be processed"
SAFE_CLEANUP_ERROR = "retry cleanup failed"
SAFE_LEASE_ERROR = "worker execution lease expired"
logger = logging.getLogger(__name__)
class IngestionJobNotFoundError(LookupError):
@ -295,47 +293,17 @@ class IngestionJobRunner:
claim = self._claim(job_id)
if not claim.run:
assert claim.result is not None
logger.info(
"Ingestion delivery deferred or already terminal",
extra={
"event": "ingestion_delivery_not_claimed",
"job_id": str(job_id),
"job_status": claim.result.status.value,
"retry_delay_seconds": claim.result.retry_delay_seconds,
},
)
return claim.result
if claim.cleanup_required:
logger.info(
"Retry cleanup started",
extra={"event": "ingestion_retry_cleanup_started", "job_id": str(job_id)},
)
try:
document_id = self._job_document_id(job_id)
self._cleanup.cleanup(document_id)
except Exception:
self._finish_cleanup_failure(job_id)
logger.error(
"Retry cleanup failed",
extra={"event": "ingestion_retry_cleanup_failed", "job_id": str(job_id)},
)
return JobRunResult(JobRunAction.COMPLETE, IngestionJobStatus.FAILED)
logger.info(
"Retry cleanup completed",
extra={"event": "ingestion_retry_cleanup_completed", "job_id": str(job_id)},
)
attempt_number, document_id = self._start_attempt(job_id)
logger.info(
"Ingestion attempt started",
extra={
"event": "ingestion_attempt_started",
"job_id": str(job_id),
"document_id": str(document_id),
"attempt_number": attempt_number,
},
)
def stage_callback(stage: IngestionStage) -> None:
"""Bind this attempt while pipeline stages remain Job-agnostic."""
@ -349,16 +317,6 @@ class IngestionJobRunner:
return self._finish_failure(job_id, attempt_number, document_id, error)
self._finish_success(job_id, attempt_number, document_id)
logger.info(
"Ingestion attempt succeeded",
extra={
"event": "ingestion_attempt_succeeded",
"job_id": str(job_id),
"document_id": str(document_id),
"attempt_number": attempt_number,
"job_status": IngestionJobStatus.SUCCEEDED.value,
},
)
return JobRunResult(JobRunAction.COMPLETE, IngestionJobStatus.SUCCEEDED)
def _claim(self, job_id: UUID) -> _Claim:
@ -493,15 +451,6 @@ class IngestionJobRunner:
job.lease_expires_at = datetime.now(UTC) + timedelta(seconds=self._lease_seconds)
attempt.stage = stage
session.commit()
logger.info(
"Ingestion stage advanced",
extra={
"event": "ingestion_stage_advanced",
"job_id": str(job_id),
"attempt_number": attempt_number,
"stage": stage.value,
},
)
def _finish_success(self, job_id: UUID, attempt_number: int, document_id: UUID) -> None:
"""Atomically close Job/Attempt after the document becomes ready."""
@ -553,17 +502,6 @@ class IngestionJobRunner:
document.status = DocumentStatus.PROCESSING
document.error_message = safe_error
session.commit()
logger.warning(
"Ingestion retry scheduled",
extra={
"event": "ingestion_retry_scheduled",
"job_id": str(job_id),
"document_id": str(document_id),
"attempt_number": attempt_number,
"job_status": job.status.value,
"retry_delay_seconds": delay,
},
)
return JobRunResult(JobRunAction.RETRY, job.status, delay)
job.status = IngestionJobStatus.FAILED
@ -573,16 +511,6 @@ class IngestionJobRunner:
document.status = DocumentStatus.FAILED
document.error_message = safe_error
session.commit()
logger.error(
"Ingestion attempt failed terminally",
extra={
"event": "ingestion_attempt_failed",
"job_id": str(job_id),
"document_id": str(document_id),
"attempt_number": attempt_number,
"job_status": job.status.value,
},
)
return JobRunResult(JobRunAction.COMPLETE, job.status)
def _finish_cleanup_failure(self, job_id: UUID) -> None:
@ -626,10 +554,6 @@ def dispatch_pending_jobs(jobs: IngestionJobService, dispatcher: JobDispatcher)
try:
dispatcher.enqueue(job_id)
except Exception:
logger.error(
"Durable Job redelivery failed",
extra={"event": "ingestion_recovery_dispatch_failed", "job_id": str(job_id)},
)
continue
dispatched += 1
return dispatched

View File

@ -1,75 +0,0 @@
"""Secret-safe structured logging shared by every runtime process."""
import json
import logging
import sys
from contextvars import ContextVar, Token
from datetime import UTC, datetime
from typing import Final
_request_id: ContextVar[str | None] = ContextVar("ekp_request_id", default=None)
_EVENT_FIELDS: Final[tuple[str, ...]] = (
"event",
"request_id",
"method",
"path",
"status_code",
"duration_ms",
"job_id",
"document_id",
"batch_id",
"attempt_number",
"stage",
"job_status",
"retry_delay_seconds",
"dispatch_count",
)
class JsonLogFormatter(logging.Formatter):
"""Render an allowlisted JSON event without serializing exception details.
Provider exceptions may contain endpoints or credentials. Callers therefore
log stable event names and identifiers, while this formatter deliberately
ignores ``exc_info`` and arbitrary record attributes.
"""
def format(self, record: logging.LogRecord) -> str:
payload: dict[str, object] = {
"timestamp": datetime.now(UTC).isoformat(),
"level": record.levelname,
"logger": record.name,
"message": record.getMessage(),
}
context_request_id = _request_id.get()
if context_request_id is not None:
payload["request_id"] = context_request_id
for field in _EVENT_FIELDS:
value = getattr(record, field, None)
if value is not None:
payload[field] = value
return json.dumps(payload, ensure_ascii=False, separators=(",", ":"))
def configure_json_logging(level: str) -> None:
"""Install one stdout JSON handler and preserve idempotent app factories."""
root = logging.getLogger()
existing = next(
(handler for handler in root.handlers if isinstance(handler.formatter, JsonLogFormatter)),
None,
)
if existing is None:
handler = logging.StreamHandler(sys.stdout)
handler.setFormatter(JsonLogFormatter())
root.addHandler(handler)
root.setLevel(level)
def bind_request_id(request_id: str) -> Token[str | None]:
"""Bind a validated correlation identifier to the current request context."""
return _request_id.set(request_id)
def reset_request_id(token: Token[str | None]) -> None:
"""Restore the previous request context after response logging completes."""
_request_id.reset(token)

View File

@ -1,201 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
# DEPLOY-003 validates one explicitly selected cluster context. Secret contents
# never enter this process; only HTTPS public endpoints and Kubernetes metadata
# are inspected. Full mode performs controlled rollout and worker-failure tests.
values_file="${1:-}"
chart_dir="${CHART:-deploy/helm/knowledge-platform}"
release_name="${RELEASE:-knowledge-platform}"
namespace="${NAMESPACE:-knowledge-platform}"
acceptance_mode="${ACCEPTANCE_MODE:-read-only}"
expected_context="${ACCEPTANCE_KUBE_CONTEXT:-}"
rollout_timeout="${ACCEPTANCE_ROLLOUT_TIMEOUT:-10m}"
worker_wait_seconds="${ACCEPTANCE_WORKER_WAIT_SECONDS:-720}"
fail() {
echo "DEPLOY-003 acceptance failed: $1" >&2
exit 1
}
if [[ -z "${values_file}" || ! -f "${values_file}" ]]; then
fail "VALUES must point to an existing environment values file"
fi
if [[ "${acceptance_mode}" != "read-only" && "${acceptance_mode}" != "full" ]]; then
fail "ACCEPTANCE_MODE must be read-only or full"
fi
for tool in helm kubectl curl python3; do
command -v "${tool}" >/dev/null 2>&1 || fail "required tool is missing: ${tool}"
done
current_context="$(kubectl config current-context 2>/dev/null || true)"
if [[ -z "${expected_context}" || "${current_context}" != "${expected_context}" ]]; then
fail "ACCEPTANCE_KUBE_CONTEXT must exactly match the active context"
fi
preflight_yaml="$({
helm template "${release_name}" "${chart_dir}" \
--namespace "${namespace}" \
--values "${values_file}" \
--show-only templates/tests/preflight-references.yaml
})"
reference() {
local field="$1"
printf '%s\n' "${preflight_yaml}" |
awk -v key="${field}:" '$1 == key {gsub(/"/, "", $2); print $2}'
}
ingress_enabled="$(reference ingressEnabled)"
api_host="$(reference apiHost)"
mcp_host="$(reference mcpHost)"
api_deployment="$(reference apiDeployment)"
mcp_deployment="$(reference mcpDeployment)"
worker_deployment="$(reference workerDeployment)"
scheduler_deployment="$(reference schedulerDeployment)"
metrics_path="$(reference metricsPath)"
if [[ "${ingress_enabled}" != "true" ]]; then
fail "target acceptance requires ingress.enabled=true"
fi
for value in \
"${api_host}" \
"${mcp_host}" \
"${api_deployment}" \
"${mcp_deployment}" \
"${worker_deployment}" \
"${scheduler_deployment}" \
"${metrics_path}"; do
[[ -n "${value}" ]] || fail "Helm acceptance references are incomplete"
done
if [[ "${acceptance_mode}" == "full" ]]; then
expected_confirmation="DEPLOY-003:${current_context}:${namespace}:${release_name}"
if [[ "${ACCEPTANCE_CONFIRM:-}" != "${expected_confirmation}" ]]; then
fail "set ACCEPTANCE_CONFIRM=${expected_confirmation} for controlled mutations"
fi
# Running the same entrypoint twice proves install/upgrade idempotency against
# the selected environment before disruption tests begin.
"${PWD}/scripts/deploy.sh" "${values_file}"
"${PWD}/scripts/deploy.sh" "${values_file}"
fi
kubectl -n "${namespace}" get deployment "${api_deployment}" >/dev/null
kubectl -n "${namespace}" get deployment "${mcp_deployment}" >/dev/null
kubectl -n "${namespace}" get deployment "${worker_deployment}" >/dev/null
kubectl -n "${namespace}" get deployment "${scheduler_deployment}" >/dev/null
for deployment in \
"${api_deployment}" \
"${mcp_deployment}" \
"${worker_deployment}" \
"${scheduler_deployment}"; do
kubectl -n "${namespace}" rollout status "deployment/${deployment}" \
--timeout "${rollout_timeout}"
done
assert_multi_replica_ready() {
local deployment="$1"
local desired ready available
desired="$(kubectl -n "${namespace}" get deployment "${deployment}" -o jsonpath='{.spec.replicas}')"
ready="$(kubectl -n "${namespace}" get deployment "${deployment}" -o jsonpath='{.status.readyReplicas}')"
available="$(kubectl -n "${namespace}" get deployment "${deployment}" -o jsonpath='{.status.availableReplicas}')"
[[ "${desired}" -ge 2 ]] || fail "${deployment} must have at least two replicas"
[[ "${ready:-0}" -eq "${desired}" ]] || fail "${deployment} does not have all replicas ready"
[[ "${available:-0}" -eq "${desired}" ]] || fail "${deployment} is not fully available"
}
assert_multi_replica_ready "${api_deployment}"
assert_multi_replica_ready "${mcp_deployment}"
for disruption_budget in "${api_deployment}" "${mcp_deployment}"; do
disruptions_allowed="0"
for attempt in $(seq 1 30); do
disruptions_allowed="$(kubectl -n "${namespace}" get poddisruptionbudget \
"${disruption_budget}" -o jsonpath='{.status.disruptionsAllowed}')"
[[ "${disruptions_allowed:-0}" -ge 1 ]] && break
sleep 1
done
[[ "${disruptions_allowed:-0}" -ge 1 ]] ||
fail "${disruption_budget} does not currently allow one voluntary disruption"
done
api_base_url="https://${api_host}"
mcp_url="https://${mcp_host}/mcp"
curl --fail --silent --show-error --connect-timeout 5 --max-time 10 \
--proto '=https' "${api_base_url}/healthz" >/dev/null
curl --fail --silent --show-error --connect-timeout 5 --max-time 10 \
--proto '=https' "${api_base_url}/admin/upload" >/dev/null
metrics_output="$({
curl --fail --silent --show-error --connect-timeout 5 --max-time 10 \
--proto '=https' "${api_base_url}${metrics_path}"
})"
printf '%s\n' "${metrics_output}" | grep -q '^ekp_process_uptime_seconds ' ||
fail "API metrics endpoint does not expose process uptime"
printf '%s\n' "${metrics_output}" | grep -q '^ekp_http_requests_total{' ||
fail "API metrics endpoint does not expose request totals"
# A credential-free request may be rejected, but it must complete TLS and reach
# the MCP application rather than fail DNS, certificate, or gateway routing.
mcp_status="$({
curl --silent --show-error --output /dev/null --write-out '%{http_code}' \
--connect-timeout 5 --max-time 10 --proto '=https' "${mcp_url}"
})"
case "${mcp_status}" in
200 | 400 | 401 | 403 | 405 | 406) ;;
*) fail "MCP HTTPS endpoint returned unexpected status ${mcp_status}" ;;
esac
continuous_probe() {
local stop_file="$1"
local failure_file="$2"
local current_mcp_status
while [[ ! -f "${stop_file}" ]]; do
if ! curl --fail --silent --show-error --connect-timeout 5 --max-time 10 --proto '=https' \
"${api_base_url}/healthz" >/dev/null; then
printf 'failed\n' >"${failure_file}"
return
fi
current_mcp_status="$({
curl --silent --show-error --output /dev/null --write-out '%{http_code}' \
--connect-timeout 5 --max-time 10 --proto '=https' "${mcp_url}"
})"
case "${current_mcp_status}" in
200 | 400 | 401 | 403 | 405 | 406) ;;
*)
printf 'failed\n' >"${failure_file}"
return
;;
esac
sleep 1
done
}
if [[ "${acceptance_mode}" == "full" ]]; then
probe_dir="$(mktemp -d)"
stop_file="${probe_dir}/stop"
failure_file="${probe_dir}/failure"
cleanup_probe() {
touch "${stop_file}" 2>/dev/null || true
wait "${probe_pid:-}" 2>/dev/null || true
rm -rf "${probe_dir}"
}
trap cleanup_probe EXIT
continuous_probe "${stop_file}" "${failure_file}" &
probe_pid="$!"
kubectl -n "${namespace}" rollout restart "deployment/${api_deployment}"
kubectl -n "${namespace}" rollout restart "deployment/${mcp_deployment}"
kubectl -n "${namespace}" rollout status "deployment/${api_deployment}" \
--timeout "${rollout_timeout}"
kubectl -n "${namespace}" rollout status "deployment/${mcp_deployment}" \
--timeout "${rollout_timeout}"
touch "${stop_file}"
wait "${probe_pid}"
[[ ! -f "${failure_file}" ]] || fail "API health probe failed during rolling restart"
rm -rf "${probe_dir}"
trap - EXIT
"${PWD}/scripts/worker_recovery_acceptance.sh" "${api_base_url}" \
"${namespace}" "${release_name}" "${worker_deployment}" "${worker_wait_seconds}"
fi
echo "DEPLOY-003 automated cluster acceptance passed (${acceptance_mode})"

View File

@ -1,56 +0,0 @@
"""Secret-safe remote MCP protocol acceptance used before WorkBuddy sign-off."""
import asyncio
import os
import sys
import httpx2
from mcp import ClientSession
from mcp.client.streamable_http import streamable_http_client
from mcp.types import CallToolResult
async def _accept() -> None:
"""Discover exactly one approved tool and execute a real remote search."""
url = os.environ.get("EKP_MCP_URL", "")
token = os.environ.get("EKP_MCP_TOKEN", "")
query = os.environ.get("EKP_MCP_ACCEPTANCE_QUERY", "")
if not url.startswith("https://") or len(token) < 32 or not query.strip():
raise ValueError(
"EKP_MCP_URL (HTTPS), EKP_MCP_TOKEN and EKP_MCP_ACCEPTANCE_QUERY are required"
)
# The credential is read in process and is never included in CLI arguments,
# output, exceptions, or the checked-in WorkBuddy example configuration.
headers = {"Authorization": f"Bearer {token}"}
async with httpx2.AsyncClient(headers=headers, timeout=30.0) as client:
async with streamable_http_client(url, http_client=client) as (read, write):
async with ClientSession(read, write) as session:
await session.initialize()
tools = await session.list_tools()
tool_names = [tool.name for tool in tools.tools]
if tool_names != ["search_knowledge"]:
raise RuntimeError("remote MCP exposed an unexpected tool set")
result = await session.call_tool(
"search_knowledge",
{"query": query, "top_k": 1},
)
if not isinstance(result, CallToolResult) or result.is_error:
raise RuntimeError("remote MCP search returned a tool error")
structured = result.structured_content
if not isinstance(structured, dict) or not structured.get("results"):
raise RuntimeError("remote MCP search returned no structured source")
print("remote MCP protocol acceptance passed: one tool, one grounded result")
def main() -> int:
"""Return a stable non-zero code without reflecting sensitive exceptions."""
try:
asyncio.run(_accept())
except Exception:
print("remote MCP protocol acceptance failed", file=sys.stderr)
return 1
return 0
if __name__ == "__main__":
raise SystemExit(main())

View File

@ -1,92 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
# This helper is called only by cluster_acceptance.sh full mode. It interrupts
# one exact worker Pod after a durable Job reaches RUNNING, then verifies that
# the lease/recovery scheduler converges the same Job without a second upload.
api_base_url="${1:?API base URL is required}"
namespace="${2:?namespace is required}"
release_name="${3:?release name is required}"
worker_deployment="${4:?worker deployment is required}"
wait_seconds="${5:-720}"
poll_interval="${ACCEPTANCE_JOB_POLL_INTERVAL_SECONDS:-0.2}"
work_dir="$(mktemp -d)"
source_file="${work_dir}/deploy-003-worker-recovery.md"
response_file="${work_dir}/response.json"
document_id=""
cleanup() {
if [[ -n "${document_id}" ]]; then
curl --silent --show-error --connect-timeout 5 --max-time 15 \
--request DELETE --proto '=https' \
"${api_base_url}/api/v1/documents/${document_id}" >/dev/null || true
fi
rm -rf "${work_dir}"
}
trap cleanup EXIT
[[ "${wait_seconds}" =~ ^[0-9]+$ && "${wait_seconds}" -gt 0 ]] || exit 2
[[ "${poll_interval}" =~ ^[0-9]+([.][0-9]+)?$ ]] || exit 2
# Enough independent paragraphs make RUNNING observable without introducing a
# large or business-derived acceptance document.
for number in $(seq 1 80); do
printf 'DEPLOY-003 worker recovery synthetic paragraph %s.\n\n' "${number}" >>"${source_file}"
done
curl --fail --silent --show-error --connect-timeout 5 --max-time 60 --proto '=https' \
--form "file=@${source_file};type=text/markdown" \
"${api_base_url}/api/v1/documents" >"${response_file}"
document_id="$(python3 -c 'import json,sys; print(json.load(sys.stdin)["document_id"])' <"${response_file}")"
job_id="$(python3 -c 'import json,sys; print(json.load(sys.stdin)["job_id"])' <"${response_file}")"
[[ -n "${document_id}" && -n "${job_id}" ]] || exit 1
deadline=$((SECONDS + wait_seconds))
interrupted="false"
while ((SECONDS < deadline)); do
job_json="$({
curl --fail --silent --show-error --connect-timeout 5 --max-time 10 --proto '=https' \
"${api_base_url}/api/v1/jobs/${job_id}"
})"
job_status="$(printf '%s' "${job_json}" | python3 -c 'import json,sys; print(json.load(sys.stdin)["status"])')"
if [[ "${job_status}" == "running" ]]; then
worker_pod="$(kubectl -n "${namespace}" get pods \
-l "app.kubernetes.io/instance=${release_name},app.kubernetes.io/component=worker" \
--field-selector=status.phase=Running \
-o jsonpath='{.items[0].metadata.name}')"
[[ -n "${worker_pod}" ]] || exit 1
# Zero grace simulates process/node loss instead of a cooperative drain.
kubectl -n "${namespace}" delete pod "${worker_pod}" \
--grace-period=0 --force --wait=false >/dev/null
interrupted="true"
break
fi
if [[ "${job_status}" == "succeeded" || "${job_status}" == "failed" ]]; then
echo "worker recovery acceptance could not interrupt the fast Job" >&2
exit 1
fi
sleep "${poll_interval}"
done
[[ "${interrupted}" == "true" ]] || exit 1
kubectl -n "${namespace}" rollout status "deployment/${worker_deployment}" --timeout 10m
while ((SECONDS < deadline)); do
job_json="$({
curl --fail --silent --show-error --connect-timeout 5 --max-time 10 --proto '=https' \
"${api_base_url}/api/v1/jobs/${job_id}"
})"
job_status="$(printf '%s' "${job_json}" | python3 -c 'import json,sys; print(json.load(sys.stdin)["status"])')"
if [[ "${job_status}" == "succeeded" ]]; then
echo "durable worker recovery passed"
exit 0
fi
if [[ "${job_status}" == "failed" || "${job_status}" == "cancelled" ]]; then
echo "worker recovery Job reached terminal status ${job_status}" >&2
exit 1
fi
sleep 1
done
echo "worker recovery Job did not complete within ${wait_seconds} seconds" >&2
exit 1

View File

@ -191,10 +191,7 @@ def test_tool_failure_is_sanitized_for_client_and_logs() -> None:
assert "knowledge search failed" in rendered
assert "internal.example.test" not in rendered
assert "secret-provider-token" not in rendered
log_error.assert_called_once_with(
"MCP knowledge search failed",
extra={"event": "mcp_knowledge_search_failed"},
)
log_error.assert_called_once_with("MCP knowledge search failed")
def test_transport_rejects_unapproved_origin() -> None:

View File

@ -82,8 +82,6 @@ def test_workloads_render_availability_and_restricted_pod_defaults() -> None:
assert "readinessProbe:" in rendered
assert "livenessProbe:" in rendered
assert "startupProbe:" in rendered
assert rendered.count("kind: PodDisruptionBudget") == 2
assert 'prometheus.io/path: "/metrics"' in rendered
def test_scheduler_is_singleton_and_workers_do_not_embed_beat() -> None:
@ -143,33 +141,6 @@ def test_cross_field_validation_rejects_unsafe_ingestion_limits() -> None:
assert "batchMaxTotalBytes must be >= maxUploadBytes" in (result.stdout + result.stderr)
def test_cross_field_validation_rejects_pdb_that_blocks_all_evictions() -> None:
"""A PDB must leave room to replace one of the required replicas."""
result = _helm(
"template",
"knowledge-platform",
str(CHART),
"--set",
"podDisruptionBudget.apiMinAvailable=2",
expect_success=False,
)
assert result.returncode != 0
assert "apiMinAvailable must be < api.replicaCount" in (result.stdout + result.stderr)
def test_cluster_acceptance_requires_explicit_context_and_mutation_confirmation() -> None:
"""The target-cluster harness cannot silently mutate an ambient context."""
cluster_script = (ROOT / "scripts" / "cluster_acceptance.sh").read_text()
worker_script = (ROOT / "scripts" / "worker_recovery_acceptance.sh").read_text()
assert "ACCEPTANCE_KUBE_CONTEXT" in cluster_script
assert "ACCEPTANCE_CONFIRM" in cluster_script
assert 'acceptance_mode="${ACCEPTANCE_MODE:-read-only}"' in cluster_script
assert "--grace-period=0 --force --wait=false" in worker_script
assert "get secret" not in worker_script
def test_deploy_entrypoint_uses_atomic_upgrade_and_never_reads_secret_values() -> None:
"""The one-command wrapper checks references and performs an atomic wait."""
script = (ROOT / "scripts" / "deploy.sh").read_text()

View File

@ -1,72 +0,0 @@
"""Unit contracts for secret-safe logs and bounded API metrics."""
import json
import logging
from fastapi.testclient import TestClient
from pydantic import SecretStr
from ekp_api.main import create_app
from ekp_core.config import Settings
from ekp_core.observability import JsonLogFormatter, bind_request_id, reset_request_id
def _settings() -> Settings:
"""Use an isolated test database while exercising provider-free endpoints."""
return Settings(
app_env="test",
database_url=SecretStr("sqlite+pysqlite:///:memory:"),
)
def test_json_formatter_keeps_allowlisted_context_and_drops_arbitrary_fields() -> None:
"""Structured events must not serialize provider exception or custom secrets."""
record = logging.LogRecord(
name="ekp.test",
level=logging.ERROR,
pathname=__file__,
lineno=1,
msg="safe failure",
args=(),
exc_info=None,
)
record.event = "safe_event"
record.job_id = "job-1"
record.provider_secret = "must-not-appear"
token = bind_request_id("request-1")
try:
payload = json.loads(JsonLogFormatter().format(record))
finally:
reset_request_id(token)
assert payload["event"] == "safe_event"
assert payload["job_id"] == "job-1"
assert payload["request_id"] == "request-1"
assert "provider_secret" not in payload
assert "must-not-appear" not in json.dumps(payload)
def test_api_exposes_low_cardinality_metrics_and_request_correlation() -> None:
"""Metrics use route templates, and callers receive a safe correlation id."""
client = TestClient(create_app(_settings()))
response = client.get("/healthz", headers={"x-request-id": "acceptance-request-1"})
metrics = client.get("/metrics")
assert response.status_code == 200
assert response.headers["x-request-id"] == "acceptance-request-1"
assert metrics.status_code == 200
assert "ekp_process_uptime_seconds " in metrics.text
assert 'path="/healthz",status="200"' in metrics.text
assert "?" not in metrics.text
def test_api_replaces_unsafe_request_identifier() -> None:
"""Untrusted correlation headers cannot inject log structure or new lines."""
client = TestClient(create_app(_settings()))
response = client.get("/healthz", headers={"x-request-id": "unsafe\nvalue"})
request_id = response.headers["x-request-id"]
assert request_id != "unsafe\nvalue"
assert len(request_id) == 32