From 6835a21f914edc6edd1bcac533433f3bc069ada0 Mon Sep 17 00:00:00 2001 From: d-robotics Date: Thu, 3 Sep 2026 14:33:34 +0800 Subject: [PATCH] deploy: checkpoint DEPLOY-003 cluster acceptance --- Makefile | 8 +- README.md | 5 + apps/api/src/ekp_api/celery_queue.py | 3 + apps/api/src/ekp_api/main.py | 65 +++++- apps/api/src/ekp_api/metrics.py | 59 +++++ apps/api/src/ekp_api/routes.py | 41 ++++ apps/api/src/ekp_api/worker.py | 27 ++- apps/mcp-server/src/ekp_mcp/main.py | 2 + apps/mcp-server/src/ekp_mcp/server.py | 5 +- deploy/helm/environments/values.example.yaml | 10 + .../knowledge-platform/templates/_helpers.tpl | 8 + .../templates/api-deployment.yaml | 5 + .../templates/poddisruptionbudgets.yaml | 31 +++ .../templates/tests/preflight-references.yaml | 7 + .../knowledge-platform/values.schema.json | 28 +++ deploy/helm/knowledge-platform/values.yaml | 12 ++ docs/api/API_CONTRACT.md | 6 + docs/api/MCP_CONTRACT.md | 4 + docs/architecture/ARCHITECTURE.md | 5 + docs/development/CLUSTER_OPERATIONS.md | 147 +++++++++++++ docs/development/DEPLOY_003_ACCEPTANCE.md | 52 +++++ docs/development/KUBERNETES_DEPLOYMENT.md | 7 + docs/plans/PROGRESS.md | 32 ++- docs/plans/PUBLIC_KNOWLEDGE_CLUSTER_PLAN.md | 6 +- docs/plans/TASKS.md | 4 +- docs/plans/V1_EXECUTION_PLAN.md | 3 +- docs/product/PRD.md | 4 + packages/core/src/ekp_core/ingestion_jobs.py | 76 +++++++ packages/core/src/ekp_core/observability.py | 75 +++++++ scripts/cluster_acceptance.sh | 201 ++++++++++++++++++ scripts/mcp_remote_acceptance.py | 56 +++++ scripts/worker_recovery_acceptance.sh | 92 ++++++++ tests/integration/test_mcp_server.py | 5 +- tests/unit/test_helm_deployment.py | 29 +++ tests/unit/test_observability.py | 72 +++++++ 35 files changed, 1177 insertions(+), 15 deletions(-) create mode 100644 apps/api/src/ekp_api/metrics.py create mode 100644 deploy/helm/knowledge-platform/templates/poddisruptionbudgets.yaml create mode 100644 docs/development/CLUSTER_OPERATIONS.md create mode 100644 docs/development/DEPLOY_003_ACCEPTANCE.md create mode 100644 packages/core/src/ekp_core/observability.py create mode 100755 scripts/cluster_acceptance.sh create mode 100644 scripts/mcp_remote_acceptance.py create mode 100755 scripts/worker_recovery_acceptance.sh create mode 100644 tests/unit/test_observability.py diff --git a/Makefile b/Makefile index 9ee587b..e980e5f 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: help install run run-mcp run-worker images images-print images-push deploy deploy-lint deploy-render format format-check lint typecheck test test-unit test-integration migration-check eval live-acceptance up down +.PHONY: help install run run-mcp run-worker images images-print images-push deploy deploy-lint deploy-render cluster-acceptance mcp-remote-acceptance format format-check lint typecheck test test-unit test-integration migration-check eval live-acceptance up down UV_CACHE_DIR ?= /tmp/ekp-uv-cache export UV_CACHE_DIR @@ -15,6 +15,8 @@ help: @echo " deploy Install or upgrade Kubernetes with VALUES=" @echo " deploy-lint Validate Helm values and templates" @echo " deploy-render Render non-secret Kubernetes manifests" + @echo " cluster-acceptance Validate the explicitly selected target cluster" + @echo " mcp-remote-acceptance Verify remote MCP protocol and real search" @echo " format Format code" @echo " format-check Check formatting without modifying files" @echo " lint Run linters" @@ -50,6 +52,10 @@ deploy-lint: deploy-render: @test -n "$(VALUES)" || (echo "VALUES must point to an environment values file" >&2; exit 2) helm template "$${RELEASE:-knowledge-platform}" deploy/helm/knowledge-platform --namespace "$${NAMESPACE:-knowledge-platform}" --values "$(VALUES)" +cluster-acceptance: + ./scripts/cluster_acceptance.sh "$(VALUES)" +mcp-remote-acceptance: + uv run python scripts/mcp_remote_acceptance.py format: uv run ruff format . format-check: diff --git a/README.md b/README.md index 6fa12c0..107b08e 100644 --- a/README.md +++ b/README.md @@ -69,6 +69,11 @@ The Streamable HTTP endpoint defaults to `http://127.0.0.1:8766/mcp` and exposes only `search_knowledge`. See `docs/development/MCP_SETUP.md` for WorkBuddy configuration, identity boundaries, remote allowlists, and protocol verification. +Kubernetes installation, target-cluster acceptance, rolling recovery, metrics and backup +responsibility are documented in `docs/development/KUBERNETES_DEPLOYMENT.md` and +`docs/development/CLUSTER_OPERATIONS.md`. Cluster acceptance always requires an explicitly named +Kubernetes context; credentials remain in Kubernetes Secret or the client process environment. + The health endpoint is available at `http://localhost:8000/healthz`, and the versioned OpenAPI document is available at `http://localhost:8000/api/v1/openapi.json`. diff --git a/apps/api/src/ekp_api/celery_queue.py b/apps/api/src/ekp_api/celery_queue.py index bde1bcc..f9a260c 100644 --- a/apps/api/src/ekp_api/celery_queue.py +++ b/apps/api/src/ekp_api/celery_queue.py @@ -24,6 +24,9 @@ def create_celery_app(settings: Settings) -> Celery: task_reject_on_worker_lost=True, worker_prefetch_multiplier=1, broker_connection_retry_on_startup=True, + # Preserve the shared JSON handler instead of letting Celery replace it + # with process-specific plain-text formatting. + worker_hijack_root_logger=False, timezone="UTC", enable_utc=True, task_routes={ diff --git a/apps/api/src/ekp_api/main.py b/apps/api/src/ekp_api/main.py index 06797f6..8cc73c1 100644 --- a/apps/api/src/ekp_api/main.py +++ b/apps/api/src/ekp_api/main.py @@ -1,17 +1,26 @@ """FastAPI application entry point.""" +import logging +import re +from collections.abc import Awaitable, Callable from pathlib import Path +from time import monotonic +from uuid import uuid4 -from fastapi import FastAPI -from fastapi.responses import FileResponse +from fastapi import FastAPI, Request, Response +from fastapi.responses import FileResponse, PlainTextResponse from fastapi.staticfiles import StaticFiles from pydantic import BaseModel +from ekp_api.metrics import HttpMetrics from ekp_api.providers import ApiServices from ekp_api.routes import router from ekp_core.config import Settings, get_settings +from ekp_core.observability import bind_request_id, configure_json_logging, reset_request_id ADMIN_WEB_ROOT = Path(__file__).resolve().parents[3] / "admin-web" +REQUEST_ID_PATTERN = re.compile(r"^[A-Za-z0-9._:-]{1,64}$") +logger = logging.getLogger(__name__) class HealthResponse(BaseModel): @@ -28,6 +37,7 @@ def create_app( ) -> FastAPI: """Create the API application with explicit, testable settings.""" resolved_settings = settings or get_settings() + configure_json_logging(resolved_settings.log_level) application = FastAPI( title=resolved_settings.app_name, version="0.1.0", @@ -36,6 +46,7 @@ def create_app( ) application.state.settings = resolved_settings application.state.api_services = api_services + application.state.http_metrics = HttpMetrics() application.include_router(router, prefix=resolved_settings.api_prefix) admin_web_root = (resolved_settings.admin_web_root or ADMIN_WEB_ROOT).resolve() @@ -47,6 +58,48 @@ def create_app( name="admin-assets", ) + @application.middleware("http") + async def observe_http( + request: Request, + call_next: Callable[[Request], Awaitable[Response]], + ) -> Response: + """Correlate requests and emit bounded metrics without body/query data.""" + incoming_request_id = request.headers.get("x-request-id", "") + request_id = ( + incoming_request_id + if REQUEST_ID_PATTERN.fullmatch(incoming_request_id) + else uuid4().hex + ) + token = bind_request_id(request_id) + started_at = monotonic() + status_code = 500 + try: + response = await call_next(request) + status_code = response.status_code + response.headers["x-request-id"] = request_id + return response + finally: + duration_seconds = monotonic() - started_at + route = request.scope.get("route") + route_path = getattr(route, "path", "unmatched") + application.state.http_metrics.observe( + request.method, + route_path, + status_code, + duration_seconds, + ) + logger.info( + "HTTP request completed", + extra={ + "event": "http_request_completed", + "method": request.method, + "path": route_path, + "status_code": status_code, + "duration_ms": round(duration_seconds * 1000, 3), + }, + ) + reset_request_id(token) + @application.get("/admin", include_in_schema=False) @application.get("/admin/upload", include_in_schema=False) @application.get("/admin/documents", include_in_schema=False) @@ -63,6 +116,14 @@ def create_app( environment=resolved_settings.app_env, ) + @application.get("/metrics", include_in_schema=False) + async def metrics() -> PlainTextResponse: + """Expose aggregate process/HTTP metrics without provider or user data.""" + return PlainTextResponse( + application.state.http_metrics.render(), + media_type="text/plain; version=0.0.4", + ) + return application diff --git a/apps/api/src/ekp_api/metrics.py b/apps/api/src/ekp_api/metrics.py new file mode 100644 index 0000000..0c794e9 --- /dev/null +++ b/apps/api/src/ekp_api/metrics.py @@ -0,0 +1,59 @@ +"""Small dependency-free Prometheus endpoint for cluster smoke monitoring.""" + +from collections import defaultdict +from threading import Lock +from time import monotonic + + +class HttpMetrics: + """Keep bounded per-process HTTP counters using route templates as labels.""" + + def __init__(self) -> None: + self._started_at = monotonic() + self._lock = Lock() + self._counts: defaultdict[tuple[str, str, int], int] = defaultdict(int) + self._durations: defaultdict[tuple[str, str], tuple[int, float]] = defaultdict( + lambda: (0, 0.0) + ) + + def observe(self, method: str, path: str, status_code: int, duration_seconds: float) -> None: + """Record one request without retaining query strings or user content.""" + with self._lock: + self._counts[(method, path, status_code)] += 1 + count, total = self._durations[(method, path)] + self._durations[(method, path)] = (count + 1, total + duration_seconds) + + def render(self) -> str: + """Return Prometheus text exposition with stable low-cardinality labels.""" + with self._lock: + counts = tuple(sorted(self._counts.items())) + durations = tuple(sorted(self._durations.items())) + lines = [ + "# HELP ekp_process_uptime_seconds Process uptime in seconds.", + "# TYPE ekp_process_uptime_seconds gauge", + f"ekp_process_uptime_seconds {monotonic() - self._started_at:.6f}", + "# HELP ekp_http_requests_total HTTP requests handled by this API process.", + "# TYPE ekp_http_requests_total counter", + ] + for (method, path, status_code), value in counts: + labels = _labels(method=method, path=path, status=str(status_code)) + lines.append(f"ekp_http_requests_total{{{labels}}} {value}") + lines.extend( + ( + "# HELP ekp_http_request_duration_seconds HTTP request duration.", + "# TYPE ekp_http_request_duration_seconds summary", + ) + ) + for (method, path), (count, total) in durations: + labels = _labels(method=method, path=path) + lines.append(f"ekp_http_request_duration_seconds_count{{{labels}}} {count}") + lines.append(f"ekp_http_request_duration_seconds_sum{{{labels}}} {total:.6f}") + return "\n".join(lines) + "\n" + + +def _labels(**values: str) -> str: + """Escape the limited label set according to Prometheus text rules.""" + return ",".join( + f'{name}="{value.replace(chr(92), chr(92) * 2).replace(chr(10), chr(92) + "n").replace(chr(34), chr(92) + chr(34))}"' + for name, value in values.items() + ) diff --git a/apps/api/src/ekp_api/routes.py b/apps/api/src/ekp_api/routes.py index b12b481..2c912b2 100644 --- a/apps/api/src/ekp_api/routes.py +++ b/apps/api/src/ekp_api/routes.py @@ -1,5 +1,6 @@ """Versioned document and search routes for the retrieval Demo.""" +import logging from typing import Annotated from uuid import UUID @@ -30,6 +31,7 @@ from ekp_core.ingestion_batches import BatchFileCountLimitError, BatchNotFoundEr from ekp_core.ingestion_jobs import IngestionJobNotFoundError router = APIRouter() +logger = logging.getLogger(__name__) def get_api_services(request: Request) -> ApiServices: @@ -40,6 +42,10 @@ def get_api_services(request: Request) -> ApiServices: try: services = create_api_services(request.app.state.settings) except ProviderConfigurationError as error: + logger.error( + "API provider composition failed", + extra={"event": "api_provider_configuration_failed"}, + ) raise HTTPException( status_code=status.HTTP_503_SERVICE_UNAVAILABLE, detail="API providers are not fully configured", @@ -81,10 +87,22 @@ def upload_document( ) from error except Exception as error: # Provider and persistence exceptions are intentionally not reflected to clients. + logger.error( + "Document acceptance failed", + extra={"event": "document_acceptance_failed"}, + ) raise HTTPException( status_code=status.HTTP_502_BAD_GATEWAY, detail="document acceptance failed", ) from error + logger.info( + "Document accepted for ingestion", + extra={ + "event": "document_accepted", + "document_id": str(submission.document.id), + "job_id": str(submission.job.id), + }, + ) return IngestionAcceptedResponse.from_submission(submission) @@ -131,10 +149,18 @@ def upload_ingestion_batch( detail="batch file count exceeds the configured limit", ) from error except Exception as error: + logger.error( + "Batch acceptance failed", + extra={"event": "ingestion_batch_acceptance_failed"}, + ) raise HTTPException( status_code=status.HTTP_502_BAD_GATEWAY, detail="batch could not be accepted", ) from error + logger.info( + "Ingestion batch accepted", + extra={"event": "ingestion_batch_accepted", "batch_id": str(view.batch.id)}, + ) return IngestionBatchResponse.from_view(view) @@ -194,11 +220,22 @@ def delete_document(document_id: UUID, services: Services) -> Response: detail="document ingestion is still active", ) from error except DocumentArtifactCleanupError as error: + logger.error( + "Document artifact cleanup failed", + extra={ + "event": "document_artifact_cleanup_failed", + "document_id": str(document_id), + }, + ) raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail="document metadata was deleted but artifact cleanup failed", ) from error except Exception as error: + logger.error( + "Document deletion failed", + extra={"event": "document_deletion_failed", "document_id": str(document_id)}, + ) raise HTTPException( status_code=status.HTTP_502_BAD_GATEWAY, detail="document deletion failed", @@ -217,6 +254,10 @@ def search_knowledge(payload: SearchRequest, services: Services) -> SearchRespon detail="search query must contain non-whitespace text", ) from error except Exception as error: + logger.error( + "Knowledge search provider chain failed", + extra={"event": "knowledge_search_failed"}, + ) raise HTTPException( status_code=status.HTTP_502_BAD_GATEWAY, detail="search providers failed", diff --git a/apps/api/src/ekp_api/worker.py b/apps/api/src/ekp_api/worker.py index e610459..835495f 100644 --- a/apps/api/src/ekp_api/worker.py +++ b/apps/api/src/ekp_api/worker.py @@ -1,5 +1,6 @@ """Celery worker entry point for durable ingestion Jobs.""" +import logging from functools import lru_cache from typing import Any from uuid import UUID @@ -12,9 +13,12 @@ from ekp_api.celery_queue import ( from ekp_api.providers import WorkerServices, create_worker_services from ekp_core.config import get_settings from ekp_core.ingestion_jobs import JobRunAction, dispatch_pending_jobs +from ekp_core.observability import configure_json_logging _settings = get_settings() +configure_json_logging(_settings.log_level) celery_app = create_celery_app(_settings) +logger = logging.getLogger(__name__) @lru_cache @@ -34,7 +38,20 @@ def run_ingestion(self: Any, job_id: str) -> None: Celery delivery retries are unbounded because valid-lease deferrals are not business Attempts; the durable runner independently enforces max_attempts. """ + logger.info( + "Ingestion delivery received", + extra={"event": "ingestion_delivery_received", "job_id": job_id}, + ) result = _worker_services().runner.run(UUID(job_id)) + logger.info( + "Ingestion delivery completed", + extra={ + "event": "ingestion_delivery_completed", + "job_id": job_id, + "job_status": result.status.value, + "retry_delay_seconds": result.retry_delay_seconds, + }, + ) if result.action == JobRunAction.RETRY: raise self.retry(countdown=result.retry_delay_seconds or 1, max_retries=None) @@ -43,7 +60,15 @@ def run_ingestion(self: Any, job_id: str) -> None: def recover_ingestion_jobs() -> int: """Redeliver due durable Jobs after API/broker or worker interruptions.""" services = _worker_services() - return dispatch_pending_jobs(services.jobs, services.dispatcher) + dispatch_count = dispatch_pending_jobs(services.jobs, services.dispatcher) + logger.info( + "Durable ingestion recovery scan completed", + extra={ + "event": "ingestion_recovery_scan_completed", + "dispatch_count": dispatch_count, + }, + ) + return dispatch_count def main() -> None: diff --git a/apps/mcp-server/src/ekp_mcp/main.py b/apps/mcp-server/src/ekp_mcp/main.py index 4efbcd2..5a0e0c8 100644 --- a/apps/mcp-server/src/ekp_mcp/main.py +++ b/apps/mcp-server/src/ekp_mcp/main.py @@ -3,12 +3,14 @@ from mcp.server.transport_security import TransportSecuritySettings from ekp_core.config import get_settings +from ekp_core.observability import configure_json_logging from ekp_mcp.server import create_mcp_server def main() -> None: """Run the authenticated Streamable HTTP server from environment settings.""" settings = get_settings() + configure_json_logging(settings.log_level) server = create_mcp_server(settings) server.run( transport="streamable-http", diff --git a/apps/mcp-server/src/ekp_mcp/server.py b/apps/mcp-server/src/ekp_mcp/server.py index 1d2680b..dc823c8 100644 --- a/apps/mcp-server/src/ekp_mcp/server.py +++ b/apps/mcp-server/src/ekp_mcp/server.py @@ -100,7 +100,10 @@ def create_mcp_server( # Do not reflect provider, database, endpoint, or credential details # into MCP content or logs. Operations can correlate this generic # event with upstream sanitized service telemetry. - logger.error("MCP knowledge search failed") + logger.error( + "MCP knowledge search failed", + extra={"event": "mcp_knowledge_search_failed"}, + ) raise ToolError("knowledge search failed") from None return server diff --git a/deploy/helm/environments/values.example.yaml b/deploy/helm/environments/values.example.yaml index 38e09ad..d97cbf8 100644 --- a/deploy/helm/environments/values.example.yaml +++ b/deploy/helm/environments/values.example.yaml @@ -17,6 +17,16 @@ images: tag: "0.1.0" digest: "" +podDisruptionBudget: + enabled: true + apiMinAvailable: 1 + mcpMinAvailable: 1 + +observability: + metrics: + enabled: true + path: /metrics + ingress: enabled: true className: nginx diff --git a/deploy/helm/knowledge-platform/templates/_helpers.tpl b/deploy/helm/knowledge-platform/templates/_helpers.tpl index 6c18f44..7aa4d2e 100644 --- a/deploy/helm/knowledge-platform/templates/_helpers.tpl +++ b/deploy/helm/knowledge-platform/templates/_helpers.tpl @@ -64,6 +64,14 @@ app.kubernetes.io/instance: {{ .Release.Name }} {{- if gt (float64 .Values.config.retrieval.searchTopK) (float64 .Values.config.retrieval.denseTopN) -}} {{- fail "config.retrieval.searchTopK must be <= denseTopN" -}} {{- end -}} +{{- if .Values.podDisruptionBudget.enabled -}} +{{- if ge (int .Values.podDisruptionBudget.apiMinAvailable) (int .Values.api.replicaCount) -}} +{{- fail "podDisruptionBudget.apiMinAvailable must be < api.replicaCount" -}} +{{- end -}} +{{- if ge (int .Values.podDisruptionBudget.mcpMinAvailable) (int .Values.mcp.replicaCount) -}} +{{- fail "podDisruptionBudget.mcpMinAvailable must be < mcp.replicaCount" -}} +{{- end -}} +{{- end -}} {{- end -}} {{- define "knowledge-platform.podSecurityContext" -}} diff --git a/deploy/helm/knowledge-platform/templates/api-deployment.yaml b/deploy/helm/knowledge-platform/templates/api-deployment.yaml index 9eca02d..90361d5 100644 --- a/deploy/helm/knowledge-platform/templates/api-deployment.yaml +++ b/deploy/helm/knowledge-platform/templates/api-deployment.yaml @@ -28,6 +28,11 @@ spec: annotations: checksum/runtime-config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} checksum/runtime-secret-reference: {{ .Values.existingSecret | sha256sum }} + {{- if .Values.observability.metrics.enabled }} + prometheus.io/scrape: "true" + prometheus.io/port: "8000" + prometheus.io/path: {{ .Values.observability.metrics.path | quote }} + {{- end }} {{- with .Values.podAnnotations }} {{- toYaml . | nindent 8 }} {{- end }} diff --git a/deploy/helm/knowledge-platform/templates/poddisruptionbudgets.yaml b/deploy/helm/knowledge-platform/templates/poddisruptionbudgets.yaml new file mode 100644 index 0000000..a9d52cd --- /dev/null +++ b/deploy/helm/knowledge-platform/templates/poddisruptionbudgets.yaml @@ -0,0 +1,31 @@ +{{- if .Values.podDisruptionBudget.enabled }} +# DEPLOY-003 protects both user-facing paths during voluntary disruptions. PDBs +# do not cover involuntary failures, which remain handled by replica recovery. +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: {{ include "knowledge-platform.fullname" . }}-api + labels: + {{- include "knowledge-platform.labels" . | nindent 4 }} + app.kubernetes.io/component: api +spec: + minAvailable: {{ .Values.podDisruptionBudget.apiMinAvailable }} + selector: + matchLabels: + {{- include "knowledge-platform.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: api +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: {{ include "knowledge-platform.fullname" . }}-mcp + labels: + {{- include "knowledge-platform.labels" . | nindent 4 }} + app.kubernetes.io/component: mcp +spec: + minAvailable: {{ .Values.podDisruptionBudget.mcpMinAvailable }} + selector: + matchLabels: + {{- include "knowledge-platform.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: mcp +{{- end }} diff --git a/deploy/helm/knowledge-platform/templates/tests/preflight-references.yaml b/deploy/helm/knowledge-platform/templates/tests/preflight-references.yaml index 8ead8f3..75567a6 100644 --- a/deploy/helm/knowledge-platform/templates/tests/preflight-references.yaml +++ b/deploy/helm/knowledge-platform/templates/tests/preflight-references.yaml @@ -11,6 +11,13 @@ metadata: data: runtimeSecret: {{ .Values.existingSecret | quote }} ingressEnabled: {{ .Values.ingress.enabled | quote }} + apiHost: {{ .Values.ingress.api.host | quote }} + mcpHost: {{ .Values.ingress.mcp.host | quote }} + apiDeployment: {{ printf "%s-api" (include "knowledge-platform.fullname" .) | quote }} + mcpDeployment: {{ printf "%s-mcp" (include "knowledge-platform.fullname" .) | quote }} + workerDeployment: {{ printf "%s-worker" (include "knowledge-platform.fullname" .) | quote }} + schedulerDeployment: {{ printf "%s-scheduler" (include "knowledge-platform.fullname" .) | quote }} + metricsPath: {{ .Values.observability.metrics.path | quote }} apiTlsSecret: {{ .Values.ingress.api.tlsSecretName | quote }} mcpTlsSecret: {{ .Values.ingress.mcp.tlsSecretName | quote }} rabbitmqEnabled: {{ .Values.rabbitmq.enabled | quote }} diff --git a/deploy/helm/knowledge-platform/values.schema.json b/deploy/helm/knowledge-platform/values.schema.json index 6162933..f0c53bf 100644 --- a/deploy/helm/knowledge-platform/values.schema.json +++ b/deploy/helm/knowledge-platform/values.schema.json @@ -7,6 +7,8 @@ "existingSecret", "images", "serviceAccount", + "podDisruptionBudget", + "observability", "api", "mcp", "worker", @@ -51,6 +53,32 @@ }, "podAnnotations": {"$ref": "#/definitions/stringMap"}, "podLabels": {"$ref": "#/definitions/stringMap"}, + "podDisruptionBudget": { + "type": "object", + "additionalProperties": false, + "required": ["enabled", "apiMinAvailable", "mcpMinAvailable"], + "properties": { + "enabled": {"type": "boolean"}, + "apiMinAvailable": {"type": "integer", "minimum": 1}, + "mcpMinAvailable": {"type": "integer", "minimum": 1} + } + }, + "observability": { + "type": "object", + "additionalProperties": false, + "required": ["metrics"], + "properties": { + "metrics": { + "type": "object", + "additionalProperties": false, + "required": ["enabled", "path"], + "properties": { + "enabled": {"type": "boolean"}, + "path": {"const": "/metrics"} + } + } + } + }, "api": {"$ref": "#/definitions/httpWorkload"}, "mcp": {"$ref": "#/definitions/httpWorkload"}, "worker": { diff --git a/deploy/helm/knowledge-platform/values.yaml b/deploy/helm/knowledge-platform/values.yaml index 9a255ae..908ebae 100644 --- a/deploy/helm/knowledge-platform/values.yaml +++ b/deploy/helm/knowledge-platform/values.yaml @@ -31,6 +31,18 @@ serviceAccount: podAnnotations: {} podLabels: {} +# DEPLOY-003 keeps one API and MCP replica available during voluntary node +# maintenance. This complements, rather than replaces, RollingUpdate settings. +podDisruptionBudget: + enabled: true + apiMinAvailable: 1 + mcpMinAvailable: 1 + +observability: + metrics: + enabled: true + path: /metrics + api: replicaCount: 2 service: diff --git a/docs/api/API_CONTRACT.md b/docs/api/API_CONTRACT.md index 2ba44ce..d800828 100644 --- a/docs/api/API_CONTRACT.md +++ b/docs/api/API_CONTRACT.md @@ -76,6 +76,12 @@ Ingress 是否进一步认证属于部署配置,但前端隐藏按钮或隐藏 集群重启、扩缩容和滚动部署不得改变本契约或要求浏览器重新注入 Provider 配置。数据库、BOS、 模型与队列凭据由服务端 Secret 提供,任何 API 响应都不得返回这些值。 +DEPLOY-003 增加两个不进入 `/api/v1` OpenAPI 的运维端点:`GET /healthz` 只表示 API 进程可用; +`GET /metrics` 使用 Prometheus text format 返回进程存活时间、按 HTTP method/路由模板/status +聚合的请求量和按 method/路由模板聚合的耗时。两者不得包含 Query、文件名、正文、身份、 +Provider Endpoint 或凭据。响应附带安全 `X-Request-ID`,客户端提供的值只有匹配 +`[A-Za-z0-9._:-]{1,64}` 时才会沿用。 + ## 异步入库契约(DEMO-015~016 已实现) | Method | Path | 行为 | diff --git a/docs/api/MCP_CONTRACT.md b/docs/api/MCP_CONTRACT.md index 7734e7e..e9b3f65 100644 --- a/docs/api/MCP_CONTRACT.md +++ b/docs/api/MCP_CONTRACT.md @@ -49,3 +49,7 @@ ACL 或用户委托授权。 Credential 由 Kubernetes Secret 在服务端运行时注入,客户端只保存管理员分发的最小 `knowledge:search` Token;Pod 重启或滚动升级不得要求重新生成配置。公共知识域不等于匿名 访问或超级管理员身份,未来引入非公共知识前必须升级为用户委托身份与 ACL-aware Search。 + +DEPLOY-003 的远程验收必须从最终 HTTPS 地址只发现 `search_knowledge` 并完成一次真实、有来源 +结果的调用。Gateway 失败日志只记录固定事件名,不写请求 Query、异常正文、Provider Endpoint +或 Token;具体请求量与 Pod 可用性由集群入口/Kubernetes 指标补充,当前不新增 MCP 业务 Tool。 diff --git a/docs/architecture/ARCHITECTURE.md b/docs/architecture/ARCHITECTURE.md index 13a0aae..d71a5ff 100644 --- a/docs/architecture/ARCHITECTURE.md +++ b/docs/architecture/ARCHITECTURE.md @@ -768,6 +768,11 @@ DEPLOY-002 的运行拓扑将 Celery Worker 与 Celery Beat 恢复调度器拆 应用服务与持久化 Job/Attempt 状态,不形成新业务服务,也不改变 ADR-002。Migration 使用 Helm `pre-install,pre-upgrade` Hook;失败会在业务 Deployment 更新前终止原子升级。 +DEPLOY-003 在上述拓扑内为 API/MCP 增加 PodDisruptionBudget,并保持双副本和零不可用滚动 +策略;Worker 继续依赖 OceanBase 的持久化 Job/Attempt、执行租约和独立恢复调度器收敛故障。 +应用进程统一输出字段白名单约束的 JSON 事件,API 暴露低基数 `/metrics` 供集群抓取。该最小 +可观测性不引入新的业务服务或完整 Observability 平台。 + --- ## 17. V1 验收重点 diff --git a/docs/development/CLUSTER_OPERATIONS.md b/docs/development/CLUSTER_OPERATIONS.md new file mode 100644 index 0000000..27a233d --- /dev/null +++ b/docs/development/CLUSTER_OPERATIONS.md @@ -0,0 +1,147 @@ +# 公共知识库集群运行与验收手册 + +## 1. 范围与安全边界 + +本手册覆盖 DEPLOY-003 的目标集群安装、幂等升级、滚动恢复、Worker 故障恢复、最小日志/指标、 +备份责任和远程 MCP/WorkBuddy 验收。当前仍是企业受控网络内的单一公共知识库,不增加用户、 +租户或 Document ACL,也不提供完整 Observability 平台和跨区域灾备。 + +所有敏感值必须由 Secret 管理系统写入 Kubernetes Secret。Values、Git、镜像、命令行参数、 +验收记录和日志中不得出现数据库 URL、RabbitMQ URL、云密钥、Provider Key 或 MCP Token。 + +## 2. 交付责任边界 + +| 资源 | 平台仓库负责 | 环境/运维负责 | +|---|---|---| +| 应用镜像 | 可重复构建的 API、MCP、Worker target | 目标 Registry、不可变 tag/digest、镜像扫描和保留策略 | +| Kubernetes | Helm Chart、Migration、探针、PDB、资源和 Pod 安全默认值 | 集群、Namespace、Ingress Controller、配额和网络策略 | +| 敏感配置 | Secret 引用和必需键名预检 | Secret 创建、轮换、审计和访问控制 | +| HTTPS | Ingress Host/TLS Secret 引用 | DNS、证书签发/续期与 TLS Secret | +| 数据 | canonical/可重建边界和恢复顺序 | OceanBase、BOS、RabbitMQ 的备份、容量、HA 和恢复演练 | +| 监控 | JSON 事件、API `/metrics`、验收检查 | 日志/指标采集、留存、查询、告警和 On-call | + +## 3. 部署前检查 + +操作者需要准备仓库外的环境 Values、Namespace、运行时 Secret、TLS Secret、ImagePullSecret, +并确认集群可访问 OceanBase、BOS、RabbitMQ、Embedding 和 Reranker。先显式选择并核对 context: + +```bash +kubectl config current-context +kubectl auth can-i get deployments -n knowledge-platform +kubectl auth can-i patch deployments -n knowledge-platform +``` + +镜像必须使用 Registry 中已存在的不可变 tag 或 digest,禁止 `latest`。部署前执行: + +```bash +make deploy-lint VALUES=/secure/path/environment-values.yaml +make deploy-render VALUES=/secure/path/environment-values.yaml >/tmp/ekp-rendered.yaml +``` + +渲染文件只能包含非敏感 ConfigMap 和 Secret 引用;完成检查后应删除临时文件。 + +## 4. 安装、幂等升级与只读验收 + +首次安装或普通升级使用同一入口: + +```bash +NAMESPACE=knowledge-platform RELEASE=knowledge-platform \ + make deploy VALUES=/secure/path/environment-values.yaml +``` + +只读验收不会触发安装、重启或故障注入,但强制 context 与操作者声明完全一致: + +```bash +ACCEPTANCE_KUBE_CONTEXT="$(kubectl config current-context)" \ +NAMESPACE=knowledge-platform RELEASE=knowledge-platform \ + make cluster-acceptance VALUES=/secure/path/environment-values.yaml +``` + +## 5. 完整可用性验收 + +完整模式会连续执行两次相同 Helm 入口、滚动重启 API/MCP,并在一个合成入库 Job 运行时强制 +删除一个明确选中的 Worker Pod。只能在已批准的验收 Namespace 执行: + +```bash +context="$(kubectl config current-context)" +ACCEPTANCE_MODE=full \ +ACCEPTANCE_KUBE_CONTEXT="${context}" \ +ACCEPTANCE_CONFIRM="DEPLOY-003:${context}:knowledge-platform:knowledge-platform" \ +NAMESPACE=knowledge-platform RELEASE=knowledge-platform \ + make cluster-acceptance VALUES=/secure/path/environment-values.yaml +``` + +脚本验证以下内容: + +- 相同 Values 的安装/升级可以重复成功,Migration Hook 先于工作负载更新完成; +- API/MCP 至少两个副本全部 Ready/Available,PDB 存在; +- Admin、健康检查、指标和 MCP 均通过稳定 HTTPS Host 到达; +- API/MCP 滚动期间连续健康探测无中断; +- Worker 被非优雅中断后,同一 canonical Job 经租约到期和恢复调度器完成,不要求重新上传; +- 合成文档验收完成后调用公开删除 API 清理,脚本不访问 Secret 值。 + +若合成文档处理过快,脚本会失败而不是虚报 Worker 中断已验证。可在受控验收环境调整 Provider +延迟或使用批准的非敏感验收文件重跑,不得加入测试后门。 + +## 6. 远程 MCP 与 WorkBuddy + +先用官方 MCP Python Client 验证远程协议、唯一 Tool 和一次真实检索。Token 通过静默输入进入 +当前进程环境,不写入命令参数或配置文件: + +```bash +export EKP_MCP_URL=https://mcp.example.internal/mcp +export EKP_MCP_ACCEPTANCE_QUERY='已批准的公共知识验收问题' +read -rsp 'MCP token: ' EKP_MCP_TOKEN && export EKP_MCP_TOKEN && echo +make mcp-remote-acceptance +unset EKP_MCP_TOKEN +``` + +随后在与最终用户相同网络路径的 WorkBuddy 客户端加载 +[`workbuddy-mcp.example.json`](../../deploy/workbuddy-mcp.example.json),只通过客户端安全环境注入 +`EKP_MCP_URL` 和 `EKP_MCP_TOKEN`。验收必须确认:连接状态正常、只发现 +`search_knowledge`、真实问题返回至少一个带来源字段的结果。记录客户端版本、时间、网络区域、 +Tool 名和成功/失败即可,禁止复制 Token、正文或内部 Provider 信息。 + +## 7. 日志、指标与最小告警 + +应用事件为单行 JSON,允许字段包括 `event`、`request_id`、`job_id`、`attempt_number`、`stage`、 +`job_status` 和耗时;异常正文不会被格式化器写出。常用检查: + +```bash +kubectl -n knowledge-platform logs deployment/knowledge-platform-api --since=15m +kubectl -n knowledge-platform logs deployment/knowledge-platform-worker --since=15m +curl --fail https://knowledge.example.internal/metrics +kubectl -n knowledge-platform get deployments,pods,poddisruptionbudgets +``` + +环境监控至少采集并告警:API/MCP Ready 副本不足、Pod 重启持续增加、HTTP 5xx、P95 请求耗时、 +Worker 无 Ready 副本、`ingestion_attempt_failed`/`ingestion_retry_cleanup_failed`、恢复扫描持续 +失败、Migration Job 失败,以及 OceanBase/BOS/RabbitMQ/模型服务在应用日志中的安全失败事件。 +阈值由真实流量基线确定,不在本任务中固化完整监控平台。 + +## 8. 故障处置 + +API/MCP 不可用时先检查 Deployment/Endpoint/Ingress/TLS,再按单个 Deployment 滚动重启;禁止 +直接打印 Pod 环境变量。Job 卡住时使用 Admin/API 的 Job ID 查询状态,再按相同 Job ID 搜索 +Worker JSON 事件。处于有效租约的 Job 不应手工重复提交;租约到期后恢复调度器会重投递。 + +更新同名 Secret 不会自动改变已运行进程。由运维完成 Secret 轮换后,依次滚动重启 API、MCP、 +Worker 和 Scheduler,并复跑只读验收。稳定 Host 和客户端 Token 未轮换时,Admin/WorkBuddy +无需重新配置;MCP Token 轮换时必须通过客户端 Secret 分发流程更新。 + +## 9. 备份与恢复顺序 + +OceanBase 关系表和 BOS 不可变源文件共同构成事实数据,必须由资源所有者执行一致性备份策略。 +OceanBase 向量表是可重建索引,不得代替 canonical 备份。RabbitMQ 消息只提供执行提示, +canonical Job/Attempt 才是恢复依据,但生产 Broker 仍应按其 SLA 做持久化和 HA。 + +恢复顺序为:恢复/验证 OceanBase canonical 表 → 恢复并核对 BOS 对象 → 部署相同或兼容应用版本 +并执行 forward-only Migration → 恢复 RabbitMQ 连通 → 启动 Worker/Scheduler → 按 active Index +Profile 重建缺失向量 → 验证来源追踪和真实检索。Helm 回滚只能回退兼容的应用清单,不能反向 +撤销数据库 Migration。完整灾备演练仍是后置任务。 + +## 10. 验收记录 + +每次目标集群验收按 [`DEPLOY_003_ACCEPTANCE.md`](DEPLOY_003_ACCEPTANCE.md) 记录非敏感结果。 +只有完整自动化验收、真实 MCP 协议调用和远程 WorkBuddy 三部分全部通过,DEPLOY-003 才能 +标记完成并合并到 `main`。 diff --git a/docs/development/DEPLOY_003_ACCEPTANCE.md b/docs/development/DEPLOY_003_ACCEPTANCE.md new file mode 100644 index 0000000..8e7549b --- /dev/null +++ b/docs/development/DEPLOY_003_ACCEPTANCE.md @@ -0,0 +1,52 @@ +# DEPLOY-003 目标集群验收记录 + +**状态:** 等待目标环境 + +**Task:** DEPLOY-003 + +**记录规则:** 只记录非敏感标识和聚合结果,不记录 Secret、业务正文或内部 Provider 响应。 + +## 环境信息 + +- Kubernetes context:待提供 +- Namespace / Release:待提供 +- API HTTPS Host:待提供(只记录公开受控 Host) +- MCP HTTPS Host:待提供(只记录公开受控 Host) +- 镜像 tag/digest:待提供 +- 验收时间与操作者:待提供 + +## 自动化验收 + +- [ ] 同一 Values 连续两次 `upgrade --install` 成功 +- [ ] Migration Hook 成功且失败保护已核对 +- [ ] API/MCP 各至少两个 Ready/Available 副本 +- [ ] API/MCP PodDisruptionBudget 生效 +- [ ] Admin、健康检查、`/metrics` 和 MCP 使用有效 HTTPS +- [ ] API/MCP 滚动重启期间连续健康探测无失败 +- [ ] Worker 非优雅中断后同一 Job 自动恢复成功 +- [ ] 验收合成文档已清理 +- [ ] Manifest、镜像与验收日志凭据扫描为 0 + +执行结果与命令退出码:待执行。 + +## 远程 MCP / WorkBuddy + +- [ ] 官方 MCP Client 只发现 `search_knowledge` +- [ ] 官方 MCP Client 完成一次真实、有来源的检索 +- [ ] 最终用户网络路径上的 WorkBuddy 连接成功 +- [ ] WorkBuddy 只发现批准 Tool 并完成真实检索 + +客户端版本、网络区域和非敏感结果:待执行。 + +## 运行保障 + +- [ ] 日志采集可按 `request_id` / `job_id` 定位 +- [ ] 最小指标已被环境监控系统抓取 +- [ ] Ready 副本、5xx、耗时、Pod 重启和 Job 失败告警责任人已确认 +- [ ] OceanBase/BOS/RabbitMQ 备份责任人、周期和恢复顺序已确认 + +## 结论 + +当前仓库实现已提供 PDB、结构化日志、最小指标、完整验收脚本和运行手册;本机没有目标 +Kubernetes context、Registry、DNS/TLS 或远程 WorkBuddy 环境参数,因此所有目标环境复选项 +保持未完成,DEPLOY-003 不得提前闭环。 diff --git a/docs/development/KUBERNETES_DEPLOYMENT.md b/docs/development/KUBERNETES_DEPLOYMENT.md index c1a1ecc..f687a2b 100644 --- a/docs/development/KUBERNETES_DEPLOYMENT.md +++ b/docs/development/KUBERNETES_DEPLOYMENT.md @@ -96,10 +96,17 @@ Controller 托管。证书只通过预建 TLS Secret 引用,不进入 Values `RuntimeDefault` seccomp 和只读根文件系统; - `/tmp` 使用有大小上限的 `emptyDir`,ServiceAccount 不挂载 API Token; - API/MCP 滚动更新使用 `maxUnavailable: 0`,未 Ready Pod 不进入 Service Endpoint。 +- API/MCP 默认各有一个 `minAvailable: 1` 的 PodDisruptionBudget,避免节点维护同时驱逐全部 + 用户入口;不可用故障仍由 Deployment 副本恢复处理。 +- API Pod 默认以标准注解暴露 `/metrics`,只包含进程存活时间、按路由模板聚合的请求量和 + 请求耗时,不包含 Query、文件名、正文、Endpoint 或凭据。 `/healthz` 当前只表示 API 进程存活,不探测所有外部 Provider;真实依赖故障、滚动重启和 Worker 恢复行为由 DEPLOY-003 在目标集群验收。 +运行、滚动更新、Worker 故障恢复、备份责任与远程 MCP/WorkBuddy 验收流程见 +[`CLUSTER_OPERATIONS.md`](CLUSTER_OPERATIONS.md)。 + ## 6. 可选 Demo RabbitMQ 生产默认 `rabbitmq.enabled=false`,`RABBITMQ_URL` 指向外部受管 Broker。仅开发/演示时可开启 diff --git a/docs/plans/PROGRESS.md b/docs/plans/PROGRESS.md index 2f4da57..d2c11f4 100644 --- a/docs/plans/PROGRESS.md +++ b/docs/plans/PROGRESS.md @@ -1,12 +1,12 @@ # 项目进度 -当前阶段:生产容器镜像闭环完成,准备实施 Kubernetes/Helm 一键部署 +当前阶段:DEPLOY-003 集群可用性与远程验收进行中 -已完成:DEMO-001~DEMO-010、DEMO-012、DEMO-014~DEMO-016、ARCH-001~ARCH-006、PLAN-001~PLAN-002、DEPLOY-001 -进行中:无 -阻塞:无 +已完成:DEMO-001~DEMO-010、DEMO-012、DEMO-014~DEMO-016、ARCH-001~ARCH-006、PLAN-001~PLAN-002、DEPLOY-001~DEPLOY-002 +进行中:DEPLOY-003 +阻塞:目标 Kubernetes context、Registry 镜像坐标、DNS/TLS 与远程 WorkBuddy 环境参数未提供 -下一任务:`DEPLOY-002`,完成并合并后执行 `DEPLOY-003`。 +下一门槛:在明确目标集群执行 DEPLOY-003 完整验收;通过前不开始后续任务。 Provider 架构迁移按 `ARCH-001 → ARCH-002 → ARCH-003 → ARCH-004 → ARCH-005 → ARCH-006` 执行并已完成。DEMO-010 在迁移闭环验收后实施,未扩大为完整 V1 Admin Console。 @@ -424,3 +424,25 @@ Provider 架构迁移按 `ARCH-001 → ARCH-002 → ARCH-003 → ARCH-004 → AR - 目标企业集群的稳定 HTTPS、滚动重启、Worker 故障恢复、真实外部依赖监测与远程 WorkBuddy 仍属于 `DEPLOY-003`;本任务没有把这些项提前标记完成,也没有改变 Accepted ADR 或公共知识库 范围。下一任务:`DEPLOY-003`。 + +### DEPLOY-003(进行中) + +- 新增 API/MCP PodDisruptionBudget,继续使用双副本和 `maxUnavailable: 0`,覆盖节点维护与滚动 + 更新的最小可用性;没有引入新微服务或扩大公共知识库范围。 +- 新增字段白名单 JSON 日志、请求相关 ID、API 低基数 `/metrics`,并为上传、批次、检索、MCP、 + Worker Attempt/Stage/Retry/Cleanup/Recovery 补充安全事件;异常正文、Query、文件名、Provider + Endpoint 和凭据不进入结构化字段。 +- 新增显式 context 核对和 `ACCEPTANCE_CONFIRM` 二次确认的集群验收入口;完整模式验证相同 Values + 两次升级、HTTPS、PDB、连续滚动探测,并在合成 Job 运行时非优雅删除单个 Worker Pod,等待 + canonical 租约与恢复调度器收敛后清理验收文档。 +- 新增官方 MCP Client 远程验收脚本;Token 只从进程环境读取,输出只包含 Tool/结果存在性, + 不包含 Token 或检索正文。远程 WorkBuddy 仍需在最终用户网络路径现场确认。 +- 新增中文集群运行手册和验收记录,明确 OceanBase canonical/BOS、可重建向量、RabbitMQ、TLS、 + Registry、日志指标与备份恢复的责任边界。完整 Observability 平台和灾备演练保持后置。 +- 仓库侧验证已通过 Ruff Format/Lint、Mypy、Bash 语法、三种 Helm lint、13 个渲染资源的 + kubeconform strict、Alembic Head/离线 SQL、Unit + Integration `200 passed` 和 RAG Evaluation + `5 passed`;本地运行 API 的 `/healthz`、`/metrics` 与 JSON 事件烟测通过。 +- 新生产镜像的首次本地重建被 Docker Hub 匿名授权连接重置阻塞,两次重试结果一致,没有改动 + 固定 digest 或镜像来源规避;当前机器同时没有目标 Kubernetes context、目标 Registry 坐标、 + DNS/TLS 或远程 WorkBuddy 参数。因此现场复选项保持未完成;当前任务分支仅保存并推送仓库侧 + 阶段性实现,在现场验收闭环前不合并 `main`。 diff --git a/docs/plans/PUBLIC_KNOWLEDGE_CLUSTER_PLAN.md b/docs/plans/PUBLIC_KNOWLEDGE_CLUSTER_PLAN.md index ebfbea2..1c691a1 100644 --- a/docs/plans/PUBLIC_KNOWLEDGE_CLUSTER_PLAN.md +++ b/docs/plans/PUBLIC_KNOWLEDGE_CLUSTER_PLAN.md @@ -100,8 +100,10 @@ PostgreSQL/MinIO 假设与 ADR-012 不一致,不得作为集群交付模板。 ### DEPLOY-003 -在目标集群完成安装、幂等升级、滚动重启、Worker 故障恢复和远程 WorkBuddy 验收;形成运行 -手册、备份/恢复责任边界和最小日志/指标检查。灾备演练与完整 Observability 平台仍属后置项。 +**进行中。** 仓库侧已实现 PDB、结构化日志、最小 HTTP 指标、显式 context/二次确认保护的 +验收脚本,并形成运行手册和备份/恢复责任边界。仍须在目标集群完成安装、幂等升级、滚动重启、 +Worker 故障恢复、日志/指标接入和远程 WorkBuddy 验收;在这些现场项完成前不得闭环。灾备演练 +与完整 Observability 平台仍属后置项。 ## 5. 最小验收标准 diff --git a/docs/plans/TASKS.md b/docs/plans/TASKS.md index 4710f5c..4ed657d 100644 --- a/docs/plans/TASKS.md +++ b/docs/plans/TASKS.md @@ -32,7 +32,9 @@ validation. Provider credentials and internal endpoints are runtime inputs and n - [x] DEMO-016 Batch async upload API + per-file Job status + Admin Console - [x] DEPLOY-001 Production images for API/Admin, MCP Gateway and Async Worker - [x] DEPLOY-002 Kubernetes/Helm one-click deployment + Secret/Migration/Ingress baseline -- [ ] DEPLOY-003 Availability, rolling upgrade and remote WorkBuddy acceptance +- [ ] DEPLOY-003 Availability, rolling upgrade and remote WorkBuddy acceptance(进行中:仓库侧 + PDB、结构化日志、最小指标、验收脚本与运行手册已实现;等待目标集群/Registry/DNS/TLS 和 + 远程 WorkBuddy 实测,不得提前勾选) Dependencies and acceptance boundaries are defined in [`ASYNC_INGESTION_MCP_PLAN.md`](ASYNC_INGESTION_MCP_PLAN.md) and diff --git a/docs/plans/V1_EXECUTION_PLAN.md b/docs/plans/V1_EXECUTION_PLAN.md index bb754b0..93d3989 100644 --- a/docs/plans/V1_EXECUTION_PLAN.md +++ b/docs/plans/V1_EXECUTION_PLAN.md @@ -20,7 +20,8 @@ 3. `DEMO-016`(已完成):批量异步上传、批次/Job 查询和 Admin Console 状态展示; 4. `DEPLOY-001`(已完成):API/Admin、MCP Gateway、Async Worker 生产镜像; 5. `DEPLOY-002`(已完成):Kubernetes/Helm、Secret、Migration、Ingress/TLS 和一键部署入口; -6. `DEPLOY-003`:多副本可用性、滚动升级、Worker 恢复和远程 WorkBuddy 验收。 +6. `DEPLOY-003`(进行中):仓库侧可用性保护、结构化日志、最小指标和验收入口已实现;仍需在 + 明确目标集群完成幂等升级、滚动升级、Worker 恢复和远程 WorkBuddy 现场验收。 严格依赖为: diff --git a/docs/product/PRD.md b/docs/product/PRD.md index 783dee6..4f6e71c 100644 --- a/docs/product/PRD.md +++ b/docs/product/PRD.md @@ -23,6 +23,10 @@ 过滤。实施顺序和集群验收边界见 [`PUBLIC_KNOWLEDGE_CLUSTER_PLAN.md`](../plans/PUBLIC_KNOWLEDGE_CLUSTER_PLAN.md)。 +公共知识库交付还要求 API/Admin 与 MCP 各至少两个副本、滚动升级期间稳定 HTTPS 地址可用、 +Worker 中断后以 canonical Job/Attempt 恢复,并提供不含业务正文或凭据的结构化事件日志与最小 +HTTP 指标。完整监控平台、跨区域灾备和自动云资源编排不随本阶段扩入 V1 Scope。 + --- # 1. 项目背景 diff --git a/packages/core/src/ekp_core/ingestion_jobs.py b/packages/core/src/ekp_core/ingestion_jobs.py index 94b9f44..051bde4 100644 --- a/packages/core/src/ekp_core/ingestion_jobs.py +++ b/packages/core/src/ekp_core/ingestion_jobs.py @@ -1,5 +1,6 @@ """Durable asynchronous ingestion orchestration and retry cleanup.""" +import logging from dataclasses import dataclass from datetime import UTC, datetime, timedelta from enum import StrEnum @@ -57,6 +58,7 @@ SAFE_TRANSIENT_ERROR = "temporary ingestion failure" SAFE_PERMANENT_ERROR = "document content could not be processed" SAFE_CLEANUP_ERROR = "retry cleanup failed" SAFE_LEASE_ERROR = "worker execution lease expired" +logger = logging.getLogger(__name__) class IngestionJobNotFoundError(LookupError): @@ -293,17 +295,47 @@ class IngestionJobRunner: claim = self._claim(job_id) if not claim.run: assert claim.result is not None + logger.info( + "Ingestion delivery deferred or already terminal", + extra={ + "event": "ingestion_delivery_not_claimed", + "job_id": str(job_id), + "job_status": claim.result.status.value, + "retry_delay_seconds": claim.result.retry_delay_seconds, + }, + ) return claim.result if claim.cleanup_required: + logger.info( + "Retry cleanup started", + extra={"event": "ingestion_retry_cleanup_started", "job_id": str(job_id)}, + ) try: document_id = self._job_document_id(job_id) self._cleanup.cleanup(document_id) except Exception: self._finish_cleanup_failure(job_id) + logger.error( + "Retry cleanup failed", + extra={"event": "ingestion_retry_cleanup_failed", "job_id": str(job_id)}, + ) return JobRunResult(JobRunAction.COMPLETE, IngestionJobStatus.FAILED) + logger.info( + "Retry cleanup completed", + extra={"event": "ingestion_retry_cleanup_completed", "job_id": str(job_id)}, + ) attempt_number, document_id = self._start_attempt(job_id) + logger.info( + "Ingestion attempt started", + extra={ + "event": "ingestion_attempt_started", + "job_id": str(job_id), + "document_id": str(document_id), + "attempt_number": attempt_number, + }, + ) def stage_callback(stage: IngestionStage) -> None: """Bind this attempt while pipeline stages remain Job-agnostic.""" @@ -317,6 +349,16 @@ class IngestionJobRunner: return self._finish_failure(job_id, attempt_number, document_id, error) self._finish_success(job_id, attempt_number, document_id) + logger.info( + "Ingestion attempt succeeded", + extra={ + "event": "ingestion_attempt_succeeded", + "job_id": str(job_id), + "document_id": str(document_id), + "attempt_number": attempt_number, + "job_status": IngestionJobStatus.SUCCEEDED.value, + }, + ) return JobRunResult(JobRunAction.COMPLETE, IngestionJobStatus.SUCCEEDED) def _claim(self, job_id: UUID) -> _Claim: @@ -451,6 +493,15 @@ class IngestionJobRunner: job.lease_expires_at = datetime.now(UTC) + timedelta(seconds=self._lease_seconds) attempt.stage = stage session.commit() + logger.info( + "Ingestion stage advanced", + extra={ + "event": "ingestion_stage_advanced", + "job_id": str(job_id), + "attempt_number": attempt_number, + "stage": stage.value, + }, + ) def _finish_success(self, job_id: UUID, attempt_number: int, document_id: UUID) -> None: """Atomically close Job/Attempt after the document becomes ready.""" @@ -502,6 +553,17 @@ class IngestionJobRunner: document.status = DocumentStatus.PROCESSING document.error_message = safe_error session.commit() + logger.warning( + "Ingestion retry scheduled", + extra={ + "event": "ingestion_retry_scheduled", + "job_id": str(job_id), + "document_id": str(document_id), + "attempt_number": attempt_number, + "job_status": job.status.value, + "retry_delay_seconds": delay, + }, + ) return JobRunResult(JobRunAction.RETRY, job.status, delay) job.status = IngestionJobStatus.FAILED @@ -511,6 +573,16 @@ class IngestionJobRunner: document.status = DocumentStatus.FAILED document.error_message = safe_error session.commit() + logger.error( + "Ingestion attempt failed terminally", + extra={ + "event": "ingestion_attempt_failed", + "job_id": str(job_id), + "document_id": str(document_id), + "attempt_number": attempt_number, + "job_status": job.status.value, + }, + ) return JobRunResult(JobRunAction.COMPLETE, job.status) def _finish_cleanup_failure(self, job_id: UUID) -> None: @@ -554,6 +626,10 @@ def dispatch_pending_jobs(jobs: IngestionJobService, dispatcher: JobDispatcher) try: dispatcher.enqueue(job_id) except Exception: + logger.error( + "Durable Job redelivery failed", + extra={"event": "ingestion_recovery_dispatch_failed", "job_id": str(job_id)}, + ) continue dispatched += 1 return dispatched diff --git a/packages/core/src/ekp_core/observability.py b/packages/core/src/ekp_core/observability.py new file mode 100644 index 0000000..eeef4cc --- /dev/null +++ b/packages/core/src/ekp_core/observability.py @@ -0,0 +1,75 @@ +"""Secret-safe structured logging shared by every runtime process.""" + +import json +import logging +import sys +from contextvars import ContextVar, Token +from datetime import UTC, datetime +from typing import Final + +_request_id: ContextVar[str | None] = ContextVar("ekp_request_id", default=None) +_EVENT_FIELDS: Final[tuple[str, ...]] = ( + "event", + "request_id", + "method", + "path", + "status_code", + "duration_ms", + "job_id", + "document_id", + "batch_id", + "attempt_number", + "stage", + "job_status", + "retry_delay_seconds", + "dispatch_count", +) + + +class JsonLogFormatter(logging.Formatter): + """Render an allowlisted JSON event without serializing exception details. + + Provider exceptions may contain endpoints or credentials. Callers therefore + log stable event names and identifiers, while this formatter deliberately + ignores ``exc_info`` and arbitrary record attributes. + """ + + def format(self, record: logging.LogRecord) -> str: + payload: dict[str, object] = { + "timestamp": datetime.now(UTC).isoformat(), + "level": record.levelname, + "logger": record.name, + "message": record.getMessage(), + } + context_request_id = _request_id.get() + if context_request_id is not None: + payload["request_id"] = context_request_id + for field in _EVENT_FIELDS: + value = getattr(record, field, None) + if value is not None: + payload[field] = value + return json.dumps(payload, ensure_ascii=False, separators=(",", ":")) + + +def configure_json_logging(level: str) -> None: + """Install one stdout JSON handler and preserve idempotent app factories.""" + root = logging.getLogger() + existing = next( + (handler for handler in root.handlers if isinstance(handler.formatter, JsonLogFormatter)), + None, + ) + if existing is None: + handler = logging.StreamHandler(sys.stdout) + handler.setFormatter(JsonLogFormatter()) + root.addHandler(handler) + root.setLevel(level) + + +def bind_request_id(request_id: str) -> Token[str | None]: + """Bind a validated correlation identifier to the current request context.""" + return _request_id.set(request_id) + + +def reset_request_id(token: Token[str | None]) -> None: + """Restore the previous request context after response logging completes.""" + _request_id.reset(token) diff --git a/scripts/cluster_acceptance.sh b/scripts/cluster_acceptance.sh new file mode 100755 index 0000000..84ddc50 --- /dev/null +++ b/scripts/cluster_acceptance.sh @@ -0,0 +1,201 @@ +#!/usr/bin/env bash +set -euo pipefail + +# DEPLOY-003 validates one explicitly selected cluster context. Secret contents +# never enter this process; only HTTPS public endpoints and Kubernetes metadata +# are inspected. Full mode performs controlled rollout and worker-failure tests. +values_file="${1:-}" +chart_dir="${CHART:-deploy/helm/knowledge-platform}" +release_name="${RELEASE:-knowledge-platform}" +namespace="${NAMESPACE:-knowledge-platform}" +acceptance_mode="${ACCEPTANCE_MODE:-read-only}" +expected_context="${ACCEPTANCE_KUBE_CONTEXT:-}" +rollout_timeout="${ACCEPTANCE_ROLLOUT_TIMEOUT:-10m}" +worker_wait_seconds="${ACCEPTANCE_WORKER_WAIT_SECONDS:-720}" + +fail() { + echo "DEPLOY-003 acceptance failed: $1" >&2 + exit 1 +} + +if [[ -z "${values_file}" || ! -f "${values_file}" ]]; then + fail "VALUES must point to an existing environment values file" +fi +if [[ "${acceptance_mode}" != "read-only" && "${acceptance_mode}" != "full" ]]; then + fail "ACCEPTANCE_MODE must be read-only or full" +fi +for tool in helm kubectl curl python3; do + command -v "${tool}" >/dev/null 2>&1 || fail "required tool is missing: ${tool}" +done + +current_context="$(kubectl config current-context 2>/dev/null || true)" +if [[ -z "${expected_context}" || "${current_context}" != "${expected_context}" ]]; then + fail "ACCEPTANCE_KUBE_CONTEXT must exactly match the active context" +fi + +preflight_yaml="$({ + helm template "${release_name}" "${chart_dir}" \ + --namespace "${namespace}" \ + --values "${values_file}" \ + --show-only templates/tests/preflight-references.yaml +})" + +reference() { + local field="$1" + printf '%s\n' "${preflight_yaml}" | + awk -v key="${field}:" '$1 == key {gsub(/"/, "", $2); print $2}' +} + +ingress_enabled="$(reference ingressEnabled)" +api_host="$(reference apiHost)" +mcp_host="$(reference mcpHost)" +api_deployment="$(reference apiDeployment)" +mcp_deployment="$(reference mcpDeployment)" +worker_deployment="$(reference workerDeployment)" +scheduler_deployment="$(reference schedulerDeployment)" +metrics_path="$(reference metricsPath)" + +if [[ "${ingress_enabled}" != "true" ]]; then + fail "target acceptance requires ingress.enabled=true" +fi +for value in \ + "${api_host}" \ + "${mcp_host}" \ + "${api_deployment}" \ + "${mcp_deployment}" \ + "${worker_deployment}" \ + "${scheduler_deployment}" \ + "${metrics_path}"; do + [[ -n "${value}" ]] || fail "Helm acceptance references are incomplete" +done + +if [[ "${acceptance_mode}" == "full" ]]; then + expected_confirmation="DEPLOY-003:${current_context}:${namespace}:${release_name}" + if [[ "${ACCEPTANCE_CONFIRM:-}" != "${expected_confirmation}" ]]; then + fail "set ACCEPTANCE_CONFIRM=${expected_confirmation} for controlled mutations" + fi + # Running the same entrypoint twice proves install/upgrade idempotency against + # the selected environment before disruption tests begin. + "${PWD}/scripts/deploy.sh" "${values_file}" + "${PWD}/scripts/deploy.sh" "${values_file}" +fi + +kubectl -n "${namespace}" get deployment "${api_deployment}" >/dev/null +kubectl -n "${namespace}" get deployment "${mcp_deployment}" >/dev/null +kubectl -n "${namespace}" get deployment "${worker_deployment}" >/dev/null +kubectl -n "${namespace}" get deployment "${scheduler_deployment}" >/dev/null + +for deployment in \ + "${api_deployment}" \ + "${mcp_deployment}" \ + "${worker_deployment}" \ + "${scheduler_deployment}"; do + kubectl -n "${namespace}" rollout status "deployment/${deployment}" \ + --timeout "${rollout_timeout}" +done + +assert_multi_replica_ready() { + local deployment="$1" + local desired ready available + desired="$(kubectl -n "${namespace}" get deployment "${deployment}" -o jsonpath='{.spec.replicas}')" + ready="$(kubectl -n "${namespace}" get deployment "${deployment}" -o jsonpath='{.status.readyReplicas}')" + available="$(kubectl -n "${namespace}" get deployment "${deployment}" -o jsonpath='{.status.availableReplicas}')" + [[ "${desired}" -ge 2 ]] || fail "${deployment} must have at least two replicas" + [[ "${ready:-0}" -eq "${desired}" ]] || fail "${deployment} does not have all replicas ready" + [[ "${available:-0}" -eq "${desired}" ]] || fail "${deployment} is not fully available" +} + +assert_multi_replica_ready "${api_deployment}" +assert_multi_replica_ready "${mcp_deployment}" +for disruption_budget in "${api_deployment}" "${mcp_deployment}"; do + disruptions_allowed="0" + for attempt in $(seq 1 30); do + disruptions_allowed="$(kubectl -n "${namespace}" get poddisruptionbudget \ + "${disruption_budget}" -o jsonpath='{.status.disruptionsAllowed}')" + [[ "${disruptions_allowed:-0}" -ge 1 ]] && break + sleep 1 + done + [[ "${disruptions_allowed:-0}" -ge 1 ]] || + fail "${disruption_budget} does not currently allow one voluntary disruption" +done + +api_base_url="https://${api_host}" +mcp_url="https://${mcp_host}/mcp" +curl --fail --silent --show-error --connect-timeout 5 --max-time 10 \ + --proto '=https' "${api_base_url}/healthz" >/dev/null +curl --fail --silent --show-error --connect-timeout 5 --max-time 10 \ + --proto '=https' "${api_base_url}/admin/upload" >/dev/null +metrics_output="$({ + curl --fail --silent --show-error --connect-timeout 5 --max-time 10 \ + --proto '=https' "${api_base_url}${metrics_path}" +})" +printf '%s\n' "${metrics_output}" | grep -q '^ekp_process_uptime_seconds ' || + fail "API metrics endpoint does not expose process uptime" +printf '%s\n' "${metrics_output}" | grep -q '^ekp_http_requests_total{' || + fail "API metrics endpoint does not expose request totals" + +# A credential-free request may be rejected, but it must complete TLS and reach +# the MCP application rather than fail DNS, certificate, or gateway routing. +mcp_status="$({ + curl --silent --show-error --output /dev/null --write-out '%{http_code}' \ + --connect-timeout 5 --max-time 10 --proto '=https' "${mcp_url}" +})" +case "${mcp_status}" in + 200 | 400 | 401 | 403 | 405 | 406) ;; + *) fail "MCP HTTPS endpoint returned unexpected status ${mcp_status}" ;; +esac + +continuous_probe() { + local stop_file="$1" + local failure_file="$2" + local current_mcp_status + while [[ ! -f "${stop_file}" ]]; do + if ! curl --fail --silent --show-error --connect-timeout 5 --max-time 10 --proto '=https' \ + "${api_base_url}/healthz" >/dev/null; then + printf 'failed\n' >"${failure_file}" + return + fi + current_mcp_status="$({ + curl --silent --show-error --output /dev/null --write-out '%{http_code}' \ + --connect-timeout 5 --max-time 10 --proto '=https' "${mcp_url}" + })" + case "${current_mcp_status}" in + 200 | 400 | 401 | 403 | 405 | 406) ;; + *) + printf 'failed\n' >"${failure_file}" + return + ;; + esac + sleep 1 + done +} + +if [[ "${acceptance_mode}" == "full" ]]; then + probe_dir="$(mktemp -d)" + stop_file="${probe_dir}/stop" + failure_file="${probe_dir}/failure" + cleanup_probe() { + touch "${stop_file}" 2>/dev/null || true + wait "${probe_pid:-}" 2>/dev/null || true + rm -rf "${probe_dir}" + } + trap cleanup_probe EXIT + continuous_probe "${stop_file}" "${failure_file}" & + probe_pid="$!" + kubectl -n "${namespace}" rollout restart "deployment/${api_deployment}" + kubectl -n "${namespace}" rollout restart "deployment/${mcp_deployment}" + kubectl -n "${namespace}" rollout status "deployment/${api_deployment}" \ + --timeout "${rollout_timeout}" + kubectl -n "${namespace}" rollout status "deployment/${mcp_deployment}" \ + --timeout "${rollout_timeout}" + touch "${stop_file}" + wait "${probe_pid}" + [[ ! -f "${failure_file}" ]] || fail "API health probe failed during rolling restart" + rm -rf "${probe_dir}" + trap - EXIT + + "${PWD}/scripts/worker_recovery_acceptance.sh" "${api_base_url}" \ + "${namespace}" "${release_name}" "${worker_deployment}" "${worker_wait_seconds}" +fi + +echo "DEPLOY-003 automated cluster acceptance passed (${acceptance_mode})" diff --git a/scripts/mcp_remote_acceptance.py b/scripts/mcp_remote_acceptance.py new file mode 100644 index 0000000..6145e22 --- /dev/null +++ b/scripts/mcp_remote_acceptance.py @@ -0,0 +1,56 @@ +"""Secret-safe remote MCP protocol acceptance used before WorkBuddy sign-off.""" + +import asyncio +import os +import sys + +import httpx2 +from mcp import ClientSession +from mcp.client.streamable_http import streamable_http_client +from mcp.types import CallToolResult + + +async def _accept() -> None: + """Discover exactly one approved tool and execute a real remote search.""" + url = os.environ.get("EKP_MCP_URL", "") + token = os.environ.get("EKP_MCP_TOKEN", "") + query = os.environ.get("EKP_MCP_ACCEPTANCE_QUERY", "") + if not url.startswith("https://") or len(token) < 32 or not query.strip(): + raise ValueError( + "EKP_MCP_URL (HTTPS), EKP_MCP_TOKEN and EKP_MCP_ACCEPTANCE_QUERY are required" + ) + # The credential is read in process and is never included in CLI arguments, + # output, exceptions, or the checked-in WorkBuddy example configuration. + headers = {"Authorization": f"Bearer {token}"} + async with httpx2.AsyncClient(headers=headers, timeout=30.0) as client: + async with streamable_http_client(url, http_client=client) as (read, write): + async with ClientSession(read, write) as session: + await session.initialize() + tools = await session.list_tools() + tool_names = [tool.name for tool in tools.tools] + if tool_names != ["search_knowledge"]: + raise RuntimeError("remote MCP exposed an unexpected tool set") + result = await session.call_tool( + "search_knowledge", + {"query": query, "top_k": 1}, + ) + if not isinstance(result, CallToolResult) or result.is_error: + raise RuntimeError("remote MCP search returned a tool error") + structured = result.structured_content + if not isinstance(structured, dict) or not structured.get("results"): + raise RuntimeError("remote MCP search returned no structured source") + print("remote MCP protocol acceptance passed: one tool, one grounded result") + + +def main() -> int: + """Return a stable non-zero code without reflecting sensitive exceptions.""" + try: + asyncio.run(_accept()) + except Exception: + print("remote MCP protocol acceptance failed", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/worker_recovery_acceptance.sh b/scripts/worker_recovery_acceptance.sh new file mode 100755 index 0000000..1887b96 --- /dev/null +++ b/scripts/worker_recovery_acceptance.sh @@ -0,0 +1,92 @@ +#!/usr/bin/env bash +set -euo pipefail + +# This helper is called only by cluster_acceptance.sh full mode. It interrupts +# one exact worker Pod after a durable Job reaches RUNNING, then verifies that +# the lease/recovery scheduler converges the same Job without a second upload. +api_base_url="${1:?API base URL is required}" +namespace="${2:?namespace is required}" +release_name="${3:?release name is required}" +worker_deployment="${4:?worker deployment is required}" +wait_seconds="${5:-720}" +poll_interval="${ACCEPTANCE_JOB_POLL_INTERVAL_SECONDS:-0.2}" +work_dir="$(mktemp -d)" +source_file="${work_dir}/deploy-003-worker-recovery.md" +response_file="${work_dir}/response.json" +document_id="" + +cleanup() { + if [[ -n "${document_id}" ]]; then + curl --silent --show-error --connect-timeout 5 --max-time 15 \ + --request DELETE --proto '=https' \ + "${api_base_url}/api/v1/documents/${document_id}" >/dev/null || true + fi + rm -rf "${work_dir}" +} +trap cleanup EXIT + +[[ "${wait_seconds}" =~ ^[0-9]+$ && "${wait_seconds}" -gt 0 ]] || exit 2 +[[ "${poll_interval}" =~ ^[0-9]+([.][0-9]+)?$ ]] || exit 2 + +# Enough independent paragraphs make RUNNING observable without introducing a +# large or business-derived acceptance document. +for number in $(seq 1 80); do + printf 'DEPLOY-003 worker recovery synthetic paragraph %s.\n\n' "${number}" >>"${source_file}" +done + +curl --fail --silent --show-error --connect-timeout 5 --max-time 60 --proto '=https' \ + --form "file=@${source_file};type=text/markdown" \ + "${api_base_url}/api/v1/documents" >"${response_file}" + +document_id="$(python3 -c 'import json,sys; print(json.load(sys.stdin)["document_id"])' <"${response_file}")" +job_id="$(python3 -c 'import json,sys; print(json.load(sys.stdin)["job_id"])' <"${response_file}")" +[[ -n "${document_id}" && -n "${job_id}" ]] || exit 1 + +deadline=$((SECONDS + wait_seconds)) +interrupted="false" +while ((SECONDS < deadline)); do + job_json="$({ + curl --fail --silent --show-error --connect-timeout 5 --max-time 10 --proto '=https' \ + "${api_base_url}/api/v1/jobs/${job_id}" + })" + job_status="$(printf '%s' "${job_json}" | python3 -c 'import json,sys; print(json.load(sys.stdin)["status"])')" + if [[ "${job_status}" == "running" ]]; then + worker_pod="$(kubectl -n "${namespace}" get pods \ + -l "app.kubernetes.io/instance=${release_name},app.kubernetes.io/component=worker" \ + --field-selector=status.phase=Running \ + -o jsonpath='{.items[0].metadata.name}')" + [[ -n "${worker_pod}" ]] || exit 1 + # Zero grace simulates process/node loss instead of a cooperative drain. + kubectl -n "${namespace}" delete pod "${worker_pod}" \ + --grace-period=0 --force --wait=false >/dev/null + interrupted="true" + break + fi + if [[ "${job_status}" == "succeeded" || "${job_status}" == "failed" ]]; then + echo "worker recovery acceptance could not interrupt the fast Job" >&2 + exit 1 + fi + sleep "${poll_interval}" +done +[[ "${interrupted}" == "true" ]] || exit 1 + +kubectl -n "${namespace}" rollout status "deployment/${worker_deployment}" --timeout 10m +while ((SECONDS < deadline)); do + job_json="$({ + curl --fail --silent --show-error --connect-timeout 5 --max-time 10 --proto '=https' \ + "${api_base_url}/api/v1/jobs/${job_id}" + })" + job_status="$(printf '%s' "${job_json}" | python3 -c 'import json,sys; print(json.load(sys.stdin)["status"])')" + if [[ "${job_status}" == "succeeded" ]]; then + echo "durable worker recovery passed" + exit 0 + fi + if [[ "${job_status}" == "failed" || "${job_status}" == "cancelled" ]]; then + echo "worker recovery Job reached terminal status ${job_status}" >&2 + exit 1 + fi + sleep 1 +done + +echo "worker recovery Job did not complete within ${wait_seconds} seconds" >&2 +exit 1 diff --git a/tests/integration/test_mcp_server.py b/tests/integration/test_mcp_server.py index 2d19145..e2a240b 100644 --- a/tests/integration/test_mcp_server.py +++ b/tests/integration/test_mcp_server.py @@ -191,7 +191,10 @@ def test_tool_failure_is_sanitized_for_client_and_logs() -> None: assert "knowledge search failed" in rendered assert "internal.example.test" not in rendered assert "secret-provider-token" not in rendered - log_error.assert_called_once_with("MCP knowledge search failed") + log_error.assert_called_once_with( + "MCP knowledge search failed", + extra={"event": "mcp_knowledge_search_failed"}, + ) def test_transport_rejects_unapproved_origin() -> None: diff --git a/tests/unit/test_helm_deployment.py b/tests/unit/test_helm_deployment.py index b3a187d..003ca84 100644 --- a/tests/unit/test_helm_deployment.py +++ b/tests/unit/test_helm_deployment.py @@ -82,6 +82,8 @@ def test_workloads_render_availability_and_restricted_pod_defaults() -> None: assert "readinessProbe:" in rendered assert "livenessProbe:" in rendered assert "startupProbe:" in rendered + assert rendered.count("kind: PodDisruptionBudget") == 2 + assert 'prometheus.io/path: "/metrics"' in rendered def test_scheduler_is_singleton_and_workers_do_not_embed_beat() -> None: @@ -141,6 +143,33 @@ def test_cross_field_validation_rejects_unsafe_ingestion_limits() -> None: assert "batchMaxTotalBytes must be >= maxUploadBytes" in (result.stdout + result.stderr) +def test_cross_field_validation_rejects_pdb_that_blocks_all_evictions() -> None: + """A PDB must leave room to replace one of the required replicas.""" + result = _helm( + "template", + "knowledge-platform", + str(CHART), + "--set", + "podDisruptionBudget.apiMinAvailable=2", + expect_success=False, + ) + + assert result.returncode != 0 + assert "apiMinAvailable must be < api.replicaCount" in (result.stdout + result.stderr) + + +def test_cluster_acceptance_requires_explicit_context_and_mutation_confirmation() -> None: + """The target-cluster harness cannot silently mutate an ambient context.""" + cluster_script = (ROOT / "scripts" / "cluster_acceptance.sh").read_text() + worker_script = (ROOT / "scripts" / "worker_recovery_acceptance.sh").read_text() + + assert "ACCEPTANCE_KUBE_CONTEXT" in cluster_script + assert "ACCEPTANCE_CONFIRM" in cluster_script + assert 'acceptance_mode="${ACCEPTANCE_MODE:-read-only}"' in cluster_script + assert "--grace-period=0 --force --wait=false" in worker_script + assert "get secret" not in worker_script + + def test_deploy_entrypoint_uses_atomic_upgrade_and_never_reads_secret_values() -> None: """The one-command wrapper checks references and performs an atomic wait.""" script = (ROOT / "scripts" / "deploy.sh").read_text() diff --git a/tests/unit/test_observability.py b/tests/unit/test_observability.py new file mode 100644 index 0000000..067ddea --- /dev/null +++ b/tests/unit/test_observability.py @@ -0,0 +1,72 @@ +"""Unit contracts for secret-safe logs and bounded API metrics.""" + +import json +import logging + +from fastapi.testclient import TestClient +from pydantic import SecretStr + +from ekp_api.main import create_app +from ekp_core.config import Settings +from ekp_core.observability import JsonLogFormatter, bind_request_id, reset_request_id + + +def _settings() -> Settings: + """Use an isolated test database while exercising provider-free endpoints.""" + return Settings( + app_env="test", + database_url=SecretStr("sqlite+pysqlite:///:memory:"), + ) + + +def test_json_formatter_keeps_allowlisted_context_and_drops_arbitrary_fields() -> None: + """Structured events must not serialize provider exception or custom secrets.""" + record = logging.LogRecord( + name="ekp.test", + level=logging.ERROR, + pathname=__file__, + lineno=1, + msg="safe failure", + args=(), + exc_info=None, + ) + record.event = "safe_event" + record.job_id = "job-1" + record.provider_secret = "must-not-appear" + token = bind_request_id("request-1") + try: + payload = json.loads(JsonLogFormatter().format(record)) + finally: + reset_request_id(token) + + assert payload["event"] == "safe_event" + assert payload["job_id"] == "job-1" + assert payload["request_id"] == "request-1" + assert "provider_secret" not in payload + assert "must-not-appear" not in json.dumps(payload) + + +def test_api_exposes_low_cardinality_metrics_and_request_correlation() -> None: + """Metrics use route templates, and callers receive a safe correlation id.""" + client = TestClient(create_app(_settings())) + + response = client.get("/healthz", headers={"x-request-id": "acceptance-request-1"}) + metrics = client.get("/metrics") + + assert response.status_code == 200 + assert response.headers["x-request-id"] == "acceptance-request-1" + assert metrics.status_code == 200 + assert "ekp_process_uptime_seconds " in metrics.text + assert 'path="/healthz",status="200"' in metrics.text + assert "?" not in metrics.text + + +def test_api_replaces_unsafe_request_identifier() -> None: + """Untrusted correlation headers cannot inject log structure or new lines.""" + client = TestClient(create_app(_settings())) + + response = client.get("/healthz", headers={"x-request-id": "unsafe\nvalue"}) + + request_id = response.headers["x-request-id"] + assert request_id != "unsafe\nvalue" + assert len(request_id) == 32