Compare commits
No commits in common. "codex/deploy-003-cluster-acceptance" and "main" have entirely different histories.
codex/depl
...
main
8
Makefile
8
Makefile
@ -1,4 +1,4 @@
|
||||
.PHONY: help install run run-mcp run-worker images images-print images-push deploy deploy-lint deploy-render cluster-acceptance mcp-remote-acceptance format format-check lint typecheck test test-unit test-integration migration-check eval live-acceptance up down
|
||||
.PHONY: help install run run-mcp run-worker images images-print images-push deploy deploy-lint deploy-render format format-check lint typecheck test test-unit test-integration migration-check eval live-acceptance up down
|
||||
|
||||
UV_CACHE_DIR ?= /tmp/ekp-uv-cache
|
||||
export UV_CACHE_DIR
|
||||
@ -15,8 +15,6 @@ help:
|
||||
@echo " deploy Install or upgrade Kubernetes with VALUES=<file>"
|
||||
@echo " deploy-lint Validate Helm values and templates"
|
||||
@echo " deploy-render Render non-secret Kubernetes manifests"
|
||||
@echo " cluster-acceptance Validate the explicitly selected target cluster"
|
||||
@echo " mcp-remote-acceptance Verify remote MCP protocol and real search"
|
||||
@echo " format Format code"
|
||||
@echo " format-check Check formatting without modifying files"
|
||||
@echo " lint Run linters"
|
||||
@ -52,10 +50,6 @@ deploy-lint:
|
||||
deploy-render:
|
||||
@test -n "$(VALUES)" || (echo "VALUES must point to an environment values file" >&2; exit 2)
|
||||
helm template "$${RELEASE:-knowledge-platform}" deploy/helm/knowledge-platform --namespace "$${NAMESPACE:-knowledge-platform}" --values "$(VALUES)"
|
||||
cluster-acceptance:
|
||||
./scripts/cluster_acceptance.sh "$(VALUES)"
|
||||
mcp-remote-acceptance:
|
||||
uv run python scripts/mcp_remote_acceptance.py
|
||||
format:
|
||||
uv run ruff format .
|
||||
format-check:
|
||||
|
||||
@ -69,11 +69,6 @@ The Streamable HTTP endpoint defaults to `http://127.0.0.1:8766/mcp` and exposes
|
||||
only `search_knowledge`. See `docs/development/MCP_SETUP.md` for WorkBuddy configuration,
|
||||
identity boundaries, remote allowlists, and protocol verification.
|
||||
|
||||
Kubernetes installation, target-cluster acceptance, rolling recovery, metrics and backup
|
||||
responsibility are documented in `docs/development/KUBERNETES_DEPLOYMENT.md` and
|
||||
`docs/development/CLUSTER_OPERATIONS.md`. Cluster acceptance always requires an explicitly named
|
||||
Kubernetes context; credentials remain in Kubernetes Secret or the client process environment.
|
||||
|
||||
The health endpoint is available at `http://localhost:8000/healthz`, and the
|
||||
versioned OpenAPI document is available at
|
||||
`http://localhost:8000/api/v1/openapi.json`.
|
||||
|
||||
@ -24,9 +24,6 @@ def create_celery_app(settings: Settings) -> Celery:
|
||||
task_reject_on_worker_lost=True,
|
||||
worker_prefetch_multiplier=1,
|
||||
broker_connection_retry_on_startup=True,
|
||||
# Preserve the shared JSON handler instead of letting Celery replace it
|
||||
# with process-specific plain-text formatting.
|
||||
worker_hijack_root_logger=False,
|
||||
timezone="UTC",
|
||||
enable_utc=True,
|
||||
task_routes={
|
||||
|
||||
@ -1,26 +1,17 @@
|
||||
"""FastAPI application entry point."""
|
||||
|
||||
import logging
|
||||
import re
|
||||
from collections.abc import Awaitable, Callable
|
||||
from pathlib import Path
|
||||
from time import monotonic
|
||||
from uuid import uuid4
|
||||
|
||||
from fastapi import FastAPI, Request, Response
|
||||
from fastapi.responses import FileResponse, PlainTextResponse
|
||||
from fastapi import FastAPI
|
||||
from fastapi.responses import FileResponse
|
||||
from fastapi.staticfiles import StaticFiles
|
||||
from pydantic import BaseModel
|
||||
|
||||
from ekp_api.metrics import HttpMetrics
|
||||
from ekp_api.providers import ApiServices
|
||||
from ekp_api.routes import router
|
||||
from ekp_core.config import Settings, get_settings
|
||||
from ekp_core.observability import bind_request_id, configure_json_logging, reset_request_id
|
||||
|
||||
ADMIN_WEB_ROOT = Path(__file__).resolve().parents[3] / "admin-web"
|
||||
REQUEST_ID_PATTERN = re.compile(r"^[A-Za-z0-9._:-]{1,64}$")
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class HealthResponse(BaseModel):
|
||||
@ -37,7 +28,6 @@ def create_app(
|
||||
) -> FastAPI:
|
||||
"""Create the API application with explicit, testable settings."""
|
||||
resolved_settings = settings or get_settings()
|
||||
configure_json_logging(resolved_settings.log_level)
|
||||
application = FastAPI(
|
||||
title=resolved_settings.app_name,
|
||||
version="0.1.0",
|
||||
@ -46,7 +36,6 @@ def create_app(
|
||||
)
|
||||
application.state.settings = resolved_settings
|
||||
application.state.api_services = api_services
|
||||
application.state.http_metrics = HttpMetrics()
|
||||
application.include_router(router, prefix=resolved_settings.api_prefix)
|
||||
admin_web_root = (resolved_settings.admin_web_root or ADMIN_WEB_ROOT).resolve()
|
||||
|
||||
@ -58,48 +47,6 @@ def create_app(
|
||||
name="admin-assets",
|
||||
)
|
||||
|
||||
@application.middleware("http")
|
||||
async def observe_http(
|
||||
request: Request,
|
||||
call_next: Callable[[Request], Awaitable[Response]],
|
||||
) -> Response:
|
||||
"""Correlate requests and emit bounded metrics without body/query data."""
|
||||
incoming_request_id = request.headers.get("x-request-id", "")
|
||||
request_id = (
|
||||
incoming_request_id
|
||||
if REQUEST_ID_PATTERN.fullmatch(incoming_request_id)
|
||||
else uuid4().hex
|
||||
)
|
||||
token = bind_request_id(request_id)
|
||||
started_at = monotonic()
|
||||
status_code = 500
|
||||
try:
|
||||
response = await call_next(request)
|
||||
status_code = response.status_code
|
||||
response.headers["x-request-id"] = request_id
|
||||
return response
|
||||
finally:
|
||||
duration_seconds = monotonic() - started_at
|
||||
route = request.scope.get("route")
|
||||
route_path = getattr(route, "path", "unmatched")
|
||||
application.state.http_metrics.observe(
|
||||
request.method,
|
||||
route_path,
|
||||
status_code,
|
||||
duration_seconds,
|
||||
)
|
||||
logger.info(
|
||||
"HTTP request completed",
|
||||
extra={
|
||||
"event": "http_request_completed",
|
||||
"method": request.method,
|
||||
"path": route_path,
|
||||
"status_code": status_code,
|
||||
"duration_ms": round(duration_seconds * 1000, 3),
|
||||
},
|
||||
)
|
||||
reset_request_id(token)
|
||||
|
||||
@application.get("/admin", include_in_schema=False)
|
||||
@application.get("/admin/upload", include_in_schema=False)
|
||||
@application.get("/admin/documents", include_in_schema=False)
|
||||
@ -116,14 +63,6 @@ def create_app(
|
||||
environment=resolved_settings.app_env,
|
||||
)
|
||||
|
||||
@application.get("/metrics", include_in_schema=False)
|
||||
async def metrics() -> PlainTextResponse:
|
||||
"""Expose aggregate process/HTTP metrics without provider or user data."""
|
||||
return PlainTextResponse(
|
||||
application.state.http_metrics.render(),
|
||||
media_type="text/plain; version=0.0.4",
|
||||
)
|
||||
|
||||
return application
|
||||
|
||||
|
||||
|
||||
@ -1,59 +0,0 @@
|
||||
"""Small dependency-free Prometheus endpoint for cluster smoke monitoring."""
|
||||
|
||||
from collections import defaultdict
|
||||
from threading import Lock
|
||||
from time import monotonic
|
||||
|
||||
|
||||
class HttpMetrics:
|
||||
"""Keep bounded per-process HTTP counters using route templates as labels."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._started_at = monotonic()
|
||||
self._lock = Lock()
|
||||
self._counts: defaultdict[tuple[str, str, int], int] = defaultdict(int)
|
||||
self._durations: defaultdict[tuple[str, str], tuple[int, float]] = defaultdict(
|
||||
lambda: (0, 0.0)
|
||||
)
|
||||
|
||||
def observe(self, method: str, path: str, status_code: int, duration_seconds: float) -> None:
|
||||
"""Record one request without retaining query strings or user content."""
|
||||
with self._lock:
|
||||
self._counts[(method, path, status_code)] += 1
|
||||
count, total = self._durations[(method, path)]
|
||||
self._durations[(method, path)] = (count + 1, total + duration_seconds)
|
||||
|
||||
def render(self) -> str:
|
||||
"""Return Prometheus text exposition with stable low-cardinality labels."""
|
||||
with self._lock:
|
||||
counts = tuple(sorted(self._counts.items()))
|
||||
durations = tuple(sorted(self._durations.items()))
|
||||
lines = [
|
||||
"# HELP ekp_process_uptime_seconds Process uptime in seconds.",
|
||||
"# TYPE ekp_process_uptime_seconds gauge",
|
||||
f"ekp_process_uptime_seconds {monotonic() - self._started_at:.6f}",
|
||||
"# HELP ekp_http_requests_total HTTP requests handled by this API process.",
|
||||
"# TYPE ekp_http_requests_total counter",
|
||||
]
|
||||
for (method, path, status_code), value in counts:
|
||||
labels = _labels(method=method, path=path, status=str(status_code))
|
||||
lines.append(f"ekp_http_requests_total{{{labels}}} {value}")
|
||||
lines.extend(
|
||||
(
|
||||
"# HELP ekp_http_request_duration_seconds HTTP request duration.",
|
||||
"# TYPE ekp_http_request_duration_seconds summary",
|
||||
)
|
||||
)
|
||||
for (method, path), (count, total) in durations:
|
||||
labels = _labels(method=method, path=path)
|
||||
lines.append(f"ekp_http_request_duration_seconds_count{{{labels}}} {count}")
|
||||
lines.append(f"ekp_http_request_duration_seconds_sum{{{labels}}} {total:.6f}")
|
||||
return "\n".join(lines) + "\n"
|
||||
|
||||
|
||||
def _labels(**values: str) -> str:
|
||||
"""Escape the limited label set according to Prometheus text rules."""
|
||||
return ",".join(
|
||||
f'{name}="{value.replace(chr(92), chr(92) * 2).replace(chr(10), chr(92) + "n").replace(chr(34), chr(92) + chr(34))}"'
|
||||
for name, value in values.items()
|
||||
)
|
||||
@ -1,6 +1,5 @@
|
||||
"""Versioned document and search routes for the retrieval Demo."""
|
||||
|
||||
import logging
|
||||
from typing import Annotated
|
||||
from uuid import UUID
|
||||
|
||||
@ -31,7 +30,6 @@ from ekp_core.ingestion_batches import BatchFileCountLimitError, BatchNotFoundEr
|
||||
from ekp_core.ingestion_jobs import IngestionJobNotFoundError
|
||||
|
||||
router = APIRouter()
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def get_api_services(request: Request) -> ApiServices:
|
||||
@ -42,10 +40,6 @@ def get_api_services(request: Request) -> ApiServices:
|
||||
try:
|
||||
services = create_api_services(request.app.state.settings)
|
||||
except ProviderConfigurationError as error:
|
||||
logger.error(
|
||||
"API provider composition failed",
|
||||
extra={"event": "api_provider_configuration_failed"},
|
||||
)
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
|
||||
detail="API providers are not fully configured",
|
||||
@ -87,22 +81,10 @@ def upload_document(
|
||||
) from error
|
||||
except Exception as error:
|
||||
# Provider and persistence exceptions are intentionally not reflected to clients.
|
||||
logger.error(
|
||||
"Document acceptance failed",
|
||||
extra={"event": "document_acceptance_failed"},
|
||||
)
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_502_BAD_GATEWAY,
|
||||
detail="document acceptance failed",
|
||||
) from error
|
||||
logger.info(
|
||||
"Document accepted for ingestion",
|
||||
extra={
|
||||
"event": "document_accepted",
|
||||
"document_id": str(submission.document.id),
|
||||
"job_id": str(submission.job.id),
|
||||
},
|
||||
)
|
||||
return IngestionAcceptedResponse.from_submission(submission)
|
||||
|
||||
|
||||
@ -149,18 +131,10 @@ def upload_ingestion_batch(
|
||||
detail="batch file count exceeds the configured limit",
|
||||
) from error
|
||||
except Exception as error:
|
||||
logger.error(
|
||||
"Batch acceptance failed",
|
||||
extra={"event": "ingestion_batch_acceptance_failed"},
|
||||
)
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_502_BAD_GATEWAY,
|
||||
detail="batch could not be accepted",
|
||||
) from error
|
||||
logger.info(
|
||||
"Ingestion batch accepted",
|
||||
extra={"event": "ingestion_batch_accepted", "batch_id": str(view.batch.id)},
|
||||
)
|
||||
return IngestionBatchResponse.from_view(view)
|
||||
|
||||
|
||||
@ -220,22 +194,11 @@ def delete_document(document_id: UUID, services: Services) -> Response:
|
||||
detail="document ingestion is still active",
|
||||
) from error
|
||||
except DocumentArtifactCleanupError as error:
|
||||
logger.error(
|
||||
"Document artifact cleanup failed",
|
||||
extra={
|
||||
"event": "document_artifact_cleanup_failed",
|
||||
"document_id": str(document_id),
|
||||
},
|
||||
)
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail="document metadata was deleted but artifact cleanup failed",
|
||||
) from error
|
||||
except Exception as error:
|
||||
logger.error(
|
||||
"Document deletion failed",
|
||||
extra={"event": "document_deletion_failed", "document_id": str(document_id)},
|
||||
)
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_502_BAD_GATEWAY,
|
||||
detail="document deletion failed",
|
||||
@ -254,10 +217,6 @@ def search_knowledge(payload: SearchRequest, services: Services) -> SearchRespon
|
||||
detail="search query must contain non-whitespace text",
|
||||
) from error
|
||||
except Exception as error:
|
||||
logger.error(
|
||||
"Knowledge search provider chain failed",
|
||||
extra={"event": "knowledge_search_failed"},
|
||||
)
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_502_BAD_GATEWAY,
|
||||
detail="search providers failed",
|
||||
|
||||
@ -1,6 +1,5 @@
|
||||
"""Celery worker entry point for durable ingestion Jobs."""
|
||||
|
||||
import logging
|
||||
from functools import lru_cache
|
||||
from typing import Any
|
||||
from uuid import UUID
|
||||
@ -13,12 +12,9 @@ from ekp_api.celery_queue import (
|
||||
from ekp_api.providers import WorkerServices, create_worker_services
|
||||
from ekp_core.config import get_settings
|
||||
from ekp_core.ingestion_jobs import JobRunAction, dispatch_pending_jobs
|
||||
from ekp_core.observability import configure_json_logging
|
||||
|
||||
_settings = get_settings()
|
||||
configure_json_logging(_settings.log_level)
|
||||
celery_app = create_celery_app(_settings)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@lru_cache
|
||||
@ -38,20 +34,7 @@ def run_ingestion(self: Any, job_id: str) -> None:
|
||||
Celery delivery retries are unbounded because valid-lease deferrals are not
|
||||
business Attempts; the durable runner independently enforces max_attempts.
|
||||
"""
|
||||
logger.info(
|
||||
"Ingestion delivery received",
|
||||
extra={"event": "ingestion_delivery_received", "job_id": job_id},
|
||||
)
|
||||
result = _worker_services().runner.run(UUID(job_id))
|
||||
logger.info(
|
||||
"Ingestion delivery completed",
|
||||
extra={
|
||||
"event": "ingestion_delivery_completed",
|
||||
"job_id": job_id,
|
||||
"job_status": result.status.value,
|
||||
"retry_delay_seconds": result.retry_delay_seconds,
|
||||
},
|
||||
)
|
||||
if result.action == JobRunAction.RETRY:
|
||||
raise self.retry(countdown=result.retry_delay_seconds or 1, max_retries=None)
|
||||
|
||||
@ -60,15 +43,7 @@ def run_ingestion(self: Any, job_id: str) -> None:
|
||||
def recover_ingestion_jobs() -> int:
|
||||
"""Redeliver due durable Jobs after API/broker or worker interruptions."""
|
||||
services = _worker_services()
|
||||
dispatch_count = dispatch_pending_jobs(services.jobs, services.dispatcher)
|
||||
logger.info(
|
||||
"Durable ingestion recovery scan completed",
|
||||
extra={
|
||||
"event": "ingestion_recovery_scan_completed",
|
||||
"dispatch_count": dispatch_count,
|
||||
},
|
||||
)
|
||||
return dispatch_count
|
||||
return dispatch_pending_jobs(services.jobs, services.dispatcher)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
|
||||
@ -3,14 +3,12 @@
|
||||
from mcp.server.transport_security import TransportSecuritySettings
|
||||
|
||||
from ekp_core.config import get_settings
|
||||
from ekp_core.observability import configure_json_logging
|
||||
from ekp_mcp.server import create_mcp_server
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""Run the authenticated Streamable HTTP server from environment settings."""
|
||||
settings = get_settings()
|
||||
configure_json_logging(settings.log_level)
|
||||
server = create_mcp_server(settings)
|
||||
server.run(
|
||||
transport="streamable-http",
|
||||
|
||||
@ -100,10 +100,7 @@ def create_mcp_server(
|
||||
# Do not reflect provider, database, endpoint, or credential details
|
||||
# into MCP content or logs. Operations can correlate this generic
|
||||
# event with upstream sanitized service telemetry.
|
||||
logger.error(
|
||||
"MCP knowledge search failed",
|
||||
extra={"event": "mcp_knowledge_search_failed"},
|
||||
)
|
||||
logger.error("MCP knowledge search failed")
|
||||
raise ToolError("knowledge search failed") from None
|
||||
|
||||
return server
|
||||
|
||||
@ -17,16 +17,6 @@ images:
|
||||
tag: "0.1.0"
|
||||
digest: ""
|
||||
|
||||
podDisruptionBudget:
|
||||
enabled: true
|
||||
apiMinAvailable: 1
|
||||
mcpMinAvailable: 1
|
||||
|
||||
observability:
|
||||
metrics:
|
||||
enabled: true
|
||||
path: /metrics
|
||||
|
||||
ingress:
|
||||
enabled: true
|
||||
className: nginx
|
||||
|
||||
@ -64,14 +64,6 @@ app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
{{- if gt (float64 .Values.config.retrieval.searchTopK) (float64 .Values.config.retrieval.denseTopN) -}}
|
||||
{{- fail "config.retrieval.searchTopK must be <= denseTopN" -}}
|
||||
{{- end -}}
|
||||
{{- if .Values.podDisruptionBudget.enabled -}}
|
||||
{{- if ge (int .Values.podDisruptionBudget.apiMinAvailable) (int .Values.api.replicaCount) -}}
|
||||
{{- fail "podDisruptionBudget.apiMinAvailable must be < api.replicaCount" -}}
|
||||
{{- end -}}
|
||||
{{- if ge (int .Values.podDisruptionBudget.mcpMinAvailable) (int .Values.mcp.replicaCount) -}}
|
||||
{{- fail "podDisruptionBudget.mcpMinAvailable must be < mcp.replicaCount" -}}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "knowledge-platform.podSecurityContext" -}}
|
||||
|
||||
@ -28,11 +28,6 @@ spec:
|
||||
annotations:
|
||||
checksum/runtime-config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }}
|
||||
checksum/runtime-secret-reference: {{ .Values.existingSecret | sha256sum }}
|
||||
{{- if .Values.observability.metrics.enabled }}
|
||||
prometheus.io/scrape: "true"
|
||||
prometheus.io/port: "8000"
|
||||
prometheus.io/path: {{ .Values.observability.metrics.path | quote }}
|
||||
{{- end }}
|
||||
{{- with .Values.podAnnotations }}
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
|
||||
@ -1,31 +0,0 @@
|
||||
{{- if .Values.podDisruptionBudget.enabled }}
|
||||
# DEPLOY-003 protects both user-facing paths during voluntary disruptions. PDBs
|
||||
# do not cover involuntary failures, which remain handled by replica recovery.
|
||||
apiVersion: policy/v1
|
||||
kind: PodDisruptionBudget
|
||||
metadata:
|
||||
name: {{ include "knowledge-platform.fullname" . }}-api
|
||||
labels:
|
||||
{{- include "knowledge-platform.labels" . | nindent 4 }}
|
||||
app.kubernetes.io/component: api
|
||||
spec:
|
||||
minAvailable: {{ .Values.podDisruptionBudget.apiMinAvailable }}
|
||||
selector:
|
||||
matchLabels:
|
||||
{{- include "knowledge-platform.selectorLabels" . | nindent 6 }}
|
||||
app.kubernetes.io/component: api
|
||||
---
|
||||
apiVersion: policy/v1
|
||||
kind: PodDisruptionBudget
|
||||
metadata:
|
||||
name: {{ include "knowledge-platform.fullname" . }}-mcp
|
||||
labels:
|
||||
{{- include "knowledge-platform.labels" . | nindent 4 }}
|
||||
app.kubernetes.io/component: mcp
|
||||
spec:
|
||||
minAvailable: {{ .Values.podDisruptionBudget.mcpMinAvailable }}
|
||||
selector:
|
||||
matchLabels:
|
||||
{{- include "knowledge-platform.selectorLabels" . | nindent 6 }}
|
||||
app.kubernetes.io/component: mcp
|
||||
{{- end }}
|
||||
@ -11,13 +11,6 @@ metadata:
|
||||
data:
|
||||
runtimeSecret: {{ .Values.existingSecret | quote }}
|
||||
ingressEnabled: {{ .Values.ingress.enabled | quote }}
|
||||
apiHost: {{ .Values.ingress.api.host | quote }}
|
||||
mcpHost: {{ .Values.ingress.mcp.host | quote }}
|
||||
apiDeployment: {{ printf "%s-api" (include "knowledge-platform.fullname" .) | quote }}
|
||||
mcpDeployment: {{ printf "%s-mcp" (include "knowledge-platform.fullname" .) | quote }}
|
||||
workerDeployment: {{ printf "%s-worker" (include "knowledge-platform.fullname" .) | quote }}
|
||||
schedulerDeployment: {{ printf "%s-scheduler" (include "knowledge-platform.fullname" .) | quote }}
|
||||
metricsPath: {{ .Values.observability.metrics.path | quote }}
|
||||
apiTlsSecret: {{ .Values.ingress.api.tlsSecretName | quote }}
|
||||
mcpTlsSecret: {{ .Values.ingress.mcp.tlsSecretName | quote }}
|
||||
rabbitmqEnabled: {{ .Values.rabbitmq.enabled | quote }}
|
||||
|
||||
@ -7,8 +7,6 @@
|
||||
"existingSecret",
|
||||
"images",
|
||||
"serviceAccount",
|
||||
"podDisruptionBudget",
|
||||
"observability",
|
||||
"api",
|
||||
"mcp",
|
||||
"worker",
|
||||
@ -53,32 +51,6 @@
|
||||
},
|
||||
"podAnnotations": {"$ref": "#/definitions/stringMap"},
|
||||
"podLabels": {"$ref": "#/definitions/stringMap"},
|
||||
"podDisruptionBudget": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["enabled", "apiMinAvailable", "mcpMinAvailable"],
|
||||
"properties": {
|
||||
"enabled": {"type": "boolean"},
|
||||
"apiMinAvailable": {"type": "integer", "minimum": 1},
|
||||
"mcpMinAvailable": {"type": "integer", "minimum": 1}
|
||||
}
|
||||
},
|
||||
"observability": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["metrics"],
|
||||
"properties": {
|
||||
"metrics": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["enabled", "path"],
|
||||
"properties": {
|
||||
"enabled": {"type": "boolean"},
|
||||
"path": {"const": "/metrics"}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"api": {"$ref": "#/definitions/httpWorkload"},
|
||||
"mcp": {"$ref": "#/definitions/httpWorkload"},
|
||||
"worker": {
|
||||
|
||||
@ -31,18 +31,6 @@ serviceAccount:
|
||||
podAnnotations: {}
|
||||
podLabels: {}
|
||||
|
||||
# DEPLOY-003 keeps one API and MCP replica available during voluntary node
|
||||
# maintenance. This complements, rather than replaces, RollingUpdate settings.
|
||||
podDisruptionBudget:
|
||||
enabled: true
|
||||
apiMinAvailable: 1
|
||||
mcpMinAvailable: 1
|
||||
|
||||
observability:
|
||||
metrics:
|
||||
enabled: true
|
||||
path: /metrics
|
||||
|
||||
api:
|
||||
replicaCount: 2
|
||||
service:
|
||||
|
||||
@ -76,12 +76,6 @@ Ingress 是否进一步认证属于部署配置,但前端隐藏按钮或隐藏
|
||||
集群重启、扩缩容和滚动部署不得改变本契约或要求浏览器重新注入 Provider 配置。数据库、BOS、
|
||||
模型与队列凭据由服务端 Secret 提供,任何 API 响应都不得返回这些值。
|
||||
|
||||
DEPLOY-003 增加两个不进入 `/api/v1` OpenAPI 的运维端点:`GET /healthz` 只表示 API 进程可用;
|
||||
`GET /metrics` 使用 Prometheus text format 返回进程存活时间、按 HTTP method/路由模板/status
|
||||
聚合的请求量和按 method/路由模板聚合的耗时。两者不得包含 Query、文件名、正文、身份、
|
||||
Provider Endpoint 或凭据。响应附带安全 `X-Request-ID`,客户端提供的值只有匹配
|
||||
`[A-Za-z0-9._:-]{1,64}` 时才会沿用。
|
||||
|
||||
## 异步入库契约(DEMO-015~016 已实现)
|
||||
|
||||
| Method | Path | 行为 |
|
||||
|
||||
@ -49,7 +49,3 @@ ACL 或用户委托授权。
|
||||
Credential 由 Kubernetes Secret 在服务端运行时注入,客户端只保存管理员分发的最小
|
||||
`knowledge:search` Token;Pod 重启或滚动升级不得要求重新生成配置。公共知识域不等于匿名
|
||||
访问或超级管理员身份,未来引入非公共知识前必须升级为用户委托身份与 ACL-aware Search。
|
||||
|
||||
DEPLOY-003 的远程验收必须从最终 HTTPS 地址只发现 `search_knowledge` 并完成一次真实、有来源
|
||||
结果的调用。Gateway 失败日志只记录固定事件名,不写请求 Query、异常正文、Provider Endpoint
|
||||
或 Token;具体请求量与 Pod 可用性由集群入口/Kubernetes 指标补充,当前不新增 MCP 业务 Tool。
|
||||
|
||||
@ -768,11 +768,6 @@ DEPLOY-002 的运行拓扑将 Celery Worker 与 Celery Beat 恢复调度器拆
|
||||
应用服务与持久化 Job/Attempt 状态,不形成新业务服务,也不改变 ADR-002。Migration 使用 Helm
|
||||
`pre-install,pre-upgrade` Hook;失败会在业务 Deployment 更新前终止原子升级。
|
||||
|
||||
DEPLOY-003 在上述拓扑内为 API/MCP 增加 PodDisruptionBudget,并保持双副本和零不可用滚动
|
||||
策略;Worker 继续依赖 OceanBase 的持久化 Job/Attempt、执行租约和独立恢复调度器收敛故障。
|
||||
应用进程统一输出字段白名单约束的 JSON 事件,API 暴露低基数 `/metrics` 供集群抓取。该最小
|
||||
可观测性不引入新的业务服务或完整 Observability 平台。
|
||||
|
||||
---
|
||||
|
||||
## 17. V1 验收重点
|
||||
|
||||
@ -1,147 +0,0 @@
|
||||
# 公共知识库集群运行与验收手册
|
||||
|
||||
## 1. 范围与安全边界
|
||||
|
||||
本手册覆盖 DEPLOY-003 的目标集群安装、幂等升级、滚动恢复、Worker 故障恢复、最小日志/指标、
|
||||
备份责任和远程 MCP/WorkBuddy 验收。当前仍是企业受控网络内的单一公共知识库,不增加用户、
|
||||
租户或 Document ACL,也不提供完整 Observability 平台和跨区域灾备。
|
||||
|
||||
所有敏感值必须由 Secret 管理系统写入 Kubernetes Secret。Values、Git、镜像、命令行参数、
|
||||
验收记录和日志中不得出现数据库 URL、RabbitMQ URL、云密钥、Provider Key 或 MCP Token。
|
||||
|
||||
## 2. 交付责任边界
|
||||
|
||||
| 资源 | 平台仓库负责 | 环境/运维负责 |
|
||||
|---|---|---|
|
||||
| 应用镜像 | 可重复构建的 API、MCP、Worker target | 目标 Registry、不可变 tag/digest、镜像扫描和保留策略 |
|
||||
| Kubernetes | Helm Chart、Migration、探针、PDB、资源和 Pod 安全默认值 | 集群、Namespace、Ingress Controller、配额和网络策略 |
|
||||
| 敏感配置 | Secret 引用和必需键名预检 | Secret 创建、轮换、审计和访问控制 |
|
||||
| HTTPS | Ingress Host/TLS Secret 引用 | DNS、证书签发/续期与 TLS Secret |
|
||||
| 数据 | canonical/可重建边界和恢复顺序 | OceanBase、BOS、RabbitMQ 的备份、容量、HA 和恢复演练 |
|
||||
| 监控 | JSON 事件、API `/metrics`、验收检查 | 日志/指标采集、留存、查询、告警和 On-call |
|
||||
|
||||
## 3. 部署前检查
|
||||
|
||||
操作者需要准备仓库外的环境 Values、Namespace、运行时 Secret、TLS Secret、ImagePullSecret,
|
||||
并确认集群可访问 OceanBase、BOS、RabbitMQ、Embedding 和 Reranker。先显式选择并核对 context:
|
||||
|
||||
```bash
|
||||
kubectl config current-context
|
||||
kubectl auth can-i get deployments -n knowledge-platform
|
||||
kubectl auth can-i patch deployments -n knowledge-platform
|
||||
```
|
||||
|
||||
镜像必须使用 Registry 中已存在的不可变 tag 或 digest,禁止 `latest`。部署前执行:
|
||||
|
||||
```bash
|
||||
make deploy-lint VALUES=/secure/path/environment-values.yaml
|
||||
make deploy-render VALUES=/secure/path/environment-values.yaml >/tmp/ekp-rendered.yaml
|
||||
```
|
||||
|
||||
渲染文件只能包含非敏感 ConfigMap 和 Secret 引用;完成检查后应删除临时文件。
|
||||
|
||||
## 4. 安装、幂等升级与只读验收
|
||||
|
||||
首次安装或普通升级使用同一入口:
|
||||
|
||||
```bash
|
||||
NAMESPACE=knowledge-platform RELEASE=knowledge-platform \
|
||||
make deploy VALUES=/secure/path/environment-values.yaml
|
||||
```
|
||||
|
||||
只读验收不会触发安装、重启或故障注入,但强制 context 与操作者声明完全一致:
|
||||
|
||||
```bash
|
||||
ACCEPTANCE_KUBE_CONTEXT="$(kubectl config current-context)" \
|
||||
NAMESPACE=knowledge-platform RELEASE=knowledge-platform \
|
||||
make cluster-acceptance VALUES=/secure/path/environment-values.yaml
|
||||
```
|
||||
|
||||
## 5. 完整可用性验收
|
||||
|
||||
完整模式会连续执行两次相同 Helm 入口、滚动重启 API/MCP,并在一个合成入库 Job 运行时强制
|
||||
删除一个明确选中的 Worker Pod。只能在已批准的验收 Namespace 执行:
|
||||
|
||||
```bash
|
||||
context="$(kubectl config current-context)"
|
||||
ACCEPTANCE_MODE=full \
|
||||
ACCEPTANCE_KUBE_CONTEXT="${context}" \
|
||||
ACCEPTANCE_CONFIRM="DEPLOY-003:${context}:knowledge-platform:knowledge-platform" \
|
||||
NAMESPACE=knowledge-platform RELEASE=knowledge-platform \
|
||||
make cluster-acceptance VALUES=/secure/path/environment-values.yaml
|
||||
```
|
||||
|
||||
脚本验证以下内容:
|
||||
|
||||
- 相同 Values 的安装/升级可以重复成功,Migration Hook 先于工作负载更新完成;
|
||||
- API/MCP 至少两个副本全部 Ready/Available,PDB 存在;
|
||||
- Admin、健康检查、指标和 MCP 均通过稳定 HTTPS Host 到达;
|
||||
- API/MCP 滚动期间连续健康探测无中断;
|
||||
- Worker 被非优雅中断后,同一 canonical Job 经租约到期和恢复调度器完成,不要求重新上传;
|
||||
- 合成文档验收完成后调用公开删除 API 清理,脚本不访问 Secret 值。
|
||||
|
||||
若合成文档处理过快,脚本会失败而不是虚报 Worker 中断已验证。可在受控验收环境调整 Provider
|
||||
延迟或使用批准的非敏感验收文件重跑,不得加入测试后门。
|
||||
|
||||
## 6. 远程 MCP 与 WorkBuddy
|
||||
|
||||
先用官方 MCP Python Client 验证远程协议、唯一 Tool 和一次真实检索。Token 通过静默输入进入
|
||||
当前进程环境,不写入命令参数或配置文件:
|
||||
|
||||
```bash
|
||||
export EKP_MCP_URL=https://mcp.example.internal/mcp
|
||||
export EKP_MCP_ACCEPTANCE_QUERY='已批准的公共知识验收问题'
|
||||
read -rsp 'MCP token: ' EKP_MCP_TOKEN && export EKP_MCP_TOKEN && echo
|
||||
make mcp-remote-acceptance
|
||||
unset EKP_MCP_TOKEN
|
||||
```
|
||||
|
||||
随后在与最终用户相同网络路径的 WorkBuddy 客户端加载
|
||||
[`workbuddy-mcp.example.json`](../../deploy/workbuddy-mcp.example.json),只通过客户端安全环境注入
|
||||
`EKP_MCP_URL` 和 `EKP_MCP_TOKEN`。验收必须确认:连接状态正常、只发现
|
||||
`search_knowledge`、真实问题返回至少一个带来源字段的结果。记录客户端版本、时间、网络区域、
|
||||
Tool 名和成功/失败即可,禁止复制 Token、正文或内部 Provider 信息。
|
||||
|
||||
## 7. 日志、指标与最小告警
|
||||
|
||||
应用事件为单行 JSON,允许字段包括 `event`、`request_id`、`job_id`、`attempt_number`、`stage`、
|
||||
`job_status` 和耗时;异常正文不会被格式化器写出。常用检查:
|
||||
|
||||
```bash
|
||||
kubectl -n knowledge-platform logs deployment/knowledge-platform-api --since=15m
|
||||
kubectl -n knowledge-platform logs deployment/knowledge-platform-worker --since=15m
|
||||
curl --fail https://knowledge.example.internal/metrics
|
||||
kubectl -n knowledge-platform get deployments,pods,poddisruptionbudgets
|
||||
```
|
||||
|
||||
环境监控至少采集并告警:API/MCP Ready 副本不足、Pod 重启持续增加、HTTP 5xx、P95 请求耗时、
|
||||
Worker 无 Ready 副本、`ingestion_attempt_failed`/`ingestion_retry_cleanup_failed`、恢复扫描持续
|
||||
失败、Migration Job 失败,以及 OceanBase/BOS/RabbitMQ/模型服务在应用日志中的安全失败事件。
|
||||
阈值由真实流量基线确定,不在本任务中固化完整监控平台。
|
||||
|
||||
## 8. 故障处置
|
||||
|
||||
API/MCP 不可用时先检查 Deployment/Endpoint/Ingress/TLS,再按单个 Deployment 滚动重启;禁止
|
||||
直接打印 Pod 环境变量。Job 卡住时使用 Admin/API 的 Job ID 查询状态,再按相同 Job ID 搜索
|
||||
Worker JSON 事件。处于有效租约的 Job 不应手工重复提交;租约到期后恢复调度器会重投递。
|
||||
|
||||
更新同名 Secret 不会自动改变已运行进程。由运维完成 Secret 轮换后,依次滚动重启 API、MCP、
|
||||
Worker 和 Scheduler,并复跑只读验收。稳定 Host 和客户端 Token 未轮换时,Admin/WorkBuddy
|
||||
无需重新配置;MCP Token 轮换时必须通过客户端 Secret 分发流程更新。
|
||||
|
||||
## 9. 备份与恢复顺序
|
||||
|
||||
OceanBase 关系表和 BOS 不可变源文件共同构成事实数据,必须由资源所有者执行一致性备份策略。
|
||||
OceanBase 向量表是可重建索引,不得代替 canonical 备份。RabbitMQ 消息只提供执行提示,
|
||||
canonical Job/Attempt 才是恢复依据,但生产 Broker 仍应按其 SLA 做持久化和 HA。
|
||||
|
||||
恢复顺序为:恢复/验证 OceanBase canonical 表 → 恢复并核对 BOS 对象 → 部署相同或兼容应用版本
|
||||
并执行 forward-only Migration → 恢复 RabbitMQ 连通 → 启动 Worker/Scheduler → 按 active Index
|
||||
Profile 重建缺失向量 → 验证来源追踪和真实检索。Helm 回滚只能回退兼容的应用清单,不能反向
|
||||
撤销数据库 Migration。完整灾备演练仍是后置任务。
|
||||
|
||||
## 10. 验收记录
|
||||
|
||||
每次目标集群验收按 [`DEPLOY_003_ACCEPTANCE.md`](DEPLOY_003_ACCEPTANCE.md) 记录非敏感结果。
|
||||
只有完整自动化验收、真实 MCP 协议调用和远程 WorkBuddy 三部分全部通过,DEPLOY-003 才能
|
||||
标记完成并合并到 `main`。
|
||||
@ -1,52 +0,0 @@
|
||||
# DEPLOY-003 目标集群验收记录
|
||||
|
||||
**状态:** 等待目标环境
|
||||
|
||||
**Task:** DEPLOY-003
|
||||
|
||||
**记录规则:** 只记录非敏感标识和聚合结果,不记录 Secret、业务正文或内部 Provider 响应。
|
||||
|
||||
## 环境信息
|
||||
|
||||
- Kubernetes context:待提供
|
||||
- Namespace / Release:待提供
|
||||
- API HTTPS Host:待提供(只记录公开受控 Host)
|
||||
- MCP HTTPS Host:待提供(只记录公开受控 Host)
|
||||
- 镜像 tag/digest:待提供
|
||||
- 验收时间与操作者:待提供
|
||||
|
||||
## 自动化验收
|
||||
|
||||
- [ ] 同一 Values 连续两次 `upgrade --install` 成功
|
||||
- [ ] Migration Hook 成功且失败保护已核对
|
||||
- [ ] API/MCP 各至少两个 Ready/Available 副本
|
||||
- [ ] API/MCP PodDisruptionBudget 生效
|
||||
- [ ] Admin、健康检查、`/metrics` 和 MCP 使用有效 HTTPS
|
||||
- [ ] API/MCP 滚动重启期间连续健康探测无失败
|
||||
- [ ] Worker 非优雅中断后同一 Job 自动恢复成功
|
||||
- [ ] 验收合成文档已清理
|
||||
- [ ] Manifest、镜像与验收日志凭据扫描为 0
|
||||
|
||||
执行结果与命令退出码:待执行。
|
||||
|
||||
## 远程 MCP / WorkBuddy
|
||||
|
||||
- [ ] 官方 MCP Client 只发现 `search_knowledge`
|
||||
- [ ] 官方 MCP Client 完成一次真实、有来源的检索
|
||||
- [ ] 最终用户网络路径上的 WorkBuddy 连接成功
|
||||
- [ ] WorkBuddy 只发现批准 Tool 并完成真实检索
|
||||
|
||||
客户端版本、网络区域和非敏感结果:待执行。
|
||||
|
||||
## 运行保障
|
||||
|
||||
- [ ] 日志采集可按 `request_id` / `job_id` 定位
|
||||
- [ ] 最小指标已被环境监控系统抓取
|
||||
- [ ] Ready 副本、5xx、耗时、Pod 重启和 Job 失败告警责任人已确认
|
||||
- [ ] OceanBase/BOS/RabbitMQ 备份责任人、周期和恢复顺序已确认
|
||||
|
||||
## 结论
|
||||
|
||||
当前仓库实现已提供 PDB、结构化日志、最小指标、完整验收脚本和运行手册;本机没有目标
|
||||
Kubernetes context、Registry、DNS/TLS 或远程 WorkBuddy 环境参数,因此所有目标环境复选项
|
||||
保持未完成,DEPLOY-003 不得提前闭环。
|
||||
@ -96,17 +96,10 @@ Controller 托管。证书只通过预建 TLS Secret 引用,不进入 Values
|
||||
`RuntimeDefault` seccomp 和只读根文件系统;
|
||||
- `/tmp` 使用有大小上限的 `emptyDir`,ServiceAccount 不挂载 API Token;
|
||||
- API/MCP 滚动更新使用 `maxUnavailable: 0`,未 Ready Pod 不进入 Service Endpoint。
|
||||
- API/MCP 默认各有一个 `minAvailable: 1` 的 PodDisruptionBudget,避免节点维护同时驱逐全部
|
||||
用户入口;不可用故障仍由 Deployment 副本恢复处理。
|
||||
- API Pod 默认以标准注解暴露 `/metrics`,只包含进程存活时间、按路由模板聚合的请求量和
|
||||
请求耗时,不包含 Query、文件名、正文、Endpoint 或凭据。
|
||||
|
||||
`/healthz` 当前只表示 API 进程存活,不探测所有外部 Provider;真实依赖故障、滚动重启和
|
||||
Worker 恢复行为由 DEPLOY-003 在目标集群验收。
|
||||
|
||||
运行、滚动更新、Worker 故障恢复、备份责任与远程 MCP/WorkBuddy 验收流程见
|
||||
[`CLUSTER_OPERATIONS.md`](CLUSTER_OPERATIONS.md)。
|
||||
|
||||
## 6. 可选 Demo RabbitMQ
|
||||
|
||||
生产默认 `rabbitmq.enabled=false`,`RABBITMQ_URL` 指向外部受管 Broker。仅开发/演示时可开启
|
||||
|
||||
@ -1,12 +1,12 @@
|
||||
# 项目进度
|
||||
|
||||
当前阶段:DEPLOY-003 集群可用性与远程验收进行中
|
||||
当前阶段:生产容器镜像闭环完成,准备实施 Kubernetes/Helm 一键部署
|
||||
|
||||
已完成:DEMO-001~DEMO-010、DEMO-012、DEMO-014~DEMO-016、ARCH-001~ARCH-006、PLAN-001~PLAN-002、DEPLOY-001~DEPLOY-002
|
||||
进行中:DEPLOY-003
|
||||
阻塞:目标 Kubernetes context、Registry 镜像坐标、DNS/TLS 与远程 WorkBuddy 环境参数未提供
|
||||
已完成:DEMO-001~DEMO-010、DEMO-012、DEMO-014~DEMO-016、ARCH-001~ARCH-006、PLAN-001~PLAN-002、DEPLOY-001
|
||||
进行中:无
|
||||
阻塞:无
|
||||
|
||||
下一门槛:在明确目标集群执行 DEPLOY-003 完整验收;通过前不开始后续任务。
|
||||
下一任务:`DEPLOY-002`,完成并合并后执行 `DEPLOY-003`。
|
||||
|
||||
Provider 架构迁移按 `ARCH-001 → ARCH-002 → ARCH-003 → ARCH-004 → ARCH-005 → ARCH-006`
|
||||
执行并已完成。DEMO-010 在迁移闭环验收后实施,未扩大为完整 V1 Admin Console。
|
||||
@ -424,25 +424,3 @@ Provider 架构迁移按 `ARCH-001 → ARCH-002 → ARCH-003 → ARCH-004 → AR
|
||||
- 目标企业集群的稳定 HTTPS、滚动重启、Worker 故障恢复、真实外部依赖监测与远程 WorkBuddy
|
||||
仍属于 `DEPLOY-003`;本任务没有把这些项提前标记完成,也没有改变 Accepted ADR 或公共知识库
|
||||
范围。下一任务:`DEPLOY-003`。
|
||||
|
||||
### DEPLOY-003(进行中)
|
||||
|
||||
- 新增 API/MCP PodDisruptionBudget,继续使用双副本和 `maxUnavailable: 0`,覆盖节点维护与滚动
|
||||
更新的最小可用性;没有引入新微服务或扩大公共知识库范围。
|
||||
- 新增字段白名单 JSON 日志、请求相关 ID、API 低基数 `/metrics`,并为上传、批次、检索、MCP、
|
||||
Worker Attempt/Stage/Retry/Cleanup/Recovery 补充安全事件;异常正文、Query、文件名、Provider
|
||||
Endpoint 和凭据不进入结构化字段。
|
||||
- 新增显式 context 核对和 `ACCEPTANCE_CONFIRM` 二次确认的集群验收入口;完整模式验证相同 Values
|
||||
两次升级、HTTPS、PDB、连续滚动探测,并在合成 Job 运行时非优雅删除单个 Worker Pod,等待
|
||||
canonical 租约与恢复调度器收敛后清理验收文档。
|
||||
- 新增官方 MCP Client 远程验收脚本;Token 只从进程环境读取,输出只包含 Tool/结果存在性,
|
||||
不包含 Token 或检索正文。远程 WorkBuddy 仍需在最终用户网络路径现场确认。
|
||||
- 新增中文集群运行手册和验收记录,明确 OceanBase canonical/BOS、可重建向量、RabbitMQ、TLS、
|
||||
Registry、日志指标与备份恢复的责任边界。完整 Observability 平台和灾备演练保持后置。
|
||||
- 仓库侧验证已通过 Ruff Format/Lint、Mypy、Bash 语法、三种 Helm lint、13 个渲染资源的
|
||||
kubeconform strict、Alembic Head/离线 SQL、Unit + Integration `200 passed` 和 RAG Evaluation
|
||||
`5 passed`;本地运行 API 的 `/healthz`、`/metrics` 与 JSON 事件烟测通过。
|
||||
- 新生产镜像的首次本地重建被 Docker Hub 匿名授权连接重置阻塞,两次重试结果一致,没有改动
|
||||
固定 digest 或镜像来源规避;当前机器同时没有目标 Kubernetes context、目标 Registry 坐标、
|
||||
DNS/TLS 或远程 WorkBuddy 参数。因此现场复选项保持未完成;当前任务分支仅保存并推送仓库侧
|
||||
阶段性实现,在现场验收闭环前不合并 `main`。
|
||||
|
||||
@ -100,10 +100,8 @@ PostgreSQL/MinIO 假设与 ADR-012 不一致,不得作为集群交付模板。
|
||||
|
||||
### DEPLOY-003
|
||||
|
||||
**进行中。** 仓库侧已实现 PDB、结构化日志、最小 HTTP 指标、显式 context/二次确认保护的
|
||||
验收脚本,并形成运行手册和备份/恢复责任边界。仍须在目标集群完成安装、幂等升级、滚动重启、
|
||||
Worker 故障恢复、日志/指标接入和远程 WorkBuddy 验收;在这些现场项完成前不得闭环。灾备演练
|
||||
与完整 Observability 平台仍属后置项。
|
||||
在目标集群完成安装、幂等升级、滚动重启、Worker 故障恢复和远程 WorkBuddy 验收;形成运行
|
||||
手册、备份/恢复责任边界和最小日志/指标检查。灾备演练与完整 Observability 平台仍属后置项。
|
||||
|
||||
## 5. 最小验收标准
|
||||
|
||||
|
||||
@ -32,9 +32,7 @@ validation. Provider credentials and internal endpoints are runtime inputs and n
|
||||
- [x] DEMO-016 Batch async upload API + per-file Job status + Admin Console
|
||||
- [x] DEPLOY-001 Production images for API/Admin, MCP Gateway and Async Worker
|
||||
- [x] DEPLOY-002 Kubernetes/Helm one-click deployment + Secret/Migration/Ingress baseline
|
||||
- [ ] DEPLOY-003 Availability, rolling upgrade and remote WorkBuddy acceptance(进行中:仓库侧
|
||||
PDB、结构化日志、最小指标、验收脚本与运行手册已实现;等待目标集群/Registry/DNS/TLS 和
|
||||
远程 WorkBuddy 实测,不得提前勾选)
|
||||
- [ ] DEPLOY-003 Availability, rolling upgrade and remote WorkBuddy acceptance
|
||||
|
||||
Dependencies and acceptance boundaries are defined in
|
||||
[`ASYNC_INGESTION_MCP_PLAN.md`](ASYNC_INGESTION_MCP_PLAN.md) and
|
||||
|
||||
@ -20,8 +20,7 @@
|
||||
3. `DEMO-016`(已完成):批量异步上传、批次/Job 查询和 Admin Console 状态展示;
|
||||
4. `DEPLOY-001`(已完成):API/Admin、MCP Gateway、Async Worker 生产镜像;
|
||||
5. `DEPLOY-002`(已完成):Kubernetes/Helm、Secret、Migration、Ingress/TLS 和一键部署入口;
|
||||
6. `DEPLOY-003`(进行中):仓库侧可用性保护、结构化日志、最小指标和验收入口已实现;仍需在
|
||||
明确目标集群完成幂等升级、滚动升级、Worker 恢复和远程 WorkBuddy 现场验收。
|
||||
6. `DEPLOY-003`:多副本可用性、滚动升级、Worker 恢复和远程 WorkBuddy 验收。
|
||||
|
||||
严格依赖为:
|
||||
|
||||
|
||||
@ -23,10 +23,6 @@
|
||||
过滤。实施顺序和集群验收边界见
|
||||
[`PUBLIC_KNOWLEDGE_CLUSTER_PLAN.md`](../plans/PUBLIC_KNOWLEDGE_CLUSTER_PLAN.md)。
|
||||
|
||||
公共知识库交付还要求 API/Admin 与 MCP 各至少两个副本、滚动升级期间稳定 HTTPS 地址可用、
|
||||
Worker 中断后以 canonical Job/Attempt 恢复,并提供不含业务正文或凭据的结构化事件日志与最小
|
||||
HTTP 指标。完整监控平台、跨区域灾备和自动云资源编排不随本阶段扩入 V1 Scope。
|
||||
|
||||
---
|
||||
|
||||
# 1. 项目背景
|
||||
|
||||
@ -1,6 +1,5 @@
|
||||
"""Durable asynchronous ingestion orchestration and retry cleanup."""
|
||||
|
||||
import logging
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from enum import StrEnum
|
||||
@ -58,7 +57,6 @@ SAFE_TRANSIENT_ERROR = "temporary ingestion failure"
|
||||
SAFE_PERMANENT_ERROR = "document content could not be processed"
|
||||
SAFE_CLEANUP_ERROR = "retry cleanup failed"
|
||||
SAFE_LEASE_ERROR = "worker execution lease expired"
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class IngestionJobNotFoundError(LookupError):
|
||||
@ -295,47 +293,17 @@ class IngestionJobRunner:
|
||||
claim = self._claim(job_id)
|
||||
if not claim.run:
|
||||
assert claim.result is not None
|
||||
logger.info(
|
||||
"Ingestion delivery deferred or already terminal",
|
||||
extra={
|
||||
"event": "ingestion_delivery_not_claimed",
|
||||
"job_id": str(job_id),
|
||||
"job_status": claim.result.status.value,
|
||||
"retry_delay_seconds": claim.result.retry_delay_seconds,
|
||||
},
|
||||
)
|
||||
return claim.result
|
||||
|
||||
if claim.cleanup_required:
|
||||
logger.info(
|
||||
"Retry cleanup started",
|
||||
extra={"event": "ingestion_retry_cleanup_started", "job_id": str(job_id)},
|
||||
)
|
||||
try:
|
||||
document_id = self._job_document_id(job_id)
|
||||
self._cleanup.cleanup(document_id)
|
||||
except Exception:
|
||||
self._finish_cleanup_failure(job_id)
|
||||
logger.error(
|
||||
"Retry cleanup failed",
|
||||
extra={"event": "ingestion_retry_cleanup_failed", "job_id": str(job_id)},
|
||||
)
|
||||
return JobRunResult(JobRunAction.COMPLETE, IngestionJobStatus.FAILED)
|
||||
logger.info(
|
||||
"Retry cleanup completed",
|
||||
extra={"event": "ingestion_retry_cleanup_completed", "job_id": str(job_id)},
|
||||
)
|
||||
|
||||
attempt_number, document_id = self._start_attempt(job_id)
|
||||
logger.info(
|
||||
"Ingestion attempt started",
|
||||
extra={
|
||||
"event": "ingestion_attempt_started",
|
||||
"job_id": str(job_id),
|
||||
"document_id": str(document_id),
|
||||
"attempt_number": attempt_number,
|
||||
},
|
||||
)
|
||||
|
||||
def stage_callback(stage: IngestionStage) -> None:
|
||||
"""Bind this attempt while pipeline stages remain Job-agnostic."""
|
||||
@ -349,16 +317,6 @@ class IngestionJobRunner:
|
||||
return self._finish_failure(job_id, attempt_number, document_id, error)
|
||||
|
||||
self._finish_success(job_id, attempt_number, document_id)
|
||||
logger.info(
|
||||
"Ingestion attempt succeeded",
|
||||
extra={
|
||||
"event": "ingestion_attempt_succeeded",
|
||||
"job_id": str(job_id),
|
||||
"document_id": str(document_id),
|
||||
"attempt_number": attempt_number,
|
||||
"job_status": IngestionJobStatus.SUCCEEDED.value,
|
||||
},
|
||||
)
|
||||
return JobRunResult(JobRunAction.COMPLETE, IngestionJobStatus.SUCCEEDED)
|
||||
|
||||
def _claim(self, job_id: UUID) -> _Claim:
|
||||
@ -493,15 +451,6 @@ class IngestionJobRunner:
|
||||
job.lease_expires_at = datetime.now(UTC) + timedelta(seconds=self._lease_seconds)
|
||||
attempt.stage = stage
|
||||
session.commit()
|
||||
logger.info(
|
||||
"Ingestion stage advanced",
|
||||
extra={
|
||||
"event": "ingestion_stage_advanced",
|
||||
"job_id": str(job_id),
|
||||
"attempt_number": attempt_number,
|
||||
"stage": stage.value,
|
||||
},
|
||||
)
|
||||
|
||||
def _finish_success(self, job_id: UUID, attempt_number: int, document_id: UUID) -> None:
|
||||
"""Atomically close Job/Attempt after the document becomes ready."""
|
||||
@ -553,17 +502,6 @@ class IngestionJobRunner:
|
||||
document.status = DocumentStatus.PROCESSING
|
||||
document.error_message = safe_error
|
||||
session.commit()
|
||||
logger.warning(
|
||||
"Ingestion retry scheduled",
|
||||
extra={
|
||||
"event": "ingestion_retry_scheduled",
|
||||
"job_id": str(job_id),
|
||||
"document_id": str(document_id),
|
||||
"attempt_number": attempt_number,
|
||||
"job_status": job.status.value,
|
||||
"retry_delay_seconds": delay,
|
||||
},
|
||||
)
|
||||
return JobRunResult(JobRunAction.RETRY, job.status, delay)
|
||||
|
||||
job.status = IngestionJobStatus.FAILED
|
||||
@ -573,16 +511,6 @@ class IngestionJobRunner:
|
||||
document.status = DocumentStatus.FAILED
|
||||
document.error_message = safe_error
|
||||
session.commit()
|
||||
logger.error(
|
||||
"Ingestion attempt failed terminally",
|
||||
extra={
|
||||
"event": "ingestion_attempt_failed",
|
||||
"job_id": str(job_id),
|
||||
"document_id": str(document_id),
|
||||
"attempt_number": attempt_number,
|
||||
"job_status": job.status.value,
|
||||
},
|
||||
)
|
||||
return JobRunResult(JobRunAction.COMPLETE, job.status)
|
||||
|
||||
def _finish_cleanup_failure(self, job_id: UUID) -> None:
|
||||
@ -626,10 +554,6 @@ def dispatch_pending_jobs(jobs: IngestionJobService, dispatcher: JobDispatcher)
|
||||
try:
|
||||
dispatcher.enqueue(job_id)
|
||||
except Exception:
|
||||
logger.error(
|
||||
"Durable Job redelivery failed",
|
||||
extra={"event": "ingestion_recovery_dispatch_failed", "job_id": str(job_id)},
|
||||
)
|
||||
continue
|
||||
dispatched += 1
|
||||
return dispatched
|
||||
|
||||
@ -1,75 +0,0 @@
|
||||
"""Secret-safe structured logging shared by every runtime process."""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import sys
|
||||
from contextvars import ContextVar, Token
|
||||
from datetime import UTC, datetime
|
||||
from typing import Final
|
||||
|
||||
_request_id: ContextVar[str | None] = ContextVar("ekp_request_id", default=None)
|
||||
_EVENT_FIELDS: Final[tuple[str, ...]] = (
|
||||
"event",
|
||||
"request_id",
|
||||
"method",
|
||||
"path",
|
||||
"status_code",
|
||||
"duration_ms",
|
||||
"job_id",
|
||||
"document_id",
|
||||
"batch_id",
|
||||
"attempt_number",
|
||||
"stage",
|
||||
"job_status",
|
||||
"retry_delay_seconds",
|
||||
"dispatch_count",
|
||||
)
|
||||
|
||||
|
||||
class JsonLogFormatter(logging.Formatter):
|
||||
"""Render an allowlisted JSON event without serializing exception details.
|
||||
|
||||
Provider exceptions may contain endpoints or credentials. Callers therefore
|
||||
log stable event names and identifiers, while this formatter deliberately
|
||||
ignores ``exc_info`` and arbitrary record attributes.
|
||||
"""
|
||||
|
||||
def format(self, record: logging.LogRecord) -> str:
|
||||
payload: dict[str, object] = {
|
||||
"timestamp": datetime.now(UTC).isoformat(),
|
||||
"level": record.levelname,
|
||||
"logger": record.name,
|
||||
"message": record.getMessage(),
|
||||
}
|
||||
context_request_id = _request_id.get()
|
||||
if context_request_id is not None:
|
||||
payload["request_id"] = context_request_id
|
||||
for field in _EVENT_FIELDS:
|
||||
value = getattr(record, field, None)
|
||||
if value is not None:
|
||||
payload[field] = value
|
||||
return json.dumps(payload, ensure_ascii=False, separators=(",", ":"))
|
||||
|
||||
|
||||
def configure_json_logging(level: str) -> None:
|
||||
"""Install one stdout JSON handler and preserve idempotent app factories."""
|
||||
root = logging.getLogger()
|
||||
existing = next(
|
||||
(handler for handler in root.handlers if isinstance(handler.formatter, JsonLogFormatter)),
|
||||
None,
|
||||
)
|
||||
if existing is None:
|
||||
handler = logging.StreamHandler(sys.stdout)
|
||||
handler.setFormatter(JsonLogFormatter())
|
||||
root.addHandler(handler)
|
||||
root.setLevel(level)
|
||||
|
||||
|
||||
def bind_request_id(request_id: str) -> Token[str | None]:
|
||||
"""Bind a validated correlation identifier to the current request context."""
|
||||
return _request_id.set(request_id)
|
||||
|
||||
|
||||
def reset_request_id(token: Token[str | None]) -> None:
|
||||
"""Restore the previous request context after response logging completes."""
|
||||
_request_id.reset(token)
|
||||
@ -1,201 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
# DEPLOY-003 validates one explicitly selected cluster context. Secret contents
|
||||
# never enter this process; only HTTPS public endpoints and Kubernetes metadata
|
||||
# are inspected. Full mode performs controlled rollout and worker-failure tests.
|
||||
values_file="${1:-}"
|
||||
chart_dir="${CHART:-deploy/helm/knowledge-platform}"
|
||||
release_name="${RELEASE:-knowledge-platform}"
|
||||
namespace="${NAMESPACE:-knowledge-platform}"
|
||||
acceptance_mode="${ACCEPTANCE_MODE:-read-only}"
|
||||
expected_context="${ACCEPTANCE_KUBE_CONTEXT:-}"
|
||||
rollout_timeout="${ACCEPTANCE_ROLLOUT_TIMEOUT:-10m}"
|
||||
worker_wait_seconds="${ACCEPTANCE_WORKER_WAIT_SECONDS:-720}"
|
||||
|
||||
fail() {
|
||||
echo "DEPLOY-003 acceptance failed: $1" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
if [[ -z "${values_file}" || ! -f "${values_file}" ]]; then
|
||||
fail "VALUES must point to an existing environment values file"
|
||||
fi
|
||||
if [[ "${acceptance_mode}" != "read-only" && "${acceptance_mode}" != "full" ]]; then
|
||||
fail "ACCEPTANCE_MODE must be read-only or full"
|
||||
fi
|
||||
for tool in helm kubectl curl python3; do
|
||||
command -v "${tool}" >/dev/null 2>&1 || fail "required tool is missing: ${tool}"
|
||||
done
|
||||
|
||||
current_context="$(kubectl config current-context 2>/dev/null || true)"
|
||||
if [[ -z "${expected_context}" || "${current_context}" != "${expected_context}" ]]; then
|
||||
fail "ACCEPTANCE_KUBE_CONTEXT must exactly match the active context"
|
||||
fi
|
||||
|
||||
preflight_yaml="$({
|
||||
helm template "${release_name}" "${chart_dir}" \
|
||||
--namespace "${namespace}" \
|
||||
--values "${values_file}" \
|
||||
--show-only templates/tests/preflight-references.yaml
|
||||
})"
|
||||
|
||||
reference() {
|
||||
local field="$1"
|
||||
printf '%s\n' "${preflight_yaml}" |
|
||||
awk -v key="${field}:" '$1 == key {gsub(/"/, "", $2); print $2}'
|
||||
}
|
||||
|
||||
ingress_enabled="$(reference ingressEnabled)"
|
||||
api_host="$(reference apiHost)"
|
||||
mcp_host="$(reference mcpHost)"
|
||||
api_deployment="$(reference apiDeployment)"
|
||||
mcp_deployment="$(reference mcpDeployment)"
|
||||
worker_deployment="$(reference workerDeployment)"
|
||||
scheduler_deployment="$(reference schedulerDeployment)"
|
||||
metrics_path="$(reference metricsPath)"
|
||||
|
||||
if [[ "${ingress_enabled}" != "true" ]]; then
|
||||
fail "target acceptance requires ingress.enabled=true"
|
||||
fi
|
||||
for value in \
|
||||
"${api_host}" \
|
||||
"${mcp_host}" \
|
||||
"${api_deployment}" \
|
||||
"${mcp_deployment}" \
|
||||
"${worker_deployment}" \
|
||||
"${scheduler_deployment}" \
|
||||
"${metrics_path}"; do
|
||||
[[ -n "${value}" ]] || fail "Helm acceptance references are incomplete"
|
||||
done
|
||||
|
||||
if [[ "${acceptance_mode}" == "full" ]]; then
|
||||
expected_confirmation="DEPLOY-003:${current_context}:${namespace}:${release_name}"
|
||||
if [[ "${ACCEPTANCE_CONFIRM:-}" != "${expected_confirmation}" ]]; then
|
||||
fail "set ACCEPTANCE_CONFIRM=${expected_confirmation} for controlled mutations"
|
||||
fi
|
||||
# Running the same entrypoint twice proves install/upgrade idempotency against
|
||||
# the selected environment before disruption tests begin.
|
||||
"${PWD}/scripts/deploy.sh" "${values_file}"
|
||||
"${PWD}/scripts/deploy.sh" "${values_file}"
|
||||
fi
|
||||
|
||||
kubectl -n "${namespace}" get deployment "${api_deployment}" >/dev/null
|
||||
kubectl -n "${namespace}" get deployment "${mcp_deployment}" >/dev/null
|
||||
kubectl -n "${namespace}" get deployment "${worker_deployment}" >/dev/null
|
||||
kubectl -n "${namespace}" get deployment "${scheduler_deployment}" >/dev/null
|
||||
|
||||
for deployment in \
|
||||
"${api_deployment}" \
|
||||
"${mcp_deployment}" \
|
||||
"${worker_deployment}" \
|
||||
"${scheduler_deployment}"; do
|
||||
kubectl -n "${namespace}" rollout status "deployment/${deployment}" \
|
||||
--timeout "${rollout_timeout}"
|
||||
done
|
||||
|
||||
assert_multi_replica_ready() {
|
||||
local deployment="$1"
|
||||
local desired ready available
|
||||
desired="$(kubectl -n "${namespace}" get deployment "${deployment}" -o jsonpath='{.spec.replicas}')"
|
||||
ready="$(kubectl -n "${namespace}" get deployment "${deployment}" -o jsonpath='{.status.readyReplicas}')"
|
||||
available="$(kubectl -n "${namespace}" get deployment "${deployment}" -o jsonpath='{.status.availableReplicas}')"
|
||||
[[ "${desired}" -ge 2 ]] || fail "${deployment} must have at least two replicas"
|
||||
[[ "${ready:-0}" -eq "${desired}" ]] || fail "${deployment} does not have all replicas ready"
|
||||
[[ "${available:-0}" -eq "${desired}" ]] || fail "${deployment} is not fully available"
|
||||
}
|
||||
|
||||
assert_multi_replica_ready "${api_deployment}"
|
||||
assert_multi_replica_ready "${mcp_deployment}"
|
||||
for disruption_budget in "${api_deployment}" "${mcp_deployment}"; do
|
||||
disruptions_allowed="0"
|
||||
for attempt in $(seq 1 30); do
|
||||
disruptions_allowed="$(kubectl -n "${namespace}" get poddisruptionbudget \
|
||||
"${disruption_budget}" -o jsonpath='{.status.disruptionsAllowed}')"
|
||||
[[ "${disruptions_allowed:-0}" -ge 1 ]] && break
|
||||
sleep 1
|
||||
done
|
||||
[[ "${disruptions_allowed:-0}" -ge 1 ]] ||
|
||||
fail "${disruption_budget} does not currently allow one voluntary disruption"
|
||||
done
|
||||
|
||||
api_base_url="https://${api_host}"
|
||||
mcp_url="https://${mcp_host}/mcp"
|
||||
curl --fail --silent --show-error --connect-timeout 5 --max-time 10 \
|
||||
--proto '=https' "${api_base_url}/healthz" >/dev/null
|
||||
curl --fail --silent --show-error --connect-timeout 5 --max-time 10 \
|
||||
--proto '=https' "${api_base_url}/admin/upload" >/dev/null
|
||||
metrics_output="$({
|
||||
curl --fail --silent --show-error --connect-timeout 5 --max-time 10 \
|
||||
--proto '=https' "${api_base_url}${metrics_path}"
|
||||
})"
|
||||
printf '%s\n' "${metrics_output}" | grep -q '^ekp_process_uptime_seconds ' ||
|
||||
fail "API metrics endpoint does not expose process uptime"
|
||||
printf '%s\n' "${metrics_output}" | grep -q '^ekp_http_requests_total{' ||
|
||||
fail "API metrics endpoint does not expose request totals"
|
||||
|
||||
# A credential-free request may be rejected, but it must complete TLS and reach
|
||||
# the MCP application rather than fail DNS, certificate, or gateway routing.
|
||||
mcp_status="$({
|
||||
curl --silent --show-error --output /dev/null --write-out '%{http_code}' \
|
||||
--connect-timeout 5 --max-time 10 --proto '=https' "${mcp_url}"
|
||||
})"
|
||||
case "${mcp_status}" in
|
||||
200 | 400 | 401 | 403 | 405 | 406) ;;
|
||||
*) fail "MCP HTTPS endpoint returned unexpected status ${mcp_status}" ;;
|
||||
esac
|
||||
|
||||
continuous_probe() {
|
||||
local stop_file="$1"
|
||||
local failure_file="$2"
|
||||
local current_mcp_status
|
||||
while [[ ! -f "${stop_file}" ]]; do
|
||||
if ! curl --fail --silent --show-error --connect-timeout 5 --max-time 10 --proto '=https' \
|
||||
"${api_base_url}/healthz" >/dev/null; then
|
||||
printf 'failed\n' >"${failure_file}"
|
||||
return
|
||||
fi
|
||||
current_mcp_status="$({
|
||||
curl --silent --show-error --output /dev/null --write-out '%{http_code}' \
|
||||
--connect-timeout 5 --max-time 10 --proto '=https' "${mcp_url}"
|
||||
})"
|
||||
case "${current_mcp_status}" in
|
||||
200 | 400 | 401 | 403 | 405 | 406) ;;
|
||||
*)
|
||||
printf 'failed\n' >"${failure_file}"
|
||||
return
|
||||
;;
|
||||
esac
|
||||
sleep 1
|
||||
done
|
||||
}
|
||||
|
||||
if [[ "${acceptance_mode}" == "full" ]]; then
|
||||
probe_dir="$(mktemp -d)"
|
||||
stop_file="${probe_dir}/stop"
|
||||
failure_file="${probe_dir}/failure"
|
||||
cleanup_probe() {
|
||||
touch "${stop_file}" 2>/dev/null || true
|
||||
wait "${probe_pid:-}" 2>/dev/null || true
|
||||
rm -rf "${probe_dir}"
|
||||
}
|
||||
trap cleanup_probe EXIT
|
||||
continuous_probe "${stop_file}" "${failure_file}" &
|
||||
probe_pid="$!"
|
||||
kubectl -n "${namespace}" rollout restart "deployment/${api_deployment}"
|
||||
kubectl -n "${namespace}" rollout restart "deployment/${mcp_deployment}"
|
||||
kubectl -n "${namespace}" rollout status "deployment/${api_deployment}" \
|
||||
--timeout "${rollout_timeout}"
|
||||
kubectl -n "${namespace}" rollout status "deployment/${mcp_deployment}" \
|
||||
--timeout "${rollout_timeout}"
|
||||
touch "${stop_file}"
|
||||
wait "${probe_pid}"
|
||||
[[ ! -f "${failure_file}" ]] || fail "API health probe failed during rolling restart"
|
||||
rm -rf "${probe_dir}"
|
||||
trap - EXIT
|
||||
|
||||
"${PWD}/scripts/worker_recovery_acceptance.sh" "${api_base_url}" \
|
||||
"${namespace}" "${release_name}" "${worker_deployment}" "${worker_wait_seconds}"
|
||||
fi
|
||||
|
||||
echo "DEPLOY-003 automated cluster acceptance passed (${acceptance_mode})"
|
||||
@ -1,56 +0,0 @@
|
||||
"""Secret-safe remote MCP protocol acceptance used before WorkBuddy sign-off."""
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
import sys
|
||||
|
||||
import httpx2
|
||||
from mcp import ClientSession
|
||||
from mcp.client.streamable_http import streamable_http_client
|
||||
from mcp.types import CallToolResult
|
||||
|
||||
|
||||
async def _accept() -> None:
|
||||
"""Discover exactly one approved tool and execute a real remote search."""
|
||||
url = os.environ.get("EKP_MCP_URL", "")
|
||||
token = os.environ.get("EKP_MCP_TOKEN", "")
|
||||
query = os.environ.get("EKP_MCP_ACCEPTANCE_QUERY", "")
|
||||
if not url.startswith("https://") or len(token) < 32 or not query.strip():
|
||||
raise ValueError(
|
||||
"EKP_MCP_URL (HTTPS), EKP_MCP_TOKEN and EKP_MCP_ACCEPTANCE_QUERY are required"
|
||||
)
|
||||
# The credential is read in process and is never included in CLI arguments,
|
||||
# output, exceptions, or the checked-in WorkBuddy example configuration.
|
||||
headers = {"Authorization": f"Bearer {token}"}
|
||||
async with httpx2.AsyncClient(headers=headers, timeout=30.0) as client:
|
||||
async with streamable_http_client(url, http_client=client) as (read, write):
|
||||
async with ClientSession(read, write) as session:
|
||||
await session.initialize()
|
||||
tools = await session.list_tools()
|
||||
tool_names = [tool.name for tool in tools.tools]
|
||||
if tool_names != ["search_knowledge"]:
|
||||
raise RuntimeError("remote MCP exposed an unexpected tool set")
|
||||
result = await session.call_tool(
|
||||
"search_knowledge",
|
||||
{"query": query, "top_k": 1},
|
||||
)
|
||||
if not isinstance(result, CallToolResult) or result.is_error:
|
||||
raise RuntimeError("remote MCP search returned a tool error")
|
||||
structured = result.structured_content
|
||||
if not isinstance(structured, dict) or not structured.get("results"):
|
||||
raise RuntimeError("remote MCP search returned no structured source")
|
||||
print("remote MCP protocol acceptance passed: one tool, one grounded result")
|
||||
|
||||
|
||||
def main() -> int:
|
||||
"""Return a stable non-zero code without reflecting sensitive exceptions."""
|
||||
try:
|
||||
asyncio.run(_accept())
|
||||
except Exception:
|
||||
print("remote MCP protocol acceptance failed", file=sys.stderr)
|
||||
return 1
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@ -1,92 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
# This helper is called only by cluster_acceptance.sh full mode. It interrupts
|
||||
# one exact worker Pod after a durable Job reaches RUNNING, then verifies that
|
||||
# the lease/recovery scheduler converges the same Job without a second upload.
|
||||
api_base_url="${1:?API base URL is required}"
|
||||
namespace="${2:?namespace is required}"
|
||||
release_name="${3:?release name is required}"
|
||||
worker_deployment="${4:?worker deployment is required}"
|
||||
wait_seconds="${5:-720}"
|
||||
poll_interval="${ACCEPTANCE_JOB_POLL_INTERVAL_SECONDS:-0.2}"
|
||||
work_dir="$(mktemp -d)"
|
||||
source_file="${work_dir}/deploy-003-worker-recovery.md"
|
||||
response_file="${work_dir}/response.json"
|
||||
document_id=""
|
||||
|
||||
cleanup() {
|
||||
if [[ -n "${document_id}" ]]; then
|
||||
curl --silent --show-error --connect-timeout 5 --max-time 15 \
|
||||
--request DELETE --proto '=https' \
|
||||
"${api_base_url}/api/v1/documents/${document_id}" >/dev/null || true
|
||||
fi
|
||||
rm -rf "${work_dir}"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
[[ "${wait_seconds}" =~ ^[0-9]+$ && "${wait_seconds}" -gt 0 ]] || exit 2
|
||||
[[ "${poll_interval}" =~ ^[0-9]+([.][0-9]+)?$ ]] || exit 2
|
||||
|
||||
# Enough independent paragraphs make RUNNING observable without introducing a
|
||||
# large or business-derived acceptance document.
|
||||
for number in $(seq 1 80); do
|
||||
printf 'DEPLOY-003 worker recovery synthetic paragraph %s.\n\n' "${number}" >>"${source_file}"
|
||||
done
|
||||
|
||||
curl --fail --silent --show-error --connect-timeout 5 --max-time 60 --proto '=https' \
|
||||
--form "file=@${source_file};type=text/markdown" \
|
||||
"${api_base_url}/api/v1/documents" >"${response_file}"
|
||||
|
||||
document_id="$(python3 -c 'import json,sys; print(json.load(sys.stdin)["document_id"])' <"${response_file}")"
|
||||
job_id="$(python3 -c 'import json,sys; print(json.load(sys.stdin)["job_id"])' <"${response_file}")"
|
||||
[[ -n "${document_id}" && -n "${job_id}" ]] || exit 1
|
||||
|
||||
deadline=$((SECONDS + wait_seconds))
|
||||
interrupted="false"
|
||||
while ((SECONDS < deadline)); do
|
||||
job_json="$({
|
||||
curl --fail --silent --show-error --connect-timeout 5 --max-time 10 --proto '=https' \
|
||||
"${api_base_url}/api/v1/jobs/${job_id}"
|
||||
})"
|
||||
job_status="$(printf '%s' "${job_json}" | python3 -c 'import json,sys; print(json.load(sys.stdin)["status"])')"
|
||||
if [[ "${job_status}" == "running" ]]; then
|
||||
worker_pod="$(kubectl -n "${namespace}" get pods \
|
||||
-l "app.kubernetes.io/instance=${release_name},app.kubernetes.io/component=worker" \
|
||||
--field-selector=status.phase=Running \
|
||||
-o jsonpath='{.items[0].metadata.name}')"
|
||||
[[ -n "${worker_pod}" ]] || exit 1
|
||||
# Zero grace simulates process/node loss instead of a cooperative drain.
|
||||
kubectl -n "${namespace}" delete pod "${worker_pod}" \
|
||||
--grace-period=0 --force --wait=false >/dev/null
|
||||
interrupted="true"
|
||||
break
|
||||
fi
|
||||
if [[ "${job_status}" == "succeeded" || "${job_status}" == "failed" ]]; then
|
||||
echo "worker recovery acceptance could not interrupt the fast Job" >&2
|
||||
exit 1
|
||||
fi
|
||||
sleep "${poll_interval}"
|
||||
done
|
||||
[[ "${interrupted}" == "true" ]] || exit 1
|
||||
|
||||
kubectl -n "${namespace}" rollout status "deployment/${worker_deployment}" --timeout 10m
|
||||
while ((SECONDS < deadline)); do
|
||||
job_json="$({
|
||||
curl --fail --silent --show-error --connect-timeout 5 --max-time 10 --proto '=https' \
|
||||
"${api_base_url}/api/v1/jobs/${job_id}"
|
||||
})"
|
||||
job_status="$(printf '%s' "${job_json}" | python3 -c 'import json,sys; print(json.load(sys.stdin)["status"])')"
|
||||
if [[ "${job_status}" == "succeeded" ]]; then
|
||||
echo "durable worker recovery passed"
|
||||
exit 0
|
||||
fi
|
||||
if [[ "${job_status}" == "failed" || "${job_status}" == "cancelled" ]]; then
|
||||
echo "worker recovery Job reached terminal status ${job_status}" >&2
|
||||
exit 1
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
|
||||
echo "worker recovery Job did not complete within ${wait_seconds} seconds" >&2
|
||||
exit 1
|
||||
@ -191,10 +191,7 @@ def test_tool_failure_is_sanitized_for_client_and_logs() -> None:
|
||||
assert "knowledge search failed" in rendered
|
||||
assert "internal.example.test" not in rendered
|
||||
assert "secret-provider-token" not in rendered
|
||||
log_error.assert_called_once_with(
|
||||
"MCP knowledge search failed",
|
||||
extra={"event": "mcp_knowledge_search_failed"},
|
||||
)
|
||||
log_error.assert_called_once_with("MCP knowledge search failed")
|
||||
|
||||
|
||||
def test_transport_rejects_unapproved_origin() -> None:
|
||||
|
||||
@ -82,8 +82,6 @@ def test_workloads_render_availability_and_restricted_pod_defaults() -> None:
|
||||
assert "readinessProbe:" in rendered
|
||||
assert "livenessProbe:" in rendered
|
||||
assert "startupProbe:" in rendered
|
||||
assert rendered.count("kind: PodDisruptionBudget") == 2
|
||||
assert 'prometheus.io/path: "/metrics"' in rendered
|
||||
|
||||
|
||||
def test_scheduler_is_singleton_and_workers_do_not_embed_beat() -> None:
|
||||
@ -143,33 +141,6 @@ def test_cross_field_validation_rejects_unsafe_ingestion_limits() -> None:
|
||||
assert "batchMaxTotalBytes must be >= maxUploadBytes" in (result.stdout + result.stderr)
|
||||
|
||||
|
||||
def test_cross_field_validation_rejects_pdb_that_blocks_all_evictions() -> None:
|
||||
"""A PDB must leave room to replace one of the required replicas."""
|
||||
result = _helm(
|
||||
"template",
|
||||
"knowledge-platform",
|
||||
str(CHART),
|
||||
"--set",
|
||||
"podDisruptionBudget.apiMinAvailable=2",
|
||||
expect_success=False,
|
||||
)
|
||||
|
||||
assert result.returncode != 0
|
||||
assert "apiMinAvailable must be < api.replicaCount" in (result.stdout + result.stderr)
|
||||
|
||||
|
||||
def test_cluster_acceptance_requires_explicit_context_and_mutation_confirmation() -> None:
|
||||
"""The target-cluster harness cannot silently mutate an ambient context."""
|
||||
cluster_script = (ROOT / "scripts" / "cluster_acceptance.sh").read_text()
|
||||
worker_script = (ROOT / "scripts" / "worker_recovery_acceptance.sh").read_text()
|
||||
|
||||
assert "ACCEPTANCE_KUBE_CONTEXT" in cluster_script
|
||||
assert "ACCEPTANCE_CONFIRM" in cluster_script
|
||||
assert 'acceptance_mode="${ACCEPTANCE_MODE:-read-only}"' in cluster_script
|
||||
assert "--grace-period=0 --force --wait=false" in worker_script
|
||||
assert "get secret" not in worker_script
|
||||
|
||||
|
||||
def test_deploy_entrypoint_uses_atomic_upgrade_and_never_reads_secret_values() -> None:
|
||||
"""The one-command wrapper checks references and performs an atomic wait."""
|
||||
script = (ROOT / "scripts" / "deploy.sh").read_text()
|
||||
|
||||
@ -1,72 +0,0 @@
|
||||
"""Unit contracts for secret-safe logs and bounded API metrics."""
|
||||
|
||||
import json
|
||||
import logging
|
||||
|
||||
from fastapi.testclient import TestClient
|
||||
from pydantic import SecretStr
|
||||
|
||||
from ekp_api.main import create_app
|
||||
from ekp_core.config import Settings
|
||||
from ekp_core.observability import JsonLogFormatter, bind_request_id, reset_request_id
|
||||
|
||||
|
||||
def _settings() -> Settings:
|
||||
"""Use an isolated test database while exercising provider-free endpoints."""
|
||||
return Settings(
|
||||
app_env="test",
|
||||
database_url=SecretStr("sqlite+pysqlite:///:memory:"),
|
||||
)
|
||||
|
||||
|
||||
def test_json_formatter_keeps_allowlisted_context_and_drops_arbitrary_fields() -> None:
|
||||
"""Structured events must not serialize provider exception or custom secrets."""
|
||||
record = logging.LogRecord(
|
||||
name="ekp.test",
|
||||
level=logging.ERROR,
|
||||
pathname=__file__,
|
||||
lineno=1,
|
||||
msg="safe failure",
|
||||
args=(),
|
||||
exc_info=None,
|
||||
)
|
||||
record.event = "safe_event"
|
||||
record.job_id = "job-1"
|
||||
record.provider_secret = "must-not-appear"
|
||||
token = bind_request_id("request-1")
|
||||
try:
|
||||
payload = json.loads(JsonLogFormatter().format(record))
|
||||
finally:
|
||||
reset_request_id(token)
|
||||
|
||||
assert payload["event"] == "safe_event"
|
||||
assert payload["job_id"] == "job-1"
|
||||
assert payload["request_id"] == "request-1"
|
||||
assert "provider_secret" not in payload
|
||||
assert "must-not-appear" not in json.dumps(payload)
|
||||
|
||||
|
||||
def test_api_exposes_low_cardinality_metrics_and_request_correlation() -> None:
|
||||
"""Metrics use route templates, and callers receive a safe correlation id."""
|
||||
client = TestClient(create_app(_settings()))
|
||||
|
||||
response = client.get("/healthz", headers={"x-request-id": "acceptance-request-1"})
|
||||
metrics = client.get("/metrics")
|
||||
|
||||
assert response.status_code == 200
|
||||
assert response.headers["x-request-id"] == "acceptance-request-1"
|
||||
assert metrics.status_code == 200
|
||||
assert "ekp_process_uptime_seconds " in metrics.text
|
||||
assert 'path="/healthz",status="200"' in metrics.text
|
||||
assert "?" not in metrics.text
|
||||
|
||||
|
||||
def test_api_replaces_unsafe_request_identifier() -> None:
|
||||
"""Untrusted correlation headers cannot inject log structure or new lines."""
|
||||
client = TestClient(create_app(_settings()))
|
||||
|
||||
response = client.get("/healthz", headers={"x-request-id": "unsafe\nvalue"})
|
||||
|
||||
request_id = response.headers["x-request-id"]
|
||||
assert request_id != "unsafe\nvalue"
|
||||
assert len(request_id) == 32
|
||||
Loading…
x
Reference in New Issue
Block a user