#!/usr/bin/env bash set -euo pipefail # DEPLOY-003 validates one explicitly selected cluster context. Secret contents # never enter this process; only HTTPS public endpoints and Kubernetes metadata # are inspected. Full mode performs controlled rollout and worker-failure tests. values_file="${1:-}" chart_dir="${CHART:-deploy/helm/knowledge-platform}" release_name="${RELEASE:-knowledge-platform}" namespace="${NAMESPACE:-knowledge-platform}" acceptance_mode="${ACCEPTANCE_MODE:-read-only}" expected_context="${ACCEPTANCE_KUBE_CONTEXT:-}" rollout_timeout="${ACCEPTANCE_ROLLOUT_TIMEOUT:-10m}" worker_wait_seconds="${ACCEPTANCE_WORKER_WAIT_SECONDS:-720}" fail() { echo "DEPLOY-003 acceptance failed: $1" >&2 exit 1 } if [[ -z "${values_file}" || ! -f "${values_file}" ]]; then fail "VALUES must point to an existing environment values file" fi if [[ "${acceptance_mode}" != "read-only" && "${acceptance_mode}" != "full" ]]; then fail "ACCEPTANCE_MODE must be read-only or full" fi for tool in helm kubectl curl python3; do command -v "${tool}" >/dev/null 2>&1 || fail "required tool is missing: ${tool}" done current_context="$(kubectl config current-context 2>/dev/null || true)" if [[ -z "${expected_context}" || "${current_context}" != "${expected_context}" ]]; then fail "ACCEPTANCE_KUBE_CONTEXT must exactly match the active context" fi preflight_yaml="$({ helm template "${release_name}" "${chart_dir}" \ --namespace "${namespace}" \ --values "${values_file}" \ --show-only templates/tests/preflight-references.yaml })" reference() { local field="$1" printf '%s\n' "${preflight_yaml}" | awk -v key="${field}:" '$1 == key {gsub(/"/, "", $2); print $2}' } ingress_enabled="$(reference ingressEnabled)" api_host="$(reference apiHost)" mcp_host="$(reference mcpHost)" api_deployment="$(reference apiDeployment)" mcp_deployment="$(reference mcpDeployment)" worker_deployment="$(reference workerDeployment)" scheduler_deployment="$(reference schedulerDeployment)" metrics_path="$(reference metricsPath)" if [[ "${ingress_enabled}" != "true" ]]; then fail "target acceptance requires ingress.enabled=true" fi for value in \ "${api_host}" \ "${mcp_host}" \ "${api_deployment}" \ "${mcp_deployment}" \ "${worker_deployment}" \ "${scheduler_deployment}" \ "${metrics_path}"; do [[ -n "${value}" ]] || fail "Helm acceptance references are incomplete" done if [[ "${acceptance_mode}" == "full" ]]; then expected_confirmation="DEPLOY-003:${current_context}:${namespace}:${release_name}" if [[ "${ACCEPTANCE_CONFIRM:-}" != "${expected_confirmation}" ]]; then fail "set ACCEPTANCE_CONFIRM=${expected_confirmation} for controlled mutations" fi # Running the same entrypoint twice proves install/upgrade idempotency against # the selected environment before disruption tests begin. "${PWD}/scripts/deploy.sh" "${values_file}" "${PWD}/scripts/deploy.sh" "${values_file}" fi kubectl -n "${namespace}" get deployment "${api_deployment}" >/dev/null kubectl -n "${namespace}" get deployment "${mcp_deployment}" >/dev/null kubectl -n "${namespace}" get deployment "${worker_deployment}" >/dev/null kubectl -n "${namespace}" get deployment "${scheduler_deployment}" >/dev/null for deployment in \ "${api_deployment}" \ "${mcp_deployment}" \ "${worker_deployment}" \ "${scheduler_deployment}"; do kubectl -n "${namespace}" rollout status "deployment/${deployment}" \ --timeout "${rollout_timeout}" done assert_multi_replica_ready() { local deployment="$1" local desired ready available desired="$(kubectl -n "${namespace}" get deployment "${deployment}" -o jsonpath='{.spec.replicas}')" ready="$(kubectl -n "${namespace}" get deployment "${deployment}" -o jsonpath='{.status.readyReplicas}')" available="$(kubectl -n "${namespace}" get deployment "${deployment}" -o jsonpath='{.status.availableReplicas}')" [[ "${desired}" -ge 2 ]] || fail "${deployment} must have at least two replicas" [[ "${ready:-0}" -eq "${desired}" ]] || fail "${deployment} does not have all replicas ready" [[ "${available:-0}" -eq "${desired}" ]] || fail "${deployment} is not fully available" } assert_multi_replica_ready "${api_deployment}" assert_multi_replica_ready "${mcp_deployment}" for disruption_budget in "${api_deployment}" "${mcp_deployment}"; do disruptions_allowed="0" for attempt in $(seq 1 30); do disruptions_allowed="$(kubectl -n "${namespace}" get poddisruptionbudget \ "${disruption_budget}" -o jsonpath='{.status.disruptionsAllowed}')" [[ "${disruptions_allowed:-0}" -ge 1 ]] && break sleep 1 done [[ "${disruptions_allowed:-0}" -ge 1 ]] || fail "${disruption_budget} does not currently allow one voluntary disruption" done api_base_url="https://${api_host}" mcp_url="https://${mcp_host}/mcp" curl --fail --silent --show-error --connect-timeout 5 --max-time 10 \ --proto '=https' "${api_base_url}/healthz" >/dev/null curl --fail --silent --show-error --connect-timeout 5 --max-time 10 \ --proto '=https' "${api_base_url}/admin/upload" >/dev/null metrics_output="$({ curl --fail --silent --show-error --connect-timeout 5 --max-time 10 \ --proto '=https' "${api_base_url}${metrics_path}" })" printf '%s\n' "${metrics_output}" | grep -q '^ekp_process_uptime_seconds ' || fail "API metrics endpoint does not expose process uptime" printf '%s\n' "${metrics_output}" | grep -q '^ekp_http_requests_total{' || fail "API metrics endpoint does not expose request totals" # A credential-free request may be rejected, but it must complete TLS and reach # the MCP application rather than fail DNS, certificate, or gateway routing. mcp_status="$({ curl --silent --show-error --output /dev/null --write-out '%{http_code}' \ --connect-timeout 5 --max-time 10 --proto '=https' "${mcp_url}" })" case "${mcp_status}" in 200 | 400 | 401 | 403 | 405 | 406) ;; *) fail "MCP HTTPS endpoint returned unexpected status ${mcp_status}" ;; esac continuous_probe() { local stop_file="$1" local failure_file="$2" local current_mcp_status while [[ ! -f "${stop_file}" ]]; do if ! curl --fail --silent --show-error --connect-timeout 5 --max-time 10 --proto '=https' \ "${api_base_url}/healthz" >/dev/null; then printf 'failed\n' >"${failure_file}" return fi current_mcp_status="$({ curl --silent --show-error --output /dev/null --write-out '%{http_code}' \ --connect-timeout 5 --max-time 10 --proto '=https' "${mcp_url}" })" case "${current_mcp_status}" in 200 | 400 | 401 | 403 | 405 | 406) ;; *) printf 'failed\n' >"${failure_file}" return ;; esac sleep 1 done } if [[ "${acceptance_mode}" == "full" ]]; then probe_dir="$(mktemp -d)" stop_file="${probe_dir}/stop" failure_file="${probe_dir}/failure" cleanup_probe() { touch "${stop_file}" 2>/dev/null || true wait "${probe_pid:-}" 2>/dev/null || true rm -rf "${probe_dir}" } trap cleanup_probe EXIT continuous_probe "${stop_file}" "${failure_file}" & probe_pid="$!" kubectl -n "${namespace}" rollout restart "deployment/${api_deployment}" kubectl -n "${namespace}" rollout restart "deployment/${mcp_deployment}" kubectl -n "${namespace}" rollout status "deployment/${api_deployment}" \ --timeout "${rollout_timeout}" kubectl -n "${namespace}" rollout status "deployment/${mcp_deployment}" \ --timeout "${rollout_timeout}" touch "${stop_file}" wait "${probe_pid}" [[ ! -f "${failure_file}" ]] || fail "API health probe failed during rolling restart" rm -rf "${probe_dir}" trap - EXIT "${PWD}/scripts/worker_recovery_acceptance.sh" "${api_base_url}" \ "${namespace}" "${release_name}" "${worker_deployment}" "${worker_wait_seconds}" fi echo "DEPLOY-003 automated cluster acceptance passed (${acceptance_mode})"