knowledge-platform/scripts/cluster_acceptance.sh

202 lines
7.7 KiB
Bash
Executable File

#!/usr/bin/env bash
set -euo pipefail
# DEPLOY-003 validates one explicitly selected cluster context. Secret contents
# never enter this process; only HTTPS public endpoints and Kubernetes metadata
# are inspected. Full mode performs controlled rollout and worker-failure tests.
values_file="${1:-}"
chart_dir="${CHART:-deploy/helm/knowledge-platform}"
release_name="${RELEASE:-knowledge-platform}"
namespace="${NAMESPACE:-knowledge-platform}"
acceptance_mode="${ACCEPTANCE_MODE:-read-only}"
expected_context="${ACCEPTANCE_KUBE_CONTEXT:-}"
rollout_timeout="${ACCEPTANCE_ROLLOUT_TIMEOUT:-10m}"
worker_wait_seconds="${ACCEPTANCE_WORKER_WAIT_SECONDS:-720}"
fail() {
echo "DEPLOY-003 acceptance failed: $1" >&2
exit 1
}
if [[ -z "${values_file}" || ! -f "${values_file}" ]]; then
fail "VALUES must point to an existing environment values file"
fi
if [[ "${acceptance_mode}" != "read-only" && "${acceptance_mode}" != "full" ]]; then
fail "ACCEPTANCE_MODE must be read-only or full"
fi
for tool in helm kubectl curl python3; do
command -v "${tool}" >/dev/null 2>&1 || fail "required tool is missing: ${tool}"
done
current_context="$(kubectl config current-context 2>/dev/null || true)"
if [[ -z "${expected_context}" || "${current_context}" != "${expected_context}" ]]; then
fail "ACCEPTANCE_KUBE_CONTEXT must exactly match the active context"
fi
preflight_yaml="$({
helm template "${release_name}" "${chart_dir}" \
--namespace "${namespace}" \
--values "${values_file}" \
--show-only templates/tests/preflight-references.yaml
})"
reference() {
local field="$1"
printf '%s\n' "${preflight_yaml}" |
awk -v key="${field}:" '$1 == key {gsub(/"/, "", $2); print $2}'
}
ingress_enabled="$(reference ingressEnabled)"
api_host="$(reference apiHost)"
mcp_host="$(reference mcpHost)"
api_deployment="$(reference apiDeployment)"
mcp_deployment="$(reference mcpDeployment)"
worker_deployment="$(reference workerDeployment)"
scheduler_deployment="$(reference schedulerDeployment)"
metrics_path="$(reference metricsPath)"
if [[ "${ingress_enabled}" != "true" ]]; then
fail "target acceptance requires ingress.enabled=true"
fi
for value in \
"${api_host}" \
"${mcp_host}" \
"${api_deployment}" \
"${mcp_deployment}" \
"${worker_deployment}" \
"${scheduler_deployment}" \
"${metrics_path}"; do
[[ -n "${value}" ]] || fail "Helm acceptance references are incomplete"
done
if [[ "${acceptance_mode}" == "full" ]]; then
expected_confirmation="DEPLOY-003:${current_context}:${namespace}:${release_name}"
if [[ "${ACCEPTANCE_CONFIRM:-}" != "${expected_confirmation}" ]]; then
fail "set ACCEPTANCE_CONFIRM=${expected_confirmation} for controlled mutations"
fi
# Running the same entrypoint twice proves install/upgrade idempotency against
# the selected environment before disruption tests begin.
"${PWD}/scripts/deploy.sh" "${values_file}"
"${PWD}/scripts/deploy.sh" "${values_file}"
fi
kubectl -n "${namespace}" get deployment "${api_deployment}" >/dev/null
kubectl -n "${namespace}" get deployment "${mcp_deployment}" >/dev/null
kubectl -n "${namespace}" get deployment "${worker_deployment}" >/dev/null
kubectl -n "${namespace}" get deployment "${scheduler_deployment}" >/dev/null
for deployment in \
"${api_deployment}" \
"${mcp_deployment}" \
"${worker_deployment}" \
"${scheduler_deployment}"; do
kubectl -n "${namespace}" rollout status "deployment/${deployment}" \
--timeout "${rollout_timeout}"
done
assert_multi_replica_ready() {
local deployment="$1"
local desired ready available
desired="$(kubectl -n "${namespace}" get deployment "${deployment}" -o jsonpath='{.spec.replicas}')"
ready="$(kubectl -n "${namespace}" get deployment "${deployment}" -o jsonpath='{.status.readyReplicas}')"
available="$(kubectl -n "${namespace}" get deployment "${deployment}" -o jsonpath='{.status.availableReplicas}')"
[[ "${desired}" -ge 2 ]] || fail "${deployment} must have at least two replicas"
[[ "${ready:-0}" -eq "${desired}" ]] || fail "${deployment} does not have all replicas ready"
[[ "${available:-0}" -eq "${desired}" ]] || fail "${deployment} is not fully available"
}
assert_multi_replica_ready "${api_deployment}"
assert_multi_replica_ready "${mcp_deployment}"
for disruption_budget in "${api_deployment}" "${mcp_deployment}"; do
disruptions_allowed="0"
for attempt in $(seq 1 30); do
disruptions_allowed="$(kubectl -n "${namespace}" get poddisruptionbudget \
"${disruption_budget}" -o jsonpath='{.status.disruptionsAllowed}')"
[[ "${disruptions_allowed:-0}" -ge 1 ]] && break
sleep 1
done
[[ "${disruptions_allowed:-0}" -ge 1 ]] ||
fail "${disruption_budget} does not currently allow one voluntary disruption"
done
api_base_url="https://${api_host}"
mcp_url="https://${mcp_host}/mcp"
curl --fail --silent --show-error --connect-timeout 5 --max-time 10 \
--proto '=https' "${api_base_url}/healthz" >/dev/null
curl --fail --silent --show-error --connect-timeout 5 --max-time 10 \
--proto '=https' "${api_base_url}/admin/upload" >/dev/null
metrics_output="$({
curl --fail --silent --show-error --connect-timeout 5 --max-time 10 \
--proto '=https' "${api_base_url}${metrics_path}"
})"
printf '%s\n' "${metrics_output}" | grep -q '^ekp_process_uptime_seconds ' ||
fail "API metrics endpoint does not expose process uptime"
printf '%s\n' "${metrics_output}" | grep -q '^ekp_http_requests_total{' ||
fail "API metrics endpoint does not expose request totals"
# A credential-free request may be rejected, but it must complete TLS and reach
# the MCP application rather than fail DNS, certificate, or gateway routing.
mcp_status="$({
curl --silent --show-error --output /dev/null --write-out '%{http_code}' \
--connect-timeout 5 --max-time 10 --proto '=https' "${mcp_url}"
})"
case "${mcp_status}" in
200 | 400 | 401 | 403 | 405 | 406) ;;
*) fail "MCP HTTPS endpoint returned unexpected status ${mcp_status}" ;;
esac
continuous_probe() {
local stop_file="$1"
local failure_file="$2"
local current_mcp_status
while [[ ! -f "${stop_file}" ]]; do
if ! curl --fail --silent --show-error --connect-timeout 5 --max-time 10 --proto '=https' \
"${api_base_url}/healthz" >/dev/null; then
printf 'failed\n' >"${failure_file}"
return
fi
current_mcp_status="$({
curl --silent --show-error --output /dev/null --write-out '%{http_code}' \
--connect-timeout 5 --max-time 10 --proto '=https' "${mcp_url}"
})"
case "${current_mcp_status}" in
200 | 400 | 401 | 403 | 405 | 406) ;;
*)
printf 'failed\n' >"${failure_file}"
return
;;
esac
sleep 1
done
}
if [[ "${acceptance_mode}" == "full" ]]; then
probe_dir="$(mktemp -d)"
stop_file="${probe_dir}/stop"
failure_file="${probe_dir}/failure"
cleanup_probe() {
touch "${stop_file}" 2>/dev/null || true
wait "${probe_pid:-}" 2>/dev/null || true
rm -rf "${probe_dir}"
}
trap cleanup_probe EXIT
continuous_probe "${stop_file}" "${failure_file}" &
probe_pid="$!"
kubectl -n "${namespace}" rollout restart "deployment/${api_deployment}"
kubectl -n "${namespace}" rollout restart "deployment/${mcp_deployment}"
kubectl -n "${namespace}" rollout status "deployment/${api_deployment}" \
--timeout "${rollout_timeout}"
kubectl -n "${namespace}" rollout status "deployment/${mcp_deployment}" \
--timeout "${rollout_timeout}"
touch "${stop_file}"
wait "${probe_pid}"
[[ ! -f "${failure_file}" ]] || fail "API health probe failed during rolling restart"
rm -rf "${probe_dir}"
trap - EXIT
"${PWD}/scripts/worker_recovery_acceptance.sh" "${api_base_url}" \
"${namespace}" "${release_name}" "${worker_deployment}" "${worker_wait_seconds}"
fi
echo "DEPLOY-003 automated cluster acceptance passed (${acceptance_mode})"