202 lines
7.7 KiB
Bash
Executable File
202 lines
7.7 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
# DEPLOY-003 validates one explicitly selected cluster context. Secret contents
|
|
# never enter this process; only HTTPS public endpoints and Kubernetes metadata
|
|
# are inspected. Full mode performs controlled rollout and worker-failure tests.
|
|
values_file="${1:-}"
|
|
chart_dir="${CHART:-deploy/helm/knowledge-platform}"
|
|
release_name="${RELEASE:-knowledge-platform}"
|
|
namespace="${NAMESPACE:-knowledge-platform}"
|
|
acceptance_mode="${ACCEPTANCE_MODE:-read-only}"
|
|
expected_context="${ACCEPTANCE_KUBE_CONTEXT:-}"
|
|
rollout_timeout="${ACCEPTANCE_ROLLOUT_TIMEOUT:-10m}"
|
|
worker_wait_seconds="${ACCEPTANCE_WORKER_WAIT_SECONDS:-720}"
|
|
|
|
fail() {
|
|
echo "DEPLOY-003 acceptance failed: $1" >&2
|
|
exit 1
|
|
}
|
|
|
|
if [[ -z "${values_file}" || ! -f "${values_file}" ]]; then
|
|
fail "VALUES must point to an existing environment values file"
|
|
fi
|
|
if [[ "${acceptance_mode}" != "read-only" && "${acceptance_mode}" != "full" ]]; then
|
|
fail "ACCEPTANCE_MODE must be read-only or full"
|
|
fi
|
|
for tool in helm kubectl curl python3; do
|
|
command -v "${tool}" >/dev/null 2>&1 || fail "required tool is missing: ${tool}"
|
|
done
|
|
|
|
current_context="$(kubectl config current-context 2>/dev/null || true)"
|
|
if [[ -z "${expected_context}" || "${current_context}" != "${expected_context}" ]]; then
|
|
fail "ACCEPTANCE_KUBE_CONTEXT must exactly match the active context"
|
|
fi
|
|
|
|
preflight_yaml="$({
|
|
helm template "${release_name}" "${chart_dir}" \
|
|
--namespace "${namespace}" \
|
|
--values "${values_file}" \
|
|
--show-only templates/tests/preflight-references.yaml
|
|
})"
|
|
|
|
reference() {
|
|
local field="$1"
|
|
printf '%s\n' "${preflight_yaml}" |
|
|
awk -v key="${field}:" '$1 == key {gsub(/"/, "", $2); print $2}'
|
|
}
|
|
|
|
ingress_enabled="$(reference ingressEnabled)"
|
|
api_host="$(reference apiHost)"
|
|
mcp_host="$(reference mcpHost)"
|
|
api_deployment="$(reference apiDeployment)"
|
|
mcp_deployment="$(reference mcpDeployment)"
|
|
worker_deployment="$(reference workerDeployment)"
|
|
scheduler_deployment="$(reference schedulerDeployment)"
|
|
metrics_path="$(reference metricsPath)"
|
|
|
|
if [[ "${ingress_enabled}" != "true" ]]; then
|
|
fail "target acceptance requires ingress.enabled=true"
|
|
fi
|
|
for value in \
|
|
"${api_host}" \
|
|
"${mcp_host}" \
|
|
"${api_deployment}" \
|
|
"${mcp_deployment}" \
|
|
"${worker_deployment}" \
|
|
"${scheduler_deployment}" \
|
|
"${metrics_path}"; do
|
|
[[ -n "${value}" ]] || fail "Helm acceptance references are incomplete"
|
|
done
|
|
|
|
if [[ "${acceptance_mode}" == "full" ]]; then
|
|
expected_confirmation="DEPLOY-003:${current_context}:${namespace}:${release_name}"
|
|
if [[ "${ACCEPTANCE_CONFIRM:-}" != "${expected_confirmation}" ]]; then
|
|
fail "set ACCEPTANCE_CONFIRM=${expected_confirmation} for controlled mutations"
|
|
fi
|
|
# Running the same entrypoint twice proves install/upgrade idempotency against
|
|
# the selected environment before disruption tests begin.
|
|
"${PWD}/scripts/deploy.sh" "${values_file}"
|
|
"${PWD}/scripts/deploy.sh" "${values_file}"
|
|
fi
|
|
|
|
kubectl -n "${namespace}" get deployment "${api_deployment}" >/dev/null
|
|
kubectl -n "${namespace}" get deployment "${mcp_deployment}" >/dev/null
|
|
kubectl -n "${namespace}" get deployment "${worker_deployment}" >/dev/null
|
|
kubectl -n "${namespace}" get deployment "${scheduler_deployment}" >/dev/null
|
|
|
|
for deployment in \
|
|
"${api_deployment}" \
|
|
"${mcp_deployment}" \
|
|
"${worker_deployment}" \
|
|
"${scheduler_deployment}"; do
|
|
kubectl -n "${namespace}" rollout status "deployment/${deployment}" \
|
|
--timeout "${rollout_timeout}"
|
|
done
|
|
|
|
assert_multi_replica_ready() {
|
|
local deployment="$1"
|
|
local desired ready available
|
|
desired="$(kubectl -n "${namespace}" get deployment "${deployment}" -o jsonpath='{.spec.replicas}')"
|
|
ready="$(kubectl -n "${namespace}" get deployment "${deployment}" -o jsonpath='{.status.readyReplicas}')"
|
|
available="$(kubectl -n "${namespace}" get deployment "${deployment}" -o jsonpath='{.status.availableReplicas}')"
|
|
[[ "${desired}" -ge 2 ]] || fail "${deployment} must have at least two replicas"
|
|
[[ "${ready:-0}" -eq "${desired}" ]] || fail "${deployment} does not have all replicas ready"
|
|
[[ "${available:-0}" -eq "${desired}" ]] || fail "${deployment} is not fully available"
|
|
}
|
|
|
|
assert_multi_replica_ready "${api_deployment}"
|
|
assert_multi_replica_ready "${mcp_deployment}"
|
|
for disruption_budget in "${api_deployment}" "${mcp_deployment}"; do
|
|
disruptions_allowed="0"
|
|
for attempt in $(seq 1 30); do
|
|
disruptions_allowed="$(kubectl -n "${namespace}" get poddisruptionbudget \
|
|
"${disruption_budget}" -o jsonpath='{.status.disruptionsAllowed}')"
|
|
[[ "${disruptions_allowed:-0}" -ge 1 ]] && break
|
|
sleep 1
|
|
done
|
|
[[ "${disruptions_allowed:-0}" -ge 1 ]] ||
|
|
fail "${disruption_budget} does not currently allow one voluntary disruption"
|
|
done
|
|
|
|
api_base_url="https://${api_host}"
|
|
mcp_url="https://${mcp_host}/mcp"
|
|
curl --fail --silent --show-error --connect-timeout 5 --max-time 10 \
|
|
--proto '=https' "${api_base_url}/healthz" >/dev/null
|
|
curl --fail --silent --show-error --connect-timeout 5 --max-time 10 \
|
|
--proto '=https' "${api_base_url}/admin/upload" >/dev/null
|
|
metrics_output="$({
|
|
curl --fail --silent --show-error --connect-timeout 5 --max-time 10 \
|
|
--proto '=https' "${api_base_url}${metrics_path}"
|
|
})"
|
|
printf '%s\n' "${metrics_output}" | grep -q '^ekp_process_uptime_seconds ' ||
|
|
fail "API metrics endpoint does not expose process uptime"
|
|
printf '%s\n' "${metrics_output}" | grep -q '^ekp_http_requests_total{' ||
|
|
fail "API metrics endpoint does not expose request totals"
|
|
|
|
# A credential-free request may be rejected, but it must complete TLS and reach
|
|
# the MCP application rather than fail DNS, certificate, or gateway routing.
|
|
mcp_status="$({
|
|
curl --silent --show-error --output /dev/null --write-out '%{http_code}' \
|
|
--connect-timeout 5 --max-time 10 --proto '=https' "${mcp_url}"
|
|
})"
|
|
case "${mcp_status}" in
|
|
200 | 400 | 401 | 403 | 405 | 406) ;;
|
|
*) fail "MCP HTTPS endpoint returned unexpected status ${mcp_status}" ;;
|
|
esac
|
|
|
|
continuous_probe() {
|
|
local stop_file="$1"
|
|
local failure_file="$2"
|
|
local current_mcp_status
|
|
while [[ ! -f "${stop_file}" ]]; do
|
|
if ! curl --fail --silent --show-error --connect-timeout 5 --max-time 10 --proto '=https' \
|
|
"${api_base_url}/healthz" >/dev/null; then
|
|
printf 'failed\n' >"${failure_file}"
|
|
return
|
|
fi
|
|
current_mcp_status="$({
|
|
curl --silent --show-error --output /dev/null --write-out '%{http_code}' \
|
|
--connect-timeout 5 --max-time 10 --proto '=https' "${mcp_url}"
|
|
})"
|
|
case "${current_mcp_status}" in
|
|
200 | 400 | 401 | 403 | 405 | 406) ;;
|
|
*)
|
|
printf 'failed\n' >"${failure_file}"
|
|
return
|
|
;;
|
|
esac
|
|
sleep 1
|
|
done
|
|
}
|
|
|
|
if [[ "${acceptance_mode}" == "full" ]]; then
|
|
probe_dir="$(mktemp -d)"
|
|
stop_file="${probe_dir}/stop"
|
|
failure_file="${probe_dir}/failure"
|
|
cleanup_probe() {
|
|
touch "${stop_file}" 2>/dev/null || true
|
|
wait "${probe_pid:-}" 2>/dev/null || true
|
|
rm -rf "${probe_dir}"
|
|
}
|
|
trap cleanup_probe EXIT
|
|
continuous_probe "${stop_file}" "${failure_file}" &
|
|
probe_pid="$!"
|
|
kubectl -n "${namespace}" rollout restart "deployment/${api_deployment}"
|
|
kubectl -n "${namespace}" rollout restart "deployment/${mcp_deployment}"
|
|
kubectl -n "${namespace}" rollout status "deployment/${api_deployment}" \
|
|
--timeout "${rollout_timeout}"
|
|
kubectl -n "${namespace}" rollout status "deployment/${mcp_deployment}" \
|
|
--timeout "${rollout_timeout}"
|
|
touch "${stop_file}"
|
|
wait "${probe_pid}"
|
|
[[ ! -f "${failure_file}" ]] || fail "API health probe failed during rolling restart"
|
|
rm -rf "${probe_dir}"
|
|
trap - EXIT
|
|
|
|
"${PWD}/scripts/worker_recovery_acceptance.sh" "${api_base_url}" \
|
|
"${namespace}" "${release_name}" "${worker_deployment}" "${worker_wait_seconds}"
|
|
fi
|
|
|
|
echo "DEPLOY-003 automated cluster acceptance passed (${acceptance_mode})"
|