knowledge-platform/scripts/worker_recovery_acceptance.sh

93 lines
3.7 KiB
Bash
Executable File

#!/usr/bin/env bash
set -euo pipefail
# This helper is called only by cluster_acceptance.sh full mode. It interrupts
# one exact worker Pod after a durable Job reaches RUNNING, then verifies that
# the lease/recovery scheduler converges the same Job without a second upload.
api_base_url="${1:?API base URL is required}"
namespace="${2:?namespace is required}"
release_name="${3:?release name is required}"
worker_deployment="${4:?worker deployment is required}"
wait_seconds="${5:-720}"
poll_interval="${ACCEPTANCE_JOB_POLL_INTERVAL_SECONDS:-0.2}"
work_dir="$(mktemp -d)"
source_file="${work_dir}/deploy-003-worker-recovery.md"
response_file="${work_dir}/response.json"
document_id=""
cleanup() {
if [[ -n "${document_id}" ]]; then
curl --silent --show-error --connect-timeout 5 --max-time 15 \
--request DELETE --proto '=https' \
"${api_base_url}/api/v1/documents/${document_id}" >/dev/null || true
fi
rm -rf "${work_dir}"
}
trap cleanup EXIT
[[ "${wait_seconds}" =~ ^[0-9]+$ && "${wait_seconds}" -gt 0 ]] || exit 2
[[ "${poll_interval}" =~ ^[0-9]+([.][0-9]+)?$ ]] || exit 2
# Enough independent paragraphs make RUNNING observable without introducing a
# large or business-derived acceptance document.
for number in $(seq 1 80); do
printf 'DEPLOY-003 worker recovery synthetic paragraph %s.\n\n' "${number}" >>"${source_file}"
done
curl --fail --silent --show-error --connect-timeout 5 --max-time 60 --proto '=https' \
--form "file=@${source_file};type=text/markdown" \
"${api_base_url}/api/v1/documents" >"${response_file}"
document_id="$(python3 -c 'import json,sys; print(json.load(sys.stdin)["document_id"])' <"${response_file}")"
job_id="$(python3 -c 'import json,sys; print(json.load(sys.stdin)["job_id"])' <"${response_file}")"
[[ -n "${document_id}" && -n "${job_id}" ]] || exit 1
deadline=$((SECONDS + wait_seconds))
interrupted="false"
while ((SECONDS < deadline)); do
job_json="$({
curl --fail --silent --show-error --connect-timeout 5 --max-time 10 --proto '=https' \
"${api_base_url}/api/v1/jobs/${job_id}"
})"
job_status="$(printf '%s' "${job_json}" | python3 -c 'import json,sys; print(json.load(sys.stdin)["status"])')"
if [[ "${job_status}" == "running" ]]; then
worker_pod="$(kubectl -n "${namespace}" get pods \
-l "app.kubernetes.io/instance=${release_name},app.kubernetes.io/component=worker" \
--field-selector=status.phase=Running \
-o jsonpath='{.items[0].metadata.name}')"
[[ -n "${worker_pod}" ]] || exit 1
# Zero grace simulates process/node loss instead of a cooperative drain.
kubectl -n "${namespace}" delete pod "${worker_pod}" \
--grace-period=0 --force --wait=false >/dev/null
interrupted="true"
break
fi
if [[ "${job_status}" == "succeeded" || "${job_status}" == "failed" ]]; then
echo "worker recovery acceptance could not interrupt the fast Job" >&2
exit 1
fi
sleep "${poll_interval}"
done
[[ "${interrupted}" == "true" ]] || exit 1
kubectl -n "${namespace}" rollout status "deployment/${worker_deployment}" --timeout 10m
while ((SECONDS < deadline)); do
job_json="$({
curl --fail --silent --show-error --connect-timeout 5 --max-time 10 --proto '=https' \
"${api_base_url}/api/v1/jobs/${job_id}"
})"
job_status="$(printf '%s' "${job_json}" | python3 -c 'import json,sys; print(json.load(sys.stdin)["status"])')"
if [[ "${job_status}" == "succeeded" ]]; then
echo "durable worker recovery passed"
exit 0
fi
if [[ "${job_status}" == "failed" || "${job_status}" == "cancelled" ]]; then
echo "worker recovery Job reached terminal status ${job_status}" >&2
exit 1
fi
sleep 1
done
echo "worker recovery Job did not complete within ${wait_seconds} seconds" >&2
exit 1