93 lines
3.7 KiB
Bash
Executable File
93 lines
3.7 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
# This helper is called only by cluster_acceptance.sh full mode. It interrupts
|
|
# one exact worker Pod after a durable Job reaches RUNNING, then verifies that
|
|
# the lease/recovery scheduler converges the same Job without a second upload.
|
|
api_base_url="${1:?API base URL is required}"
|
|
namespace="${2:?namespace is required}"
|
|
release_name="${3:?release name is required}"
|
|
worker_deployment="${4:?worker deployment is required}"
|
|
wait_seconds="${5:-720}"
|
|
poll_interval="${ACCEPTANCE_JOB_POLL_INTERVAL_SECONDS:-0.2}"
|
|
work_dir="$(mktemp -d)"
|
|
source_file="${work_dir}/deploy-003-worker-recovery.md"
|
|
response_file="${work_dir}/response.json"
|
|
document_id=""
|
|
|
|
cleanup() {
|
|
if [[ -n "${document_id}" ]]; then
|
|
curl --silent --show-error --connect-timeout 5 --max-time 15 \
|
|
--request DELETE --proto '=https' \
|
|
"${api_base_url}/api/v1/documents/${document_id}" >/dev/null || true
|
|
fi
|
|
rm -rf "${work_dir}"
|
|
}
|
|
trap cleanup EXIT
|
|
|
|
[[ "${wait_seconds}" =~ ^[0-9]+$ && "${wait_seconds}" -gt 0 ]] || exit 2
|
|
[[ "${poll_interval}" =~ ^[0-9]+([.][0-9]+)?$ ]] || exit 2
|
|
|
|
# Enough independent paragraphs make RUNNING observable without introducing a
|
|
# large or business-derived acceptance document.
|
|
for number in $(seq 1 80); do
|
|
printf 'DEPLOY-003 worker recovery synthetic paragraph %s.\n\n' "${number}" >>"${source_file}"
|
|
done
|
|
|
|
curl --fail --silent --show-error --connect-timeout 5 --max-time 60 --proto '=https' \
|
|
--form "file=@${source_file};type=text/markdown" \
|
|
"${api_base_url}/api/v1/documents" >"${response_file}"
|
|
|
|
document_id="$(python3 -c 'import json,sys; print(json.load(sys.stdin)["document_id"])' <"${response_file}")"
|
|
job_id="$(python3 -c 'import json,sys; print(json.load(sys.stdin)["job_id"])' <"${response_file}")"
|
|
[[ -n "${document_id}" && -n "${job_id}" ]] || exit 1
|
|
|
|
deadline=$((SECONDS + wait_seconds))
|
|
interrupted="false"
|
|
while ((SECONDS < deadline)); do
|
|
job_json="$({
|
|
curl --fail --silent --show-error --connect-timeout 5 --max-time 10 --proto '=https' \
|
|
"${api_base_url}/api/v1/jobs/${job_id}"
|
|
})"
|
|
job_status="$(printf '%s' "${job_json}" | python3 -c 'import json,sys; print(json.load(sys.stdin)["status"])')"
|
|
if [[ "${job_status}" == "running" ]]; then
|
|
worker_pod="$(kubectl -n "${namespace}" get pods \
|
|
-l "app.kubernetes.io/instance=${release_name},app.kubernetes.io/component=worker" \
|
|
--field-selector=status.phase=Running \
|
|
-o jsonpath='{.items[0].metadata.name}')"
|
|
[[ -n "${worker_pod}" ]] || exit 1
|
|
# Zero grace simulates process/node loss instead of a cooperative drain.
|
|
kubectl -n "${namespace}" delete pod "${worker_pod}" \
|
|
--grace-period=0 --force --wait=false >/dev/null
|
|
interrupted="true"
|
|
break
|
|
fi
|
|
if [[ "${job_status}" == "succeeded" || "${job_status}" == "failed" ]]; then
|
|
echo "worker recovery acceptance could not interrupt the fast Job" >&2
|
|
exit 1
|
|
fi
|
|
sleep "${poll_interval}"
|
|
done
|
|
[[ "${interrupted}" == "true" ]] || exit 1
|
|
|
|
kubectl -n "${namespace}" rollout status "deployment/${worker_deployment}" --timeout 10m
|
|
while ((SECONDS < deadline)); do
|
|
job_json="$({
|
|
curl --fail --silent --show-error --connect-timeout 5 --max-time 10 --proto '=https' \
|
|
"${api_base_url}/api/v1/jobs/${job_id}"
|
|
})"
|
|
job_status="$(printf '%s' "${job_json}" | python3 -c 'import json,sys; print(json.load(sys.stdin)["status"])')"
|
|
if [[ "${job_status}" == "succeeded" ]]; then
|
|
echo "durable worker recovery passed"
|
|
exit 0
|
|
fi
|
|
if [[ "${job_status}" == "failed" || "${job_status}" == "cancelled" ]]; then
|
|
echo "worker recovery Job reached terminal status ${job_status}" >&2
|
|
exit 1
|
|
fi
|
|
sleep 1
|
|
done
|
|
|
|
echo "worker recovery Job did not complete within ${wait_seconds} seconds" >&2
|
|
exit 1
|