droboticflow/scripts/deploy-k8s.sh

262 lines
11 KiB
Bash
Executable File

#!/usr/bin/env bash
set -euo pipefail
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
DEPLOY_ENV="${DEPLOY_ENV:-}"
if [[ $# -gt 0 && -z "$DEPLOY_ENV" && "${1:-}" != -* ]]; then
DEPLOY_ENV="$1"
shift
fi
export DEPLOY_ENV
# shellcheck source=scripts/load-deploy-env.sh
source "$ROOT_DIR/scripts/load-deploy-env.sh"
DEPLOY_ENV="${DEPLOY_ENV:-}"
# shellcheck source=scripts/lib/build-metadata.sh
source "$ROOT_DIR/scripts/lib/build-metadata.sh"
K8S_NAMESPACE="${K8S_NAMESPACE:-droboticflow}"
REGISTRY_HOST="${REGISTRY_HOST:-}"
REGISTRY_NAMESPACE="${REGISTRY_NAMESPACE:-}"
REGISTRY_SERVER="${REGISTRY_SERVER:-$REGISTRY_HOST}"
REGISTRY_USERNAME="${REGISTRY_USERNAME:-}"
REGISTRY_PASSWORD="${REGISTRY_PASSWORD:-}"
CREATE_REGISTRY_SECRET="${CREATE_REGISTRY_SECRET:-true}"
MONGO_DB="${MONGO_DB:-droboticflow}"
MONGO_URI="${MONGO_URI:-}"
MONGO_CONNECT_RETRIES="${MONGO_CONNECT_RETRIES:-30}"
MONGO_CONNECT_RETRY_DELAY_MS="${MONGO_CONNECT_RETRY_DELAY_MS:-2000}"
WORKER_POLL_INTERVAL_MS="${WORKER_POLL_INTERVAL_MS:-2000}"
DROBOTICFLOW_SHARED_MEMORY_SIZE="${DROBOTICFLOW_SHARED_MEMORY_SIZE:-8Gi}"
WORKER_API_BASE_URL="${WORKER_API_BASE_URL:-http://droboticflow-worker:3002}"
BOS_ENDPOINT="${BOS_ENDPOINT:-}"
K8S_IMAGE_PULL_SECRET_NAME="${K8S_IMAGE_PULL_SECRET_NAME:-droboticflow-registry}"
WORKSPACE_PVC_NAME="${WORKSPACE_PVC_NAME:-droboticflow-workspace}"
WORKSPACE_MOUNT_PATH="${WORKSPACE_MOUNT_PATH:-/workspace}"
API_NODE_PORT="${API_NODE_PORT:-30080}"
API_SERVICE_TYPE="${API_SERVICE_TYPE:-NodePort}"
API_PUBLIC_LOAD_BALANCER="${API_PUBLIC_LOAD_BALANCER:-false}"
API_PUBLIC_SERVICE_NAME="${API_PUBLIC_SERVICE_NAME:-droboticflow-api-public}"
API_REPLICAS="${API_REPLICAS:-2}"
WORKER_REPLICAS="${WORKER_REPLICAS:-2}"
WORKER_ORPHAN_CLEANUP_ENABLED="${WORKER_ORPHAN_CLEANUP_ENABLED:-false}"
WORKER_ORPHAN_MIN_AGE_SECONDS="${WORKER_ORPHAN_MIN_AGE_SECONDS:-86400}"
WORKER_ORPHAN_CHECK_INTERVAL_SECONDS="${WORKER_ORPHAN_CHECK_INTERVAL_SECONDS:-300}"
WORKER_PENDING_FAILURE_SECONDS="${WORKER_PENDING_FAILURE_SECONDS:-600}"
WORKER_MISSING_JOB_MAX_RECOVERIES="${WORKER_MISSING_JOB_MAX_RECOVERIES:-3}"
# shellcheck source=scripts/lib/image-config.sh
source "$ROOT_DIR/scripts/lib/image-config.sh"
droboticflow_resolve_image_configuration
DROBOTICFLOW_BUILD_COMMIT="${DROBOTICFLOW_BUILD_COMMIT:-$(git -C "$ROOT_DIR" rev-parse HEAD)}"
DROBOTICFLOW_BUILD_TIME="${DROBOTICFLOW_BUILD_TIME:-}"
OPERATOR_RUNNER_BUILD_COMMIT="${OPERATOR_RUNNER_BUILD_COMMIT:-$DROBOTICFLOW_BUILD_COMMIT}"
OPERATOR_RUNNER_BUILD_TIME="${OPERATOR_RUNNER_BUILD_TIME:-}"
droboticflow_resolve_build_metadata
OPERATOR_GPU_NODE_SELECTOR="${OPERATOR_GPU_NODE_SELECTOR:-}"
OPERATOR_GPU_TOLERATIONS_JSON="${OPERATOR_GPU_TOLERATIONS_JSON:-}"
OPERATOR_GPU_RUNTIME_CLASS="${OPERATOR_GPU_RUNTIME_CLASS:-}"
OPERATOR_JOB_TTL_SECONDS="${OPERATOR_JOB_TTL_SECONDS:-86400}"
CUSTOM_OPERATOR_IMAGE_REGISTRY="${CUSTOM_OPERATOR_IMAGE_REGISTRY:-}"
if [[ -z "$CUSTOM_OPERATOR_IMAGE_REGISTRY" && -n "$REGISTRY_HOST" && -n "$REGISTRY_NAMESPACE" ]]; then
CUSTOM_OPERATOR_IMAGE_REGISTRY="$REGISTRY_HOST/$REGISTRY_NAMESPACE/custom-operators"
fi
CUSTOM_OPERATOR_IMAGE_IMPORTER_IMAGE="${CUSTOM_OPERATOR_IMAGE_IMPORTER_IMAGE:-quay.io/skopeo/stable:v1.16.1}"
CUSTOM_OPERATOR_IMAGE_IMPORT_TIMEOUT_SECONDS="${CUSTOM_OPERATOR_IMAGE_IMPORT_TIMEOUT_SECONDS:-1800}"
CUSTOM_OPERATOR_IMAGE_ARCHIVE_MAX_MIB="${CUSTOM_OPERATOR_IMAGE_ARCHIVE_MAX_MIB:-8192}"
OPERATOR_ALLOW_SHELL_COMMANDS="${OPERATOR_ALLOW_SHELL_COMMANDS:-false}"
OPERATOR_COMMAND_ALLOWLIST_REGEX="${OPERATOR_COMMAND_ALLOWLIST_REGEX:-}"
OPERATOR_COMMAND_DENYLIST_REGEX="${OPERATOR_COMMAND_DENYLIST_REGEX:-}"
DROBOTICFLOW_SESSION_COOKIE_SECURE="${DROBOTICFLOW_SESSION_COOKIE_SECURE:-$API_PUBLIC_LOAD_BALANCER}"
DROBOTICFLOW_TRUSTED_PROXY_CIDRS="${DROBOTICFLOW_TRUSTED_PROXY_CIDRS:-}"
AI_API_BASE_URL="${AI_API_BASE_URL:-}"
AI_MODEL="${AI_MODEL:-}"
AI_API_KEY="${AI_API_KEY:-}"
DROBOTICFLOW_API_TOKEN="${DROBOTICFLOW_API_TOKEN:-}"
# shellcheck source=scripts/lib/deploy-validation.sh
source "$ROOT_DIR/scripts/lib/deploy-validation.sh"
validate_deploy_startup_configuration
if boolean_true CREATE_REGISTRY_SECRET "$CREATE_REGISTRY_SECRET"; then
: "${REGISTRY_SERVER:?REGISTRY_SERVER is required when CREATE_REGISTRY_SECRET=true}"
: "${REGISTRY_USERNAME:?REGISTRY_USERNAME is required when CREATE_REGISTRY_SECRET=true}"
: "${REGISTRY_PASSWORD:?REGISTRY_PASSWORD is required when CREATE_REGISTRY_SECRET=true}"
fi
read_secret_value() {
local secret_name="$1"
local key="$2"
kubectl --request-timeout=5s -n "$K8S_NAMESPACE" get secret "$secret_name" \
-o "jsonpath={.data.$key}" 2>/dev/null | base64 --decode 2>/dev/null || true
}
read_config_value() {
local configmap_name="$1"
local key="$2"
kubectl --request-timeout=5s -n "$K8S_NAMESPACE" get configmap "$configmap_name" \
-o "jsonpath={.data.$key}" 2>/dev/null || true
}
validate_external_mongo_uri() {
local uri="$1"
if [[ -z "$uri" ]]; then
echo "ERROR: MONGO_URI is required and must point to an external MongoDB instance. Export MONGO_URI before running scripts/deploy-k8s.sh." >&2
exit 1
fi
shopt -s nocasematch
if [[ "$uri" =~ (^|[@/])mongodb:27017([/?]|$) ]] || \
[[ "$uri" =~ localhost ]] || \
[[ "$uri" =~ 127\.0\.0\.1 ]] || \
[[ "$uri" =~ \.svc([:/?]|$) ]]; then
echo "ERROR: MONGO_URI must reference an external MongoDB instance. Cluster-local or localhost targets are not allowed in the main Kubernetes deployment path." >&2
exit 1
fi
shopt -u nocasematch
}
if [[ -z "$MONGO_URI" ]] && kubectl --request-timeout=5s -n "$K8S_NAMESPACE" get secret droboticflow-mongo-secret >/dev/null 2>&1; then
MONGO_URI="$(read_secret_value droboticflow-mongo-secret MONGO_URI)"
fi
validate_external_mongo_uri "$MONGO_URI"
kubectl get namespace "$K8S_NAMESPACE" >/dev/null 2>&1 || kubectl create namespace "$K8S_NAMESPACE"
if boolean_true CREATE_REGISTRY_SECRET "$CREATE_REGISTRY_SECRET"; then
kubectl -n "$K8S_NAMESPACE" create secret docker-registry "$K8S_IMAGE_PULL_SECRET_NAME" \
--docker-server="$REGISTRY_SERVER" \
--docker-username="$REGISTRY_USERNAME" \
--docker-password="$REGISTRY_PASSWORD" \
--dry-run=client \
-o yaml | kubectl apply -f -
fi
if kubectl -n "$K8S_NAMESPACE" get secret droboticflow-ai-secret >/dev/null 2>&1; then
AI_API_KEY="${AI_API_KEY:-$(read_secret_value droboticflow-ai-secret AI_API_KEY)}"
AI_API_BASE_URL="${AI_API_BASE_URL:-$(read_config_value droboticflow-config AI_API_BASE_URL)}"
AI_MODEL="${AI_MODEL:-$(read_config_value droboticflow-config AI_MODEL)}"
fi
validate_ai_runtime_configuration
if kubectl -n "$K8S_NAMESPACE" get secret droboticflow-auth-secret >/dev/null 2>&1; then
DROBOTICFLOW_API_TOKEN="${DROBOTICFLOW_API_TOKEN:-$(read_secret_value droboticflow-auth-secret DROBOTICFLOW_API_TOKEN)}"
fi
if boolean_true API_PUBLIC_LOAD_BALANCER "$API_PUBLIC_LOAD_BALANCER" && [[ -z "$DROBOTICFLOW_API_TOKEN" ]]; then
echo "ERROR: DROBOTICFLOW_API_TOKEN is required when API_PUBLIC_LOAD_BALANCER=true." >&2
exit 1
fi
case "$API_SERVICE_TYPE" in
NodePort|ClusterIP)
;;
*)
echo "ERROR: API_SERVICE_TYPE must be NodePort or ClusterIP, got $API_SERVICE_TYPE" >&2
exit 1
;;
esac
kubectl -n "$K8S_NAMESPACE" create secret generic droboticflow-mongo-secret \
--from-literal=MONGO_URI="$MONGO_URI" \
--dry-run=client \
-o yaml | kubectl apply -f -
kubectl -n "$K8S_NAMESPACE" create secret generic droboticflow-ai-secret \
--from-literal=AI_API_KEY="$AI_API_KEY" \
--dry-run=client \
-o yaml | kubectl apply -f -
kubectl -n "$K8S_NAMESPACE" create secret generic droboticflow-auth-secret \
--from-literal=DROBOTICFLOW_API_TOKEN="$DROBOTICFLOW_API_TOKEN" \
--dry-run=client \
-o yaml | kubectl apply -f -
rendered_manifest="$(mktemp)"
cleanup() {
rm -f "$rendered_manifest"
}
trap cleanup EXIT
env \
K8S_NAMESPACE="$K8S_NAMESPACE" \
MONGO_DB="$MONGO_DB" \
MONGO_CONNECT_RETRIES="$MONGO_CONNECT_RETRIES" \
MONGO_CONNECT_RETRY_DELAY_MS="$MONGO_CONNECT_RETRY_DELAY_MS" \
WORKER_POLL_INTERVAL_MS="$WORKER_POLL_INTERVAL_MS" \
DROBOTICFLOW_SHARED_MEMORY_SIZE="$DROBOTICFLOW_SHARED_MEMORY_SIZE" \
WORKER_API_BASE_URL="$WORKER_API_BASE_URL" \
BOS_ENDPOINT="$BOS_ENDPOINT" \
K8S_IMAGE_PULL_SECRET_NAME="$K8S_IMAGE_PULL_SECRET_NAME" \
WORKSPACE_PVC_NAME="$WORKSPACE_PVC_NAME" \
WORKSPACE_MOUNT_PATH="$WORKSPACE_MOUNT_PATH" \
API_NODE_PORT="$API_NODE_PORT" \
API_SERVICE_TYPE="$API_SERVICE_TYPE" \
API_REPLICAS="$API_REPLICAS" \
WORKER_REPLICAS="$WORKER_REPLICAS" \
WORKER_ORPHAN_CLEANUP_ENABLED="$WORKER_ORPHAN_CLEANUP_ENABLED" \
WORKER_ORPHAN_MIN_AGE_SECONDS="$WORKER_ORPHAN_MIN_AGE_SECONDS" \
WORKER_ORPHAN_CHECK_INTERVAL_SECONDS="$WORKER_ORPHAN_CHECK_INTERVAL_SECONDS" \
WORKER_PENDING_FAILURE_SECONDS="$WORKER_PENDING_FAILURE_SECONDS" \
WORKER_MISSING_JOB_MAX_RECOVERIES="$WORKER_MISSING_JOB_MAX_RECOVERIES" \
AI_API_BASE_URL="$AI_API_BASE_URL" \
AI_MODEL="$AI_MODEL" \
AI_API_KEY="$AI_API_KEY" \
DROBOTICFLOW_API_TOKEN="$DROBOTICFLOW_API_TOKEN" \
API_IMAGE="$API_IMAGE" \
WORKER_IMAGE="$WORKER_IMAGE" \
OPERATOR_RUNNER_IMAGE="$OPERATOR_RUNNER_IMAGE" \
OPERATOR_RUNNER_BUILD_COMMIT="$OPERATOR_RUNNER_BUILD_COMMIT" \
OPERATOR_RUNNER_BUILD_TIME="$OPERATOR_RUNNER_BUILD_TIME" \
DROBOTICFLOW_BUILD_COMMIT="$DROBOTICFLOW_BUILD_COMMIT" \
DROBOTICFLOW_BUILD_TIME="$DROBOTICFLOW_BUILD_TIME" \
OPERATOR_GPU_NODE_SELECTOR="$OPERATOR_GPU_NODE_SELECTOR" \
OPERATOR_GPU_TOLERATIONS_JSON="$OPERATOR_GPU_TOLERATIONS_JSON" \
OPERATOR_GPU_RUNTIME_CLASS="$OPERATOR_GPU_RUNTIME_CLASS" \
OPERATOR_JOB_TTL_SECONDS="$OPERATOR_JOB_TTL_SECONDS" \
CUSTOM_OPERATOR_IMAGE_REGISTRY="$CUSTOM_OPERATOR_IMAGE_REGISTRY" \
CUSTOM_OPERATOR_IMAGE_IMPORTER_IMAGE="$CUSTOM_OPERATOR_IMAGE_IMPORTER_IMAGE" \
CUSTOM_OPERATOR_IMAGE_IMPORT_TIMEOUT_SECONDS="$CUSTOM_OPERATOR_IMAGE_IMPORT_TIMEOUT_SECONDS" \
CUSTOM_OPERATOR_IMAGE_ARCHIVE_MAX_MIB="$CUSTOM_OPERATOR_IMAGE_ARCHIVE_MAX_MIB" \
OPERATOR_ALLOW_SHELL_COMMANDS="$OPERATOR_ALLOW_SHELL_COMMANDS" \
OPERATOR_COMMAND_ALLOWLIST_REGEX="$OPERATOR_COMMAND_ALLOWLIST_REGEX" \
OPERATOR_COMMAND_DENYLIST_REGEX="$OPERATOR_COMMAND_DENYLIST_REGEX" \
DROBOTICFLOW_SESSION_COOKIE_SECURE="$DROBOTICFLOW_SESSION_COOKIE_SECURE" \
DROBOTICFLOW_TRUSTED_PROXY_CIDRS="$DROBOTICFLOW_TRUSTED_PROXY_CIDRS" \
"$ROOT_DIR/scripts/render-manifests.sh" >"$rendered_manifest"
kubectl apply --dry-run=client -f "$rendered_manifest" >/dev/null
kubectl apply -f "$rendered_manifest"
if boolean_true API_PUBLIC_LOAD_BALANCER "$API_PUBLIC_LOAD_BALANCER"; then
cat <<YAML | kubectl apply -f -
apiVersion: v1
kind: Service
metadata:
name: ${API_PUBLIC_SERVICE_NAME}
namespace: ${K8S_NAMESPACE}
spec:
type: LoadBalancer
selector:
app: droboticflow-api
ports:
- name: http
port: 80
targetPort: 3001
protocol: TCP
YAML
fi
kubectl -n "$K8S_NAMESPACE" rollout status deployment/droboticflow-api --timeout=300s
kubectl -n "$K8S_NAMESPACE" rollout status deployment/droboticflow-worker --timeout=300s
kubectl -n "$K8S_NAMESPACE" get pods -l 'app in (droboticflow-api,droboticflow-worker)'
kubectl -n "$K8S_NAMESPACE" get svc droboticflow-api
if boolean_true API_PUBLIC_LOAD_BALANCER "$API_PUBLIC_LOAD_BALANCER"; then
kubectl -n "$K8S_NAMESPACE" get svc "$API_PUBLIC_SERVICE_NAME"
fi