forked from longtao.wu/droboticflow
226 lines
10 KiB
Bash
Executable File
226 lines
10 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
|
TEMPLATE="$ROOT_DIR/k8s/droboticflow.template.yaml"
|
|
DEPLOY_ENV="${DEPLOY_ENV:-}"
|
|
|
|
if [[ $# -gt 0 && -z "$DEPLOY_ENV" && "${1:-}" != -* ]]; then
|
|
DEPLOY_ENV="$1"
|
|
shift
|
|
fi
|
|
|
|
export DEPLOY_ENV
|
|
# shellcheck source=scripts/load-deploy-env.sh
|
|
source "$ROOT_DIR/scripts/load-deploy-env.sh"
|
|
DEPLOY_ENV="${DEPLOY_ENV:-}"
|
|
# shellcheck source=scripts/lib/build-metadata.sh
|
|
source "$ROOT_DIR/scripts/lib/build-metadata.sh"
|
|
|
|
K8S_NAMESPACE="${K8S_NAMESPACE:-droboticflow}"
|
|
MONGO_DB="${MONGO_DB:-droboticflow}"
|
|
MONGO_CONNECT_RETRIES="${MONGO_CONNECT_RETRIES:-30}"
|
|
MONGO_CONNECT_RETRY_DELAY_MS="${MONGO_CONNECT_RETRY_DELAY_MS:-2000}"
|
|
WORKER_POLL_INTERVAL_MS="${WORKER_POLL_INTERVAL_MS:-2000}"
|
|
DROBOTICFLOW_SHARED_MEMORY_SIZE="${DROBOTICFLOW_SHARED_MEMORY_SIZE:-8Gi}"
|
|
WORKER_API_BASE_URL="${WORKER_API_BASE_URL:-http://droboticflow-worker:3002}"
|
|
BOS_ENDPOINT="${BOS_ENDPOINT:-}"
|
|
K8S_IMAGE_PULL_SECRET_NAME="${K8S_IMAGE_PULL_SECRET_NAME:-droboticflow-registry}"
|
|
WORKSPACE_PVC_NAME="${WORKSPACE_PVC_NAME:-droboticflow-workspace}"
|
|
WORKSPACE_MOUNT_PATH="${WORKSPACE_MOUNT_PATH:-/workspace}"
|
|
API_NODE_PORT="${API_NODE_PORT:-30080}"
|
|
API_SERVICE_TYPE="${API_SERVICE_TYPE:-NodePort}"
|
|
API_PUBLIC_LOAD_BALANCER="${API_PUBLIC_LOAD_BALANCER:-false}"
|
|
API_REPLICAS="${API_REPLICAS:-2}"
|
|
WORKER_REPLICAS="${WORKER_REPLICAS:-2}"
|
|
DROSYNC_COORDINATOR_REPLICAS="${DROSYNC_COORDINATOR_REPLICAS:-2}"
|
|
DROBOTICFLOW_OBJECT_SYNC_ENABLED="${DROBOTICFLOW_OBJECT_SYNC_ENABLED:-false}"
|
|
DROBOTICFLOW_BUILD_COMMIT="${DROBOTICFLOW_BUILD_COMMIT:-$(git -C "$ROOT_DIR" rev-parse HEAD)}"
|
|
DROBOTICFLOW_BUILD_TIME="${DROBOTICFLOW_BUILD_TIME:-}"
|
|
OPERATOR_RUNNER_BUILD_COMMIT="${OPERATOR_RUNNER_BUILD_COMMIT:-$DROBOTICFLOW_BUILD_COMMIT}"
|
|
OPERATOR_RUNNER_BUILD_TIME="${OPERATOR_RUNNER_BUILD_TIME:-}"
|
|
droboticflow_resolve_build_metadata
|
|
WORKER_ORPHAN_CLEANUP_ENABLED="${WORKER_ORPHAN_CLEANUP_ENABLED:-false}"
|
|
WORKER_ORPHAN_MIN_AGE_SECONDS="${WORKER_ORPHAN_MIN_AGE_SECONDS:-86400}"
|
|
WORKER_ORPHAN_CHECK_INTERVAL_SECONDS="${WORKER_ORPHAN_CHECK_INTERVAL_SECONDS:-300}"
|
|
WORKER_PENDING_FAILURE_SECONDS="${WORKER_PENDING_FAILURE_SECONDS:-600}"
|
|
WORKER_MISSING_JOB_MAX_RECOVERIES="${WORKER_MISSING_JOB_MAX_RECOVERIES:-3}"
|
|
AI_API_BASE_URL="${AI_API_BASE_URL:-}"
|
|
AI_MODEL="${AI_MODEL:-}"
|
|
AI_API_KEY="${AI_API_KEY:-}"
|
|
OPERATOR_GPU_NODE_SELECTOR="${OPERATOR_GPU_NODE_SELECTOR:-}"
|
|
OPERATOR_GPU_TOLERATIONS_JSON="${OPERATOR_GPU_TOLERATIONS_JSON:-}"
|
|
OPERATOR_GPU_RUNTIME_CLASS="${OPERATOR_GPU_RUNTIME_CLASS:-}"
|
|
OPERATOR_JOB_TTL_SECONDS="${OPERATOR_JOB_TTL_SECONDS:-86400}"
|
|
CUSTOM_OPERATOR_IMAGE_REGISTRY="${CUSTOM_OPERATOR_IMAGE_REGISTRY:-}"
|
|
CUSTOM_OPERATOR_IMAGE_IMPORTER_IMAGE="${CUSTOM_OPERATOR_IMAGE_IMPORTER_IMAGE:-quay.io/skopeo/stable:v1.16.1}"
|
|
CUSTOM_OPERATOR_IMAGE_IMPORT_TIMEOUT_SECONDS="${CUSTOM_OPERATOR_IMAGE_IMPORT_TIMEOUT_SECONDS:-1800}"
|
|
CUSTOM_OPERATOR_IMAGE_ARCHIVE_MAX_MIB="${CUSTOM_OPERATOR_IMAGE_ARCHIVE_MAX_MIB:-8192}"
|
|
OPERATOR_ALLOW_SHELL_COMMANDS="${OPERATOR_ALLOW_SHELL_COMMANDS:-false}"
|
|
OPERATOR_COMMAND_ALLOWLIST_REGEX="${OPERATOR_COMMAND_ALLOWLIST_REGEX:-}"
|
|
OPERATOR_COMMAND_DENYLIST_REGEX="${OPERATOR_COMMAND_DENYLIST_REGEX:-}"
|
|
DROBOTICFLOW_REQUIRE_SESSION_AUTH="${DROBOTICFLOW_REQUIRE_SESSION_AUTH:-$API_PUBLIC_LOAD_BALANCER}"
|
|
DROBOTICFLOW_SESSION_COOKIE_SECURE="${DROBOTICFLOW_SESSION_COOKIE_SECURE:-$API_PUBLIC_LOAD_BALANCER}"
|
|
DROBOTICFLOW_TRUSTED_PROXY_CIDRS="${DROBOTICFLOW_TRUSTED_PROXY_CIDRS:-}"
|
|
# shellcheck source=scripts/lib/image-config.sh
|
|
source "$ROOT_DIR/scripts/lib/image-config.sh"
|
|
REGISTRY_HOST="${REGISTRY_HOST:-}"
|
|
REGISTRY_NAMESPACE="${REGISTRY_NAMESPACE:-}"
|
|
droboticflow_resolve_image_configuration
|
|
if [[ -z "$CUSTOM_OPERATOR_IMAGE_REGISTRY" && -n "$REGISTRY_HOST" && -n "$REGISTRY_NAMESPACE" ]]; then
|
|
CUSTOM_OPERATOR_IMAGE_REGISTRY="$REGISTRY_HOST/$REGISTRY_NAMESPACE/custom-operators"
|
|
fi
|
|
|
|
# shellcheck source=scripts/lib/deploy-validation.sh
|
|
source "$ROOT_DIR/scripts/lib/deploy-validation.sh"
|
|
validate_manifest_startup_configuration
|
|
|
|
case "$API_SERVICE_TYPE" in
|
|
NodePort)
|
|
API_NODE_PORT_BLOCK=" nodePort: $API_NODE_PORT"
|
|
;;
|
|
ClusterIP|"")
|
|
API_SERVICE_TYPE="ClusterIP"
|
|
API_NODE_PORT_BLOCK=""
|
|
;;
|
|
*)
|
|
echo "ERROR: API_SERVICE_TYPE must be NodePort or ClusterIP, got $API_SERVICE_TYPE" >&2
|
|
exit 1
|
|
;;
|
|
esac
|
|
|
|
render_block_scalar() {
|
|
local value="$1"
|
|
BLOCK_SCALAR_VALUE="$value" python3 - <<'PY'
|
|
import os
|
|
|
|
value = os.environ.get("BLOCK_SCALAR_VALUE", "").rstrip("\n")
|
|
print(value.replace("\n", "\n "), end="")
|
|
PY
|
|
}
|
|
|
|
OPERATOR_GPU_TOLERATIONS_JSON_BLOCK="$(render_block_scalar "$OPERATOR_GPU_TOLERATIONS_JSON")"
|
|
OPERATOR_COMMAND_ALLOWLIST_REGEX_BLOCK="$(render_block_scalar "$OPERATOR_COMMAND_ALLOWLIST_REGEX")"
|
|
OPERATOR_COMMAND_DENYLIST_REGEX_BLOCK="$(render_block_scalar "$OPERATOR_COMMAND_DENYLIST_REGEX")"
|
|
|
|
export \
|
|
TEMPLATE \
|
|
K8S_NAMESPACE \
|
|
MONGO_DB \
|
|
MONGO_CONNECT_RETRIES \
|
|
MONGO_CONNECT_RETRY_DELAY_MS \
|
|
WORKER_POLL_INTERVAL_MS \
|
|
DROBOTICFLOW_SHARED_MEMORY_SIZE \
|
|
WORKER_API_BASE_URL \
|
|
BOS_ENDPOINT \
|
|
K8S_IMAGE_PULL_SECRET_NAME \
|
|
WORKSPACE_PVC_NAME \
|
|
WORKSPACE_MOUNT_PATH \
|
|
API_SERVICE_TYPE \
|
|
API_NODE_PORT_BLOCK \
|
|
API_REPLICAS \
|
|
WORKER_REPLICAS \
|
|
DROSYNC_COORDINATOR_REPLICAS \
|
|
DROBOTICFLOW_OBJECT_SYNC_ENABLED \
|
|
DROBOTICFLOW_BUILD_COMMIT \
|
|
DROBOTICFLOW_BUILD_TIME \
|
|
OPERATOR_RUNNER_BUILD_COMMIT \
|
|
OPERATOR_RUNNER_BUILD_TIME \
|
|
WORKER_ORPHAN_CLEANUP_ENABLED \
|
|
WORKER_ORPHAN_MIN_AGE_SECONDS \
|
|
WORKER_ORPHAN_CHECK_INTERVAL_SECONDS \
|
|
WORKER_PENDING_FAILURE_SECONDS \
|
|
WORKER_MISSING_JOB_MAX_RECOVERIES \
|
|
AI_API_BASE_URL \
|
|
AI_MODEL \
|
|
API_IMAGE \
|
|
WORKER_IMAGE \
|
|
DROSYNC_COORDINATOR_IMAGE \
|
|
OPERATOR_RUNNER_IMAGE \
|
|
OPERATOR_GPU_NODE_SELECTOR \
|
|
OPERATOR_GPU_TOLERATIONS_JSON_BLOCK \
|
|
OPERATOR_GPU_RUNTIME_CLASS \
|
|
OPERATOR_JOB_TTL_SECONDS \
|
|
CUSTOM_OPERATOR_IMAGE_REGISTRY \
|
|
CUSTOM_OPERATOR_IMAGE_IMPORTER_IMAGE \
|
|
CUSTOM_OPERATOR_IMAGE_IMPORT_TIMEOUT_SECONDS \
|
|
CUSTOM_OPERATOR_IMAGE_ARCHIVE_MAX_MIB \
|
|
OPERATOR_ALLOW_SHELL_COMMANDS \
|
|
OPERATOR_COMMAND_ALLOWLIST_REGEX_BLOCK \
|
|
OPERATOR_COMMAND_DENYLIST_REGEX_BLOCK \
|
|
DROBOTICFLOW_REQUIRE_SESSION_AUTH \
|
|
DROBOTICFLOW_SESSION_COOKIE_SECURE \
|
|
DROBOTICFLOW_TRUSTED_PROXY_CIDRS
|
|
|
|
python3 - <<'PY'
|
|
import os
|
|
import pathlib
|
|
import re
|
|
import sys
|
|
|
|
template = pathlib.Path(os.environ["TEMPLATE"]).read_text(encoding="utf-8")
|
|
replacements = {
|
|
"__NAMESPACE__": os.environ["K8S_NAMESPACE"],
|
|
"__MONGO_DB__": os.environ["MONGO_DB"],
|
|
"__MONGO_CONNECT_RETRIES__": os.environ["MONGO_CONNECT_RETRIES"],
|
|
"__MONGO_CONNECT_RETRY_DELAY_MS__": os.environ["MONGO_CONNECT_RETRY_DELAY_MS"],
|
|
"__WORKER_POLL_INTERVAL_MS__": os.environ["WORKER_POLL_INTERVAL_MS"],
|
|
"__DROBOTICFLOW_SHARED_MEMORY_SIZE__": os.environ["DROBOTICFLOW_SHARED_MEMORY_SIZE"],
|
|
"__WORKER_API_BASE_URL__": os.environ["WORKER_API_BASE_URL"],
|
|
"__BOS_ENDPOINT__": os.environ["BOS_ENDPOINT"],
|
|
"__K8S_IMAGE_PULL_SECRET_NAME__": os.environ["K8S_IMAGE_PULL_SECRET_NAME"],
|
|
"__WORKSPACE_PVC_NAME__": os.environ["WORKSPACE_PVC_NAME"],
|
|
"__WORKSPACE_MOUNT_PATH__": os.environ["WORKSPACE_MOUNT_PATH"],
|
|
"__API_SERVICE_TYPE__": os.environ["API_SERVICE_TYPE"],
|
|
"__API_NODE_PORT_BLOCK__": os.environ["API_NODE_PORT_BLOCK"],
|
|
"__API_REPLICAS__": os.environ["API_REPLICAS"],
|
|
"__WORKER_REPLICAS__": os.environ["WORKER_REPLICAS"],
|
|
"__DROSYNC_COORDINATOR_REPLICAS__": os.environ["DROSYNC_COORDINATOR_REPLICAS"],
|
|
"__DROBOTICFLOW_OBJECT_SYNC_ENABLED__": os.environ["DROBOTICFLOW_OBJECT_SYNC_ENABLED"],
|
|
"__DROBOTICFLOW_BUILD_COMMIT__": os.environ["DROBOTICFLOW_BUILD_COMMIT"],
|
|
"__DROBOTICFLOW_BUILD_TIME__": os.environ["DROBOTICFLOW_BUILD_TIME"],
|
|
"__OPERATOR_RUNNER_BUILD_COMMIT__": os.environ["OPERATOR_RUNNER_BUILD_COMMIT"],
|
|
"__OPERATOR_RUNNER_BUILD_TIME__": os.environ["OPERATOR_RUNNER_BUILD_TIME"],
|
|
"__WORKER_ORPHAN_CLEANUP_ENABLED__": os.environ["WORKER_ORPHAN_CLEANUP_ENABLED"],
|
|
"__WORKER_ORPHAN_MIN_AGE_SECONDS__": os.environ["WORKER_ORPHAN_MIN_AGE_SECONDS"],
|
|
"__WORKER_ORPHAN_CHECK_INTERVAL_SECONDS__": os.environ["WORKER_ORPHAN_CHECK_INTERVAL_SECONDS"],
|
|
"__WORKER_PENDING_FAILURE_SECONDS__": os.environ["WORKER_PENDING_FAILURE_SECONDS"],
|
|
"__WORKER_MISSING_JOB_MAX_RECOVERIES__": os.environ["WORKER_MISSING_JOB_MAX_RECOVERIES"],
|
|
"__AI_API_BASE_URL__": os.environ["AI_API_BASE_URL"],
|
|
"__AI_MODEL__": os.environ["AI_MODEL"],
|
|
"__API_IMAGE__": os.environ["API_IMAGE"],
|
|
"__WORKER_IMAGE__": os.environ["WORKER_IMAGE"],
|
|
"__DROSYNC_COORDINATOR_IMAGE__": os.environ["DROSYNC_COORDINATOR_IMAGE"],
|
|
"__OPERATOR_RUNNER_IMAGE__": os.environ["OPERATOR_RUNNER_IMAGE"],
|
|
"__OPERATOR_GPU_NODE_SELECTOR__": os.environ["OPERATOR_GPU_NODE_SELECTOR"],
|
|
"__OPERATOR_GPU_TOLERATIONS_JSON__": os.environ["OPERATOR_GPU_TOLERATIONS_JSON_BLOCK"],
|
|
"__OPERATOR_GPU_RUNTIME_CLASS__": os.environ["OPERATOR_GPU_RUNTIME_CLASS"],
|
|
"__OPERATOR_JOB_TTL_SECONDS__": os.environ["OPERATOR_JOB_TTL_SECONDS"],
|
|
"__CUSTOM_OPERATOR_IMAGE_REGISTRY__": os.environ["CUSTOM_OPERATOR_IMAGE_REGISTRY"],
|
|
"__CUSTOM_OPERATOR_IMAGE_IMPORTER_IMAGE__": os.environ["CUSTOM_OPERATOR_IMAGE_IMPORTER_IMAGE"],
|
|
"__CUSTOM_OPERATOR_IMAGE_IMPORT_TIMEOUT_SECONDS__": os.environ["CUSTOM_OPERATOR_IMAGE_IMPORT_TIMEOUT_SECONDS"],
|
|
"__CUSTOM_OPERATOR_IMAGE_ARCHIVE_MAX_MIB__": os.environ["CUSTOM_OPERATOR_IMAGE_ARCHIVE_MAX_MIB"],
|
|
"__OPERATOR_ALLOW_SHELL_COMMANDS__": os.environ["OPERATOR_ALLOW_SHELL_COMMANDS"],
|
|
"__OPERATOR_COMMAND_ALLOWLIST_REGEX__": os.environ["OPERATOR_COMMAND_ALLOWLIST_REGEX_BLOCK"],
|
|
"__OPERATOR_COMMAND_DENYLIST_REGEX__": os.environ["OPERATOR_COMMAND_DENYLIST_REGEX_BLOCK"],
|
|
"__DROBOTICFLOW_REQUIRE_SESSION_AUTH__": os.environ["DROBOTICFLOW_REQUIRE_SESSION_AUTH"],
|
|
"__DROBOTICFLOW_SESSION_COOKIE_SECURE__": os.environ["DROBOTICFLOW_SESSION_COOKIE_SECURE"],
|
|
"__DROBOTICFLOW_TRUSTED_PROXY_CIDRS__": os.environ["DROBOTICFLOW_TRUSTED_PROXY_CIDRS"],
|
|
}
|
|
|
|
rendered = template
|
|
for placeholder, value in replacements.items():
|
|
rendered = rendered.replace(placeholder, value)
|
|
|
|
unresolved = sorted(set(re.findall(r"__[A-Z0-9_]+__", rendered)))
|
|
if unresolved:
|
|
print("ERROR: rendered manifest still contains unresolved placeholders:", file=sys.stderr)
|
|
for placeholder in unresolved:
|
|
print(placeholder, file=sys.stderr)
|
|
raise SystemExit(1)
|
|
|
|
sys.stdout.write(rendered)
|
|
if not rendered.endswith("\n"):
|
|
sys.stdout.write("\n")
|
|
PY
|