#!/usr/bin/env bash set -euo pipefail ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" DEPLOY_ENV="${DEPLOY_ENV:-}" export DEPLOY_ENV # shellcheck source=scripts/load-deploy-env.sh source "$ROOT_DIR/scripts/load-deploy-env.sh" K8S_NAMESPACE="${K8S_NAMESPACE:-droboticflow}" POC_GPU_CAPACITY_NAME="${POC_GPU_CAPACITY_NAME:-droboticflow-gpu-capacity-$(date +%s)}" POC_GPU_CAPACITY_TIMEOUT_SECONDS="${POC_GPU_CAPACITY_TIMEOUT_SECONDS:-120}" POC_GPU_CAPACITY_HTTP_TIMEOUT_SECONDS="${POC_GPU_CAPACITY_HTTP_TIMEOUT_SECONDS:-30}" POC_GPU_CAPACITY_EXPECT_AVAILABLE="${POC_GPU_CAPACITY_EXPECT_AVAILABLE:-true}" POC_GPU_CAPACITY_WORKER_URL="${POC_GPU_CAPACITY_WORKER_URL:-http://droboticflow-worker:3002/worker-api/capacity}" validate_dns_label() { local name="$1" local value="$2" if [[ ${#value} -gt 63 ]] || ! [[ "$value" =~ ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ ]]; then echo "ERROR: $name must be a lowercase DNS label" >&2 exit 1 fi } validate_image_reference() { local name="$1" local value="$2" if [[ -z "$value" ]] || [[ "$value" =~ [[:space:]] ]]; then echo "ERROR: $name must be a non-empty container image reference without whitespace" >&2 exit 1 fi } validate_positive_integer() { local name="$1" local value="$2" if ! [[ "$value" =~ ^[1-9][0-9]*$ ]]; then echo "ERROR: $name must be a positive integer" >&2 exit 1 fi } normalize_bool() { local name="$1" local value="$2" local normalized normalized="$(printf '%s' "$value" | tr '[:upper:]' '[:lower:]')" case "$normalized" in true|1|yes|y|on) printf 'true\n' ;; false|0|no|n|off) printf 'false\n' ;; *) echo "ERROR: $name must be true or false" >&2 exit 1 ;; esac } yaml_string() { python3 - "$1" <<'PY' import json import sys print(json.dumps(sys.argv[1])) PY } config_value() { local key="$1" kubectl -n "$K8S_NAMESPACE" get configmap droboticflow-config -o "jsonpath={.data.$key}" 2>/dev/null || true } validate_dns_label "K8S_NAMESPACE" "$K8S_NAMESPACE" validate_dns_label "POC_GPU_CAPACITY_NAME" "$POC_GPU_CAPACITY_NAME" validate_positive_integer "POC_GPU_CAPACITY_TIMEOUT_SECONDS" "$POC_GPU_CAPACITY_TIMEOUT_SECONDS" validate_positive_integer "POC_GPU_CAPACITY_HTTP_TIMEOUT_SECONDS" "$POC_GPU_CAPACITY_HTTP_TIMEOUT_SECONDS" POC_GPU_CAPACITY_EXPECT_AVAILABLE="$(normalize_bool "POC_GPU_CAPACITY_EXPECT_AVAILABLE" "$POC_GPU_CAPACITY_EXPECT_AVAILABLE")" if [[ -z "$POC_GPU_CAPACITY_WORKER_URL" || "$POC_GPU_CAPACITY_WORKER_URL" =~ [[:space:]] ]]; then echo "ERROR: POC_GPU_CAPACITY_WORKER_URL must be a non-empty URL without whitespace" >&2 exit 1 fi if ! kubectl get namespace "$K8S_NAMESPACE" >/dev/null 2>&1; then echo "ERROR: namespace $K8S_NAMESPACE does not exist" >&2 exit 1 fi OPERATOR_RUNNER_IMAGE="${OPERATOR_RUNNER_IMAGE:-$(config_value OPERATOR_RUNNER_IMAGE)}" POC_GPU_CAPACITY_IMAGE="${POC_GPU_CAPACITY_IMAGE:-$OPERATOR_RUNNER_IMAGE}" K8S_IMAGE_PULL_SECRET_NAME="${K8S_IMAGE_PULL_SECRET_NAME:-$(config_value K8S_IMAGE_PULL_SECRET_NAME)}" validate_image_reference "POC_GPU_CAPACITY_IMAGE" "$POC_GPU_CAPACITY_IMAGE" if [[ -n "$K8S_IMAGE_PULL_SECRET_NAME" ]]; then validate_dns_label "K8S_IMAGE_PULL_SECRET_NAME" "$K8S_IMAGE_PULL_SECRET_NAME" fi manifest="$(mktemp)" cleanup() { kubectl -n "$K8S_NAMESPACE" delete pod "$POC_GPU_CAPACITY_NAME" --ignore-not-found >/dev/null 2>&1 || true rm -f "$manifest" } trap cleanup EXIT K8S_NAMESPACE_YAML="$(yaml_string "$K8S_NAMESPACE")" POC_GPU_CAPACITY_NAME_YAML="$(yaml_string "$POC_GPU_CAPACITY_NAME")" POC_GPU_CAPACITY_IMAGE_YAML="$(yaml_string "$POC_GPU_CAPACITY_IMAGE")" POC_GPU_CAPACITY_WORKER_URL_YAML="$(yaml_string "$POC_GPU_CAPACITY_WORKER_URL")" POC_GPU_CAPACITY_HTTP_TIMEOUT_SECONDS_YAML="$(yaml_string "$POC_GPU_CAPACITY_HTTP_TIMEOUT_SECONDS")" image_pull_secrets_yaml="" if [[ -n "$K8S_IMAGE_PULL_SECRET_NAME" ]]; then K8S_IMAGE_PULL_SECRET_NAME_YAML="$(yaml_string "$K8S_IMAGE_PULL_SECRET_NAME")" image_pull_secrets_yaml=" imagePullSecrets: - name: $K8S_IMAGE_PULL_SECRET_NAME_YAML" fi cat >"$manifest" </dev/null || true)" case "$phase" in Succeeded|Failed) break ;; esac if (( SECONDS >= deadline )); then echo "ERROR: timed out waiting for Pod $POC_GPU_CAPACITY_NAME to finish; last phase=${phase:-unknown}" >&2 kubectl -n "$K8S_NAMESPACE" describe pod "$POC_GPU_CAPACITY_NAME" >&2 || true kubectl -n "$K8S_NAMESPACE" logs "$POC_GPU_CAPACITY_NAME" >&2 || true exit 1 fi sleep 2 done logs="$(kubectl -n "$K8S_NAMESPACE" logs "$POC_GPU_CAPACITY_NAME" 2>&1 || true)" if [[ "$phase" != "Succeeded" ]]; then echo "ERROR: Pod $POC_GPU_CAPACITY_NAME finished with phase=$phase" >&2 printf '%s\n' "$logs" >&2 exit 1 fi CAPACITY_JSON="$logs" EXPECT_AVAILABLE="$POC_GPU_CAPACITY_EXPECT_AVAILABLE" python3 - <<'PY' import json import os import sys raw = os.environ.get("CAPACITY_JSON", "") payload = None last_error: Exception | None = None for line in reversed(raw.splitlines()): candidate = line.strip() if not candidate.startswith("{"): continue try: payload = json.loads(candidate) break except json.JSONDecodeError as exc: last_error = exc if payload is None: detail = f": {last_error}" if last_error else "" print(f"ERROR: worker capacity response did not contain a JSON object{detail}", file=sys.stderr) raise SystemExit(1) gpu = payload.get("gpu") if not isinstance(gpu, dict): print("ERROR: worker capacity response is missing gpu object", file=sys.stderr) raise SystemExit(1) resource_name = str(gpu.get("resourceName") or "") available = bool(gpu.get("available")) nodes = gpu.get("nodes") if not isinstance(nodes, list): print("ERROR: gpu.nodes must be an array", file=sys.stderr) raise SystemExit(1) def int_field(name: str) -> int: value = gpu.get(name, 0) try: return int(value) except (TypeError, ValueError): print(f"ERROR: gpu.{name} must be an integer-compatible value", file=sys.stderr) raise SystemExit(1) schedulable = int_field("schedulableAllocatable") max_allocatable = int_field("maxAllocatable") expected_available = os.environ.get("EXPECT_AVAILABLE") == "true" if not resource_name: print("ERROR: gpu.resourceName is required", file=sys.stderr) raise SystemExit(1) if not nodes: print("ERROR: gpu.nodes is empty; capacity endpoint did not report checked nodes", file=sys.stderr) raise SystemExit(1) if expected_available: if not available or schedulable <= 0 or max_allocatable <= 0: print( "ERROR: expected POC GPU capacity to be available, " f"got available={available} schedulableAllocatable={schedulable} maxAllocatable={max_allocatable}", file=sys.stderr, ) raise SystemExit(1) else: if available or schedulable != 0 or max_allocatable != 0: print( "ERROR: expected CPU-only POC GPU capacity, " f"got available={available} schedulableAllocatable={schedulable} maxAllocatable={max_allocatable}", file=sys.stderr, ) raise SystemExit(1) print( "gpu.available=" + str(available).lower() + f" resource={resource_name} schedulableAllocatable={schedulable} maxAllocatable={max_allocatable} nodes={len(nodes)}" ) for node in nodes: if not isinstance(node, dict): continue name = node.get("name") or "" allocatable = node.get("allocatable", 0) selector_matched = node.get("selectorMatched", False) print(f"node {name} {resource_name}={allocatable} selectorMatched={str(bool(selector_matched)).lower()}") PY