#!/usr/bin/env bash set -euo pipefail ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" DEPLOY_ENV="${DEPLOY_ENV:-}" export DEPLOY_ENV # shellcheck source=scripts/load-deploy-env.sh source "$ROOT_DIR/scripts/load-deploy-env.sh" K8S_NAMESPACE="${K8S_NAMESPACE:-droboticflow}" GPU_ACCEPTANCE_IMAGE="${GPU_ACCEPTANCE_IMAGE:-nvidia/cuda:12.4.1-base-ubuntu22.04}" GPU_ACCEPTANCE_TIMEOUT="${GPU_ACCEPTANCE_TIMEOUT:-300s}" GPU_ACCEPTANCE_NAME="${GPU_ACCEPTANCE_NAME:-droboticflow-gpu-acceptance-$(date +%s)}" GPU_RESOURCE_NAME="${GPU_RESOURCE_NAME:-nvidia.com/gpu}" GPU_RESOURCE_COUNT="${GPU_RESOURCE_COUNT:-1}" OPERATOR_GPU_NODE_SELECTOR="${OPERATOR_GPU_NODE_SELECTOR:-}" OPERATOR_GPU_TOLERATIONS_JSON="${OPERATOR_GPU_TOLERATIONS_JSON:-}" OPERATOR_GPU_RUNTIME_CLASS="${OPERATOR_GPU_RUNTIME_CLASS:-}" validate_dns_label() { local name="$1" local value="$2" if [[ ${#value} -gt 63 ]] || ! [[ "$value" =~ ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ ]]; then echo "ERROR: $name must be a lowercase DNS label" >&2 exit 1 fi } validate_resource_name() { local value="$1" if [[ -z "$value" ]] || ! [[ "$value" =~ ^([A-Za-z0-9]([-A-Za-z0-9_.]*[A-Za-z0-9])?/)?[A-Za-z0-9]([-A-Za-z0-9_.]*[A-Za-z0-9])?$ ]]; then echo "ERROR: GPU_RESOURCE_NAME must be a Kubernetes resource name" >&2 exit 1 fi } validate_image_reference() { local value="$1" if [[ -z "$value" ]] || [[ "$value" =~ [[:space:]] ]]; then echo "ERROR: GPU_ACCEPTANCE_IMAGE must be a non-empty container image reference without whitespace" >&2 exit 1 fi } validate_positive_duration() { local value="$1" GPU_ACCEPTANCE_TIMEOUT="$value" python3 - <<'PY' import os import re import sys value = os.environ.get("GPU_ACCEPTANCE_TIMEOUT", "").strip() if not re.fullmatch(r"(?:[1-9][0-9]*(?:ns|us|ms|s|m|h))+", value): print("ERROR: GPU_ACCEPTANCE_TIMEOUT must be a positive Go-style duration such as 300s, 5m, or 1h30m", file=sys.stderr) raise SystemExit(1) PY } yaml_string() { python3 - "$1" <<'PY' import json import sys print(json.dumps(sys.argv[1])) PY } if [[ -z "$GPU_ACCEPTANCE_IMAGE" ]]; then echo "ERROR: GPU_ACCEPTANCE_IMAGE is required" >&2 exit 1 fi validate_dns_label "K8S_NAMESPACE" "$K8S_NAMESPACE" validate_dns_label "GPU_ACCEPTANCE_NAME" "$GPU_ACCEPTANCE_NAME" validate_resource_name "$GPU_RESOURCE_NAME" validate_image_reference "$GPU_ACCEPTANCE_IMAGE" validate_positive_duration "$GPU_ACCEPTANCE_TIMEOUT" if ! [[ "$GPU_RESOURCE_COUNT" =~ ^[1-9][0-9]*$ ]]; then echo "ERROR: GPU_RESOURCE_COUNT must be a positive integer" >&2 exit 1 fi selector_yaml="$( OPERATOR_GPU_NODE_SELECTOR="$OPERATOR_GPU_NODE_SELECTOR" python3 - <<'PY' import json import os import re import sys raw = os.environ.get("OPERATOR_GPU_NODE_SELECTOR", "").strip() if not raw: raise SystemExit(0) dns_label = r"[A-Za-z0-9]([-A-Za-z0-9_.]*[A-Za-z0-9])?" label_key = re.compile(rf"^({dns_label}/)?{dns_label}$") label_value = re.compile(rf"^({dns_label})?$") items: dict[str, str] = {} for item in raw.split(","): item = item.strip() if not item: continue if "=" not in item: print(f"ERROR: invalid OPERATOR_GPU_NODE_SELECTOR item {item!r}: expected key=value", file=sys.stderr) raise SystemExit(1) key, value = (part.strip() for part in item.split("=", 1)) if not key or not value: print(f"ERROR: invalid OPERATOR_GPU_NODE_SELECTOR item {item!r}: key and value are required", file=sys.stderr) raise SystemExit(1) if not label_key.fullmatch(key): print(f"ERROR: invalid OPERATOR_GPU_NODE_SELECTOR key {key!r}", file=sys.stderr) raise SystemExit(1) if not label_value.fullmatch(value): print(f"ERROR: invalid OPERATOR_GPU_NODE_SELECTOR value for {key!r}", file=sys.stderr) raise SystemExit(1) if key in items and items[key] != value: print(f"ERROR: duplicate OPERATOR_GPU_NODE_SELECTOR key {key!r} with conflicting values", file=sys.stderr) raise SystemExit(1) items[key] = value if not items: raise SystemExit(0) print(" nodeSelector:") for key in sorted(items): print(f" {json.dumps(key)}: {json.dumps(items[key])}") PY )" tolerations_yaml="$( OPERATOR_GPU_TOLERATIONS_JSON="$OPERATOR_GPU_TOLERATIONS_JSON" python3 - <<'PY' import json import os import sys raw = os.environ.get("OPERATOR_GPU_TOLERATIONS_JSON", "").strip() if not raw: raise SystemExit(0) try: items = json.loads(raw) except json.JSONDecodeError as exc: print(f"ERROR: invalid OPERATOR_GPU_TOLERATIONS_JSON: {exc}", file=sys.stderr) raise SystemExit(1) from exc if not isinstance(items, list): print("ERROR: OPERATOR_GPU_TOLERATIONS_JSON must be a JSON array", file=sys.stderr) raise SystemExit(1) allowed_keys = {"key", "operator", "value", "effect", "tolerationSeconds"} ordered_keys = ["key", "operator", "value", "effect", "tolerationSeconds"] allowed_operators = {"Equal", "Exists"} allowed_effects = {"NoSchedule", "PreferNoSchedule", "NoExecute"} normalized: list[dict[str, object]] = [] for index, item in enumerate(items): if not isinstance(item, dict): print(f"ERROR: OPERATOR_GPU_TOLERATIONS_JSON[{index}] must be an object", file=sys.stderr) raise SystemExit(1) unknown = sorted(set(item) - allowed_keys) if unknown: print(f"ERROR: OPERATOR_GPU_TOLERATIONS_JSON[{index}] has unsupported keys: {', '.join(unknown)}", file=sys.stderr) raise SystemExit(1) output: dict[str, object] = {} for key in ("key", "operator", "value", "effect"): if key not in item or item[key] is None: continue if not isinstance(item[key], str): print(f"ERROR: OPERATOR_GPU_TOLERATIONS_JSON[{index}].{key} must be a string", file=sys.stderr) raise SystemExit(1) value = item[key].strip() if key == "operator" and value and value not in allowed_operators: print(f"ERROR: OPERATOR_GPU_TOLERATIONS_JSON[{index}].operator must be Equal or Exists", file=sys.stderr) raise SystemExit(1) if key == "effect" and value and value not in allowed_effects: print(f"ERROR: OPERATOR_GPU_TOLERATIONS_JSON[{index}].effect is not a supported Kubernetes toleration effect", file=sys.stderr) raise SystemExit(1) if value: output[key] = value if "tolerationSeconds" in item and item["tolerationSeconds"] is not None: seconds = item["tolerationSeconds"] if not isinstance(seconds, int) or seconds < 0: print(f"ERROR: OPERATOR_GPU_TOLERATIONS_JSON[{index}].tolerationSeconds must be a non-negative integer", file=sys.stderr) raise SystemExit(1) output["tolerationSeconds"] = seconds normalized.append(output) if not normalized: raise SystemExit(0) print(" tolerations:") for item in normalized: print(" -") for key in ordered_keys: if key in item: print(f" {key}: {json.dumps(item[key])}") PY )" runtime_class_yaml="$( OPERATOR_GPU_RUNTIME_CLASS="$OPERATOR_GPU_RUNTIME_CLASS" python3 - <<'PY' import json import os import re import sys value = os.environ.get("OPERATOR_GPU_RUNTIME_CLASS", "").strip() if not value: raise SystemExit(0) if len(value) > 253 or not re.fullmatch(r"[a-z0-9]([-a-z0-9.]*[a-z0-9])?", value): print("ERROR: OPERATOR_GPU_RUNTIME_CLASS must be a lowercase DNS subdomain", file=sys.stderr) raise SystemExit(1) print(f" runtimeClassName: {json.dumps(value)}") PY )" if ! kubectl get namespace "$K8S_NAMESPACE" >/dev/null 2>&1; then echo "ERROR: namespace $K8S_NAMESPACE does not exist" >&2 exit 1 fi nodes_json="$(kubectl get nodes -o json)" if ! NODES_JSON="$nodes_json" OPERATOR_GPU_NODE_SELECTOR="$OPERATOR_GPU_NODE_SELECTOR" python3 - "$GPU_RESOURCE_NAME" "$GPU_RESOURCE_COUNT" <<'PY' import json import os import sys resource = sys.argv[1] requested = int(sys.argv[2]) payload = json.loads(os.environ["NODES_JSON"]) selector: dict[str, str] = {} raw_selector = os.environ.get("OPERATOR_GPU_NODE_SELECTOR", "").strip() if raw_selector: for item in raw_selector.split(","): item = item.strip() if not item: continue key, value = (part.strip() for part in item.split("=", 1)) selector[key] = value def selector_matches(labels: dict[str, str]) -> bool: for key, value in selector.items(): if labels.get(key) != value: return False return True checked: list[str] = [] for node in payload.get("items", []): name = node["metadata"]["name"] labels = node.get("metadata", {}).get("labels", {}) labels = labels if isinstance(labels, dict) else {} selector_matched = selector_matches(labels) unschedulable = bool(node.get("spec", {}).get("unschedulable", False)) allocatable = node.get("status", {}).get("allocatable", {}) value = str(allocatable.get(resource, "0")) try: count = int(value) except ValueError: count = 0 checked.append(f"{name} {resource}={value} selectorMatched={str(selector_matched).lower()} unschedulable={str(unschedulable).lower()}") if count >= requested and selector_matched and not unschedulable: print(f"{name} {resource}={value} selectorMatched={str(selector_matched).lower()} unschedulable={str(unschedulable).lower()}") raise SystemExit(0) selector_summary = ",".join(f"{key}={selector[key]}" for key in sorted(selector)) if selector else "" print(f"GPU readiness unavailable: no schedulable node reports allocatable {resource}>={requested} with selector {selector_summary}", file=sys.stderr) if checked: print("Checked nodes:", file=sys.stderr) for item in checked: print(f" - {item}", file=sys.stderr) else: print("Checked nodes: none", file=sys.stderr) raise SystemExit(1) PY then echo "ERROR: no Kubernetes node reports allocatable $GPU_RESOURCE_NAME >= $GPU_RESOURCE_COUNT" >&2 exit 1 fi manifest="$(mktemp)" cleanup() { kubectl -n "$K8S_NAMESPACE" delete job "$GPU_ACCEPTANCE_NAME" --ignore-not-found >/dev/null 2>&1 || true rm -f "$manifest" } trap cleanup EXIT K8S_NAMESPACE_YAML="$(yaml_string "$K8S_NAMESPACE")" GPU_ACCEPTANCE_NAME_YAML="$(yaml_string "$GPU_ACCEPTANCE_NAME")" GPU_ACCEPTANCE_IMAGE_YAML="$(yaml_string "$GPU_ACCEPTANCE_IMAGE")" GPU_RESOURCE_NAME_YAML="$(yaml_string "$GPU_RESOURCE_NAME")" GPU_RESOURCE_COUNT_YAML="$(yaml_string "$GPU_RESOURCE_COUNT")" cat >"$manifest" <&2 || true kubectl -n "$K8S_NAMESPACE" logs "job/$GPU_ACCEPTANCE_NAME" >&2 || true exit 1 fi kubectl -n "$K8S_NAMESPACE" logs "job/$GPU_ACCEPTANCE_NAME"