droboticflow/scripts/poc-gpu-capacity-acceptance.sh

271 lines
8.7 KiB
Bash
Executable File

#!/usr/bin/env bash
set -euo pipefail
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
DEPLOY_ENV="${DEPLOY_ENV:-}"
export DEPLOY_ENV
# shellcheck source=scripts/load-deploy-env.sh
source "$ROOT_DIR/scripts/load-deploy-env.sh"
K8S_NAMESPACE="${K8S_NAMESPACE:-droboticflow}"
POC_GPU_CAPACITY_NAME="${POC_GPU_CAPACITY_NAME:-droboticflow-gpu-capacity-$(date +%s)}"
POC_GPU_CAPACITY_TIMEOUT_SECONDS="${POC_GPU_CAPACITY_TIMEOUT_SECONDS:-120}"
POC_GPU_CAPACITY_HTTP_TIMEOUT_SECONDS="${POC_GPU_CAPACITY_HTTP_TIMEOUT_SECONDS:-30}"
POC_GPU_CAPACITY_EXPECT_AVAILABLE="${POC_GPU_CAPACITY_EXPECT_AVAILABLE:-true}"
POC_GPU_CAPACITY_WORKER_URL="${POC_GPU_CAPACITY_WORKER_URL:-http://droboticflow-worker:3002/worker-api/capacity}"
validate_dns_label() {
local name="$1"
local value="$2"
if [[ ${#value} -gt 63 ]] || ! [[ "$value" =~ ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ ]]; then
echo "ERROR: $name must be a lowercase DNS label" >&2
exit 1
fi
}
validate_image_reference() {
local name="$1"
local value="$2"
if [[ -z "$value" ]] || [[ "$value" =~ [[:space:]] ]]; then
echo "ERROR: $name must be a non-empty container image reference without whitespace" >&2
exit 1
fi
}
validate_positive_integer() {
local name="$1"
local value="$2"
if ! [[ "$value" =~ ^[1-9][0-9]*$ ]]; then
echo "ERROR: $name must be a positive integer" >&2
exit 1
fi
}
normalize_bool() {
local name="$1"
local value="$2"
local normalized
normalized="$(printf '%s' "$value" | tr '[:upper:]' '[:lower:]')"
case "$normalized" in
true|1|yes|y|on)
printf 'true\n'
;;
false|0|no|n|off)
printf 'false\n'
;;
*)
echo "ERROR: $name must be true or false" >&2
exit 1
;;
esac
}
yaml_string() {
python3 - "$1" <<'PY'
import json
import sys
print(json.dumps(sys.argv[1]))
PY
}
config_value() {
local key="$1"
kubectl -n "$K8S_NAMESPACE" get configmap droboticflow-config -o "jsonpath={.data.$key}" 2>/dev/null || true
}
validate_dns_label "K8S_NAMESPACE" "$K8S_NAMESPACE"
validate_dns_label "POC_GPU_CAPACITY_NAME" "$POC_GPU_CAPACITY_NAME"
validate_positive_integer "POC_GPU_CAPACITY_TIMEOUT_SECONDS" "$POC_GPU_CAPACITY_TIMEOUT_SECONDS"
validate_positive_integer "POC_GPU_CAPACITY_HTTP_TIMEOUT_SECONDS" "$POC_GPU_CAPACITY_HTTP_TIMEOUT_SECONDS"
POC_GPU_CAPACITY_EXPECT_AVAILABLE="$(normalize_bool "POC_GPU_CAPACITY_EXPECT_AVAILABLE" "$POC_GPU_CAPACITY_EXPECT_AVAILABLE")"
if [[ -z "$POC_GPU_CAPACITY_WORKER_URL" || "$POC_GPU_CAPACITY_WORKER_URL" =~ [[:space:]] ]]; then
echo "ERROR: POC_GPU_CAPACITY_WORKER_URL must be a non-empty URL without whitespace" >&2
exit 1
fi
if ! kubectl get namespace "$K8S_NAMESPACE" >/dev/null 2>&1; then
echo "ERROR: namespace $K8S_NAMESPACE does not exist" >&2
exit 1
fi
OPERATOR_RUNNER_IMAGE="${OPERATOR_RUNNER_IMAGE:-$(config_value OPERATOR_RUNNER_IMAGE)}"
POC_GPU_CAPACITY_IMAGE="${POC_GPU_CAPACITY_IMAGE:-$OPERATOR_RUNNER_IMAGE}"
K8S_IMAGE_PULL_SECRET_NAME="${K8S_IMAGE_PULL_SECRET_NAME:-$(config_value K8S_IMAGE_PULL_SECRET_NAME)}"
validate_image_reference "POC_GPU_CAPACITY_IMAGE" "$POC_GPU_CAPACITY_IMAGE"
if [[ -n "$K8S_IMAGE_PULL_SECRET_NAME" ]]; then
validate_dns_label "K8S_IMAGE_PULL_SECRET_NAME" "$K8S_IMAGE_PULL_SECRET_NAME"
fi
manifest="$(mktemp)"
cleanup() {
kubectl -n "$K8S_NAMESPACE" delete pod "$POC_GPU_CAPACITY_NAME" --ignore-not-found >/dev/null 2>&1 || true
rm -f "$manifest"
}
trap cleanup EXIT
K8S_NAMESPACE_YAML="$(yaml_string "$K8S_NAMESPACE")"
POC_GPU_CAPACITY_NAME_YAML="$(yaml_string "$POC_GPU_CAPACITY_NAME")"
POC_GPU_CAPACITY_IMAGE_YAML="$(yaml_string "$POC_GPU_CAPACITY_IMAGE")"
POC_GPU_CAPACITY_WORKER_URL_YAML="$(yaml_string "$POC_GPU_CAPACITY_WORKER_URL")"
POC_GPU_CAPACITY_HTTP_TIMEOUT_SECONDS_YAML="$(yaml_string "$POC_GPU_CAPACITY_HTTP_TIMEOUT_SECONDS")"
image_pull_secrets_yaml=""
if [[ -n "$K8S_IMAGE_PULL_SECRET_NAME" ]]; then
K8S_IMAGE_PULL_SECRET_NAME_YAML="$(yaml_string "$K8S_IMAGE_PULL_SECRET_NAME")"
image_pull_secrets_yaml=" imagePullSecrets:
- name: $K8S_IMAGE_PULL_SECRET_NAME_YAML"
fi
cat >"$manifest" <<YAML
apiVersion: v1
kind: Pod
metadata:
name: $POC_GPU_CAPACITY_NAME_YAML
namespace: $K8S_NAMESPACE_YAML
labels:
app: droboticflow
component: poc-gpu-capacity-acceptance
spec:
restartPolicy: Never
$image_pull_secrets_yaml
containers:
- name: capacity
image: $POC_GPU_CAPACITY_IMAGE_YAML
imagePullPolicy: IfNotPresent
command:
- python
- -c
- |
import json
import os
import urllib.request
url = os.environ["WORKER_CAPACITY_URL"]
timeout = int(os.environ.get("WORKER_CAPACITY_TIMEOUT_SECONDS", "30"))
with urllib.request.urlopen(url, timeout=timeout) as response:
body = response.read().decode("utf-8")
json.loads(body)
print(body)
env:
- name: WORKER_CAPACITY_URL
value: $POC_GPU_CAPACITY_WORKER_URL_YAML
- name: WORKER_CAPACITY_TIMEOUT_SECONDS
value: $POC_GPU_CAPACITY_HTTP_TIMEOUT_SECONDS_YAML
YAML
kubectl apply -f "$manifest"
deadline=$((SECONDS + POC_GPU_CAPACITY_TIMEOUT_SECONDS))
phase=""
while true; do
phase="$(kubectl -n "$K8S_NAMESPACE" get pod "$POC_GPU_CAPACITY_NAME" -o "jsonpath={.status.phase}" 2>/dev/null || true)"
case "$phase" in
Succeeded|Failed)
break
;;
esac
if (( SECONDS >= deadline )); then
echo "ERROR: timed out waiting for Pod $POC_GPU_CAPACITY_NAME to finish; last phase=${phase:-unknown}" >&2
kubectl -n "$K8S_NAMESPACE" describe pod "$POC_GPU_CAPACITY_NAME" >&2 || true
kubectl -n "$K8S_NAMESPACE" logs "$POC_GPU_CAPACITY_NAME" >&2 || true
exit 1
fi
sleep 2
done
logs="$(kubectl -n "$K8S_NAMESPACE" logs "$POC_GPU_CAPACITY_NAME" 2>&1 || true)"
if [[ "$phase" != "Succeeded" ]]; then
echo "ERROR: Pod $POC_GPU_CAPACITY_NAME finished with phase=$phase" >&2
printf '%s\n' "$logs" >&2
exit 1
fi
CAPACITY_JSON="$logs" EXPECT_AVAILABLE="$POC_GPU_CAPACITY_EXPECT_AVAILABLE" python3 - <<'PY'
import json
import os
import sys
raw = os.environ.get("CAPACITY_JSON", "")
payload = None
last_error: Exception | None = None
for line in reversed(raw.splitlines()):
candidate = line.strip()
if not candidate.startswith("{"):
continue
try:
payload = json.loads(candidate)
break
except json.JSONDecodeError as exc:
last_error = exc
if payload is None:
detail = f": {last_error}" if last_error else ""
print(f"ERROR: worker capacity response did not contain a JSON object{detail}", file=sys.stderr)
raise SystemExit(1)
gpu = payload.get("gpu")
if not isinstance(gpu, dict):
print("ERROR: worker capacity response is missing gpu object", file=sys.stderr)
raise SystemExit(1)
resource_name = str(gpu.get("resourceName") or "")
available = bool(gpu.get("available"))
nodes = gpu.get("nodes")
if not isinstance(nodes, list):
print("ERROR: gpu.nodes must be an array", file=sys.stderr)
raise SystemExit(1)
def int_field(name: str) -> int:
value = gpu.get(name, 0)
try:
return int(value)
except (TypeError, ValueError):
print(f"ERROR: gpu.{name} must be an integer-compatible value", file=sys.stderr)
raise SystemExit(1)
schedulable = int_field("schedulableAllocatable")
max_allocatable = int_field("maxAllocatable")
expected_available = os.environ.get("EXPECT_AVAILABLE") == "true"
if not resource_name:
print("ERROR: gpu.resourceName is required", file=sys.stderr)
raise SystemExit(1)
if not nodes:
print("ERROR: gpu.nodes is empty; capacity endpoint did not report checked nodes", file=sys.stderr)
raise SystemExit(1)
if expected_available:
if not available or schedulable <= 0 or max_allocatable <= 0:
print(
"ERROR: expected POC GPU capacity to be available, "
f"got available={available} schedulableAllocatable={schedulable} maxAllocatable={max_allocatable}",
file=sys.stderr,
)
raise SystemExit(1)
else:
if available or schedulable != 0 or max_allocatable != 0:
print(
"ERROR: expected CPU-only POC GPU capacity, "
f"got available={available} schedulableAllocatable={schedulable} maxAllocatable={max_allocatable}",
file=sys.stderr,
)
raise SystemExit(1)
print(
"gpu.available="
+ str(available).lower()
+ f" resource={resource_name} schedulableAllocatable={schedulable} maxAllocatable={max_allocatable} nodes={len(nodes)}"
)
for node in nodes:
if not isinstance(node, dict):
continue
name = node.get("name") or "<unknown>"
allocatable = node.get("allocatable", 0)
selector_matched = node.get("selectorMatched", False)
print(f"node {name} {resource_name}={allocatable} selectorMatched={str(bool(selector_matched)).lower()}")
PY