forked from longtao.wu/droboticflow
271 lines
8.7 KiB
Bash
Executable File
271 lines
8.7 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
|
DEPLOY_ENV="${DEPLOY_ENV:-}"
|
|
export DEPLOY_ENV
|
|
# shellcheck source=scripts/load-deploy-env.sh
|
|
source "$ROOT_DIR/scripts/load-deploy-env.sh"
|
|
|
|
K8S_NAMESPACE="${K8S_NAMESPACE:-droboticflow}"
|
|
POC_GPU_CAPACITY_NAME="${POC_GPU_CAPACITY_NAME:-droboticflow-gpu-capacity-$(date +%s)}"
|
|
POC_GPU_CAPACITY_TIMEOUT_SECONDS="${POC_GPU_CAPACITY_TIMEOUT_SECONDS:-120}"
|
|
POC_GPU_CAPACITY_HTTP_TIMEOUT_SECONDS="${POC_GPU_CAPACITY_HTTP_TIMEOUT_SECONDS:-30}"
|
|
POC_GPU_CAPACITY_EXPECT_AVAILABLE="${POC_GPU_CAPACITY_EXPECT_AVAILABLE:-true}"
|
|
POC_GPU_CAPACITY_WORKER_URL="${POC_GPU_CAPACITY_WORKER_URL:-http://droboticflow-worker:3002/worker-api/capacity}"
|
|
|
|
validate_dns_label() {
|
|
local name="$1"
|
|
local value="$2"
|
|
if [[ ${#value} -gt 63 ]] || ! [[ "$value" =~ ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ ]]; then
|
|
echo "ERROR: $name must be a lowercase DNS label" >&2
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
validate_image_reference() {
|
|
local name="$1"
|
|
local value="$2"
|
|
if [[ -z "$value" ]] || [[ "$value" =~ [[:space:]] ]]; then
|
|
echo "ERROR: $name must be a non-empty container image reference without whitespace" >&2
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
validate_positive_integer() {
|
|
local name="$1"
|
|
local value="$2"
|
|
if ! [[ "$value" =~ ^[1-9][0-9]*$ ]]; then
|
|
echo "ERROR: $name must be a positive integer" >&2
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
normalize_bool() {
|
|
local name="$1"
|
|
local value="$2"
|
|
local normalized
|
|
normalized="$(printf '%s' "$value" | tr '[:upper:]' '[:lower:]')"
|
|
case "$normalized" in
|
|
true|1|yes|y|on)
|
|
printf 'true\n'
|
|
;;
|
|
false|0|no|n|off)
|
|
printf 'false\n'
|
|
;;
|
|
*)
|
|
echo "ERROR: $name must be true or false" >&2
|
|
exit 1
|
|
;;
|
|
esac
|
|
}
|
|
|
|
yaml_string() {
|
|
python3 - "$1" <<'PY'
|
|
import json
|
|
import sys
|
|
|
|
print(json.dumps(sys.argv[1]))
|
|
PY
|
|
}
|
|
|
|
config_value() {
|
|
local key="$1"
|
|
kubectl -n "$K8S_NAMESPACE" get configmap droboticflow-config -o "jsonpath={.data.$key}" 2>/dev/null || true
|
|
}
|
|
|
|
validate_dns_label "K8S_NAMESPACE" "$K8S_NAMESPACE"
|
|
validate_dns_label "POC_GPU_CAPACITY_NAME" "$POC_GPU_CAPACITY_NAME"
|
|
validate_positive_integer "POC_GPU_CAPACITY_TIMEOUT_SECONDS" "$POC_GPU_CAPACITY_TIMEOUT_SECONDS"
|
|
validate_positive_integer "POC_GPU_CAPACITY_HTTP_TIMEOUT_SECONDS" "$POC_GPU_CAPACITY_HTTP_TIMEOUT_SECONDS"
|
|
POC_GPU_CAPACITY_EXPECT_AVAILABLE="$(normalize_bool "POC_GPU_CAPACITY_EXPECT_AVAILABLE" "$POC_GPU_CAPACITY_EXPECT_AVAILABLE")"
|
|
|
|
if [[ -z "$POC_GPU_CAPACITY_WORKER_URL" || "$POC_GPU_CAPACITY_WORKER_URL" =~ [[:space:]] ]]; then
|
|
echo "ERROR: POC_GPU_CAPACITY_WORKER_URL must be a non-empty URL without whitespace" >&2
|
|
exit 1
|
|
fi
|
|
|
|
if ! kubectl get namespace "$K8S_NAMESPACE" >/dev/null 2>&1; then
|
|
echo "ERROR: namespace $K8S_NAMESPACE does not exist" >&2
|
|
exit 1
|
|
fi
|
|
|
|
OPERATOR_RUNNER_IMAGE="${OPERATOR_RUNNER_IMAGE:-$(config_value OPERATOR_RUNNER_IMAGE)}"
|
|
POC_GPU_CAPACITY_IMAGE="${POC_GPU_CAPACITY_IMAGE:-$OPERATOR_RUNNER_IMAGE}"
|
|
K8S_IMAGE_PULL_SECRET_NAME="${K8S_IMAGE_PULL_SECRET_NAME:-$(config_value K8S_IMAGE_PULL_SECRET_NAME)}"
|
|
|
|
validate_image_reference "POC_GPU_CAPACITY_IMAGE" "$POC_GPU_CAPACITY_IMAGE"
|
|
if [[ -n "$K8S_IMAGE_PULL_SECRET_NAME" ]]; then
|
|
validate_dns_label "K8S_IMAGE_PULL_SECRET_NAME" "$K8S_IMAGE_PULL_SECRET_NAME"
|
|
fi
|
|
|
|
manifest="$(mktemp)"
|
|
cleanup() {
|
|
kubectl -n "$K8S_NAMESPACE" delete pod "$POC_GPU_CAPACITY_NAME" --ignore-not-found >/dev/null 2>&1 || true
|
|
rm -f "$manifest"
|
|
}
|
|
trap cleanup EXIT
|
|
|
|
K8S_NAMESPACE_YAML="$(yaml_string "$K8S_NAMESPACE")"
|
|
POC_GPU_CAPACITY_NAME_YAML="$(yaml_string "$POC_GPU_CAPACITY_NAME")"
|
|
POC_GPU_CAPACITY_IMAGE_YAML="$(yaml_string "$POC_GPU_CAPACITY_IMAGE")"
|
|
POC_GPU_CAPACITY_WORKER_URL_YAML="$(yaml_string "$POC_GPU_CAPACITY_WORKER_URL")"
|
|
POC_GPU_CAPACITY_HTTP_TIMEOUT_SECONDS_YAML="$(yaml_string "$POC_GPU_CAPACITY_HTTP_TIMEOUT_SECONDS")"
|
|
|
|
image_pull_secrets_yaml=""
|
|
if [[ -n "$K8S_IMAGE_PULL_SECRET_NAME" ]]; then
|
|
K8S_IMAGE_PULL_SECRET_NAME_YAML="$(yaml_string "$K8S_IMAGE_PULL_SECRET_NAME")"
|
|
image_pull_secrets_yaml=" imagePullSecrets:
|
|
- name: $K8S_IMAGE_PULL_SECRET_NAME_YAML"
|
|
fi
|
|
|
|
cat >"$manifest" <<YAML
|
|
apiVersion: v1
|
|
kind: Pod
|
|
metadata:
|
|
name: $POC_GPU_CAPACITY_NAME_YAML
|
|
namespace: $K8S_NAMESPACE_YAML
|
|
labels:
|
|
app: droboticflow
|
|
component: poc-gpu-capacity-acceptance
|
|
spec:
|
|
restartPolicy: Never
|
|
$image_pull_secrets_yaml
|
|
containers:
|
|
- name: capacity
|
|
image: $POC_GPU_CAPACITY_IMAGE_YAML
|
|
imagePullPolicy: IfNotPresent
|
|
command:
|
|
- python
|
|
- -c
|
|
- |
|
|
import json
|
|
import os
|
|
import urllib.request
|
|
|
|
url = os.environ["WORKER_CAPACITY_URL"]
|
|
timeout = int(os.environ.get("WORKER_CAPACITY_TIMEOUT_SECONDS", "30"))
|
|
with urllib.request.urlopen(url, timeout=timeout) as response:
|
|
body = response.read().decode("utf-8")
|
|
json.loads(body)
|
|
print(body)
|
|
env:
|
|
- name: WORKER_CAPACITY_URL
|
|
value: $POC_GPU_CAPACITY_WORKER_URL_YAML
|
|
- name: WORKER_CAPACITY_TIMEOUT_SECONDS
|
|
value: $POC_GPU_CAPACITY_HTTP_TIMEOUT_SECONDS_YAML
|
|
YAML
|
|
|
|
kubectl apply -f "$manifest"
|
|
|
|
deadline=$((SECONDS + POC_GPU_CAPACITY_TIMEOUT_SECONDS))
|
|
phase=""
|
|
while true; do
|
|
phase="$(kubectl -n "$K8S_NAMESPACE" get pod "$POC_GPU_CAPACITY_NAME" -o "jsonpath={.status.phase}" 2>/dev/null || true)"
|
|
case "$phase" in
|
|
Succeeded|Failed)
|
|
break
|
|
;;
|
|
esac
|
|
|
|
if (( SECONDS >= deadline )); then
|
|
echo "ERROR: timed out waiting for Pod $POC_GPU_CAPACITY_NAME to finish; last phase=${phase:-unknown}" >&2
|
|
kubectl -n "$K8S_NAMESPACE" describe pod "$POC_GPU_CAPACITY_NAME" >&2 || true
|
|
kubectl -n "$K8S_NAMESPACE" logs "$POC_GPU_CAPACITY_NAME" >&2 || true
|
|
exit 1
|
|
fi
|
|
sleep 2
|
|
done
|
|
|
|
logs="$(kubectl -n "$K8S_NAMESPACE" logs "$POC_GPU_CAPACITY_NAME" 2>&1 || true)"
|
|
if [[ "$phase" != "Succeeded" ]]; then
|
|
echo "ERROR: Pod $POC_GPU_CAPACITY_NAME finished with phase=$phase" >&2
|
|
printf '%s\n' "$logs" >&2
|
|
exit 1
|
|
fi
|
|
|
|
CAPACITY_JSON="$logs" EXPECT_AVAILABLE="$POC_GPU_CAPACITY_EXPECT_AVAILABLE" python3 - <<'PY'
|
|
import json
|
|
import os
|
|
import sys
|
|
|
|
raw = os.environ.get("CAPACITY_JSON", "")
|
|
payload = None
|
|
last_error: Exception | None = None
|
|
for line in reversed(raw.splitlines()):
|
|
candidate = line.strip()
|
|
if not candidate.startswith("{"):
|
|
continue
|
|
try:
|
|
payload = json.loads(candidate)
|
|
break
|
|
except json.JSONDecodeError as exc:
|
|
last_error = exc
|
|
|
|
if payload is None:
|
|
detail = f": {last_error}" if last_error else ""
|
|
print(f"ERROR: worker capacity response did not contain a JSON object{detail}", file=sys.stderr)
|
|
raise SystemExit(1)
|
|
|
|
gpu = payload.get("gpu")
|
|
if not isinstance(gpu, dict):
|
|
print("ERROR: worker capacity response is missing gpu object", file=sys.stderr)
|
|
raise SystemExit(1)
|
|
|
|
resource_name = str(gpu.get("resourceName") or "")
|
|
available = bool(gpu.get("available"))
|
|
nodes = gpu.get("nodes")
|
|
if not isinstance(nodes, list):
|
|
print("ERROR: gpu.nodes must be an array", file=sys.stderr)
|
|
raise SystemExit(1)
|
|
|
|
def int_field(name: str) -> int:
|
|
value = gpu.get(name, 0)
|
|
try:
|
|
return int(value)
|
|
except (TypeError, ValueError):
|
|
print(f"ERROR: gpu.{name} must be an integer-compatible value", file=sys.stderr)
|
|
raise SystemExit(1)
|
|
|
|
schedulable = int_field("schedulableAllocatable")
|
|
max_allocatable = int_field("maxAllocatable")
|
|
expected_available = os.environ.get("EXPECT_AVAILABLE") == "true"
|
|
|
|
if not resource_name:
|
|
print("ERROR: gpu.resourceName is required", file=sys.stderr)
|
|
raise SystemExit(1)
|
|
if not nodes:
|
|
print("ERROR: gpu.nodes is empty; capacity endpoint did not report checked nodes", file=sys.stderr)
|
|
raise SystemExit(1)
|
|
|
|
if expected_available:
|
|
if not available or schedulable <= 0 or max_allocatable <= 0:
|
|
print(
|
|
"ERROR: expected POC GPU capacity to be available, "
|
|
f"got available={available} schedulableAllocatable={schedulable} maxAllocatable={max_allocatable}",
|
|
file=sys.stderr,
|
|
)
|
|
raise SystemExit(1)
|
|
else:
|
|
if available or schedulable != 0 or max_allocatable != 0:
|
|
print(
|
|
"ERROR: expected CPU-only POC GPU capacity, "
|
|
f"got available={available} schedulableAllocatable={schedulable} maxAllocatable={max_allocatable}",
|
|
file=sys.stderr,
|
|
)
|
|
raise SystemExit(1)
|
|
|
|
print(
|
|
"gpu.available="
|
|
+ str(available).lower()
|
|
+ f" resource={resource_name} schedulableAllocatable={schedulable} maxAllocatable={max_allocatable} nodes={len(nodes)}"
|
|
)
|
|
for node in nodes:
|
|
if not isinstance(node, dict):
|
|
continue
|
|
name = node.get("name") or "<unknown>"
|
|
allocatable = node.get("allocatable", 0)
|
|
selector_matched = node.get("selectorMatched", False)
|
|
print(f"node {name} {resource_name}={allocatable} selectorMatched={str(bool(selector_matched)).lower()}")
|
|
PY
|