forked from longtao.wu/droboticflow
323 lines
11 KiB
Bash
Executable File
323 lines
11 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
|
DEPLOY_ENV="${DEPLOY_ENV:-}"
|
|
export DEPLOY_ENV
|
|
# shellcheck source=scripts/load-deploy-env.sh
|
|
source "$ROOT_DIR/scripts/load-deploy-env.sh"
|
|
|
|
K8S_NAMESPACE="${K8S_NAMESPACE:-droboticflow}"
|
|
GPU_ACCEPTANCE_IMAGE="${GPU_ACCEPTANCE_IMAGE:-nvidia/cuda:12.4.1-base-ubuntu22.04}"
|
|
GPU_ACCEPTANCE_TIMEOUT="${GPU_ACCEPTANCE_TIMEOUT:-300s}"
|
|
GPU_ACCEPTANCE_NAME="${GPU_ACCEPTANCE_NAME:-droboticflow-gpu-acceptance-$(date +%s)}"
|
|
GPU_RESOURCE_NAME="${GPU_RESOURCE_NAME:-nvidia.com/gpu}"
|
|
GPU_RESOURCE_COUNT="${GPU_RESOURCE_COUNT:-1}"
|
|
OPERATOR_GPU_NODE_SELECTOR="${OPERATOR_GPU_NODE_SELECTOR:-}"
|
|
OPERATOR_GPU_TOLERATIONS_JSON="${OPERATOR_GPU_TOLERATIONS_JSON:-}"
|
|
OPERATOR_GPU_RUNTIME_CLASS="${OPERATOR_GPU_RUNTIME_CLASS:-}"
|
|
|
|
validate_dns_label() {
|
|
local name="$1"
|
|
local value="$2"
|
|
if [[ ${#value} -gt 63 ]] || ! [[ "$value" =~ ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ ]]; then
|
|
echo "ERROR: $name must be a lowercase DNS label" >&2
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
validate_resource_name() {
|
|
local value="$1"
|
|
if [[ -z "$value" ]] || ! [[ "$value" =~ ^([A-Za-z0-9]([-A-Za-z0-9_.]*[A-Za-z0-9])?/)?[A-Za-z0-9]([-A-Za-z0-9_.]*[A-Za-z0-9])?$ ]]; then
|
|
echo "ERROR: GPU_RESOURCE_NAME must be a Kubernetes resource name" >&2
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
validate_image_reference() {
|
|
local value="$1"
|
|
if [[ -z "$value" ]] || [[ "$value" =~ [[:space:]] ]]; then
|
|
echo "ERROR: GPU_ACCEPTANCE_IMAGE must be a non-empty container image reference without whitespace" >&2
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
validate_positive_duration() {
|
|
local value="$1"
|
|
GPU_ACCEPTANCE_TIMEOUT="$value" python3 - <<'PY'
|
|
import os
|
|
import re
|
|
import sys
|
|
|
|
value = os.environ.get("GPU_ACCEPTANCE_TIMEOUT", "").strip()
|
|
if not re.fullmatch(r"(?:[1-9][0-9]*(?:ns|us|ms|s|m|h))+", value):
|
|
print("ERROR: GPU_ACCEPTANCE_TIMEOUT must be a positive Go-style duration such as 300s, 5m, or 1h30m", file=sys.stderr)
|
|
raise SystemExit(1)
|
|
PY
|
|
}
|
|
|
|
yaml_string() {
|
|
python3 - "$1" <<'PY'
|
|
import json
|
|
import sys
|
|
|
|
print(json.dumps(sys.argv[1]))
|
|
PY
|
|
}
|
|
|
|
if [[ -z "$GPU_ACCEPTANCE_IMAGE" ]]; then
|
|
echo "ERROR: GPU_ACCEPTANCE_IMAGE is required" >&2
|
|
exit 1
|
|
fi
|
|
validate_dns_label "K8S_NAMESPACE" "$K8S_NAMESPACE"
|
|
validate_dns_label "GPU_ACCEPTANCE_NAME" "$GPU_ACCEPTANCE_NAME"
|
|
validate_resource_name "$GPU_RESOURCE_NAME"
|
|
validate_image_reference "$GPU_ACCEPTANCE_IMAGE"
|
|
validate_positive_duration "$GPU_ACCEPTANCE_TIMEOUT"
|
|
if ! [[ "$GPU_RESOURCE_COUNT" =~ ^[1-9][0-9]*$ ]]; then
|
|
echo "ERROR: GPU_RESOURCE_COUNT must be a positive integer" >&2
|
|
exit 1
|
|
fi
|
|
|
|
selector_yaml="$(
|
|
OPERATOR_GPU_NODE_SELECTOR="$OPERATOR_GPU_NODE_SELECTOR" python3 - <<'PY'
|
|
import json
|
|
import os
|
|
import re
|
|
import sys
|
|
|
|
raw = os.environ.get("OPERATOR_GPU_NODE_SELECTOR", "").strip()
|
|
if not raw:
|
|
raise SystemExit(0)
|
|
|
|
dns_label = r"[A-Za-z0-9]([-A-Za-z0-9_.]*[A-Za-z0-9])?"
|
|
label_key = re.compile(rf"^({dns_label}/)?{dns_label}$")
|
|
label_value = re.compile(rf"^({dns_label})?$")
|
|
items: dict[str, str] = {}
|
|
for item in raw.split(","):
|
|
item = item.strip()
|
|
if not item:
|
|
continue
|
|
if "=" not in item:
|
|
print(f"ERROR: invalid OPERATOR_GPU_NODE_SELECTOR item {item!r}: expected key=value", file=sys.stderr)
|
|
raise SystemExit(1)
|
|
key, value = (part.strip() for part in item.split("=", 1))
|
|
if not key or not value:
|
|
print(f"ERROR: invalid OPERATOR_GPU_NODE_SELECTOR item {item!r}: key and value are required", file=sys.stderr)
|
|
raise SystemExit(1)
|
|
if not label_key.fullmatch(key):
|
|
print(f"ERROR: invalid OPERATOR_GPU_NODE_SELECTOR key {key!r}", file=sys.stderr)
|
|
raise SystemExit(1)
|
|
if not label_value.fullmatch(value):
|
|
print(f"ERROR: invalid OPERATOR_GPU_NODE_SELECTOR value for {key!r}", file=sys.stderr)
|
|
raise SystemExit(1)
|
|
if key in items and items[key] != value:
|
|
print(f"ERROR: duplicate OPERATOR_GPU_NODE_SELECTOR key {key!r} with conflicting values", file=sys.stderr)
|
|
raise SystemExit(1)
|
|
items[key] = value
|
|
|
|
if not items:
|
|
raise SystemExit(0)
|
|
|
|
print(" nodeSelector:")
|
|
for key in sorted(items):
|
|
print(f" {json.dumps(key)}: {json.dumps(items[key])}")
|
|
PY
|
|
)"
|
|
|
|
tolerations_yaml="$(
|
|
OPERATOR_GPU_TOLERATIONS_JSON="$OPERATOR_GPU_TOLERATIONS_JSON" python3 - <<'PY'
|
|
import json
|
|
import os
|
|
import sys
|
|
|
|
raw = os.environ.get("OPERATOR_GPU_TOLERATIONS_JSON", "").strip()
|
|
if not raw:
|
|
raise SystemExit(0)
|
|
|
|
try:
|
|
items = json.loads(raw)
|
|
except json.JSONDecodeError as exc:
|
|
print(f"ERROR: invalid OPERATOR_GPU_TOLERATIONS_JSON: {exc}", file=sys.stderr)
|
|
raise SystemExit(1) from exc
|
|
|
|
if not isinstance(items, list):
|
|
print("ERROR: OPERATOR_GPU_TOLERATIONS_JSON must be a JSON array", file=sys.stderr)
|
|
raise SystemExit(1)
|
|
|
|
allowed_keys = {"key", "operator", "value", "effect", "tolerationSeconds"}
|
|
ordered_keys = ["key", "operator", "value", "effect", "tolerationSeconds"]
|
|
allowed_operators = {"Equal", "Exists"}
|
|
allowed_effects = {"NoSchedule", "PreferNoSchedule", "NoExecute"}
|
|
normalized: list[dict[str, object]] = []
|
|
for index, item in enumerate(items):
|
|
if not isinstance(item, dict):
|
|
print(f"ERROR: OPERATOR_GPU_TOLERATIONS_JSON[{index}] must be an object", file=sys.stderr)
|
|
raise SystemExit(1)
|
|
unknown = sorted(set(item) - allowed_keys)
|
|
if unknown:
|
|
print(f"ERROR: OPERATOR_GPU_TOLERATIONS_JSON[{index}] has unsupported keys: {', '.join(unknown)}", file=sys.stderr)
|
|
raise SystemExit(1)
|
|
output: dict[str, object] = {}
|
|
for key in ("key", "operator", "value", "effect"):
|
|
if key not in item or item[key] is None:
|
|
continue
|
|
if not isinstance(item[key], str):
|
|
print(f"ERROR: OPERATOR_GPU_TOLERATIONS_JSON[{index}].{key} must be a string", file=sys.stderr)
|
|
raise SystemExit(1)
|
|
value = item[key].strip()
|
|
if key == "operator" and value and value not in allowed_operators:
|
|
print(f"ERROR: OPERATOR_GPU_TOLERATIONS_JSON[{index}].operator must be Equal or Exists", file=sys.stderr)
|
|
raise SystemExit(1)
|
|
if key == "effect" and value and value not in allowed_effects:
|
|
print(f"ERROR: OPERATOR_GPU_TOLERATIONS_JSON[{index}].effect is not a supported Kubernetes toleration effect", file=sys.stderr)
|
|
raise SystemExit(1)
|
|
if value:
|
|
output[key] = value
|
|
if "tolerationSeconds" in item and item["tolerationSeconds"] is not None:
|
|
seconds = item["tolerationSeconds"]
|
|
if not isinstance(seconds, int) or seconds < 0:
|
|
print(f"ERROR: OPERATOR_GPU_TOLERATIONS_JSON[{index}].tolerationSeconds must be a non-negative integer", file=sys.stderr)
|
|
raise SystemExit(1)
|
|
output["tolerationSeconds"] = seconds
|
|
normalized.append(output)
|
|
|
|
if not normalized:
|
|
raise SystemExit(0)
|
|
|
|
print(" tolerations:")
|
|
for item in normalized:
|
|
print(" -")
|
|
for key in ordered_keys:
|
|
if key in item:
|
|
print(f" {key}: {json.dumps(item[key])}")
|
|
PY
|
|
)"
|
|
|
|
runtime_class_yaml="$(
|
|
OPERATOR_GPU_RUNTIME_CLASS="$OPERATOR_GPU_RUNTIME_CLASS" python3 - <<'PY'
|
|
import json
|
|
import os
|
|
import re
|
|
import sys
|
|
|
|
value = os.environ.get("OPERATOR_GPU_RUNTIME_CLASS", "").strip()
|
|
if not value:
|
|
raise SystemExit(0)
|
|
if len(value) > 253 or not re.fullmatch(r"[a-z0-9]([-a-z0-9.]*[a-z0-9])?", value):
|
|
print("ERROR: OPERATOR_GPU_RUNTIME_CLASS must be a lowercase DNS subdomain", file=sys.stderr)
|
|
raise SystemExit(1)
|
|
print(f" runtimeClassName: {json.dumps(value)}")
|
|
PY
|
|
)"
|
|
|
|
if ! kubectl get namespace "$K8S_NAMESPACE" >/dev/null 2>&1; then
|
|
echo "ERROR: namespace $K8S_NAMESPACE does not exist" >&2
|
|
exit 1
|
|
fi
|
|
|
|
nodes_json="$(kubectl get nodes -o json)"
|
|
if ! NODES_JSON="$nodes_json" OPERATOR_GPU_NODE_SELECTOR="$OPERATOR_GPU_NODE_SELECTOR" python3 - "$GPU_RESOURCE_NAME" "$GPU_RESOURCE_COUNT" <<'PY'
|
|
import json
|
|
import os
|
|
import sys
|
|
|
|
resource = sys.argv[1]
|
|
requested = int(sys.argv[2])
|
|
payload = json.loads(os.environ["NODES_JSON"])
|
|
|
|
selector: dict[str, str] = {}
|
|
raw_selector = os.environ.get("OPERATOR_GPU_NODE_SELECTOR", "").strip()
|
|
if raw_selector:
|
|
for item in raw_selector.split(","):
|
|
item = item.strip()
|
|
if not item:
|
|
continue
|
|
key, value = (part.strip() for part in item.split("=", 1))
|
|
selector[key] = value
|
|
|
|
def selector_matches(labels: dict[str, str]) -> bool:
|
|
for key, value in selector.items():
|
|
if labels.get(key) != value:
|
|
return False
|
|
return True
|
|
|
|
checked: list[str] = []
|
|
for node in payload.get("items", []):
|
|
name = node["metadata"]["name"]
|
|
labels = node.get("metadata", {}).get("labels", {})
|
|
labels = labels if isinstance(labels, dict) else {}
|
|
selector_matched = selector_matches(labels)
|
|
unschedulable = bool(node.get("spec", {}).get("unschedulable", False))
|
|
allocatable = node.get("status", {}).get("allocatable", {})
|
|
value = str(allocatable.get(resource, "0"))
|
|
try:
|
|
count = int(value)
|
|
except ValueError:
|
|
count = 0
|
|
checked.append(f"{name} {resource}={value} selectorMatched={str(selector_matched).lower()} unschedulable={str(unschedulable).lower()}")
|
|
if count >= requested and selector_matched and not unschedulable:
|
|
print(f"{name} {resource}={value} selectorMatched={str(selector_matched).lower()} unschedulable={str(unschedulable).lower()}")
|
|
raise SystemExit(0)
|
|
selector_summary = ",".join(f"{key}={selector[key]}" for key in sorted(selector)) if selector else "<none>"
|
|
print(f"GPU readiness unavailable: no schedulable node reports allocatable {resource}>={requested} with selector {selector_summary}", file=sys.stderr)
|
|
if checked:
|
|
print("Checked nodes:", file=sys.stderr)
|
|
for item in checked:
|
|
print(f" - {item}", file=sys.stderr)
|
|
else:
|
|
print("Checked nodes: none", file=sys.stderr)
|
|
raise SystemExit(1)
|
|
PY
|
|
then
|
|
echo "ERROR: no Kubernetes node reports allocatable $GPU_RESOURCE_NAME >= $GPU_RESOURCE_COUNT" >&2
|
|
exit 1
|
|
fi
|
|
|
|
manifest="$(mktemp)"
|
|
cleanup() {
|
|
kubectl -n "$K8S_NAMESPACE" delete job "$GPU_ACCEPTANCE_NAME" --ignore-not-found >/dev/null 2>&1 || true
|
|
rm -f "$manifest"
|
|
}
|
|
trap cleanup EXIT
|
|
|
|
K8S_NAMESPACE_YAML="$(yaml_string "$K8S_NAMESPACE")"
|
|
GPU_ACCEPTANCE_NAME_YAML="$(yaml_string "$GPU_ACCEPTANCE_NAME")"
|
|
GPU_ACCEPTANCE_IMAGE_YAML="$(yaml_string "$GPU_ACCEPTANCE_IMAGE")"
|
|
GPU_RESOURCE_NAME_YAML="$(yaml_string "$GPU_RESOURCE_NAME")"
|
|
GPU_RESOURCE_COUNT_YAML="$(yaml_string "$GPU_RESOURCE_COUNT")"
|
|
|
|
cat >"$manifest" <<YAML
|
|
apiVersion: batch/v1
|
|
kind: Job
|
|
metadata:
|
|
name: $GPU_ACCEPTANCE_NAME_YAML
|
|
namespace: $K8S_NAMESPACE_YAML
|
|
spec:
|
|
backoffLimit: 0
|
|
template:
|
|
spec:
|
|
restartPolicy: Never
|
|
$selector_yaml
|
|
$tolerations_yaml
|
|
$runtime_class_yaml
|
|
containers:
|
|
- name: gpu-acceptance
|
|
image: $GPU_ACCEPTANCE_IMAGE_YAML
|
|
imagePullPolicy: IfNotPresent
|
|
command: ["bash", "-lc", "nvidia-smi && echo droboticflow-gpu-acceptance-ok"]
|
|
resources:
|
|
limits:
|
|
$GPU_RESOURCE_NAME_YAML: $GPU_RESOURCE_COUNT_YAML
|
|
requests:
|
|
$GPU_RESOURCE_NAME_YAML: $GPU_RESOURCE_COUNT_YAML
|
|
YAML
|
|
|
|
kubectl apply -f "$manifest"
|
|
if ! kubectl -n "$K8S_NAMESPACE" wait --for=condition=complete "job/$GPU_ACCEPTANCE_NAME" --timeout="$GPU_ACCEPTANCE_TIMEOUT"; then
|
|
kubectl -n "$K8S_NAMESPACE" describe job "$GPU_ACCEPTANCE_NAME" >&2 || true
|
|
kubectl -n "$K8S_NAMESPACE" logs "job/$GPU_ACCEPTANCE_NAME" >&2 || true
|
|
exit 1
|
|
fi
|
|
kubectl -n "$K8S_NAMESPACE" logs "job/$GPU_ACCEPTANCE_NAME"
|