droboticflow/scripts/gpu-acceptance-test.sh
2026-06-26 04:07:55 +08:00

323 lines
11 KiB
Bash
Executable File

#!/usr/bin/env bash
set -euo pipefail
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
DEPLOY_ENV="${DEPLOY_ENV:-}"
export DEPLOY_ENV
# shellcheck source=scripts/load-deploy-env.sh
source "$ROOT_DIR/scripts/load-deploy-env.sh"
K8S_NAMESPACE="${K8S_NAMESPACE:-droboticflow}"
GPU_ACCEPTANCE_IMAGE="${GPU_ACCEPTANCE_IMAGE:-nvidia/cuda:12.4.1-base-ubuntu22.04}"
GPU_ACCEPTANCE_TIMEOUT="${GPU_ACCEPTANCE_TIMEOUT:-300s}"
GPU_ACCEPTANCE_NAME="${GPU_ACCEPTANCE_NAME:-droboticflow-gpu-acceptance-$(date +%s)}"
GPU_RESOURCE_NAME="${GPU_RESOURCE_NAME:-nvidia.com/gpu}"
GPU_RESOURCE_COUNT="${GPU_RESOURCE_COUNT:-1}"
OPERATOR_GPU_NODE_SELECTOR="${OPERATOR_GPU_NODE_SELECTOR:-}"
OPERATOR_GPU_TOLERATIONS_JSON="${OPERATOR_GPU_TOLERATIONS_JSON:-}"
OPERATOR_GPU_RUNTIME_CLASS="${OPERATOR_GPU_RUNTIME_CLASS:-}"
validate_dns_label() {
local name="$1"
local value="$2"
if [[ ${#value} -gt 63 ]] || ! [[ "$value" =~ ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ ]]; then
echo "ERROR: $name must be a lowercase DNS label" >&2
exit 1
fi
}
validate_resource_name() {
local value="$1"
if [[ -z "$value" ]] || ! [[ "$value" =~ ^([A-Za-z0-9]([-A-Za-z0-9_.]*[A-Za-z0-9])?/)?[A-Za-z0-9]([-A-Za-z0-9_.]*[A-Za-z0-9])?$ ]]; then
echo "ERROR: GPU_RESOURCE_NAME must be a Kubernetes resource name" >&2
exit 1
fi
}
validate_image_reference() {
local value="$1"
if [[ -z "$value" ]] || [[ "$value" =~ [[:space:]] ]]; then
echo "ERROR: GPU_ACCEPTANCE_IMAGE must be a non-empty container image reference without whitespace" >&2
exit 1
fi
}
validate_positive_duration() {
local value="$1"
GPU_ACCEPTANCE_TIMEOUT="$value" python3 - <<'PY'
import os
import re
import sys
value = os.environ.get("GPU_ACCEPTANCE_TIMEOUT", "").strip()
if not re.fullmatch(r"(?:[1-9][0-9]*(?:ns|us|ms|s|m|h))+", value):
print("ERROR: GPU_ACCEPTANCE_TIMEOUT must be a positive Go-style duration such as 300s, 5m, or 1h30m", file=sys.stderr)
raise SystemExit(1)
PY
}
yaml_string() {
python3 - "$1" <<'PY'
import json
import sys
print(json.dumps(sys.argv[1]))
PY
}
if [[ -z "$GPU_ACCEPTANCE_IMAGE" ]]; then
echo "ERROR: GPU_ACCEPTANCE_IMAGE is required" >&2
exit 1
fi
validate_dns_label "K8S_NAMESPACE" "$K8S_NAMESPACE"
validate_dns_label "GPU_ACCEPTANCE_NAME" "$GPU_ACCEPTANCE_NAME"
validate_resource_name "$GPU_RESOURCE_NAME"
validate_image_reference "$GPU_ACCEPTANCE_IMAGE"
validate_positive_duration "$GPU_ACCEPTANCE_TIMEOUT"
if ! [[ "$GPU_RESOURCE_COUNT" =~ ^[1-9][0-9]*$ ]]; then
echo "ERROR: GPU_RESOURCE_COUNT must be a positive integer" >&2
exit 1
fi
selector_yaml="$(
OPERATOR_GPU_NODE_SELECTOR="$OPERATOR_GPU_NODE_SELECTOR" python3 - <<'PY'
import json
import os
import re
import sys
raw = os.environ.get("OPERATOR_GPU_NODE_SELECTOR", "").strip()
if not raw:
raise SystemExit(0)
dns_label = r"[A-Za-z0-9]([-A-Za-z0-9_.]*[A-Za-z0-9])?"
label_key = re.compile(rf"^({dns_label}/)?{dns_label}$")
label_value = re.compile(rf"^({dns_label})?$")
items: dict[str, str] = {}
for item in raw.split(","):
item = item.strip()
if not item:
continue
if "=" not in item:
print(f"ERROR: invalid OPERATOR_GPU_NODE_SELECTOR item {item!r}: expected key=value", file=sys.stderr)
raise SystemExit(1)
key, value = (part.strip() for part in item.split("=", 1))
if not key or not value:
print(f"ERROR: invalid OPERATOR_GPU_NODE_SELECTOR item {item!r}: key and value are required", file=sys.stderr)
raise SystemExit(1)
if not label_key.fullmatch(key):
print(f"ERROR: invalid OPERATOR_GPU_NODE_SELECTOR key {key!r}", file=sys.stderr)
raise SystemExit(1)
if not label_value.fullmatch(value):
print(f"ERROR: invalid OPERATOR_GPU_NODE_SELECTOR value for {key!r}", file=sys.stderr)
raise SystemExit(1)
if key in items and items[key] != value:
print(f"ERROR: duplicate OPERATOR_GPU_NODE_SELECTOR key {key!r} with conflicting values", file=sys.stderr)
raise SystemExit(1)
items[key] = value
if not items:
raise SystemExit(0)
print(" nodeSelector:")
for key in sorted(items):
print(f" {json.dumps(key)}: {json.dumps(items[key])}")
PY
)"
tolerations_yaml="$(
OPERATOR_GPU_TOLERATIONS_JSON="$OPERATOR_GPU_TOLERATIONS_JSON" python3 - <<'PY'
import json
import os
import sys
raw = os.environ.get("OPERATOR_GPU_TOLERATIONS_JSON", "").strip()
if not raw:
raise SystemExit(0)
try:
items = json.loads(raw)
except json.JSONDecodeError as exc:
print(f"ERROR: invalid OPERATOR_GPU_TOLERATIONS_JSON: {exc}", file=sys.stderr)
raise SystemExit(1) from exc
if not isinstance(items, list):
print("ERROR: OPERATOR_GPU_TOLERATIONS_JSON must be a JSON array", file=sys.stderr)
raise SystemExit(1)
allowed_keys = {"key", "operator", "value", "effect", "tolerationSeconds"}
ordered_keys = ["key", "operator", "value", "effect", "tolerationSeconds"]
allowed_operators = {"Equal", "Exists"}
allowed_effects = {"NoSchedule", "PreferNoSchedule", "NoExecute"}
normalized: list[dict[str, object]] = []
for index, item in enumerate(items):
if not isinstance(item, dict):
print(f"ERROR: OPERATOR_GPU_TOLERATIONS_JSON[{index}] must be an object", file=sys.stderr)
raise SystemExit(1)
unknown = sorted(set(item) - allowed_keys)
if unknown:
print(f"ERROR: OPERATOR_GPU_TOLERATIONS_JSON[{index}] has unsupported keys: {', '.join(unknown)}", file=sys.stderr)
raise SystemExit(1)
output: dict[str, object] = {}
for key in ("key", "operator", "value", "effect"):
if key not in item or item[key] is None:
continue
if not isinstance(item[key], str):
print(f"ERROR: OPERATOR_GPU_TOLERATIONS_JSON[{index}].{key} must be a string", file=sys.stderr)
raise SystemExit(1)
value = item[key].strip()
if key == "operator" and value and value not in allowed_operators:
print(f"ERROR: OPERATOR_GPU_TOLERATIONS_JSON[{index}].operator must be Equal or Exists", file=sys.stderr)
raise SystemExit(1)
if key == "effect" and value and value not in allowed_effects:
print(f"ERROR: OPERATOR_GPU_TOLERATIONS_JSON[{index}].effect is not a supported Kubernetes toleration effect", file=sys.stderr)
raise SystemExit(1)
if value:
output[key] = value
if "tolerationSeconds" in item and item["tolerationSeconds"] is not None:
seconds = item["tolerationSeconds"]
if not isinstance(seconds, int) or seconds < 0:
print(f"ERROR: OPERATOR_GPU_TOLERATIONS_JSON[{index}].tolerationSeconds must be a non-negative integer", file=sys.stderr)
raise SystemExit(1)
output["tolerationSeconds"] = seconds
normalized.append(output)
if not normalized:
raise SystemExit(0)
print(" tolerations:")
for item in normalized:
print(" -")
for key in ordered_keys:
if key in item:
print(f" {key}: {json.dumps(item[key])}")
PY
)"
runtime_class_yaml="$(
OPERATOR_GPU_RUNTIME_CLASS="$OPERATOR_GPU_RUNTIME_CLASS" python3 - <<'PY'
import json
import os
import re
import sys
value = os.environ.get("OPERATOR_GPU_RUNTIME_CLASS", "").strip()
if not value:
raise SystemExit(0)
if len(value) > 253 or not re.fullmatch(r"[a-z0-9]([-a-z0-9.]*[a-z0-9])?", value):
print("ERROR: OPERATOR_GPU_RUNTIME_CLASS must be a lowercase DNS subdomain", file=sys.stderr)
raise SystemExit(1)
print(f" runtimeClassName: {json.dumps(value)}")
PY
)"
if ! kubectl get namespace "$K8S_NAMESPACE" >/dev/null 2>&1; then
echo "ERROR: namespace $K8S_NAMESPACE does not exist" >&2
exit 1
fi
nodes_json="$(kubectl get nodes -o json)"
if ! NODES_JSON="$nodes_json" OPERATOR_GPU_NODE_SELECTOR="$OPERATOR_GPU_NODE_SELECTOR" python3 - "$GPU_RESOURCE_NAME" "$GPU_RESOURCE_COUNT" <<'PY'
import json
import os
import sys
resource = sys.argv[1]
requested = int(sys.argv[2])
payload = json.loads(os.environ["NODES_JSON"])
selector: dict[str, str] = {}
raw_selector = os.environ.get("OPERATOR_GPU_NODE_SELECTOR", "").strip()
if raw_selector:
for item in raw_selector.split(","):
item = item.strip()
if not item:
continue
key, value = (part.strip() for part in item.split("=", 1))
selector[key] = value
def selector_matches(labels: dict[str, str]) -> bool:
for key, value in selector.items():
if labels.get(key) != value:
return False
return True
checked: list[str] = []
for node in payload.get("items", []):
name = node["metadata"]["name"]
labels = node.get("metadata", {}).get("labels", {})
labels = labels if isinstance(labels, dict) else {}
selector_matched = selector_matches(labels)
unschedulable = bool(node.get("spec", {}).get("unschedulable", False))
allocatable = node.get("status", {}).get("allocatable", {})
value = str(allocatable.get(resource, "0"))
try:
count = int(value)
except ValueError:
count = 0
checked.append(f"{name} {resource}={value} selectorMatched={str(selector_matched).lower()} unschedulable={str(unschedulable).lower()}")
if count >= requested and selector_matched and not unschedulable:
print(f"{name} {resource}={value} selectorMatched={str(selector_matched).lower()} unschedulable={str(unschedulable).lower()}")
raise SystemExit(0)
selector_summary = ",".join(f"{key}={selector[key]}" for key in sorted(selector)) if selector else "<none>"
print(f"GPU readiness unavailable: no schedulable node reports allocatable {resource}>={requested} with selector {selector_summary}", file=sys.stderr)
if checked:
print("Checked nodes:", file=sys.stderr)
for item in checked:
print(f" - {item}", file=sys.stderr)
else:
print("Checked nodes: none", file=sys.stderr)
raise SystemExit(1)
PY
then
echo "ERROR: no Kubernetes node reports allocatable $GPU_RESOURCE_NAME >= $GPU_RESOURCE_COUNT" >&2
exit 1
fi
manifest="$(mktemp)"
cleanup() {
kubectl -n "$K8S_NAMESPACE" delete job "$GPU_ACCEPTANCE_NAME" --ignore-not-found >/dev/null 2>&1 || true
rm -f "$manifest"
}
trap cleanup EXIT
K8S_NAMESPACE_YAML="$(yaml_string "$K8S_NAMESPACE")"
GPU_ACCEPTANCE_NAME_YAML="$(yaml_string "$GPU_ACCEPTANCE_NAME")"
GPU_ACCEPTANCE_IMAGE_YAML="$(yaml_string "$GPU_ACCEPTANCE_IMAGE")"
GPU_RESOURCE_NAME_YAML="$(yaml_string "$GPU_RESOURCE_NAME")"
GPU_RESOURCE_COUNT_YAML="$(yaml_string "$GPU_RESOURCE_COUNT")"
cat >"$manifest" <<YAML
apiVersion: batch/v1
kind: Job
metadata:
name: $GPU_ACCEPTANCE_NAME_YAML
namespace: $K8S_NAMESPACE_YAML
spec:
backoffLimit: 0
template:
spec:
restartPolicy: Never
$selector_yaml
$tolerations_yaml
$runtime_class_yaml
containers:
- name: gpu-acceptance
image: $GPU_ACCEPTANCE_IMAGE_YAML
imagePullPolicy: IfNotPresent
command: ["bash", "-lc", "nvidia-smi && echo droboticflow-gpu-acceptance-ok"]
resources:
limits:
$GPU_RESOURCE_NAME_YAML: $GPU_RESOURCE_COUNT_YAML
requests:
$GPU_RESOURCE_NAME_YAML: $GPU_RESOURCE_COUNT_YAML
YAML
kubectl apply -f "$manifest"
if ! kubectl -n "$K8S_NAMESPACE" wait --for=condition=complete "job/$GPU_ACCEPTANCE_NAME" --timeout="$GPU_ACCEPTANCE_TIMEOUT"; then
kubectl -n "$K8S_NAMESPACE" describe job "$GPU_ACCEPTANCE_NAME" >&2 || true
kubectl -n "$K8S_NAMESPACE" logs "job/$GPU_ACCEPTANCE_NAME" >&2 || true
exit 1
fi
kubectl -n "$K8S_NAMESPACE" logs "job/$GPU_ACCEPTANCE_NAME"