forked from longtao.wu/droboticflow
284 lines
9.3 KiB
Bash
Executable File
284 lines
9.3 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
K8S_NAMESPACE="${K8S_NAMESPACE:-droboticflow}"
|
|
GPU_SMOKE_IMAGE="${GPU_SMOKE_IMAGE:-nvidia/cuda:12.4.1-base-ubuntu22.04}"
|
|
GPU_SMOKE_TIMEOUT="${GPU_SMOKE_TIMEOUT:-300s}"
|
|
GPU_SMOKE_NAME="${GPU_SMOKE_NAME:-droboticflow-gpu-smoke-$(date +%s)}"
|
|
GPU_RESOURCE_NAME="${GPU_RESOURCE_NAME:-nvidia.com/gpu}"
|
|
GPU_RESOURCE_COUNT="${GPU_RESOURCE_COUNT:-1}"
|
|
OPERATOR_GPU_NODE_SELECTOR="${OPERATOR_GPU_NODE_SELECTOR:-}"
|
|
OPERATOR_GPU_TOLERATIONS_JSON="${OPERATOR_GPU_TOLERATIONS_JSON:-}"
|
|
OPERATOR_GPU_RUNTIME_CLASS="${OPERATOR_GPU_RUNTIME_CLASS:-}"
|
|
|
|
validate_dns_label() {
|
|
local name="$1"
|
|
local value="$2"
|
|
if [[ ${#value} -gt 63 ]] || ! [[ "$value" =~ ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ ]]; then
|
|
echo "ERROR: $name must be a lowercase DNS label" >&2
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
validate_resource_name() {
|
|
local value="$1"
|
|
if [[ -z "$value" ]] || ! [[ "$value" =~ ^([A-Za-z0-9]([-A-Za-z0-9_.]*[A-Za-z0-9])?/)?[A-Za-z0-9]([-A-Za-z0-9_.]*[A-Za-z0-9])?$ ]]; then
|
|
echo "ERROR: GPU_RESOURCE_NAME must be a Kubernetes resource name" >&2
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
validate_image_reference() {
|
|
local value="$1"
|
|
if [[ -z "$value" ]] || [[ "$value" =~ [[:space:]] ]]; then
|
|
echo "ERROR: GPU_SMOKE_IMAGE must be a non-empty container image reference without whitespace" >&2
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
validate_positive_duration() {
|
|
local value="$1"
|
|
GPU_SMOKE_TIMEOUT="$value" python3 - <<'PY'
|
|
import os
|
|
import re
|
|
import sys
|
|
|
|
value = os.environ.get("GPU_SMOKE_TIMEOUT", "").strip()
|
|
if not re.fullmatch(r"(?:[1-9][0-9]*(?:ns|us|ms|s|m|h))+", value):
|
|
print("ERROR: GPU_SMOKE_TIMEOUT must be a positive Go-style duration such as 300s, 5m, or 1h30m", file=sys.stderr)
|
|
raise SystemExit(1)
|
|
PY
|
|
}
|
|
|
|
yaml_string() {
|
|
python3 - "$1" <<'PY'
|
|
import json
|
|
import sys
|
|
|
|
print(json.dumps(sys.argv[1]))
|
|
PY
|
|
}
|
|
|
|
if [[ -z "$GPU_SMOKE_IMAGE" ]]; then
|
|
echo "ERROR: GPU_SMOKE_IMAGE is required" >&2
|
|
exit 1
|
|
fi
|
|
validate_dns_label "K8S_NAMESPACE" "$K8S_NAMESPACE"
|
|
validate_dns_label "GPU_SMOKE_NAME" "$GPU_SMOKE_NAME"
|
|
validate_resource_name "$GPU_RESOURCE_NAME"
|
|
validate_image_reference "$GPU_SMOKE_IMAGE"
|
|
validate_positive_duration "$GPU_SMOKE_TIMEOUT"
|
|
if ! [[ "$GPU_RESOURCE_COUNT" =~ ^[1-9][0-9]*$ ]]; then
|
|
echo "ERROR: GPU_RESOURCE_COUNT must be a positive integer" >&2
|
|
exit 1
|
|
fi
|
|
|
|
selector_yaml="$(
|
|
OPERATOR_GPU_NODE_SELECTOR="$OPERATOR_GPU_NODE_SELECTOR" python3 - <<'PY'
|
|
import json
|
|
import os
|
|
import re
|
|
import sys
|
|
|
|
raw = os.environ.get("OPERATOR_GPU_NODE_SELECTOR", "").strip()
|
|
if not raw:
|
|
raise SystemExit(0)
|
|
|
|
dns_label = r"[A-Za-z0-9]([-A-Za-z0-9_.]*[A-Za-z0-9])?"
|
|
label_key = re.compile(rf"^({dns_label}/)?{dns_label}$")
|
|
label_value = re.compile(rf"^({dns_label})?$")
|
|
items: dict[str, str] = {}
|
|
for item in raw.split(","):
|
|
item = item.strip()
|
|
if not item:
|
|
continue
|
|
if "=" not in item:
|
|
print(f"ERROR: invalid OPERATOR_GPU_NODE_SELECTOR item {item!r}: expected key=value", file=sys.stderr)
|
|
raise SystemExit(1)
|
|
key, value = (part.strip() for part in item.split("=", 1))
|
|
if not key or not value:
|
|
print(f"ERROR: invalid OPERATOR_GPU_NODE_SELECTOR item {item!r}: key and value are required", file=sys.stderr)
|
|
raise SystemExit(1)
|
|
if not label_key.fullmatch(key):
|
|
print(f"ERROR: invalid OPERATOR_GPU_NODE_SELECTOR key {key!r}", file=sys.stderr)
|
|
raise SystemExit(1)
|
|
if not label_value.fullmatch(value):
|
|
print(f"ERROR: invalid OPERATOR_GPU_NODE_SELECTOR value for {key!r}", file=sys.stderr)
|
|
raise SystemExit(1)
|
|
if key in items and items[key] != value:
|
|
print(f"ERROR: duplicate OPERATOR_GPU_NODE_SELECTOR key {key!r} with conflicting values", file=sys.stderr)
|
|
raise SystemExit(1)
|
|
items[key] = value
|
|
|
|
if not items:
|
|
raise SystemExit(0)
|
|
|
|
print(" nodeSelector:")
|
|
for key in sorted(items):
|
|
print(f" {json.dumps(key)}: {json.dumps(items[key])}")
|
|
PY
|
|
)"
|
|
|
|
tolerations_yaml="$(
|
|
OPERATOR_GPU_TOLERATIONS_JSON="$OPERATOR_GPU_TOLERATIONS_JSON" python3 - <<'PY'
|
|
import json
|
|
import os
|
|
import sys
|
|
|
|
raw = os.environ.get("OPERATOR_GPU_TOLERATIONS_JSON", "").strip()
|
|
if not raw:
|
|
raise SystemExit(0)
|
|
|
|
try:
|
|
items = json.loads(raw)
|
|
except json.JSONDecodeError as exc:
|
|
print(f"ERROR: invalid OPERATOR_GPU_TOLERATIONS_JSON: {exc}", file=sys.stderr)
|
|
raise SystemExit(1) from exc
|
|
|
|
if not isinstance(items, list):
|
|
print("ERROR: OPERATOR_GPU_TOLERATIONS_JSON must be a JSON array", file=sys.stderr)
|
|
raise SystemExit(1)
|
|
|
|
allowed_keys = {"key", "operator", "value", "effect", "tolerationSeconds"}
|
|
ordered_keys = ["key", "operator", "value", "effect", "tolerationSeconds"]
|
|
allowed_operators = {"Equal", "Exists"}
|
|
allowed_effects = {"NoSchedule", "PreferNoSchedule", "NoExecute"}
|
|
normalized: list[dict[str, object]] = []
|
|
for index, item in enumerate(items):
|
|
if not isinstance(item, dict):
|
|
print(f"ERROR: OPERATOR_GPU_TOLERATIONS_JSON[{index}] must be an object", file=sys.stderr)
|
|
raise SystemExit(1)
|
|
unknown = sorted(set(item) - allowed_keys)
|
|
if unknown:
|
|
print(f"ERROR: OPERATOR_GPU_TOLERATIONS_JSON[{index}] has unsupported keys: {', '.join(unknown)}", file=sys.stderr)
|
|
raise SystemExit(1)
|
|
output: dict[str, object] = {}
|
|
for key in ("key", "operator", "value", "effect"):
|
|
if key not in item or item[key] is None:
|
|
continue
|
|
if not isinstance(item[key], str):
|
|
print(f"ERROR: OPERATOR_GPU_TOLERATIONS_JSON[{index}].{key} must be a string", file=sys.stderr)
|
|
raise SystemExit(1)
|
|
value = item[key].strip()
|
|
if key == "operator" and value and value not in allowed_operators:
|
|
print(f"ERROR: OPERATOR_GPU_TOLERATIONS_JSON[{index}].operator must be Equal or Exists", file=sys.stderr)
|
|
raise SystemExit(1)
|
|
if key == "effect" and value and value not in allowed_effects:
|
|
print(f"ERROR: OPERATOR_GPU_TOLERATIONS_JSON[{index}].effect is not a supported Kubernetes toleration effect", file=sys.stderr)
|
|
raise SystemExit(1)
|
|
if value:
|
|
output[key] = value
|
|
if "tolerationSeconds" in item and item["tolerationSeconds"] is not None:
|
|
seconds = item["tolerationSeconds"]
|
|
if not isinstance(seconds, int) or seconds < 0:
|
|
print(f"ERROR: OPERATOR_GPU_TOLERATIONS_JSON[{index}].tolerationSeconds must be a non-negative integer", file=sys.stderr)
|
|
raise SystemExit(1)
|
|
output["tolerationSeconds"] = seconds
|
|
normalized.append(output)
|
|
|
|
if not normalized:
|
|
raise SystemExit(0)
|
|
|
|
print(" tolerations:")
|
|
for item in normalized:
|
|
print(" -")
|
|
for key in ordered_keys:
|
|
if key in item:
|
|
print(f" {key}: {json.dumps(item[key])}")
|
|
PY
|
|
)"
|
|
|
|
runtime_class_yaml="$(
|
|
OPERATOR_GPU_RUNTIME_CLASS="$OPERATOR_GPU_RUNTIME_CLASS" python3 - <<'PY'
|
|
import json
|
|
import os
|
|
import re
|
|
import sys
|
|
|
|
value = os.environ.get("OPERATOR_GPU_RUNTIME_CLASS", "").strip()
|
|
if not value:
|
|
raise SystemExit(0)
|
|
if len(value) > 253 or not re.fullmatch(r"[a-z0-9]([-a-z0-9.]*[a-z0-9])?", value):
|
|
print("ERROR: OPERATOR_GPU_RUNTIME_CLASS must be a lowercase DNS subdomain", file=sys.stderr)
|
|
raise SystemExit(1)
|
|
print(f" runtimeClassName: {json.dumps(value)}")
|
|
PY
|
|
)"
|
|
|
|
if ! kubectl get namespace "$K8S_NAMESPACE" >/dev/null 2>&1; then
|
|
echo "ERROR: namespace $K8S_NAMESPACE does not exist" >&2
|
|
exit 1
|
|
fi
|
|
|
|
nodes_json="$(kubectl get nodes -o json)"
|
|
if ! NODES_JSON="$nodes_json" python3 - "$GPU_RESOURCE_NAME" <<'PY'
|
|
import json
|
|
import os
|
|
import sys
|
|
|
|
resource = sys.argv[1]
|
|
payload = json.loads(os.environ["NODES_JSON"])
|
|
for node in payload.get("items", []):
|
|
allocatable = node.get("status", {}).get("allocatable", {})
|
|
value = str(allocatable.get(resource, "0"))
|
|
try:
|
|
count = int(value)
|
|
except ValueError:
|
|
count = 0
|
|
if count > 0:
|
|
print(f"{node['metadata']['name']} {resource}={value}")
|
|
raise SystemExit(0)
|
|
raise SystemExit(1)
|
|
PY
|
|
then
|
|
echo "ERROR: no Kubernetes node reports allocatable $GPU_RESOURCE_NAME" >&2
|
|
exit 1
|
|
fi
|
|
|
|
manifest="$(mktemp)"
|
|
cleanup() {
|
|
kubectl -n "$K8S_NAMESPACE" delete job "$GPU_SMOKE_NAME" --ignore-not-found >/dev/null 2>&1 || true
|
|
rm -f "$manifest"
|
|
}
|
|
trap cleanup EXIT
|
|
|
|
K8S_NAMESPACE_YAML="$(yaml_string "$K8S_NAMESPACE")"
|
|
GPU_SMOKE_NAME_YAML="$(yaml_string "$GPU_SMOKE_NAME")"
|
|
GPU_SMOKE_IMAGE_YAML="$(yaml_string "$GPU_SMOKE_IMAGE")"
|
|
GPU_RESOURCE_NAME_YAML="$(yaml_string "$GPU_RESOURCE_NAME")"
|
|
GPU_RESOURCE_COUNT_YAML="$(yaml_string "$GPU_RESOURCE_COUNT")"
|
|
|
|
cat >"$manifest" <<YAML
|
|
apiVersion: batch/v1
|
|
kind: Job
|
|
metadata:
|
|
name: $GPU_SMOKE_NAME_YAML
|
|
namespace: $K8S_NAMESPACE_YAML
|
|
spec:
|
|
backoffLimit: 0
|
|
template:
|
|
spec:
|
|
restartPolicy: Never
|
|
$selector_yaml
|
|
$tolerations_yaml
|
|
$runtime_class_yaml
|
|
containers:
|
|
- name: gpu-smoke
|
|
image: $GPU_SMOKE_IMAGE_YAML
|
|
imagePullPolicy: IfNotPresent
|
|
command: ["bash", "-lc", "nvidia-smi && echo droboticflow-gpu-smoke-ok"]
|
|
resources:
|
|
limits:
|
|
$GPU_RESOURCE_NAME_YAML: $GPU_RESOURCE_COUNT_YAML
|
|
requests:
|
|
$GPU_RESOURCE_NAME_YAML: $GPU_RESOURCE_COUNT_YAML
|
|
YAML
|
|
|
|
kubectl apply -f "$manifest"
|
|
if ! kubectl -n "$K8S_NAMESPACE" wait --for=condition=complete "job/$GPU_SMOKE_NAME" --timeout="$GPU_SMOKE_TIMEOUT"; then
|
|
kubectl -n "$K8S_NAMESPACE" describe job "$GPU_SMOKE_NAME" >&2 || true
|
|
kubectl -n "$K8S_NAMESPACE" logs "job/$GPU_SMOKE_NAME" >&2 || true
|
|
exit 1
|
|
fi
|
|
kubectl -n "$K8S_NAMESPACE" logs "job/$GPU_SMOKE_NAME"
|