forked from longtao.wu/droboticflow
276 lines
11 KiB
Bash
Executable File
276 lines
11 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
|
DEPLOY_ENV="${DEPLOY_ENV:-}"
|
|
export DEPLOY_ENV
|
|
# shellcheck source=scripts/load-deploy-env.sh
|
|
source "$ROOT_DIR/scripts/load-deploy-env.sh"
|
|
# shellcheck source=scripts/lib/service-endpoints.sh
|
|
source "$ROOT_DIR/scripts/lib/service-endpoints.sh"
|
|
|
|
K8S_NAMESPACE="${K8S_NAMESPACE:-droboticflow}"
|
|
API_PUBLIC_SERVICE_NAME="${API_PUBLIC_SERVICE_NAME:-droboticflow-api-public}"
|
|
WORKER_SERVICE_NAME="${POC_GPU_WORKFLOW_WORKER_SERVICE_NAME:-droboticflow-worker}"
|
|
WORKER_SERVICE_PORT="${POC_GPU_WORKFLOW_WORKER_SERVICE_PORT:-3002}"
|
|
API_TOKEN_SECRET_NAME="${POC_GPU_WORKFLOW_API_TOKEN_SECRET_NAME:-droboticflow-auth-secret}"
|
|
API_TOKEN_SECRET_KEY="${POC_GPU_WORKFLOW_API_TOKEN_SECRET_KEY:-DROBOTICFLOW_API_TOKEN}"
|
|
|
|
normalize_bool() {
|
|
local name="$1"
|
|
local value="$2"
|
|
local normalized
|
|
normalized="$(printf '%s' "$value" | tr '[:upper:]' '[:lower:]')"
|
|
case "$normalized" in
|
|
true|1|yes|y|on)
|
|
printf 'true\n'
|
|
;;
|
|
false|0|no|n|off)
|
|
printf 'false\n'
|
|
;;
|
|
*)
|
|
echo "ERROR: $name must be true or false." >&2
|
|
exit 1
|
|
;;
|
|
esac
|
|
}
|
|
|
|
if [[ -z "${API_BASE_URL:-}" && -z "${BASE_URL:-}" ]]; then
|
|
public_host="$(
|
|
kubectl -n "$K8S_NAMESPACE" get svc "$API_PUBLIC_SERVICE_NAME" -o json | python3 -c '
|
|
import ipaddress
|
|
import json
|
|
import sys
|
|
|
|
payload = json.load(sys.stdin)
|
|
items = payload.get("status", {}).get("loadBalancer", {}).get("ingress", [])
|
|
hosts = [item.get("ip") or item.get("hostname") for item in items]
|
|
hosts = [item for item in hosts if item]
|
|
if not hosts:
|
|
raise SystemExit("no LoadBalancer ingress host is available")
|
|
|
|
def is_private(host: str) -> bool:
|
|
try:
|
|
return ipaddress.ip_address(host).is_private
|
|
except ValueError:
|
|
return False
|
|
|
|
for host in hosts:
|
|
if not is_private(host):
|
|
print(host)
|
|
raise SystemExit(0)
|
|
print(hosts[0])
|
|
'
|
|
)"
|
|
export API_BASE_URL="http://${public_host}"
|
|
fi
|
|
|
|
export WORKFLOW_ACCEPTANCE_SOURCE_PATH="${POC_GPU_WORKFLOW_SOURCE_PATH:-${WORKFLOW_ACCEPTANCE_SOURCE_PATH:-/workspace/datasets/fixtures/poc-operator-acceptance/xplatform}}"
|
|
export WORKFLOW_ACCEPTANCE_GPU_LIMIT="${WORKFLOW_ACCEPTANCE_GPU_LIMIT:-1}"
|
|
POC_GPU_WORKFLOW_EXPECT_AVAILABLE="$(normalize_bool "POC_GPU_WORKFLOW_EXPECT_AVAILABLE" "${POC_GPU_WORKFLOW_EXPECT_AVAILABLE:-${POC_GPU_CAPACITY_EXPECT_AVAILABLE:-true}}")"
|
|
if [[ "$POC_GPU_WORKFLOW_EXPECT_AVAILABLE" == "true" ]]; then
|
|
export WORKFLOW_ACCEPTANCE_EXPECT_TERMINAL_STATUS="${WORKFLOW_ACCEPTANCE_EXPECT_TERMINAL_STATUS:-completed}"
|
|
export WORKFLOW_ACCEPTANCE_EXPECT_LOG_SUBSTRING="${WORKFLOW_ACCEPTANCE_EXPECT_LOG_SUBSTRING:-}"
|
|
else
|
|
export WORKFLOW_ACCEPTANCE_EXPECT_TERMINAL_STATUS="${WORKFLOW_ACCEPTANCE_EXPECT_TERMINAL_STATUS:-failed}"
|
|
export WORKFLOW_ACCEPTANCE_EXPECT_LOG_SUBSTRING="${WORKFLOW_ACCEPTANCE_EXPECT_LOG_SUBSTRING:-gpu request cannot be scheduled}"
|
|
fi
|
|
export WORKFLOW_ACCEPTANCE_RUN_TIMEOUT_SECONDS="${WORKFLOW_ACCEPTANCE_RUN_TIMEOUT_SECONDS:-120}"
|
|
|
|
if [[ -n "${DROBOTICFLOW_ACCEPTANCE_EMAIL:-}" && -n "${DROBOTICFLOW_ACCEPTANCE_PASSWORD:-}" ]]; then
|
|
export WORKFLOW_ACCEPTANCE_REQUIRE_EXPLICIT_ENDPOINT=true
|
|
"$ROOT_DIR/scripts/live-workflow-acceptance.sh"
|
|
exit 0
|
|
fi
|
|
|
|
if [[ -n "${DROBOTICFLOW_ACCEPTANCE_EMAIL:-}" || -n "${DROBOTICFLOW_ACCEPTANCE_PASSWORD:-}" ]]; then
|
|
echo "ERROR: DROBOTICFLOW_ACCEPTANCE_EMAIL and DROBOTICFLOW_ACCEPTANCE_PASSWORD must be set together." >&2
|
|
exit 1
|
|
fi
|
|
|
|
if [[ -z "${DROBOTICFLOW_API_TOKEN:-}" ]]; then
|
|
api_token_secret_json="$(kubectl -n "$K8S_NAMESPACE" get secret "$API_TOKEN_SECRET_NAME" -o json 2>/dev/null || true)"
|
|
DROBOTICFLOW_API_TOKEN="$(
|
|
printf "%s" "$api_token_secret_json" | API_TOKEN_SECRET_KEY="$API_TOKEN_SECRET_KEY" python3 -c '
|
|
import base64
|
|
import json
|
|
import os
|
|
import sys
|
|
|
|
try:
|
|
payload = json.load(sys.stdin)
|
|
except json.JSONDecodeError:
|
|
raise SystemExit(0)
|
|
data = payload.get("data", {})
|
|
if not isinstance(data, dict):
|
|
raise SystemExit(0)
|
|
raw = data.get(os.environ["API_TOKEN_SECRET_KEY"])
|
|
if not isinstance(raw, str) or not raw:
|
|
raise SystemExit(0)
|
|
try:
|
|
print(base64.b64decode(raw, validate=True).decode("utf-8", errors="replace").strip(), end="")
|
|
except Exception:
|
|
raise SystemExit(0)
|
|
'
|
|
)"
|
|
export DROBOTICFLOW_API_TOKEN
|
|
fi
|
|
|
|
if [[ -z "${DROBOTICFLOW_API_TOKEN:-}" ]]; then
|
|
echo "ERROR: DROBOTICFLOW_API_TOKEN or ${K8S_NAMESPACE}/${API_TOKEN_SECRET_NAME}.${API_TOKEN_SECRET_KEY} is required for token-only POC GPU workflow acceptance when no session credentials are configured." >&2
|
|
exit 1
|
|
fi
|
|
|
|
if ! [[ "$WORKER_SERVICE_PORT" =~ ^[0-9]+$ ]] || (( WORKER_SERVICE_PORT < 1 || WORKER_SERVICE_PORT > 65535 )); then
|
|
echo "ERROR: POC_GPU_WORKFLOW_WORKER_SERVICE_PORT must be a valid TCP port." >&2
|
|
exit 1
|
|
fi
|
|
if [[ -z "$WORKER_SERVICE_NAME" || "$WORKER_SERVICE_NAME" =~ [[:space:]] ]]; then
|
|
echo "ERROR: POC_GPU_WORKFLOW_WORKER_SERVICE_NAME must be non-empty and contain no whitespace." >&2
|
|
exit 1
|
|
fi
|
|
if ! [[ "$WORKFLOW_ACCEPTANCE_RUN_TIMEOUT_SECONDS" =~ ^[0-9]+$ ]] || (( WORKFLOW_ACCEPTANCE_RUN_TIMEOUT_SECONDS < 1 )); then
|
|
echo "ERROR: WORKFLOW_ACCEPTANCE_RUN_TIMEOUT_SECONDS must be a positive integer." >&2
|
|
exit 1
|
|
fi
|
|
|
|
API_BASE_URL="$(droboticflow_first_service_url "${API_BASE_URL:-}" "${BASE_URL:-}")"
|
|
|
|
json_get() {
|
|
local expression="$1"
|
|
python3 -c "import json,sys; print(${expression})"
|
|
}
|
|
|
|
api_curl() {
|
|
curl --retry 6 --retry-delay 2 --retry-connrefused --retry-all-errors -fsS \
|
|
-H "Authorization: Bearer ${DROBOTICFLOW_API_TOKEN}" "$@"
|
|
}
|
|
|
|
worker_curl() {
|
|
curl --retry 3 --retry-delay 1 --retry-connrefused --retry-all-errors -fsS "$@"
|
|
}
|
|
|
|
free_local_port() {
|
|
python3 - <<'PY'
|
|
import socket
|
|
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
|
|
sock.bind(("127.0.0.1", 0))
|
|
print(sock.getsockname()[1])
|
|
PY
|
|
}
|
|
|
|
PORT_FORWARD_PID=""
|
|
PORT_FORWARD_LOG="$(mktemp)"
|
|
cleanup() {
|
|
if [[ -n "$PORT_FORWARD_PID" ]] && kill -0 "$PORT_FORWARD_PID" >/dev/null 2>&1; then
|
|
kill "$PORT_FORWARD_PID" >/dev/null 2>&1 || true
|
|
wait "$PORT_FORWARD_PID" >/dev/null 2>&1 || true
|
|
fi
|
|
rm -f "$PORT_FORWARD_LOG"
|
|
}
|
|
trap cleanup EXIT
|
|
|
|
start_worker_port_forward() {
|
|
local local_port
|
|
local_port="$(free_local_port)"
|
|
kubectl -n "$K8S_NAMESPACE" port-forward "service/${WORKER_SERVICE_NAME}" "${local_port}:${WORKER_SERVICE_PORT}" >"$PORT_FORWARD_LOG" 2>&1 &
|
|
PORT_FORWARD_PID="$!"
|
|
WORKER_LOCAL_BASE_URL="http://127.0.0.1:${local_port}"
|
|
local deadline=$((SECONDS + 20))
|
|
local last_error=""
|
|
while (( SECONDS < deadline )); do
|
|
if ! kill -0 "$PORT_FORWARD_PID" >/dev/null 2>&1; then
|
|
echo "ERROR: worker port-forward exited early." >&2
|
|
tail -n 20 "$PORT_FORWARD_LOG" >&2 || true
|
|
exit 1
|
|
fi
|
|
if worker_curl "$WORKER_LOCAL_BASE_URL/worker-api/health" >/dev/null 2>&1; then
|
|
printf 'worker port-forward: %s -> service/%s:%s\n' "$WORKER_LOCAL_BASE_URL" "$WORKER_SERVICE_NAME" "$WORKER_SERVICE_PORT"
|
|
return 0
|
|
fi
|
|
last_error="worker health not ready"
|
|
sleep 0.25
|
|
done
|
|
echo "ERROR: timed out waiting for worker port-forward: ${last_error}" >&2
|
|
tail -n 20 "$PORT_FORWARD_LOG" >&2 || true
|
|
exit 1
|
|
}
|
|
|
|
api_health="$(api_curl "$API_BASE_URL/api/health")"
|
|
printf 'api health: %s\n' "$api_health"
|
|
printf '%s' "$api_health" | grep -q '"status":"ok"'
|
|
|
|
create_payload="$(WORKFLOW_ACCEPTANCE_SOURCE_PATH="$WORKFLOW_ACCEPTANCE_SOURCE_PATH" python3 - <<'PY'
|
|
import json
|
|
import os
|
|
print(json.dumps({"sourcePath": os.environ["WORKFLOW_ACCEPTANCE_SOURCE_PATH"]}))
|
|
PY
|
|
)"
|
|
acceptance_response="$(api_curl -X POST "$API_BASE_URL/api/acceptance/poc-gpu-workflow" -H 'Content-Type: application/json' -d "$create_payload")"
|
|
run_id="$(printf '%s' "$acceptance_response" | json_get 'json.load(sys.stdin)["run"]["id"]')"
|
|
workflow_id="$(printf '%s' "$acceptance_response" | json_get 'json.load(sys.stdin)["workflow"]["id"]')"
|
|
project_id="$(printf '%s' "$acceptance_response" | json_get 'json.load(sys.stdin)["project"]["id"]')"
|
|
printf 'created POC GPU acceptance project=%s workflow=%s run=%s\n' "$project_id" "$workflow_id" "$run_id"
|
|
|
|
start_worker_port_forward
|
|
|
|
deadline=$((SECONDS + WORKFLOW_ACCEPTANCE_RUN_TIMEOUT_SECONDS))
|
|
final_run=""
|
|
final_status=""
|
|
while (( SECONDS < deadline )); do
|
|
worker_curl -X POST "$WORKER_LOCAL_BASE_URL/worker-api/process-once" >/dev/null
|
|
final_run="$(api_curl "$API_BASE_URL/api/runs/$run_id")"
|
|
final_status="$(printf '%s' "$final_run" | json_get 'json.load(sys.stdin)["status"]')"
|
|
printf 'run status: %s\n' "$final_status"
|
|
if [[ "$final_status" == "$WORKFLOW_ACCEPTANCE_EXPECT_TERMINAL_STATUS" ]]; then
|
|
break
|
|
fi
|
|
case "$final_status" in
|
|
completed|failed|cancelled|waiting_for_audit)
|
|
printf 'unexpected terminal status %s, expected %s: %s\n' "$final_status" "$WORKFLOW_ACCEPTANCE_EXPECT_TERMINAL_STATUS" "$final_run" >&2
|
|
exit 1
|
|
;;
|
|
esac
|
|
sleep 2
|
|
done
|
|
|
|
if [[ "$final_status" != "$WORKFLOW_ACCEPTANCE_EXPECT_TERMINAL_STATUS" ]]; then
|
|
printf 'run did not reach expected terminal status %s before timeout: %s\n' "$WORKFLOW_ACCEPTANCE_EXPECT_TERMINAL_STATUS" "$final_run" >&2
|
|
exit 1
|
|
fi
|
|
|
|
logs_response="$(api_curl "$API_BASE_URL/api/runs/$run_id/logs")"
|
|
printf '%s' "$logs_response" | python3 -c 'import json, sys; payload = json.load(sys.stdin); items = payload.get("items", []); print("logs summary:", json.dumps({"total": payload.get("total", len(items)), "lastMessages": [item.get("message", "") for item in items[-3:]]}, ensure_ascii=False))'
|
|
|
|
printf '%s' "$logs_response" | python3 -c 'import json, sys; payload = json.load(sys.stdin); items = payload.get("items", []);
|
|
if not items: raise SystemExit("expected at least one log entry")'
|
|
|
|
if [[ -n "$WORKFLOW_ACCEPTANCE_EXPECT_LOG_SUBSTRING" ]]; then
|
|
printf '%s' "$logs_response" | WORKFLOW_ACCEPTANCE_EXPECT_LOG_SUBSTRING="$WORKFLOW_ACCEPTANCE_EXPECT_LOG_SUBSTRING" python3 -c 'import json, os, sys; payload = json.load(sys.stdin); needle = os.environ["WORKFLOW_ACCEPTANCE_EXPECT_LOG_SUBSTRING"]; messages = "\n".join(str(item.get("message", "")) for item in payload.get("items", []));
|
|
if needle not in messages: raise SystemExit(f"expected log substring not found: {needle}")'
|
|
fi
|
|
|
|
if [[ "$WORKFLOW_ACCEPTANCE_EXPECT_TERMINAL_STATUS" == "completed" ]]; then
|
|
FINAL_RUN_JSON="$final_run" python3 - <<'PY'
|
|
import json
|
|
import os
|
|
import sys
|
|
|
|
payload = json.loads(os.environ["FINAL_RUN_JSON"])
|
|
samples = []
|
|
for item in payload.get("resourceSeries") or []:
|
|
if isinstance(item, dict):
|
|
samples.append(item)
|
|
for node in payload.get("nodeExecutions") or []:
|
|
for item in node.get("resourceSeries") or []:
|
|
if isinstance(item, dict):
|
|
samples.append(item)
|
|
if not any(int(sample.get("gpuMemoryTotalMiB") or 0) > 0 for sample in samples):
|
|
raise SystemExit("expected GPU memory metrics in completed GPU workflow resourceSeries")
|
|
PY
|
|
artifacts_response="$(api_curl "$API_BASE_URL/api/runs/$run_id/artifacts")"
|
|
printf '%s' "$artifacts_response" | python3 -c 'import json, sys; payload = json.load(sys.stdin); items = payload.get("items", []); print("artifacts summary:", json.dumps({"total": payload.get("total", len(items)), "items": [{"name": item.get("name"), "type": item.get("type"), "size": item.get("size")} for item in items[:10]]}, ensure_ascii=False))'
|
|
printf '%s' "$artifacts_response" | python3 -c 'import json, sys; payload = json.load(sys.stdin); items = payload.get("items", []);
|
|
if not items: raise SystemExit("expected at least one artifact")'
|
|
fi
|