#!/usr/bin/env bash set -euo pipefail ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" DEPLOY_ENV="${DEPLOY_ENV:-}" export DEPLOY_ENV # shellcheck source=scripts/load-deploy-env.sh source "$ROOT_DIR/scripts/load-deploy-env.sh" # shellcheck source=scripts/lib/service-endpoints.sh source "$ROOT_DIR/scripts/lib/service-endpoints.sh" K8S_NAMESPACE="${K8S_NAMESPACE:-droboticflow}" API_PUBLIC_SERVICE_NAME="${API_PUBLIC_SERVICE_NAME:-droboticflow-api-public}" WORKER_SERVICE_NAME="${POC_GPU_WORKFLOW_WORKER_SERVICE_NAME:-droboticflow-worker}" WORKER_SERVICE_PORT="${POC_GPU_WORKFLOW_WORKER_SERVICE_PORT:-3002}" API_TOKEN_SECRET_NAME="${POC_GPU_WORKFLOW_API_TOKEN_SECRET_NAME:-droboticflow-auth-secret}" API_TOKEN_SECRET_KEY="${POC_GPU_WORKFLOW_API_TOKEN_SECRET_KEY:-DROBOTICFLOW_API_TOKEN}" normalize_bool() { local name="$1" local value="$2" local normalized normalized="$(printf '%s' "$value" | tr '[:upper:]' '[:lower:]')" case "$normalized" in true|1|yes|y|on) printf 'true\n' ;; false|0|no|n|off) printf 'false\n' ;; *) echo "ERROR: $name must be true or false." >&2 exit 1 ;; esac } if [[ -z "${API_BASE_URL:-}" && -z "${BASE_URL:-}" ]]; then public_host="$( kubectl -n "$K8S_NAMESPACE" get svc "$API_PUBLIC_SERVICE_NAME" -o json | python3 -c ' import ipaddress import json import sys payload = json.load(sys.stdin) items = payload.get("status", {}).get("loadBalancer", {}).get("ingress", []) hosts = [item.get("ip") or item.get("hostname") for item in items] hosts = [item for item in hosts if item] if not hosts: raise SystemExit("no LoadBalancer ingress host is available") def is_private(host: str) -> bool: try: return ipaddress.ip_address(host).is_private except ValueError: return False for host in hosts: if not is_private(host): print(host) raise SystemExit(0) print(hosts[0]) ' )" export API_BASE_URL="http://${public_host}" fi export WORKFLOW_ACCEPTANCE_SOURCE_PATH="${POC_GPU_WORKFLOW_SOURCE_PATH:-${WORKFLOW_ACCEPTANCE_SOURCE_PATH:-/workspace/datasets/fixtures/poc-operator-acceptance/xplatform}}" export WORKFLOW_ACCEPTANCE_GPU_LIMIT="${WORKFLOW_ACCEPTANCE_GPU_LIMIT:-1}" POC_GPU_WORKFLOW_EXPECT_AVAILABLE="$(normalize_bool "POC_GPU_WORKFLOW_EXPECT_AVAILABLE" "${POC_GPU_WORKFLOW_EXPECT_AVAILABLE:-${POC_GPU_CAPACITY_EXPECT_AVAILABLE:-true}}")" if [[ "$POC_GPU_WORKFLOW_EXPECT_AVAILABLE" == "true" ]]; then export WORKFLOW_ACCEPTANCE_EXPECT_TERMINAL_STATUS="${WORKFLOW_ACCEPTANCE_EXPECT_TERMINAL_STATUS:-completed}" export WORKFLOW_ACCEPTANCE_EXPECT_LOG_SUBSTRING="${WORKFLOW_ACCEPTANCE_EXPECT_LOG_SUBSTRING:-}" else export WORKFLOW_ACCEPTANCE_EXPECT_TERMINAL_STATUS="${WORKFLOW_ACCEPTANCE_EXPECT_TERMINAL_STATUS:-failed}" export WORKFLOW_ACCEPTANCE_EXPECT_LOG_SUBSTRING="${WORKFLOW_ACCEPTANCE_EXPECT_LOG_SUBSTRING:-gpu request cannot be scheduled}" fi export WORKFLOW_ACCEPTANCE_RUN_TIMEOUT_SECONDS="${WORKFLOW_ACCEPTANCE_RUN_TIMEOUT_SECONDS:-120}" if [[ -n "${DROBOTICFLOW_ACCEPTANCE_EMAIL:-}" && -n "${DROBOTICFLOW_ACCEPTANCE_PASSWORD:-}" ]]; then export WORKFLOW_ACCEPTANCE_REQUIRE_EXPLICIT_ENDPOINT=true "$ROOT_DIR/scripts/live-workflow-acceptance.sh" exit 0 fi if [[ -n "${DROBOTICFLOW_ACCEPTANCE_EMAIL:-}" || -n "${DROBOTICFLOW_ACCEPTANCE_PASSWORD:-}" ]]; then echo "ERROR: DROBOTICFLOW_ACCEPTANCE_EMAIL and DROBOTICFLOW_ACCEPTANCE_PASSWORD must be set together." >&2 exit 1 fi if [[ -z "${DROBOTICFLOW_API_TOKEN:-}" ]]; then api_token_secret_json="$(kubectl -n "$K8S_NAMESPACE" get secret "$API_TOKEN_SECRET_NAME" -o json 2>/dev/null || true)" DROBOTICFLOW_API_TOKEN="$( printf "%s" "$api_token_secret_json" | API_TOKEN_SECRET_KEY="$API_TOKEN_SECRET_KEY" python3 -c ' import base64 import json import os import sys try: payload = json.load(sys.stdin) except json.JSONDecodeError: raise SystemExit(0) data = payload.get("data", {}) if not isinstance(data, dict): raise SystemExit(0) raw = data.get(os.environ["API_TOKEN_SECRET_KEY"]) if not isinstance(raw, str) or not raw: raise SystemExit(0) try: print(base64.b64decode(raw, validate=True).decode("utf-8", errors="replace").strip(), end="") except Exception: raise SystemExit(0) ' )" export DROBOTICFLOW_API_TOKEN fi if [[ -z "${DROBOTICFLOW_API_TOKEN:-}" ]]; then echo "ERROR: DROBOTICFLOW_API_TOKEN or ${K8S_NAMESPACE}/${API_TOKEN_SECRET_NAME}.${API_TOKEN_SECRET_KEY} is required for token-only POC GPU workflow acceptance when no session credentials are configured." >&2 exit 1 fi if ! [[ "$WORKER_SERVICE_PORT" =~ ^[0-9]+$ ]] || (( WORKER_SERVICE_PORT < 1 || WORKER_SERVICE_PORT > 65535 )); then echo "ERROR: POC_GPU_WORKFLOW_WORKER_SERVICE_PORT must be a valid TCP port." >&2 exit 1 fi if [[ -z "$WORKER_SERVICE_NAME" || "$WORKER_SERVICE_NAME" =~ [[:space:]] ]]; then echo "ERROR: POC_GPU_WORKFLOW_WORKER_SERVICE_NAME must be non-empty and contain no whitespace." >&2 exit 1 fi if ! [[ "$WORKFLOW_ACCEPTANCE_RUN_TIMEOUT_SECONDS" =~ ^[0-9]+$ ]] || (( WORKFLOW_ACCEPTANCE_RUN_TIMEOUT_SECONDS < 1 )); then echo "ERROR: WORKFLOW_ACCEPTANCE_RUN_TIMEOUT_SECONDS must be a positive integer." >&2 exit 1 fi API_BASE_URL="$(droboticflow_first_service_url "${API_BASE_URL:-}" "${BASE_URL:-}")" json_get() { local expression="$1" python3 -c "import json,sys; print(${expression})" } api_curl() { curl --retry 6 --retry-delay 2 --retry-connrefused --retry-all-errors -fsS \ -H "Authorization: Bearer ${DROBOTICFLOW_API_TOKEN}" "$@" } worker_curl() { curl --retry 3 --retry-delay 1 --retry-connrefused --retry-all-errors -fsS "$@" } free_local_port() { python3 - <<'PY' import socket with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: sock.bind(("127.0.0.1", 0)) print(sock.getsockname()[1]) PY } PORT_FORWARD_PID="" PORT_FORWARD_LOG="$(mktemp)" cleanup() { if [[ -n "$PORT_FORWARD_PID" ]] && kill -0 "$PORT_FORWARD_PID" >/dev/null 2>&1; then kill "$PORT_FORWARD_PID" >/dev/null 2>&1 || true wait "$PORT_FORWARD_PID" >/dev/null 2>&1 || true fi rm -f "$PORT_FORWARD_LOG" } trap cleanup EXIT start_worker_port_forward() { local local_port local_port="$(free_local_port)" kubectl -n "$K8S_NAMESPACE" port-forward "service/${WORKER_SERVICE_NAME}" "${local_port}:${WORKER_SERVICE_PORT}" >"$PORT_FORWARD_LOG" 2>&1 & PORT_FORWARD_PID="$!" WORKER_LOCAL_BASE_URL="http://127.0.0.1:${local_port}" local deadline=$((SECONDS + 20)) local last_error="" while (( SECONDS < deadline )); do if ! kill -0 "$PORT_FORWARD_PID" >/dev/null 2>&1; then echo "ERROR: worker port-forward exited early." >&2 tail -n 20 "$PORT_FORWARD_LOG" >&2 || true exit 1 fi if worker_curl "$WORKER_LOCAL_BASE_URL/worker-api/health" >/dev/null 2>&1; then printf 'worker port-forward: %s -> service/%s:%s\n' "$WORKER_LOCAL_BASE_URL" "$WORKER_SERVICE_NAME" "$WORKER_SERVICE_PORT" return 0 fi last_error="worker health not ready" sleep 0.25 done echo "ERROR: timed out waiting for worker port-forward: ${last_error}" >&2 tail -n 20 "$PORT_FORWARD_LOG" >&2 || true exit 1 } api_health="$(api_curl "$API_BASE_URL/api/health")" printf 'api health: %s\n' "$api_health" printf '%s' "$api_health" | grep -q '"status":"ok"' create_payload="$(WORKFLOW_ACCEPTANCE_SOURCE_PATH="$WORKFLOW_ACCEPTANCE_SOURCE_PATH" python3 - <<'PY' import json import os print(json.dumps({"sourcePath": os.environ["WORKFLOW_ACCEPTANCE_SOURCE_PATH"]})) PY )" acceptance_response="$(api_curl -X POST "$API_BASE_URL/api/acceptance/poc-gpu-workflow" -H 'Content-Type: application/json' -d "$create_payload")" run_id="$(printf '%s' "$acceptance_response" | json_get 'json.load(sys.stdin)["run"]["id"]')" workflow_id="$(printf '%s' "$acceptance_response" | json_get 'json.load(sys.stdin)["workflow"]["id"]')" project_id="$(printf '%s' "$acceptance_response" | json_get 'json.load(sys.stdin)["project"]["id"]')" printf 'created POC GPU acceptance project=%s workflow=%s run=%s\n' "$project_id" "$workflow_id" "$run_id" start_worker_port_forward deadline=$((SECONDS + WORKFLOW_ACCEPTANCE_RUN_TIMEOUT_SECONDS)) final_run="" final_status="" while (( SECONDS < deadline )); do worker_curl -X POST "$WORKER_LOCAL_BASE_URL/worker-api/process-once" >/dev/null final_run="$(api_curl "$API_BASE_URL/api/runs/$run_id")" final_status="$(printf '%s' "$final_run" | json_get 'json.load(sys.stdin)["status"]')" printf 'run status: %s\n' "$final_status" if [[ "$final_status" == "$WORKFLOW_ACCEPTANCE_EXPECT_TERMINAL_STATUS" ]]; then break fi case "$final_status" in completed|failed|cancelled|waiting_for_audit) printf 'unexpected terminal status %s, expected %s: %s\n' "$final_status" "$WORKFLOW_ACCEPTANCE_EXPECT_TERMINAL_STATUS" "$final_run" >&2 exit 1 ;; esac sleep 2 done if [[ "$final_status" != "$WORKFLOW_ACCEPTANCE_EXPECT_TERMINAL_STATUS" ]]; then printf 'run did not reach expected terminal status %s before timeout: %s\n' "$WORKFLOW_ACCEPTANCE_EXPECT_TERMINAL_STATUS" "$final_run" >&2 exit 1 fi logs_response="$(api_curl "$API_BASE_URL/api/runs/$run_id/logs")" printf '%s' "$logs_response" | python3 -c 'import json, sys; payload = json.load(sys.stdin); items = payload.get("items", []); print("logs summary:", json.dumps({"total": payload.get("total", len(items)), "lastMessages": [item.get("message", "") for item in items[-3:]]}, ensure_ascii=False))' printf '%s' "$logs_response" | python3 -c 'import json, sys; payload = json.load(sys.stdin); items = payload.get("items", []); if not items: raise SystemExit("expected at least one log entry")' if [[ -n "$WORKFLOW_ACCEPTANCE_EXPECT_LOG_SUBSTRING" ]]; then printf '%s' "$logs_response" | WORKFLOW_ACCEPTANCE_EXPECT_LOG_SUBSTRING="$WORKFLOW_ACCEPTANCE_EXPECT_LOG_SUBSTRING" python3 -c 'import json, os, sys; payload = json.load(sys.stdin); needle = os.environ["WORKFLOW_ACCEPTANCE_EXPECT_LOG_SUBSTRING"]; messages = "\n".join(str(item.get("message", "")) for item in payload.get("items", [])); if needle not in messages: raise SystemExit(f"expected log substring not found: {needle}")' fi if [[ "$WORKFLOW_ACCEPTANCE_EXPECT_TERMINAL_STATUS" == "completed" ]]; then FINAL_RUN_JSON="$final_run" python3 - <<'PY' import json import os import sys payload = json.loads(os.environ["FINAL_RUN_JSON"]) samples = [] for item in payload.get("resourceSeries") or []: if isinstance(item, dict): samples.append(item) for node in payload.get("nodeExecutions") or []: for item in node.get("resourceSeries") or []: if isinstance(item, dict): samples.append(item) if not any(int(sample.get("gpuMemoryTotalMiB") or 0) > 0 for sample in samples): raise SystemExit("expected GPU memory metrics in completed GPU workflow resourceSeries") PY artifacts_response="$(api_curl "$API_BASE_URL/api/runs/$run_id/artifacts")" printf '%s' "$artifacts_response" | python3 -c 'import json, sys; payload = json.load(sys.stdin); items = payload.get("items", []); print("artifacts summary:", json.dumps({"total": payload.get("total", len(items)), "items": [{"name": item.get("name"), "type": item.get("type"), "size": item.get("size")} for item in items[:10]]}, ensure_ascii=False))' printf '%s' "$artifacts_response" | python3 -c 'import json, sys; payload = json.load(sys.stdin); items = payload.get("items", []); if not items: raise SystemExit("expected at least one artifact")' fi