droboticflow/scripts/poc-gpu-workflow-blocked-acceptance.sh
2026-07-07 18:57:49 +08:00

276 lines
11 KiB
Bash
Executable File

#!/usr/bin/env bash
set -euo pipefail
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
DEPLOY_ENV="${DEPLOY_ENV:-}"
export DEPLOY_ENV
# shellcheck source=scripts/load-deploy-env.sh
source "$ROOT_DIR/scripts/load-deploy-env.sh"
# shellcheck source=scripts/lib/service-endpoints.sh
source "$ROOT_DIR/scripts/lib/service-endpoints.sh"
K8S_NAMESPACE="${K8S_NAMESPACE:-droboticflow}"
API_PUBLIC_SERVICE_NAME="${API_PUBLIC_SERVICE_NAME:-droboticflow-api-public}"
WORKER_SERVICE_NAME="${POC_GPU_WORKFLOW_WORKER_SERVICE_NAME:-droboticflow-worker}"
WORKER_SERVICE_PORT="${POC_GPU_WORKFLOW_WORKER_SERVICE_PORT:-3002}"
API_TOKEN_SECRET_NAME="${POC_GPU_WORKFLOW_API_TOKEN_SECRET_NAME:-droboticflow-auth-secret}"
API_TOKEN_SECRET_KEY="${POC_GPU_WORKFLOW_API_TOKEN_SECRET_KEY:-DROBOTICFLOW_API_TOKEN}"
normalize_bool() {
local name="$1"
local value="$2"
local normalized
normalized="$(printf '%s' "$value" | tr '[:upper:]' '[:lower:]')"
case "$normalized" in
true|1|yes|y|on)
printf 'true\n'
;;
false|0|no|n|off)
printf 'false\n'
;;
*)
echo "ERROR: $name must be true or false." >&2
exit 1
;;
esac
}
if [[ -z "${API_BASE_URL:-}" && -z "${BASE_URL:-}" ]]; then
public_host="$(
kubectl -n "$K8S_NAMESPACE" get svc "$API_PUBLIC_SERVICE_NAME" -o json | python3 -c '
import ipaddress
import json
import sys
payload = json.load(sys.stdin)
items = payload.get("status", {}).get("loadBalancer", {}).get("ingress", [])
hosts = [item.get("ip") or item.get("hostname") for item in items]
hosts = [item for item in hosts if item]
if not hosts:
raise SystemExit("no LoadBalancer ingress host is available")
def is_private(host: str) -> bool:
try:
return ipaddress.ip_address(host).is_private
except ValueError:
return False
for host in hosts:
if not is_private(host):
print(host)
raise SystemExit(0)
print(hosts[0])
'
)"
export API_BASE_URL="http://${public_host}"
fi
export WORKFLOW_ACCEPTANCE_SOURCE_PATH="${POC_GPU_WORKFLOW_SOURCE_PATH:-${WORKFLOW_ACCEPTANCE_SOURCE_PATH:-/workspace/datasets/fixtures/poc-operator-acceptance/xplatform}}"
export WORKFLOW_ACCEPTANCE_GPU_LIMIT="${WORKFLOW_ACCEPTANCE_GPU_LIMIT:-1}"
POC_GPU_WORKFLOW_EXPECT_AVAILABLE="$(normalize_bool "POC_GPU_WORKFLOW_EXPECT_AVAILABLE" "${POC_GPU_WORKFLOW_EXPECT_AVAILABLE:-${POC_GPU_CAPACITY_EXPECT_AVAILABLE:-true}}")"
if [[ "$POC_GPU_WORKFLOW_EXPECT_AVAILABLE" == "true" ]]; then
export WORKFLOW_ACCEPTANCE_EXPECT_TERMINAL_STATUS="${WORKFLOW_ACCEPTANCE_EXPECT_TERMINAL_STATUS:-completed}"
export WORKFLOW_ACCEPTANCE_EXPECT_LOG_SUBSTRING="${WORKFLOW_ACCEPTANCE_EXPECT_LOG_SUBSTRING:-}"
else
export WORKFLOW_ACCEPTANCE_EXPECT_TERMINAL_STATUS="${WORKFLOW_ACCEPTANCE_EXPECT_TERMINAL_STATUS:-failed}"
export WORKFLOW_ACCEPTANCE_EXPECT_LOG_SUBSTRING="${WORKFLOW_ACCEPTANCE_EXPECT_LOG_SUBSTRING:-gpu request cannot be scheduled}"
fi
export WORKFLOW_ACCEPTANCE_RUN_TIMEOUT_SECONDS="${WORKFLOW_ACCEPTANCE_RUN_TIMEOUT_SECONDS:-120}"
if [[ -n "${DROBOTICFLOW_ACCEPTANCE_EMAIL:-}" && -n "${DROBOTICFLOW_ACCEPTANCE_PASSWORD:-}" ]]; then
export WORKFLOW_ACCEPTANCE_REQUIRE_EXPLICIT_ENDPOINT=true
"$ROOT_DIR/scripts/live-workflow-acceptance.sh"
exit 0
fi
if [[ -n "${DROBOTICFLOW_ACCEPTANCE_EMAIL:-}" || -n "${DROBOTICFLOW_ACCEPTANCE_PASSWORD:-}" ]]; then
echo "ERROR: DROBOTICFLOW_ACCEPTANCE_EMAIL and DROBOTICFLOW_ACCEPTANCE_PASSWORD must be set together." >&2
exit 1
fi
if [[ -z "${DROBOTICFLOW_API_TOKEN:-}" ]]; then
api_token_secret_json="$(kubectl -n "$K8S_NAMESPACE" get secret "$API_TOKEN_SECRET_NAME" -o json 2>/dev/null || true)"
DROBOTICFLOW_API_TOKEN="$(
printf "%s" "$api_token_secret_json" | API_TOKEN_SECRET_KEY="$API_TOKEN_SECRET_KEY" python3 -c '
import base64
import json
import os
import sys
try:
payload = json.load(sys.stdin)
except json.JSONDecodeError:
raise SystemExit(0)
data = payload.get("data", {})
if not isinstance(data, dict):
raise SystemExit(0)
raw = data.get(os.environ["API_TOKEN_SECRET_KEY"])
if not isinstance(raw, str) or not raw:
raise SystemExit(0)
try:
print(base64.b64decode(raw, validate=True).decode("utf-8", errors="replace").strip(), end="")
except Exception:
raise SystemExit(0)
'
)"
export DROBOTICFLOW_API_TOKEN
fi
if [[ -z "${DROBOTICFLOW_API_TOKEN:-}" ]]; then
echo "ERROR: DROBOTICFLOW_API_TOKEN or ${K8S_NAMESPACE}/${API_TOKEN_SECRET_NAME}.${API_TOKEN_SECRET_KEY} is required for token-only POC GPU workflow acceptance when no session credentials are configured." >&2
exit 1
fi
if ! [[ "$WORKER_SERVICE_PORT" =~ ^[0-9]+$ ]] || (( WORKER_SERVICE_PORT < 1 || WORKER_SERVICE_PORT > 65535 )); then
echo "ERROR: POC_GPU_WORKFLOW_WORKER_SERVICE_PORT must be a valid TCP port." >&2
exit 1
fi
if [[ -z "$WORKER_SERVICE_NAME" || "$WORKER_SERVICE_NAME" =~ [[:space:]] ]]; then
echo "ERROR: POC_GPU_WORKFLOW_WORKER_SERVICE_NAME must be non-empty and contain no whitespace." >&2
exit 1
fi
if ! [[ "$WORKFLOW_ACCEPTANCE_RUN_TIMEOUT_SECONDS" =~ ^[0-9]+$ ]] || (( WORKFLOW_ACCEPTANCE_RUN_TIMEOUT_SECONDS < 1 )); then
echo "ERROR: WORKFLOW_ACCEPTANCE_RUN_TIMEOUT_SECONDS must be a positive integer." >&2
exit 1
fi
API_BASE_URL="$(droboticflow_first_service_url "${API_BASE_URL:-}" "${BASE_URL:-}")"
json_get() {
local expression="$1"
python3 -c "import json,sys; print(${expression})"
}
api_curl() {
curl --retry 6 --retry-delay 2 --retry-connrefused --retry-all-errors -fsS \
-H "Authorization: Bearer ${DROBOTICFLOW_API_TOKEN}" "$@"
}
worker_curl() {
curl --retry 3 --retry-delay 1 --retry-connrefused --retry-all-errors -fsS "$@"
}
free_local_port() {
python3 - <<'PY'
import socket
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
sock.bind(("127.0.0.1", 0))
print(sock.getsockname()[1])
PY
}
PORT_FORWARD_PID=""
PORT_FORWARD_LOG="$(mktemp)"
cleanup() {
if [[ -n "$PORT_FORWARD_PID" ]] && kill -0 "$PORT_FORWARD_PID" >/dev/null 2>&1; then
kill "$PORT_FORWARD_PID" >/dev/null 2>&1 || true
wait "$PORT_FORWARD_PID" >/dev/null 2>&1 || true
fi
rm -f "$PORT_FORWARD_LOG"
}
trap cleanup EXIT
start_worker_port_forward() {
local local_port
local_port="$(free_local_port)"
kubectl -n "$K8S_NAMESPACE" port-forward "service/${WORKER_SERVICE_NAME}" "${local_port}:${WORKER_SERVICE_PORT}" >"$PORT_FORWARD_LOG" 2>&1 &
PORT_FORWARD_PID="$!"
WORKER_LOCAL_BASE_URL="http://127.0.0.1:${local_port}"
local deadline=$((SECONDS + 20))
local last_error=""
while (( SECONDS < deadline )); do
if ! kill -0 "$PORT_FORWARD_PID" >/dev/null 2>&1; then
echo "ERROR: worker port-forward exited early." >&2
tail -n 20 "$PORT_FORWARD_LOG" >&2 || true
exit 1
fi
if worker_curl "$WORKER_LOCAL_BASE_URL/worker-api/health" >/dev/null 2>&1; then
printf 'worker port-forward: %s -> service/%s:%s\n' "$WORKER_LOCAL_BASE_URL" "$WORKER_SERVICE_NAME" "$WORKER_SERVICE_PORT"
return 0
fi
last_error="worker health not ready"
sleep 0.25
done
echo "ERROR: timed out waiting for worker port-forward: ${last_error}" >&2
tail -n 20 "$PORT_FORWARD_LOG" >&2 || true
exit 1
}
api_health="$(api_curl "$API_BASE_URL/api/health")"
printf 'api health: %s\n' "$api_health"
printf '%s' "$api_health" | grep -q '"status":"ok"'
create_payload="$(WORKFLOW_ACCEPTANCE_SOURCE_PATH="$WORKFLOW_ACCEPTANCE_SOURCE_PATH" python3 - <<'PY'
import json
import os
print(json.dumps({"sourcePath": os.environ["WORKFLOW_ACCEPTANCE_SOURCE_PATH"]}))
PY
)"
acceptance_response="$(api_curl -X POST "$API_BASE_URL/api/acceptance/poc-gpu-workflow" -H 'Content-Type: application/json' -d "$create_payload")"
run_id="$(printf '%s' "$acceptance_response" | json_get 'json.load(sys.stdin)["run"]["id"]')"
workflow_id="$(printf '%s' "$acceptance_response" | json_get 'json.load(sys.stdin)["workflow"]["id"]')"
project_id="$(printf '%s' "$acceptance_response" | json_get 'json.load(sys.stdin)["project"]["id"]')"
printf 'created POC GPU acceptance project=%s workflow=%s run=%s\n' "$project_id" "$workflow_id" "$run_id"
start_worker_port_forward
deadline=$((SECONDS + WORKFLOW_ACCEPTANCE_RUN_TIMEOUT_SECONDS))
final_run=""
final_status=""
while (( SECONDS < deadline )); do
worker_curl -X POST "$WORKER_LOCAL_BASE_URL/worker-api/process-once" >/dev/null
final_run="$(api_curl "$API_BASE_URL/api/runs/$run_id")"
final_status="$(printf '%s' "$final_run" | json_get 'json.load(sys.stdin)["status"]')"
printf 'run status: %s\n' "$final_status"
if [[ "$final_status" == "$WORKFLOW_ACCEPTANCE_EXPECT_TERMINAL_STATUS" ]]; then
break
fi
case "$final_status" in
completed|failed|cancelled|waiting_for_audit)
printf 'unexpected terminal status %s, expected %s: %s\n' "$final_status" "$WORKFLOW_ACCEPTANCE_EXPECT_TERMINAL_STATUS" "$final_run" >&2
exit 1
;;
esac
sleep 2
done
if [[ "$final_status" != "$WORKFLOW_ACCEPTANCE_EXPECT_TERMINAL_STATUS" ]]; then
printf 'run did not reach expected terminal status %s before timeout: %s\n' "$WORKFLOW_ACCEPTANCE_EXPECT_TERMINAL_STATUS" "$final_run" >&2
exit 1
fi
logs_response="$(api_curl "$API_BASE_URL/api/runs/$run_id/logs")"
printf '%s' "$logs_response" | python3 -c 'import json, sys; payload = json.load(sys.stdin); items = payload.get("items", []); print("logs summary:", json.dumps({"total": payload.get("total", len(items)), "lastMessages": [item.get("message", "") for item in items[-3:]]}, ensure_ascii=False))'
printf '%s' "$logs_response" | python3 -c 'import json, sys; payload = json.load(sys.stdin); items = payload.get("items", []);
if not items: raise SystemExit("expected at least one log entry")'
if [[ -n "$WORKFLOW_ACCEPTANCE_EXPECT_LOG_SUBSTRING" ]]; then
printf '%s' "$logs_response" | WORKFLOW_ACCEPTANCE_EXPECT_LOG_SUBSTRING="$WORKFLOW_ACCEPTANCE_EXPECT_LOG_SUBSTRING" python3 -c 'import json, os, sys; payload = json.load(sys.stdin); needle = os.environ["WORKFLOW_ACCEPTANCE_EXPECT_LOG_SUBSTRING"]; messages = "\n".join(str(item.get("message", "")) for item in payload.get("items", []));
if needle not in messages: raise SystemExit(f"expected log substring not found: {needle}")'
fi
if [[ "$WORKFLOW_ACCEPTANCE_EXPECT_TERMINAL_STATUS" == "completed" ]]; then
FINAL_RUN_JSON="$final_run" python3 - <<'PY'
import json
import os
import sys
payload = json.loads(os.environ["FINAL_RUN_JSON"])
samples = []
for item in payload.get("resourceSeries") or []:
if isinstance(item, dict):
samples.append(item)
for node in payload.get("nodeExecutions") or []:
for item in node.get("resourceSeries") or []:
if isinstance(item, dict):
samples.append(item)
if not any(int(sample.get("gpuMemoryTotalMiB") or 0) > 0 for sample in samples):
raise SystemExit("expected GPU memory metrics in completed GPU workflow resourceSeries")
PY
artifacts_response="$(api_curl "$API_BASE_URL/api/runs/$run_id/artifacts")"
printf '%s' "$artifacts_response" | python3 -c 'import json, sys; payload = json.load(sys.stdin); items = payload.get("items", []); print("artifacts summary:", json.dumps({"total": payload.get("total", len(items)), "items": [{"name": item.get("name"), "type": item.get("type"), "size": item.get("size")} for item in items[:10]]}, ensure_ascii=False))'
printf '%s' "$artifacts_response" | python3 -c 'import json, sys; payload = json.load(sys.stdin); items = payload.get("items", []);
if not items: raise SystemExit("expected at least one artifact")'
fi