#!/usr/bin/env bash : "${DROBOTICFLOW_ACCEPTANCE_PASSWORD:?Set DROBOTICFLOW_ACCEPTANCE_PASSWORD before running acceptance}" set -euo pipefail ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" DEPLOY_ENV="${DEPLOY_ENV:-}" export DEPLOY_ENV # shellcheck source=scripts/load-deploy-env.sh source "$ROOT_DIR/scripts/load-deploy-env.sh" # shellcheck source=scripts/lib/service-endpoints.sh source "$ROOT_DIR/scripts/lib/service-endpoints.sh" K8S_NAMESPACE="${K8S_NAMESPACE:-droboticflow}" API_PUBLIC_SERVICE_NAME="${API_PUBLIC_SERVICE_NAME:-droboticflow-api-public}" WORKER_SERVICE_NAME="${WILOR_WORKFLOW_WORKER_SERVICE_NAME:-droboticflow-worker}" WORKER_SERVICE_PORT="${WILOR_WORKFLOW_WORKER_SERVICE_PORT:-3002}" API_TOKEN_SECRET_NAME="${WILOR_WORKFLOW_API_TOKEN_SECRET_NAME:-droboticflow-auth-secret}" API_TOKEN_SECRET_KEY="${WILOR_WORKFLOW_API_TOKEN_SECRET_KEY:-DROBOTICFLOW_API_TOKEN}" DROBOTICFLOW_ACCEPTANCE_EMAIL="${DROBOTICFLOW_ACCEPTANCE_EMAIL:-poc-acceptance-admin@droboticflow.local}" WILOR_WORKFLOW_RUN_TIMEOUT_SECONDS="${WILOR_WORKFLOW_RUN_TIMEOUT_SECONDS:-900}" WILOR_API_BASE_URL="${WILOR_API_BASE_URL:-http://120.48.65.232}" WILOR_WORKFLOW_INPUT_URL="${WILOR_WORKFLOW_INPUT_URL:-http://120.48.65.232/examples/sample-hands.png}" WILOR_WORKFLOW_MEDIA_KIND="${WILOR_WORKFLOW_MEDIA_KIND:-image}" WILOR_WORKFLOW_IO_OPERATOR_IMAGE="${WILOR_WORKFLOW_IO_OPERATOR_IMAGE:-ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc/droboticflow-operator-runner:wilor-hand-pose-io}" WILOR_WORKFLOW_IO_OPERATOR_IMAGE_ID="${WILOR_WORKFLOW_IO_OPERATOR_IMAGE_ID:-ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc/droboticflow-operator-runner@sha256:8ffb868d204edcf2a31a95bd436f85e2284b2e8ba97e67b3b634337048bd3ec8}" if [[ -z "${API_BASE_URL:-}" && -z "${BASE_URL:-}" ]]; then public_host="$( kubectl -n "$K8S_NAMESPACE" get svc "$API_PUBLIC_SERVICE_NAME" -o json | python3 -c ' import ipaddress import json import sys payload = json.load(sys.stdin) hosts = [item.get("ip") or item.get("hostname") for item in payload.get("status", {}).get("loadBalancer", {}).get("ingress", [])] hosts = [item for item in hosts if item] if not hosts: raise SystemExit("no LoadBalancer ingress host is available") for host in hosts: try: if not ipaddress.ip_address(host).is_private: print(host) raise SystemExit(0) except ValueError: print(host) raise SystemExit(0) print(hosts[0]) ' )" export API_BASE_URL="http://${public_host}" fi API_BASE_URL="$(droboticflow_first_service_url "${API_BASE_URL:-}" "${BASE_URL:-}")" if [[ -z "${DROBOTICFLOW_API_TOKEN:-}" ]]; then api_token_secret_json="$(kubectl -n "$K8S_NAMESPACE" get secret "$API_TOKEN_SECRET_NAME" -o json 2>/dev/null || true)" DROBOTICFLOW_API_TOKEN="$( printf "%s" "$api_token_secret_json" | API_TOKEN_SECRET_KEY="$API_TOKEN_SECRET_KEY" python3 -c ' import base64 import json import os import sys try: payload = json.load(sys.stdin) except json.JSONDecodeError: raise SystemExit(0) raw = payload.get("data", {}).get(os.environ["API_TOKEN_SECRET_KEY"]) if isinstance(raw, str) and raw: print(base64.b64decode(raw, validate=True).decode("utf-8", errors="replace").strip(), end="") ' )" export DROBOTICFLOW_API_TOKEN fi if [[ -z "${DROBOTICFLOW_API_TOKEN:-}" ]]; then echo "ERROR: DROBOTICFLOW_API_TOKEN or ${K8S_NAMESPACE}/${API_TOKEN_SECRET_NAME}.${API_TOKEN_SECRET_KEY} is required." >&2 exit 1 fi json_get() { local expression="$1" python3 -c "import json,sys; print(${expression})" } COOKIE_JAR="$(mktemp)" PORT_FORWARD_PID="" PORT_FORWARD_LOG="$(mktemp)" WORKER_LOCAL_BASE_URL="" api_curl() { curl --retry 6 --retry-delay 2 --retry-connrefused --retry-all-errors -fsS \ -b "$COOKIE_JAR" -c "$COOKIE_JAR" -H "Authorization: Bearer ${DROBOTICFLOW_API_TOKEN}" "$@" } csrf_token() { awk '$6 == "droboticflow_csrf" { value=$7 } END { print value }' "$COOKIE_JAR" } api_curl_write() { local token token="$(csrf_token)" if [[ -n "$token" ]]; then api_curl -H "X-Droboticflow-CSRF-Token: $token" "$@" return fi api_curl "$@" } free_local_port() { python3 - <<'PY' import socket with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: sock.bind(("127.0.0.1", 0)) print(sock.getsockname()[1]) PY } stop_worker_port_forward() { if [[ -n "$PORT_FORWARD_PID" ]] && kill -0 "$PORT_FORWARD_PID" >/dev/null 2>&1; then kill "$PORT_FORWARD_PID" >/dev/null 2>&1 || true wait "$PORT_FORWARD_PID" >/dev/null 2>&1 || true fi PORT_FORWARD_PID="" WORKER_LOCAL_BASE_URL="" } cleanup() { stop_worker_port_forward rm -f "$COOKIE_JAR" "$PORT_FORWARD_LOG" } trap cleanup EXIT start_worker_port_forward() { local local_port stop_worker_port_forward : >"$PORT_FORWARD_LOG" local_port="$(free_local_port)" kubectl -n "$K8S_NAMESPACE" port-forward "service/${WORKER_SERVICE_NAME}" "${local_port}:${WORKER_SERVICE_PORT}" >"$PORT_FORWARD_LOG" 2>&1 & PORT_FORWARD_PID="$!" WORKER_LOCAL_BASE_URL="http://127.0.0.1:${local_port}" local deadline=$((SECONDS + 20)) while (( SECONDS < deadline )); do if ! kill -0 "$PORT_FORWARD_PID" >/dev/null 2>&1; then tail -n 20 "$PORT_FORWARD_LOG" >&2 || true return 1 fi if curl -fsS "$WORKER_LOCAL_BASE_URL/worker-api/health" >/dev/null 2>&1; then return 0 fi sleep 0.25 done tail -n 20 "$PORT_FORWARD_LOG" >&2 || true return 1 } run_needs_process_once() { RUN_JSON="$1" python3 - <<'PY' import datetime as dt import json import os run = json.loads(os.environ["RUN_JSON"]) if run.get("status") != "queued": raise SystemExit(1) for node in run.get("nodeExecutions") or []: if node.get("status") in {"running", "waiting_for_audit", "queued"}: raise SystemExit(1) owner = str(run.get("executionLeaseOwner") or "").strip() expires_raw = str(run.get("executionLeaseExpiresAt") or "").strip() if owner and expires_raw: try: expires = dt.datetime.fromisoformat(expires_raw.replace("Z", "+00:00")) except ValueError: expires = None if expires and expires > dt.datetime.now(dt.timezone.utc): raise SystemExit(1) raise SystemExit(0) PY } maybe_process_once() { local run_json="$1" if ! run_needs_process_once "$run_json"; then return 0 fi if [[ -n "$WORKER_LOCAL_BASE_URL" ]]; then curl -fsS -X POST "$WORKER_LOCAL_BASE_URL/worker-api/process-once" >/dev/null || true fi } api_health="$(api_curl "$API_BASE_URL/api/health")" printf 'api health: %s\n' "$api_health" printf '%s' "$api_health" | grep -q '"status":"ok"' login_payload="$( DROBOTICFLOW_ACCEPTANCE_EMAIL="$DROBOTICFLOW_ACCEPTANCE_EMAIL" \ DROBOTICFLOW_ACCEPTANCE_PASSWORD="$DROBOTICFLOW_ACCEPTANCE_PASSWORD" python3 - <<'PY' import json import os print(json.dumps({"email": os.environ["DROBOTICFLOW_ACCEPTANCE_EMAIL"], "password": os.environ["DROBOTICFLOW_ACCEPTANCE_PASSWORD"]})) PY )" api_curl -H 'Content-Type: application/json' -X POST "$API_BASE_URL/api/auth/login" -d "$login_payload" >/dev/null project_payload="$(python3 - <<'PY' import json print(json.dumps({"name": "WiLoR Operator Verification", "description": "Source -> WiLoR -> result upload verification", "status": "active"})) PY )" project_response="$(api_curl_write -X POST "$API_BASE_URL/api/projects" -H 'Content-Type: application/json' -d "$project_payload")" project_id="$(printf '%s' "$project_response" | json_get 'json.load(sys.stdin)["id"]')" printf 'created project: %s\n' "$project_id" workflow_payload="$( ROOT_DIR="$ROOT_DIR" WILOR_API_BASE_URL="$WILOR_API_BASE_URL" WILOR_WORKFLOW_INPUT_URL="$WILOR_WORKFLOW_INPUT_URL" WILOR_WORKFLOW_MEDIA_KIND="$WILOR_WORKFLOW_MEDIA_KIND" WILOR_WORKFLOW_IO_OPERATOR_IMAGE="$WILOR_WORKFLOW_IO_OPERATOR_IMAGE" WILOR_WORKFLOW_IO_OPERATOR_IMAGE_ID="$WILOR_WORKFLOW_IO_OPERATOR_IMAGE_ID" python3 - <<'PY' import json import os import pathlib import urllib.parse root = pathlib.Path(os.environ["ROOT_DIR"]) catalog = json.loads((root / "internal/api/builtin_operators_catalog.json").read_text(encoding="utf-8")) operators = {item.get("id"): item for item in catalog} source_operator = operators["operator-builtin-real-source"] wilor_operator = operators["operator-builtin-wilor-hand-pose"] upload_operator = operators["operator-builtin-upload"] input_url = os.environ["WILOR_WORKFLOW_INPUT_URL"] media_kind = os.environ["WILOR_WORKFLOW_MEDIA_KIND"] io_operator_image = os.environ["WILOR_WORKFLOW_IO_OPERATOR_IMAGE"].strip() io_operator_image_id = os.environ["WILOR_WORKFLOW_IO_OPERATOR_IMAGE_ID"].strip() source_name = pathlib.PurePosixPath(urllib.parse.urlparse(input_url).path).name or "wilor-input" def node_data(operator, hyperparams): resources = operator.get("resources") or {} data = { "label": operator["name"], "category": operator["category"], "description": operator["description"], "operatorId": operator["id"], "operatorImageId": operator["imageId"], "image": operator["image"], "containerCommand": operator.get("containerCommand", []), "containerArgs": operator.get("containerArgs", []), "runtimeContract": operator.get("runtimeContract", {}), "skillDefinition": operator.get("skillDefinition", ""), "inputs": operator.get("inputs") or [], "outputs": operator.get("outputs") or [], "hyperparams": hyperparams, } if resources.get("cpuLimit"): data["cpu"] = resources["cpuLimit"] if resources.get("memoryLimit"): data["memory"] = resources["memoryLimit"] if operator["id"] in {"operator-builtin-real-source", "operator-builtin-upload"} and io_operator_image: data["image"] = io_operator_image data["operatorImageId"] = io_operator_image_id return data source_hyperparams = [ {"key": "DATA_SOURCE", "value": "INLINE", "description": "Use an explicit media URL manifest as workflow input"}, { "key": "SOURCE_JSON", "value": json.dumps({"name": source_name, "url": input_url, "mediaKind": media_kind}, ensure_ascii=False), "description": "Input media reference consumed by the WiLoR node", }, ] wilor_hyperparams = [] overrides = { "WILOR_API_BASE_URL": os.environ["WILOR_API_BASE_URL"], "INPUT_URL": "", "INPUT_PATH": "", "MEDIA_KIND": media_kind, } for item in wilor_operator.get("hyperparams", []): next_item = dict(item) if next_item.get("key") in overrides: next_item["value"] = overrides[next_item["key"]] wilor_hyperparams.append(next_item) print(json.dumps({ "name": "WiLoR 3D Hand Pose Verification", "description": "Read an explicit media input, run WiLoR inference, and persist the produced artifacts", "type": "batch", "nodes": [ {"id": "source-1", "type": "operator", "data": node_data(source_operator, source_hyperparams), "position": {"x": 0, "y": 0}}, {"id": "wilor-1", "type": "operator", "data": node_data(wilor_operator, wilor_hyperparams), "position": {"x": 360, "y": 0}}, {"id": "upload-1", "type": "operator", "data": node_data(upload_operator, []), "position": {"x": 720, "y": 0}}, ], "edges": [ {"id": "edge-source-wilor", "source": "source-1", "sourceHandle": "out-1", "target": "wilor-1", "targetHandle": "in-1"}, {"id": "edge-wilor-upload", "source": "wilor-1", "sourceHandle": "out-1", "target": "upload-1", "targetHandle": "in-1"}, ], }, ensure_ascii=False)) PY )" workflow_response="$(api_curl_write -X POST "$API_BASE_URL/api/projects/$project_id/workflows" -H 'Content-Type: application/json' -d "$workflow_payload")" workflow_id="$(printf '%s' "$workflow_response" | json_get 'json.load(sys.stdin)["id"]')" printf 'created workflow: %s\n' "$workflow_id" queued_run="$(api_curl_write -X POST "$API_BASE_URL/api/workflows/$workflow_id/run" -H 'Content-Type: application/json' -d '{"trigger":"manual"}')" run_id="$(printf '%s' "$queued_run" | json_get 'json.load(sys.stdin)["id"]')" printf 'queued run: %s\n' "$run_id" start_worker_port_forward || printf 'warning: worker port-forward unavailable; relying on background worker polling.\n' >&2 deadline=$((SECONDS + WILOR_WORKFLOW_RUN_TIMEOUT_SECONDS)) final_run="" while (( SECONDS < deadline )); do run_json="$(api_curl "$API_BASE_URL/api/runs/$run_id")" maybe_process_once "$run_json" status="$(printf '%s' "$run_json" | json_get 'json.load(sys.stdin)["status"]')" printf 'run status: %s\n' "$status" if [[ "$status" == "completed" || "$status" == "failed" || "$status" == "cancelled" || "$status" == "waiting_for_audit" ]]; then final_run="$run_json" break fi sleep 5 done if [[ -z "$final_run" ]]; then final_run="$(api_curl "$API_BASE_URL/api/runs/$run_id")" fi artifacts_response="$(api_curl "$API_BASE_URL/api/runs/$run_id/artifacts")" manifest_artifact_id="$( printf '%s' "$artifacts_response" | python3 -c ' import json import sys payload = json.load(sys.stdin) items = payload.get("items") if isinstance(payload, dict) else payload for item in items or []: if item.get("name") == "wilor-result.json" and item.get("nodeId") == "wilor-1": print(item.get("id", "")) break ' )" if [[ -z "$manifest_artifact_id" ]]; then echo "ERROR: wilor-result.json artifact was not produced." >&2 printf '%s\n' "$artifacts_response" >&2 exit 1 fi manifest_content="$(api_curl "$API_BASE_URL/api/runs/$run_id/artifacts/$manifest_artifact_id/content")" upload_artifact_id="$( printf '%s' "$artifacts_response" | python3 -c ' import json import sys payload = json.load(sys.stdin) items = payload.get("items") if isinstance(payload, dict) else payload for item in items or []: if item.get("name") == "uploaded-results.json" and item.get("nodeId") == "upload-1": print(item.get("id", "")) break ' )" if [[ -z "$upload_artifact_id" ]]; then echo "ERROR: uploaded-results.json artifact was not produced by upload-1." >&2 printf '%s\n' "$artifacts_response" >&2 exit 1 fi upload_content="$(api_curl "$API_BASE_URL/api/runs/$run_id/artifacts/$upload_artifact_id/content")" FINAL_RUN="$final_run" ARTIFACTS_RESPONSE="$artifacts_response" MANIFEST_CONTENT="$manifest_content" UPLOAD_CONTENT="$upload_content" python3 - <<'PY' import json import os run = json.loads(os.environ["FINAL_RUN"]) artifacts_payload = json.loads(os.environ["ARTIFACTS_RESPONSE"]) manifest = json.loads(os.environ["MANIFEST_CONTENT"]) upload = json.loads(os.environ["UPLOAD_CONTENT"]) items = artifacts_payload.get("items") if isinstance(artifacts_payload, dict) else artifacts_payload items = items or [] names = sorted(item.get("name", "") for item in items) node_statuses = {item.get("nodeId"): item.get("status") for item in run.get("nodeExecutions") or []} if run.get("status") != "completed": raise SystemExit(f"run did not complete: {run.get('status')}") for node_id in ("source-1", "wilor-1", "upload-1"): if node_statuses.get(node_id) != "completed": raise SystemExit(f"{node_id} did not complete: {node_statuses}") if manifest.get("status") != "success": raise SystemExit(f"WiLoR manifest status is not success: {manifest.get('status')}") service_model = ((manifest.get("service") or {}).get("health") or {}).get("model") or {} if service_model.get("device") != "cuda": raise SystemExit(f"WiLoR service did not report cuda device: {service_model}") if "A800" not in str(service_model.get("cudaDeviceName", "")): raise SystemExit(f"WiLoR service did not report A800 GPU: {service_model}") detected = ((manifest.get("result") or {}).get("detectedHands") or 0) if int(detected) < 1: raise SystemExit(f"WiLoR detected no hands: {detected}") if "source-manifest.json" not in names: raise SystemExit("missing source-manifest.json") if "wilor-result.json" not in names: raise SystemExit("missing wilor-result.json") if not ({"wilor-annotated-image.png", "wilor-annotated-video.mp4"} & set(names)): raise SystemExit(f"missing annotated media artifact: {names}") if "uploaded-results.json" not in names: raise SystemExit("missing uploaded-results.json") if int(upload.get("inputCount") or 0) < 1: raise SystemExit(f"upload node did not receive upstream artifacts: {upload}") wilor_execution = next((item for item in run.get("nodeExecutions") or [] if item.get("nodeId") == "wilor-1"), {}) print("wilor workflow acceptance:", json.dumps({ "runId": run.get("id"), "workflowId": run.get("workflowId"), "status": run.get("status"), "nodeStatuses": node_statuses, "detectedHands": detected, "cudaDeviceName": service_model.get("cudaDeviceName"), "artifacts": names, "uploadInputCount": upload.get("inputCount"), "operatorImageId": wilor_execution.get("operatorImageId", ""), }, ensure_ascii=False)) PY