droboticflow/scripts/wilor-operator-workflow-acceptance.sh

404 lines
16 KiB
Bash
Executable File

#!/usr/bin/env bash
: "${DROBOTICFLOW_ACCEPTANCE_PASSWORD:?Set DROBOTICFLOW_ACCEPTANCE_PASSWORD before running acceptance}"
set -euo pipefail
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
DEPLOY_ENV="${DEPLOY_ENV:-}"
export DEPLOY_ENV
# shellcheck source=scripts/load-deploy-env.sh
source "$ROOT_DIR/scripts/load-deploy-env.sh"
# shellcheck source=scripts/lib/service-endpoints.sh
source "$ROOT_DIR/scripts/lib/service-endpoints.sh"
K8S_NAMESPACE="${K8S_NAMESPACE:-droboticflow}"
API_PUBLIC_SERVICE_NAME="${API_PUBLIC_SERVICE_NAME:-droboticflow-api-public}"
WORKER_SERVICE_NAME="${WILOR_WORKFLOW_WORKER_SERVICE_NAME:-droboticflow-worker}"
WORKER_SERVICE_PORT="${WILOR_WORKFLOW_WORKER_SERVICE_PORT:-3002}"
API_TOKEN_SECRET_NAME="${WILOR_WORKFLOW_API_TOKEN_SECRET_NAME:-droboticflow-auth-secret}"
API_TOKEN_SECRET_KEY="${WILOR_WORKFLOW_API_TOKEN_SECRET_KEY:-DROBOTICFLOW_API_TOKEN}"
DROBOTICFLOW_ACCEPTANCE_EMAIL="${DROBOTICFLOW_ACCEPTANCE_EMAIL:-poc-acceptance-admin@droboticflow.local}"
WILOR_WORKFLOW_RUN_TIMEOUT_SECONDS="${WILOR_WORKFLOW_RUN_TIMEOUT_SECONDS:-900}"
WILOR_API_BASE_URL="${WILOR_API_BASE_URL:-http://120.48.65.232}"
WILOR_WORKFLOW_INPUT_URL="${WILOR_WORKFLOW_INPUT_URL:-http://120.48.65.232/examples/sample-hands.png}"
WILOR_WORKFLOW_MEDIA_KIND="${WILOR_WORKFLOW_MEDIA_KIND:-image}"
WILOR_WORKFLOW_IO_OPERATOR_IMAGE="${WILOR_WORKFLOW_IO_OPERATOR_IMAGE:-ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc/droboticflow-operator-runner:wilor-hand-pose-io}"
WILOR_WORKFLOW_IO_OPERATOR_IMAGE_ID="${WILOR_WORKFLOW_IO_OPERATOR_IMAGE_ID:-ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc/droboticflow-operator-runner@sha256:8ffb868d204edcf2a31a95bd436f85e2284b2e8ba97e67b3b634337048bd3ec8}"
if [[ -z "${API_BASE_URL:-}" && -z "${BASE_URL:-}" ]]; then
public_host="$(
kubectl -n "$K8S_NAMESPACE" get svc "$API_PUBLIC_SERVICE_NAME" -o json | python3 -c '
import ipaddress
import json
import sys
payload = json.load(sys.stdin)
hosts = [item.get("ip") or item.get("hostname") for item in payload.get("status", {}).get("loadBalancer", {}).get("ingress", [])]
hosts = [item for item in hosts if item]
if not hosts:
raise SystemExit("no LoadBalancer ingress host is available")
for host in hosts:
try:
if not ipaddress.ip_address(host).is_private:
print(host)
raise SystemExit(0)
except ValueError:
print(host)
raise SystemExit(0)
print(hosts[0])
'
)"
export API_BASE_URL="http://${public_host}"
fi
API_BASE_URL="$(droboticflow_first_service_url "${API_BASE_URL:-}" "${BASE_URL:-}")"
if [[ -z "${DROBOTICFLOW_API_TOKEN:-}" ]]; then
api_token_secret_json="$(kubectl -n "$K8S_NAMESPACE" get secret "$API_TOKEN_SECRET_NAME" -o json 2>/dev/null || true)"
DROBOTICFLOW_API_TOKEN="$(
printf "%s" "$api_token_secret_json" | API_TOKEN_SECRET_KEY="$API_TOKEN_SECRET_KEY" python3 -c '
import base64
import json
import os
import sys
try:
payload = json.load(sys.stdin)
except json.JSONDecodeError:
raise SystemExit(0)
raw = payload.get("data", {}).get(os.environ["API_TOKEN_SECRET_KEY"])
if isinstance(raw, str) and raw:
print(base64.b64decode(raw, validate=True).decode("utf-8", errors="replace").strip(), end="")
'
)"
export DROBOTICFLOW_API_TOKEN
fi
if [[ -z "${DROBOTICFLOW_API_TOKEN:-}" ]]; then
echo "ERROR: DROBOTICFLOW_API_TOKEN or ${K8S_NAMESPACE}/${API_TOKEN_SECRET_NAME}.${API_TOKEN_SECRET_KEY} is required." >&2
exit 1
fi
json_get() {
local expression="$1"
python3 -c "import json,sys; print(${expression})"
}
COOKIE_JAR="$(mktemp)"
PORT_FORWARD_PID=""
PORT_FORWARD_LOG="$(mktemp)"
WORKER_LOCAL_BASE_URL=""
api_curl() {
curl --retry 6 --retry-delay 2 --retry-connrefused --retry-all-errors -fsS \
-b "$COOKIE_JAR" -c "$COOKIE_JAR" -H "Authorization: Bearer ${DROBOTICFLOW_API_TOKEN}" "$@"
}
csrf_token() {
awk '$6 == "droboticflow_csrf" { value=$7 } END { print value }' "$COOKIE_JAR"
}
api_curl_write() {
local token
token="$(csrf_token)"
if [[ -n "$token" ]]; then
api_curl -H "X-Droboticflow-CSRF-Token: $token" "$@"
return
fi
api_curl "$@"
}
free_local_port() {
python3 - <<'PY'
import socket
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
sock.bind(("127.0.0.1", 0))
print(sock.getsockname()[1])
PY
}
stop_worker_port_forward() {
if [[ -n "$PORT_FORWARD_PID" ]] && kill -0 "$PORT_FORWARD_PID" >/dev/null 2>&1; then
kill "$PORT_FORWARD_PID" >/dev/null 2>&1 || true
wait "$PORT_FORWARD_PID" >/dev/null 2>&1 || true
fi
PORT_FORWARD_PID=""
WORKER_LOCAL_BASE_URL=""
}
cleanup() {
stop_worker_port_forward
rm -f "$COOKIE_JAR" "$PORT_FORWARD_LOG"
}
trap cleanup EXIT
start_worker_port_forward() {
local local_port
stop_worker_port_forward
: >"$PORT_FORWARD_LOG"
local_port="$(free_local_port)"
kubectl -n "$K8S_NAMESPACE" port-forward "service/${WORKER_SERVICE_NAME}" "${local_port}:${WORKER_SERVICE_PORT}" >"$PORT_FORWARD_LOG" 2>&1 &
PORT_FORWARD_PID="$!"
WORKER_LOCAL_BASE_URL="http://127.0.0.1:${local_port}"
local deadline=$((SECONDS + 20))
while (( SECONDS < deadline )); do
if ! kill -0 "$PORT_FORWARD_PID" >/dev/null 2>&1; then
tail -n 20 "$PORT_FORWARD_LOG" >&2 || true
return 1
fi
if curl -fsS "$WORKER_LOCAL_BASE_URL/worker-api/health" >/dev/null 2>&1; then
return 0
fi
sleep 0.25
done
tail -n 20 "$PORT_FORWARD_LOG" >&2 || true
return 1
}
run_needs_process_once() {
RUN_JSON="$1" python3 - <<'PY'
import datetime as dt
import json
import os
run = json.loads(os.environ["RUN_JSON"])
if run.get("status") != "queued":
raise SystemExit(1)
for node in run.get("nodeExecutions") or []:
if node.get("status") in {"running", "waiting_for_audit", "queued"}:
raise SystemExit(1)
owner = str(run.get("executionLeaseOwner") or "").strip()
expires_raw = str(run.get("executionLeaseExpiresAt") or "").strip()
if owner and expires_raw:
try:
expires = dt.datetime.fromisoformat(expires_raw.replace("Z", "+00:00"))
except ValueError:
expires = None
if expires and expires > dt.datetime.now(dt.timezone.utc):
raise SystemExit(1)
raise SystemExit(0)
PY
}
maybe_process_once() {
local run_json="$1"
if ! run_needs_process_once "$run_json"; then
return 0
fi
if [[ -n "$WORKER_LOCAL_BASE_URL" ]]; then
curl -fsS -X POST "$WORKER_LOCAL_BASE_URL/worker-api/process-once" >/dev/null || true
fi
}
api_health="$(api_curl "$API_BASE_URL/api/health")"
printf 'api health: %s\n' "$api_health"
printf '%s' "$api_health" | grep -q '"status":"ok"'
login_payload="$(
DROBOTICFLOW_ACCEPTANCE_EMAIL="$DROBOTICFLOW_ACCEPTANCE_EMAIL" \
DROBOTICFLOW_ACCEPTANCE_PASSWORD="$DROBOTICFLOW_ACCEPTANCE_PASSWORD" python3 - <<'PY'
import json
import os
print(json.dumps({"email": os.environ["DROBOTICFLOW_ACCEPTANCE_EMAIL"], "password": os.environ["DROBOTICFLOW_ACCEPTANCE_PASSWORD"]}))
PY
)"
api_curl -H 'Content-Type: application/json' -X POST "$API_BASE_URL/api/auth/login" -d "$login_payload" >/dev/null
project_payload="$(python3 - <<'PY'
import json
print(json.dumps({"name": "WiLoR Operator Verification", "description": "Source -> WiLoR -> result upload verification", "status": "active"}))
PY
)"
project_response="$(api_curl_write -X POST "$API_BASE_URL/api/projects" -H 'Content-Type: application/json' -d "$project_payload")"
project_id="$(printf '%s' "$project_response" | json_get 'json.load(sys.stdin)["id"]')"
printf 'created project: %s\n' "$project_id"
workflow_payload="$(
ROOT_DIR="$ROOT_DIR" WILOR_API_BASE_URL="$WILOR_API_BASE_URL" WILOR_WORKFLOW_INPUT_URL="$WILOR_WORKFLOW_INPUT_URL" WILOR_WORKFLOW_MEDIA_KIND="$WILOR_WORKFLOW_MEDIA_KIND" WILOR_WORKFLOW_IO_OPERATOR_IMAGE="$WILOR_WORKFLOW_IO_OPERATOR_IMAGE" WILOR_WORKFLOW_IO_OPERATOR_IMAGE_ID="$WILOR_WORKFLOW_IO_OPERATOR_IMAGE_ID" python3 - <<'PY'
import json
import os
import pathlib
import urllib.parse
root = pathlib.Path(os.environ["ROOT_DIR"])
catalog = json.loads((root / "internal/api/builtin_operators_catalog.json").read_text(encoding="utf-8"))
operators = {item.get("id"): item for item in catalog}
source_operator = operators["operator-builtin-real-source"]
wilor_operator = operators["operator-builtin-wilor-hand-pose"]
upload_operator = operators["operator-builtin-upload"]
input_url = os.environ["WILOR_WORKFLOW_INPUT_URL"]
media_kind = os.environ["WILOR_WORKFLOW_MEDIA_KIND"]
io_operator_image = os.environ["WILOR_WORKFLOW_IO_OPERATOR_IMAGE"].strip()
io_operator_image_id = os.environ["WILOR_WORKFLOW_IO_OPERATOR_IMAGE_ID"].strip()
source_name = pathlib.PurePosixPath(urllib.parse.urlparse(input_url).path).name or "wilor-input"
def node_data(operator, hyperparams):
resources = operator.get("resources") or {}
data = {
"label": operator["name"],
"category": operator["category"],
"description": operator["description"],
"operatorId": operator["id"],
"operatorImageId": operator["imageId"],
"image": operator["image"],
"containerCommand": operator.get("containerCommand", []),
"containerArgs": operator.get("containerArgs", []),
"runtimeContract": operator.get("runtimeContract", {}),
"skillDefinition": operator.get("skillDefinition", ""),
"inputs": operator.get("inputs") or [],
"outputs": operator.get("outputs") or [],
"hyperparams": hyperparams,
}
if resources.get("cpuLimit"):
data["cpu"] = resources["cpuLimit"]
if resources.get("memoryLimit"):
data["memory"] = resources["memoryLimit"]
if operator["id"] in {"operator-builtin-real-source", "operator-builtin-upload"} and io_operator_image:
data["image"] = io_operator_image
data["operatorImageId"] = io_operator_image_id
return data
source_hyperparams = [
{"key": "DATA_SOURCE", "value": "INLINE", "description": "Use an explicit media URL manifest as workflow input"},
{
"key": "SOURCE_JSON",
"value": json.dumps({"name": source_name, "url": input_url, "mediaKind": media_kind}, ensure_ascii=False),
"description": "Input media reference consumed by the WiLoR node",
},
]
wilor_hyperparams = []
overrides = {
"WILOR_API_BASE_URL": os.environ["WILOR_API_BASE_URL"],
"INPUT_URL": "",
"INPUT_PATH": "",
"MEDIA_KIND": media_kind,
}
for item in wilor_operator.get("hyperparams", []):
next_item = dict(item)
if next_item.get("key") in overrides:
next_item["value"] = overrides[next_item["key"]]
wilor_hyperparams.append(next_item)
print(json.dumps({
"name": "WiLoR 3D Hand Pose Verification",
"description": "Read an explicit media input, run WiLoR inference, and persist the produced artifacts",
"type": "batch",
"nodes": [
{"id": "source-1", "type": "operator", "data": node_data(source_operator, source_hyperparams), "position": {"x": 0, "y": 0}},
{"id": "wilor-1", "type": "operator", "data": node_data(wilor_operator, wilor_hyperparams), "position": {"x": 360, "y": 0}},
{"id": "upload-1", "type": "operator", "data": node_data(upload_operator, []), "position": {"x": 720, "y": 0}},
],
"edges": [
{"id": "edge-source-wilor", "source": "source-1", "sourceHandle": "out-1", "target": "wilor-1", "targetHandle": "in-1"},
{"id": "edge-wilor-upload", "source": "wilor-1", "sourceHandle": "out-1", "target": "upload-1", "targetHandle": "in-1"},
],
}, ensure_ascii=False))
PY
)"
workflow_response="$(api_curl_write -X POST "$API_BASE_URL/api/projects/$project_id/workflows" -H 'Content-Type: application/json' -d "$workflow_payload")"
workflow_id="$(printf '%s' "$workflow_response" | json_get 'json.load(sys.stdin)["id"]')"
printf 'created workflow: %s\n' "$workflow_id"
queued_run="$(api_curl_write -X POST "$API_BASE_URL/api/workflows/$workflow_id/run" -H 'Content-Type: application/json' -d '{"trigger":"manual"}')"
run_id="$(printf '%s' "$queued_run" | json_get 'json.load(sys.stdin)["id"]')"
printf 'queued run: %s\n' "$run_id"
start_worker_port_forward || printf 'warning: worker port-forward unavailable; relying on background worker polling.\n' >&2
deadline=$((SECONDS + WILOR_WORKFLOW_RUN_TIMEOUT_SECONDS))
final_run=""
while (( SECONDS < deadline )); do
run_json="$(api_curl "$API_BASE_URL/api/runs/$run_id")"
maybe_process_once "$run_json"
status="$(printf '%s' "$run_json" | json_get 'json.load(sys.stdin)["status"]')"
printf 'run status: %s\n' "$status"
if [[ "$status" == "completed" || "$status" == "failed" || "$status" == "cancelled" || "$status" == "waiting_for_audit" ]]; then
final_run="$run_json"
break
fi
sleep 5
done
if [[ -z "$final_run" ]]; then
final_run="$(api_curl "$API_BASE_URL/api/runs/$run_id")"
fi
artifacts_response="$(api_curl "$API_BASE_URL/api/runs/$run_id/artifacts")"
manifest_artifact_id="$(
printf '%s' "$artifacts_response" | python3 -c '
import json
import sys
payload = json.load(sys.stdin)
items = payload.get("items") if isinstance(payload, dict) else payload
for item in items or []:
if item.get("name") == "wilor-result.json" and item.get("nodeId") == "wilor-1":
print(item.get("id", ""))
break
'
)"
if [[ -z "$manifest_artifact_id" ]]; then
echo "ERROR: wilor-result.json artifact was not produced." >&2
printf '%s\n' "$artifacts_response" >&2
exit 1
fi
manifest_content="$(api_curl "$API_BASE_URL/api/runs/$run_id/artifacts/$manifest_artifact_id/content")"
upload_artifact_id="$(
printf '%s' "$artifacts_response" | python3 -c '
import json
import sys
payload = json.load(sys.stdin)
items = payload.get("items") if isinstance(payload, dict) else payload
for item in items or []:
if item.get("name") == "uploaded-results.json" and item.get("nodeId") == "upload-1":
print(item.get("id", ""))
break
'
)"
if [[ -z "$upload_artifact_id" ]]; then
echo "ERROR: uploaded-results.json artifact was not produced by upload-1." >&2
printf '%s\n' "$artifacts_response" >&2
exit 1
fi
upload_content="$(api_curl "$API_BASE_URL/api/runs/$run_id/artifacts/$upload_artifact_id/content")"
FINAL_RUN="$final_run" ARTIFACTS_RESPONSE="$artifacts_response" MANIFEST_CONTENT="$manifest_content" UPLOAD_CONTENT="$upload_content" python3 - <<'PY'
import json
import os
run = json.loads(os.environ["FINAL_RUN"])
artifacts_payload = json.loads(os.environ["ARTIFACTS_RESPONSE"])
manifest = json.loads(os.environ["MANIFEST_CONTENT"])
upload = json.loads(os.environ["UPLOAD_CONTENT"])
items = artifacts_payload.get("items") if isinstance(artifacts_payload, dict) else artifacts_payload
items = items or []
names = sorted(item.get("name", "") for item in items)
node_statuses = {item.get("nodeId"): item.get("status") for item in run.get("nodeExecutions") or []}
if run.get("status") != "completed":
raise SystemExit(f"run did not complete: {run.get('status')}")
for node_id in ("source-1", "wilor-1", "upload-1"):
if node_statuses.get(node_id) != "completed":
raise SystemExit(f"{node_id} did not complete: {node_statuses}")
if manifest.get("status") != "success":
raise SystemExit(f"WiLoR manifest status is not success: {manifest.get('status')}")
service_model = ((manifest.get("service") or {}).get("health") or {}).get("model") or {}
if service_model.get("device") != "cuda":
raise SystemExit(f"WiLoR service did not report cuda device: {service_model}")
if "A800" not in str(service_model.get("cudaDeviceName", "")):
raise SystemExit(f"WiLoR service did not report A800 GPU: {service_model}")
detected = ((manifest.get("result") or {}).get("detectedHands") or 0)
if int(detected) < 1:
raise SystemExit(f"WiLoR detected no hands: {detected}")
if "source-manifest.json" not in names:
raise SystemExit("missing source-manifest.json")
if "wilor-result.json" not in names:
raise SystemExit("missing wilor-result.json")
if not ({"wilor-annotated-image.png", "wilor-annotated-video.mp4"} & set(names)):
raise SystemExit(f"missing annotated media artifact: {names}")
if "uploaded-results.json" not in names:
raise SystemExit("missing uploaded-results.json")
if int(upload.get("inputCount") or 0) < 1:
raise SystemExit(f"upload node did not receive upstream artifacts: {upload}")
wilor_execution = next((item for item in run.get("nodeExecutions") or [] if item.get("nodeId") == "wilor-1"), {})
print("wilor workflow acceptance:", json.dumps({
"runId": run.get("id"),
"workflowId": run.get("workflowId"),
"status": run.get("status"),
"nodeStatuses": node_statuses,
"detectedHands": detected,
"cudaDeviceName": service_model.get("cudaDeviceName"),
"artifacts": names,
"uploadInputCount": upload.get("inputCount"),
"operatorImageId": wilor_execution.get("operatorImageId", ""),
}, ensure_ascii=False))
PY