forked from longtao.wu/droboticflow
404 lines
16 KiB
Bash
Executable File
404 lines
16 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
|
DEPLOY_ENV="${DEPLOY_ENV:-}"
|
|
export DEPLOY_ENV
|
|
# shellcheck source=scripts/load-deploy-env.sh
|
|
source "$ROOT_DIR/scripts/load-deploy-env.sh"
|
|
# shellcheck source=scripts/lib/service-endpoints.sh
|
|
source "$ROOT_DIR/scripts/lib/service-endpoints.sh"
|
|
|
|
K8S_NAMESPACE="${K8S_NAMESPACE:-droboticflow}"
|
|
API_PUBLIC_SERVICE_NAME="${API_PUBLIC_SERVICE_NAME:-droboticflow-api-public}"
|
|
WORKER_SERVICE_NAME="${WILOR_WORKFLOW_WORKER_SERVICE_NAME:-droboticflow-worker}"
|
|
WORKER_SERVICE_PORT="${WILOR_WORKFLOW_WORKER_SERVICE_PORT:-3002}"
|
|
API_TOKEN_SECRET_NAME="${WILOR_WORKFLOW_API_TOKEN_SECRET_NAME:-droboticflow-auth-secret}"
|
|
API_TOKEN_SECRET_KEY="${WILOR_WORKFLOW_API_TOKEN_SECRET_KEY:-DROBOTICFLOW_API_TOKEN}"
|
|
DROBOTICFLOW_ACCEPTANCE_EMAIL="${DROBOTICFLOW_ACCEPTANCE_EMAIL:-poc-acceptance-admin@droboticflow.local}"
|
|
DROBOTICFLOW_ACCEPTANCE_PASSWORD="${DROBOTICFLOW_ACCEPTANCE_PASSWORD:-youbixuanpoc}"
|
|
WILOR_WORKFLOW_RUN_TIMEOUT_SECONDS="${WILOR_WORKFLOW_RUN_TIMEOUT_SECONDS:-900}"
|
|
WILOR_API_BASE_URL="${WILOR_API_BASE_URL:-http://120.48.65.232}"
|
|
WILOR_WORKFLOW_INPUT_URL="${WILOR_WORKFLOW_INPUT_URL:-http://120.48.65.232/examples/sample-hands.png}"
|
|
WILOR_WORKFLOW_MEDIA_KIND="${WILOR_WORKFLOW_MEDIA_KIND:-image}"
|
|
WILOR_WORKFLOW_IO_OPERATOR_IMAGE="${WILOR_WORKFLOW_IO_OPERATOR_IMAGE:-ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc/droboticflow-operator-runner:wilor-hand-pose-io}"
|
|
WILOR_WORKFLOW_IO_OPERATOR_IMAGE_ID="${WILOR_WORKFLOW_IO_OPERATOR_IMAGE_ID:-ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc/droboticflow-operator-runner@sha256:8ffb868d204edcf2a31a95bd436f85e2284b2e8ba97e67b3b634337048bd3ec8}"
|
|
|
|
if [[ -z "${API_BASE_URL:-}" && -z "${BASE_URL:-}" ]]; then
|
|
public_host="$(
|
|
kubectl -n "$K8S_NAMESPACE" get svc "$API_PUBLIC_SERVICE_NAME" -o json | python3 -c '
|
|
import ipaddress
|
|
import json
|
|
import sys
|
|
payload = json.load(sys.stdin)
|
|
hosts = [item.get("ip") or item.get("hostname") for item in payload.get("status", {}).get("loadBalancer", {}).get("ingress", [])]
|
|
hosts = [item for item in hosts if item]
|
|
if not hosts:
|
|
raise SystemExit("no LoadBalancer ingress host is available")
|
|
for host in hosts:
|
|
try:
|
|
if not ipaddress.ip_address(host).is_private:
|
|
print(host)
|
|
raise SystemExit(0)
|
|
except ValueError:
|
|
print(host)
|
|
raise SystemExit(0)
|
|
print(hosts[0])
|
|
'
|
|
)"
|
|
export API_BASE_URL="http://${public_host}"
|
|
fi
|
|
API_BASE_URL="$(droboticflow_first_service_url "${API_BASE_URL:-}" "${BASE_URL:-}")"
|
|
|
|
if [[ -z "${DROBOTICFLOW_API_TOKEN:-}" ]]; then
|
|
api_token_secret_json="$(kubectl -n "$K8S_NAMESPACE" get secret "$API_TOKEN_SECRET_NAME" -o json 2>/dev/null || true)"
|
|
DROBOTICFLOW_API_TOKEN="$(
|
|
printf "%s" "$api_token_secret_json" | API_TOKEN_SECRET_KEY="$API_TOKEN_SECRET_KEY" python3 -c '
|
|
import base64
|
|
import json
|
|
import os
|
|
import sys
|
|
try:
|
|
payload = json.load(sys.stdin)
|
|
except json.JSONDecodeError:
|
|
raise SystemExit(0)
|
|
raw = payload.get("data", {}).get(os.environ["API_TOKEN_SECRET_KEY"])
|
|
if isinstance(raw, str) and raw:
|
|
print(base64.b64decode(raw, validate=True).decode("utf-8", errors="replace").strip(), end="")
|
|
'
|
|
)"
|
|
export DROBOTICFLOW_API_TOKEN
|
|
fi
|
|
if [[ -z "${DROBOTICFLOW_API_TOKEN:-}" ]]; then
|
|
echo "ERROR: DROBOTICFLOW_API_TOKEN or ${K8S_NAMESPACE}/${API_TOKEN_SECRET_NAME}.${API_TOKEN_SECRET_KEY} is required." >&2
|
|
exit 1
|
|
fi
|
|
|
|
json_get() {
|
|
local expression="$1"
|
|
python3 -c "import json,sys; print(${expression})"
|
|
}
|
|
|
|
COOKIE_JAR="$(mktemp)"
|
|
PORT_FORWARD_PID=""
|
|
PORT_FORWARD_LOG="$(mktemp)"
|
|
WORKER_LOCAL_BASE_URL=""
|
|
|
|
api_curl() {
|
|
curl --retry 6 --retry-delay 2 --retry-connrefused --retry-all-errors -fsS \
|
|
-b "$COOKIE_JAR" -c "$COOKIE_JAR" -H "Authorization: Bearer ${DROBOTICFLOW_API_TOKEN}" "$@"
|
|
}
|
|
|
|
csrf_token() {
|
|
awk '$6 == "droboticflow_csrf" { value=$7 } END { print value }' "$COOKIE_JAR"
|
|
}
|
|
|
|
api_curl_write() {
|
|
local token
|
|
token="$(csrf_token)"
|
|
if [[ -n "$token" ]]; then
|
|
api_curl -H "X-Droboticflow-CSRF-Token: $token" "$@"
|
|
return
|
|
fi
|
|
api_curl "$@"
|
|
}
|
|
|
|
free_local_port() {
|
|
python3 - <<'PY'
|
|
import socket
|
|
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
|
|
sock.bind(("127.0.0.1", 0))
|
|
print(sock.getsockname()[1])
|
|
PY
|
|
}
|
|
|
|
stop_worker_port_forward() {
|
|
if [[ -n "$PORT_FORWARD_PID" ]] && kill -0 "$PORT_FORWARD_PID" >/dev/null 2>&1; then
|
|
kill "$PORT_FORWARD_PID" >/dev/null 2>&1 || true
|
|
wait "$PORT_FORWARD_PID" >/dev/null 2>&1 || true
|
|
fi
|
|
PORT_FORWARD_PID=""
|
|
WORKER_LOCAL_BASE_URL=""
|
|
}
|
|
|
|
cleanup() {
|
|
stop_worker_port_forward
|
|
rm -f "$COOKIE_JAR" "$PORT_FORWARD_LOG"
|
|
}
|
|
trap cleanup EXIT
|
|
|
|
start_worker_port_forward() {
|
|
local local_port
|
|
stop_worker_port_forward
|
|
: >"$PORT_FORWARD_LOG"
|
|
local_port="$(free_local_port)"
|
|
kubectl -n "$K8S_NAMESPACE" port-forward "service/${WORKER_SERVICE_NAME}" "${local_port}:${WORKER_SERVICE_PORT}" >"$PORT_FORWARD_LOG" 2>&1 &
|
|
PORT_FORWARD_PID="$!"
|
|
WORKER_LOCAL_BASE_URL="http://127.0.0.1:${local_port}"
|
|
local deadline=$((SECONDS + 20))
|
|
while (( SECONDS < deadline )); do
|
|
if ! kill -0 "$PORT_FORWARD_PID" >/dev/null 2>&1; then
|
|
tail -n 20 "$PORT_FORWARD_LOG" >&2 || true
|
|
return 1
|
|
fi
|
|
if curl -fsS "$WORKER_LOCAL_BASE_URL/worker-api/health" >/dev/null 2>&1; then
|
|
return 0
|
|
fi
|
|
sleep 0.25
|
|
done
|
|
tail -n 20 "$PORT_FORWARD_LOG" >&2 || true
|
|
return 1
|
|
}
|
|
|
|
run_needs_process_once() {
|
|
RUN_JSON="$1" python3 - <<'PY'
|
|
import datetime as dt
|
|
import json
|
|
import os
|
|
run = json.loads(os.environ["RUN_JSON"])
|
|
if run.get("status") != "queued":
|
|
raise SystemExit(1)
|
|
for node in run.get("nodeExecutions") or []:
|
|
if node.get("status") in {"running", "waiting_for_audit", "queued"}:
|
|
raise SystemExit(1)
|
|
owner = str(run.get("executionLeaseOwner") or "").strip()
|
|
expires_raw = str(run.get("executionLeaseExpiresAt") or "").strip()
|
|
if owner and expires_raw:
|
|
try:
|
|
expires = dt.datetime.fromisoformat(expires_raw.replace("Z", "+00:00"))
|
|
except ValueError:
|
|
expires = None
|
|
if expires and expires > dt.datetime.now(dt.timezone.utc):
|
|
raise SystemExit(1)
|
|
raise SystemExit(0)
|
|
PY
|
|
}
|
|
|
|
maybe_process_once() {
|
|
local run_json="$1"
|
|
if ! run_needs_process_once "$run_json"; then
|
|
return 0
|
|
fi
|
|
if [[ -n "$WORKER_LOCAL_BASE_URL" ]]; then
|
|
curl -fsS -X POST "$WORKER_LOCAL_BASE_URL/worker-api/process-once" >/dev/null || true
|
|
fi
|
|
}
|
|
|
|
api_health="$(api_curl "$API_BASE_URL/api/health")"
|
|
printf 'api health: %s\n' "$api_health"
|
|
printf '%s' "$api_health" | grep -q '"status":"ok"'
|
|
|
|
login_payload="$(
|
|
DROBOTICFLOW_ACCEPTANCE_EMAIL="$DROBOTICFLOW_ACCEPTANCE_EMAIL" \
|
|
DROBOTICFLOW_ACCEPTANCE_PASSWORD="$DROBOTICFLOW_ACCEPTANCE_PASSWORD" python3 - <<'PY'
|
|
import json
|
|
import os
|
|
print(json.dumps({"email": os.environ["DROBOTICFLOW_ACCEPTANCE_EMAIL"], "password": os.environ["DROBOTICFLOW_ACCEPTANCE_PASSWORD"]}))
|
|
PY
|
|
)"
|
|
api_curl -H 'Content-Type: application/json' -X POST "$API_BASE_URL/api/auth/login" -d "$login_payload" >/dev/null
|
|
|
|
project_payload="$(python3 - <<'PY'
|
|
import json
|
|
print(json.dumps({"name": "WiLoR Operator Verification", "description": "Source -> WiLoR -> result upload verification", "status": "active"}))
|
|
PY
|
|
)"
|
|
project_response="$(api_curl_write -X POST "$API_BASE_URL/api/projects" -H 'Content-Type: application/json' -d "$project_payload")"
|
|
project_id="$(printf '%s' "$project_response" | json_get 'json.load(sys.stdin)["id"]')"
|
|
printf 'created project: %s\n' "$project_id"
|
|
|
|
workflow_payload="$(
|
|
ROOT_DIR="$ROOT_DIR" WILOR_API_BASE_URL="$WILOR_API_BASE_URL" WILOR_WORKFLOW_INPUT_URL="$WILOR_WORKFLOW_INPUT_URL" WILOR_WORKFLOW_MEDIA_KIND="$WILOR_WORKFLOW_MEDIA_KIND" WILOR_WORKFLOW_IO_OPERATOR_IMAGE="$WILOR_WORKFLOW_IO_OPERATOR_IMAGE" WILOR_WORKFLOW_IO_OPERATOR_IMAGE_ID="$WILOR_WORKFLOW_IO_OPERATOR_IMAGE_ID" python3 - <<'PY'
|
|
import json
|
|
import os
|
|
import pathlib
|
|
import urllib.parse
|
|
root = pathlib.Path(os.environ["ROOT_DIR"])
|
|
catalog = json.loads((root / "internal/api/builtin_operators_catalog.json").read_text(encoding="utf-8"))
|
|
operators = {item.get("id"): item for item in catalog}
|
|
source_operator = operators["operator-builtin-real-source"]
|
|
wilor_operator = operators["operator-builtin-wilor-hand-pose"]
|
|
upload_operator = operators["operator-builtin-upload"]
|
|
input_url = os.environ["WILOR_WORKFLOW_INPUT_URL"]
|
|
media_kind = os.environ["WILOR_WORKFLOW_MEDIA_KIND"]
|
|
io_operator_image = os.environ["WILOR_WORKFLOW_IO_OPERATOR_IMAGE"].strip()
|
|
io_operator_image_id = os.environ["WILOR_WORKFLOW_IO_OPERATOR_IMAGE_ID"].strip()
|
|
source_name = pathlib.PurePosixPath(urllib.parse.urlparse(input_url).path).name or "wilor-input"
|
|
|
|
def node_data(operator, hyperparams):
|
|
resources = operator.get("resources") or {}
|
|
data = {
|
|
"label": operator["name"],
|
|
"category": operator["category"],
|
|
"description": operator["description"],
|
|
"operatorId": operator["id"],
|
|
"operatorImageId": operator["imageId"],
|
|
"image": operator["image"],
|
|
"containerCommand": operator.get("containerCommand", []),
|
|
"containerArgs": operator.get("containerArgs", []),
|
|
"runtimeContract": operator.get("runtimeContract", {}),
|
|
"skillDefinition": operator.get("skillDefinition", ""),
|
|
"inputs": operator.get("inputs") or [],
|
|
"outputs": operator.get("outputs") or [],
|
|
"hyperparams": hyperparams,
|
|
}
|
|
if resources.get("cpuLimit"):
|
|
data["cpu"] = resources["cpuLimit"]
|
|
if resources.get("memoryLimit"):
|
|
data["memory"] = resources["memoryLimit"]
|
|
if operator["id"] in {"operator-builtin-real-source", "operator-builtin-upload"} and io_operator_image:
|
|
data["image"] = io_operator_image
|
|
data["operatorImageId"] = io_operator_image_id
|
|
return data
|
|
|
|
source_hyperparams = [
|
|
{"key": "DATA_SOURCE", "value": "INLINE", "description": "Use an explicit media URL manifest as workflow input"},
|
|
{
|
|
"key": "SOURCE_JSON",
|
|
"value": json.dumps({"name": source_name, "url": input_url, "mediaKind": media_kind}, ensure_ascii=False),
|
|
"description": "Input media reference consumed by the WiLoR node",
|
|
},
|
|
]
|
|
wilor_hyperparams = []
|
|
overrides = {
|
|
"WILOR_API_BASE_URL": os.environ["WILOR_API_BASE_URL"],
|
|
"INPUT_URL": "",
|
|
"INPUT_PATH": "",
|
|
"MEDIA_KIND": media_kind,
|
|
}
|
|
for item in wilor_operator.get("hyperparams", []):
|
|
next_item = dict(item)
|
|
if next_item.get("key") in overrides:
|
|
next_item["value"] = overrides[next_item["key"]]
|
|
wilor_hyperparams.append(next_item)
|
|
print(json.dumps({
|
|
"name": "WiLoR 3D Hand Pose Verification",
|
|
"description": "Read an explicit media input, run WiLoR inference, and persist the produced artifacts",
|
|
"type": "batch",
|
|
"nodes": [
|
|
{"id": "source-1", "type": "operator", "data": node_data(source_operator, source_hyperparams), "position": {"x": 0, "y": 0}},
|
|
{"id": "wilor-1", "type": "operator", "data": node_data(wilor_operator, wilor_hyperparams), "position": {"x": 360, "y": 0}},
|
|
{"id": "upload-1", "type": "operator", "data": node_data(upload_operator, []), "position": {"x": 720, "y": 0}},
|
|
],
|
|
"edges": [
|
|
{"id": "edge-source-wilor", "source": "source-1", "sourceHandle": "out-1", "target": "wilor-1", "targetHandle": "in-1"},
|
|
{"id": "edge-wilor-upload", "source": "wilor-1", "sourceHandle": "out-1", "target": "upload-1", "targetHandle": "in-1"},
|
|
],
|
|
}, ensure_ascii=False))
|
|
PY
|
|
)"
|
|
workflow_response="$(api_curl_write -X POST "$API_BASE_URL/api/projects/$project_id/workflows" -H 'Content-Type: application/json' -d "$workflow_payload")"
|
|
workflow_id="$(printf '%s' "$workflow_response" | json_get 'json.load(sys.stdin)["id"]')"
|
|
printf 'created workflow: %s\n' "$workflow_id"
|
|
|
|
queued_run="$(api_curl_write -X POST "$API_BASE_URL/api/workflows/$workflow_id/run" -H 'Content-Type: application/json' -d '{"trigger":"manual"}')"
|
|
run_id="$(printf '%s' "$queued_run" | json_get 'json.load(sys.stdin)["id"]')"
|
|
printf 'queued run: %s\n' "$run_id"
|
|
|
|
start_worker_port_forward || printf 'warning: worker port-forward unavailable; relying on background worker polling.\n' >&2
|
|
|
|
deadline=$((SECONDS + WILOR_WORKFLOW_RUN_TIMEOUT_SECONDS))
|
|
final_run=""
|
|
while (( SECONDS < deadline )); do
|
|
run_json="$(api_curl "$API_BASE_URL/api/runs/$run_id")"
|
|
maybe_process_once "$run_json"
|
|
status="$(printf '%s' "$run_json" | json_get 'json.load(sys.stdin)["status"]')"
|
|
printf 'run status: %s\n' "$status"
|
|
if [[ "$status" == "completed" || "$status" == "failed" || "$status" == "cancelled" || "$status" == "waiting_for_audit" ]]; then
|
|
final_run="$run_json"
|
|
break
|
|
fi
|
|
sleep 5
|
|
done
|
|
if [[ -z "$final_run" ]]; then
|
|
final_run="$(api_curl "$API_BASE_URL/api/runs/$run_id")"
|
|
fi
|
|
|
|
artifacts_response="$(api_curl "$API_BASE_URL/api/runs/$run_id/artifacts")"
|
|
manifest_artifact_id="$(
|
|
printf '%s' "$artifacts_response" | python3 -c '
|
|
import json
|
|
import sys
|
|
payload = json.load(sys.stdin)
|
|
items = payload.get("items") if isinstance(payload, dict) else payload
|
|
for item in items or []:
|
|
if item.get("name") == "wilor-result.json" and item.get("nodeId") == "wilor-1":
|
|
print(item.get("id", ""))
|
|
break
|
|
'
|
|
)"
|
|
if [[ -z "$manifest_artifact_id" ]]; then
|
|
echo "ERROR: wilor-result.json artifact was not produced." >&2
|
|
printf '%s\n' "$artifacts_response" >&2
|
|
exit 1
|
|
fi
|
|
manifest_content="$(api_curl "$API_BASE_URL/api/runs/$run_id/artifacts/$manifest_artifact_id/content")"
|
|
upload_artifact_id="$(
|
|
printf '%s' "$artifacts_response" | python3 -c '
|
|
import json
|
|
import sys
|
|
payload = json.load(sys.stdin)
|
|
items = payload.get("items") if isinstance(payload, dict) else payload
|
|
for item in items or []:
|
|
if item.get("name") == "uploaded-results.json" and item.get("nodeId") == "upload-1":
|
|
print(item.get("id", ""))
|
|
break
|
|
'
|
|
)"
|
|
if [[ -z "$upload_artifact_id" ]]; then
|
|
echo "ERROR: uploaded-results.json artifact was not produced by upload-1." >&2
|
|
printf '%s\n' "$artifacts_response" >&2
|
|
exit 1
|
|
fi
|
|
upload_content="$(api_curl "$API_BASE_URL/api/runs/$run_id/artifacts/$upload_artifact_id/content")"
|
|
|
|
FINAL_RUN="$final_run" ARTIFACTS_RESPONSE="$artifacts_response" MANIFEST_CONTENT="$manifest_content" UPLOAD_CONTENT="$upload_content" python3 - <<'PY'
|
|
import json
|
|
import os
|
|
run = json.loads(os.environ["FINAL_RUN"])
|
|
artifacts_payload = json.loads(os.environ["ARTIFACTS_RESPONSE"])
|
|
manifest = json.loads(os.environ["MANIFEST_CONTENT"])
|
|
upload = json.loads(os.environ["UPLOAD_CONTENT"])
|
|
items = artifacts_payload.get("items") if isinstance(artifacts_payload, dict) else artifacts_payload
|
|
items = items or []
|
|
names = sorted(item.get("name", "") for item in items)
|
|
node_statuses = {item.get("nodeId"): item.get("status") for item in run.get("nodeExecutions") or []}
|
|
if run.get("status") != "completed":
|
|
raise SystemExit(f"run did not complete: {run.get('status')}")
|
|
for node_id in ("source-1", "wilor-1", "upload-1"):
|
|
if node_statuses.get(node_id) != "completed":
|
|
raise SystemExit(f"{node_id} did not complete: {node_statuses}")
|
|
if manifest.get("status") != "success":
|
|
raise SystemExit(f"WiLoR manifest status is not success: {manifest.get('status')}")
|
|
service_model = ((manifest.get("service") or {}).get("health") or {}).get("model") or {}
|
|
if service_model.get("device") != "cuda":
|
|
raise SystemExit(f"WiLoR service did not report cuda device: {service_model}")
|
|
if "A800" not in str(service_model.get("cudaDeviceName", "")):
|
|
raise SystemExit(f"WiLoR service did not report A800 GPU: {service_model}")
|
|
detected = ((manifest.get("result") or {}).get("detectedHands") or 0)
|
|
if int(detected) < 1:
|
|
raise SystemExit(f"WiLoR detected no hands: {detected}")
|
|
if "source-manifest.json" not in names:
|
|
raise SystemExit("missing source-manifest.json")
|
|
if "wilor-result.json" not in names:
|
|
raise SystemExit("missing wilor-result.json")
|
|
if not ({"wilor-annotated-image.png", "wilor-annotated-video.mp4"} & set(names)):
|
|
raise SystemExit(f"missing annotated media artifact: {names}")
|
|
if "uploaded-results.json" not in names:
|
|
raise SystemExit("missing uploaded-results.json")
|
|
if int(upload.get("inputCount") or 0) < 1:
|
|
raise SystemExit(f"upload node did not receive upstream artifacts: {upload}")
|
|
wilor_execution = next((item for item in run.get("nodeExecutions") or [] if item.get("nodeId") == "wilor-1"), {})
|
|
print("wilor workflow acceptance:", json.dumps({
|
|
"runId": run.get("id"),
|
|
"workflowId": run.get("workflowId"),
|
|
"status": run.get("status"),
|
|
"nodeStatuses": node_statuses,
|
|
"detectedHands": detected,
|
|
"cudaDeviceName": service_model.get("cudaDeviceName"),
|
|
"artifacts": names,
|
|
"uploadInputCount": upload.get("inputCount"),
|
|
"operatorImageId": wilor_execution.get("operatorImageId", ""),
|
|
}, ensure_ascii=False))
|
|
PY
|