#!/usr/bin/env bash set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck disable=SC1091 source "$SCRIPT_DIR/lib/service-endpoints.sh" WORKFLOW_ACCEPTANCE_REQUIRE_EXPLICIT_ENDPOINT="${WORKFLOW_ACCEPTANCE_REQUIRE_EXPLICIT_ENDPOINT:-false}" case "$(printf '%s' "$WORKFLOW_ACCEPTANCE_REQUIRE_EXPLICIT_ENDPOINT" | tr '[:upper:]' '[:lower:]')" in 1|true|yes|on) WORKFLOW_ACCEPTANCE_REQUIRE_EXPLICIT_ENDPOINT="true" ;; 0|false|no|off) WORKFLOW_ACCEPTANCE_REQUIRE_EXPLICIT_ENDPOINT="false" ;; *) echo "ERROR: WORKFLOW_ACCEPTANCE_REQUIRE_EXPLICIT_ENDPOINT must be a boolean." >&2 exit 1 ;; esac if [[ $# -gt 0 ]]; then API_BASE_URL="$(droboticflow_first_service_url "$1")" WORKER_BASE_URL="$(droboticflow_first_service_url "${2:-}" "${WORKER_BASE_URL:-}")" elif [[ "$WORKFLOW_ACCEPTANCE_REQUIRE_EXPLICIT_ENDPOINT" == "true" ]]; then API_BASE_URL="$(droboticflow_first_service_url "${API_BASE_URL:-}" "${BASE_URL:-}")" WORKER_BASE_URL="$(droboticflow_first_service_url "${WORKER_BASE_URL:-}" "${WORKER_API_BASE_URL:-}")" if [[ -z "$API_BASE_URL" ]]; then echo "ERROR: API_BASE_URL or BASE_URL is required when WORKFLOW_ACCEPTANCE_REQUIRE_EXPLICIT_ENDPOINT=true." >&2 exit 1 fi else API_BASE_URL="$(droboticflow_first_service_url "${API_BASE_URL:-}" "${BASE_URL:-}" "$(droboticflow_default_api_base_url)")" WORKER_BASE_URL="$(droboticflow_first_service_url "${WORKER_BASE_URL:-}" "${WORKER_API_BASE_URL:-}" "$(droboticflow_default_worker_base_url)")" fi WORKFLOW_ACCEPTANCE_SOURCE_PATH="${WORKFLOW_ACCEPTANCE_SOURCE_PATH:-}" WORKFLOW_ACCEPTANCE_RUN_TIMEOUT_SECONDS="${WORKFLOW_ACCEPTANCE_RUN_TIMEOUT_SECONDS:-180}" WORKFLOW_ACCEPTANCE_EMAIL="${DROBOTICFLOW_ACCEPTANCE_EMAIL:-}" WORKFLOW_ACCEPTANCE_PASSWORD="${DROBOTICFLOW_ACCEPTANCE_PASSWORD:-}" WORKFLOW_ACCEPTANCE_GPU_LIMIT="${WORKFLOW_ACCEPTANCE_GPU_LIMIT:-}" WORKFLOW_ACCEPTANCE_EXPECT_TERMINAL_STATUS="${WORKFLOW_ACCEPTANCE_EXPECT_TERMINAL_STATUS:-completed}" WORKFLOW_ACCEPTANCE_EXPECT_LOG_SUBSTRING="${WORKFLOW_ACCEPTANCE_EXPECT_LOG_SUBSTRING:-}" if [[ -z "$WORKFLOW_ACCEPTANCE_SOURCE_PATH" ]]; then echo "ERROR: WORKFLOW_ACCEPTANCE_SOURCE_PATH is required for live workflow acceptance; provide a real path visible to operator-runner Jobs." >&2 exit 1 fi if [[ -n "$WORKFLOW_ACCEPTANCE_EMAIL" && -z "$WORKFLOW_ACCEPTANCE_PASSWORD" ]] || [[ -z "$WORKFLOW_ACCEPTANCE_EMAIL" && -n "$WORKFLOW_ACCEPTANCE_PASSWORD" ]]; then echo "ERROR: DROBOTICFLOW_ACCEPTANCE_EMAIL and DROBOTICFLOW_ACCEPTANCE_PASSWORD must be set together." >&2 exit 1 fi if ! [[ "$WORKFLOW_ACCEPTANCE_RUN_TIMEOUT_SECONDS" =~ ^[0-9]+$ ]] || (( WORKFLOW_ACCEPTANCE_RUN_TIMEOUT_SECONDS < 1 )); then echo "ERROR: WORKFLOW_ACCEPTANCE_RUN_TIMEOUT_SECONDS must be a positive integer." >&2 exit 1 fi if [[ -n "$WORKFLOW_ACCEPTANCE_GPU_LIMIT" ]] && ! [[ "$WORKFLOW_ACCEPTANCE_GPU_LIMIT" =~ ^[1-9][0-9]*$ ]]; then echo "ERROR: WORKFLOW_ACCEPTANCE_GPU_LIMIT must be a positive integer when set." >&2 exit 1 fi case "$WORKFLOW_ACCEPTANCE_EXPECT_TERMINAL_STATUS" in completed|failed|cancelled|waiting_for_audit) ;; *) echo "ERROR: WORKFLOW_ACCEPTANCE_EXPECT_TERMINAL_STATUS must be completed, failed, cancelled, or waiting_for_audit." >&2 exit 1 ;; esac json_get() { local expression="$1" python3 -c "import json,sys; print(${expression})" } COOKIE_JAR="$(mktemp)" cleanup() { rm -f "$COOKIE_JAR" } trap cleanup EXIT csrf_token() { awk '$6 == "droboticflow_csrf" { value=$7 } END { print value }' "$COOKIE_JAR" } api_curl() { local headers=() if [[ -n "${DROBOTICFLOW_API_TOKEN:-}" ]]; then headers+=(-H "Authorization: Bearer ${DROBOTICFLOW_API_TOKEN}") fi local curl_args=(--retry 6 --retry-delay 2 --retry-connrefused --retry-all-errors -fsS -b "$COOKIE_JAR" -c "$COOKIE_JAR") if (( ${#headers[@]} > 0 )); then curl_args+=("${headers[@]}") fi curl "${curl_args[@]}" "$@" } api_curl_write() { local token token="$(csrf_token)" if [[ -n "$token" ]]; then api_curl -H "X-Droboticflow-CSRF-Token: $token" "$@" return fi api_curl "$@" } api_health="$(api_curl "$API_BASE_URL/api/health")" printf 'api health: %s\n' "$api_health" printf '%s' "$api_health" | grep -q '"status":"ok"' if [[ -n "$WORKFLOW_ACCEPTANCE_EMAIL" && -n "$WORKFLOW_ACCEPTANCE_PASSWORD" ]]; then login_payload="$(WORKFLOW_ACCEPTANCE_EMAIL="$WORKFLOW_ACCEPTANCE_EMAIL" WORKFLOW_ACCEPTANCE_PASSWORD="$WORKFLOW_ACCEPTANCE_PASSWORD" python3 - <<'PY' import json import os print(json.dumps({ "email": os.environ["WORKFLOW_ACCEPTANCE_EMAIL"], "password": os.environ["WORKFLOW_ACCEPTANCE_PASSWORD"], })) PY )" login_response="$(api_curl -X POST "$API_BASE_URL/api/auth/login" -H 'Content-Type: application/json' -d "$login_payload")" printf 'logged in acceptance user: %s\n' "$login_response" fi project_payload="$(python3 - <<'PY' import json print(json.dumps({ "name": "Real Source Acceptance Project", "description": "Created by scripts/live-workflow-acceptance.sh", })) PY )" project_response="$(api_curl_write -X POST "$API_BASE_URL/api/projects" -H 'Content-Type: application/json' -d "$project_payload")" project_id="$(printf '%s' "$project_response" | json_get 'json.load(sys.stdin)["id"]')" printf 'created project: %s\n' "$project_response" workflow_payload="$(WORKFLOW_ACCEPTANCE_SOURCE_PATH="$WORKFLOW_ACCEPTANCE_SOURCE_PATH" WORKFLOW_ACCEPTANCE_GPU_LIMIT="$WORKFLOW_ACCEPTANCE_GPU_LIMIT" python3 - <<'PY' import json import os source_data = { "label": "真实数据源", "category": "IO", "description": "Read WORKFLOW_ACCEPTANCE_SOURCE_PATH to emit a real manifest artifact", "inputs": [], "outputs": [{"id": "out-1", "name": "manifest", "type": "json"}], "hyperparams": [{"key": "SOURCE_PATH", "value": os.environ["WORKFLOW_ACCEPTANCE_SOURCE_PATH"]}], } gpu_limit = os.environ.get("WORKFLOW_ACCEPTANCE_GPU_LIMIT", "").strip() if gpu_limit: source_data["gpu"] = gpu_limit print(json.dumps({ "name": "Real Source Acceptance Workflow", "description": "End-to-end workflow acceptance with an explicit real source path", "type": "batch", "nodes": [ { "id": "source-1", "type": "operator", "data": source_data, "position": {"x": 0, "y": 0}, }, { "id": "upload-1", "type": "operator", "data": { "label": "结果上传", "category": "IO", "description": "Persist artifacts produced by upstream nodes", "inputs": [{"id": "in-1", "name": "manifest", "type": "json"}], "outputs": [], }, "position": {"x": 240, "y": 0}, }, ], "edges": [ { "id": "edge-source-upload", "source": "source-1", "sourceHandle": "out-1", "target": "upload-1", "targetHandle": "in-1", } ], })) PY )" workflow_response="$(api_curl_write -X POST "$API_BASE_URL/api/projects/$project_id/workflows" -H 'Content-Type: application/json' -d "$workflow_payload")" workflow_id="$(printf '%s' "$workflow_response" | json_get 'json.load(sys.stdin)["id"]')" printf 'created workflow: %s\n' "$workflow_response" queued_run="$(api_curl_write -X POST "$API_BASE_URL/api/workflows/$workflow_id/run" -H 'Content-Type: application/json' -d '{"trigger":"manual"}')" run_id="$(printf '%s' "$queued_run" | json_get 'json.load(sys.stdin)["id"]')" printf 'queued run: %s\n' "$queued_run" run_needs_process_once() { RUN_JSON="$1" python3 - <<'PY' import datetime as dt import json import os run = json.loads(os.environ["RUN_JSON"]) if run.get("status") != "queued": raise SystemExit(1) for node in run.get("nodeExecutions") or []: if node.get("status") in {"running", "waiting_for_audit", "queued"}: raise SystemExit(1) owner = str(run.get("executionLeaseOwner") or "").strip() expires_raw = str(run.get("executionLeaseExpiresAt") or "").strip() if owner and expires_raw: try: expires = dt.datetime.fromisoformat(expires_raw.replace("Z", "+00:00")) except ValueError: expires = None if expires and expires > dt.datetime.now(dt.timezone.utc): raise SystemExit(1) raise SystemExit(0) PY } maybe_process_once() { local run_json="$1" if ! run_needs_process_once "$run_json"; then return 0 fi if [[ -n "${WORKER_BASE_URL}" ]] && api_curl "$WORKER_BASE_URL/worker-api/health" >/dev/null 2>&1; then api_curl -X POST "$WORKER_BASE_URL/worker-api/process-once" >/dev/null fi } if [[ -n "${WORKER_BASE_URL}" ]] && api_curl "$WORKER_BASE_URL/worker-api/health" >/dev/null 2>&1; then worker_health="$(api_curl "$WORKER_BASE_URL/worker-api/health")" printf 'worker health: %s\n' "$worker_health" fi deadline=$((SECONDS + WORKFLOW_ACCEPTANCE_RUN_TIMEOUT_SECONDS)) final_run="" final_status="" last_process_once_at=0 while (( SECONDS < deadline )); do final_run="$(api_curl "$API_BASE_URL/api/runs/$run_id")" if (( SECONDS - last_process_once_at >= 3 )); then maybe_process_once "$final_run" last_process_once_at=$SECONDS final_run="$(api_curl "$API_BASE_URL/api/runs/$run_id")" fi final_status="$(printf '%s' "$final_run" | json_get 'json.load(sys.stdin)["status"]')" printf 'run status: %s\n' "$final_status" case "$final_status" in "$WORKFLOW_ACCEPTANCE_EXPECT_TERMINAL_STATUS") case "$final_status" in completed|failed|cancelled|waiting_for_audit) break ;; esac ;; failed|cancelled|waiting_for_audit) printf 'unexpected terminal status %s, expected %s: %s\n' "$final_status" "$WORKFLOW_ACCEPTANCE_EXPECT_TERMINAL_STATUS" "$final_run" >&2 exit 1 ;; esac sleep 2 done if [[ "$final_status" != "$WORKFLOW_ACCEPTANCE_EXPECT_TERMINAL_STATUS" ]]; then printf 'run did not reach expected terminal status %s before timeout: %s\n' "$WORKFLOW_ACCEPTANCE_EXPECT_TERMINAL_STATUS" "$final_run" >&2 exit 1 fi logs_response="$(api_curl "$API_BASE_URL/api/runs/$run_id/logs")" artifacts_response="$(api_curl "$API_BASE_URL/api/runs/$run_id/artifacts")" printf '%s' "$logs_response" | python3 -c 'import json, sys; payload = json.load(sys.stdin); items = payload.get("items", []); print("logs summary:", json.dumps({"total": payload.get("total", len(items)), "lastMessages": [item.get("message", "") for item in items[-3:]]}, ensure_ascii=False))' printf '%s' "$artifacts_response" | python3 -c 'import json, sys; payload = json.load(sys.stdin); items = payload.get("items", []); print("artifacts summary:", json.dumps({"total": payload.get("total", len(items)), "items": [{"name": item.get("name"), "type": item.get("type"), "size": item.get("size")} for item in items[:10]]}, ensure_ascii=False))' printf '%s' "$logs_response" | python3 -c 'import json, sys; payload = json.load(sys.stdin); items = payload.get("items", []); if not items: raise SystemExit("expected at least one log entry")' if [[ -n "$WORKFLOW_ACCEPTANCE_EXPECT_LOG_SUBSTRING" ]]; then printf '%s' "$logs_response" | WORKFLOW_ACCEPTANCE_EXPECT_LOG_SUBSTRING="$WORKFLOW_ACCEPTANCE_EXPECT_LOG_SUBSTRING" python3 -c 'import json, os, sys; payload = json.load(sys.stdin); needle = os.environ["WORKFLOW_ACCEPTANCE_EXPECT_LOG_SUBSTRING"]; messages = "\n".join(str(item.get("message", "")) for item in payload.get("items", [])); if needle not in messages: raise SystemExit(f"expected log substring not found: {needle}")' fi if [[ "$WORKFLOW_ACCEPTANCE_EXPECT_TERMINAL_STATUS" == "completed" ]]; then FINAL_RUN_JSON="$final_run" WORKFLOW_ACCEPTANCE_GPU_LIMIT="$WORKFLOW_ACCEPTANCE_GPU_LIMIT" python3 - <<'PY' import json import os payload = json.loads(os.environ["FINAL_RUN_JSON"]) executions = {item.get("nodeId"): item for item in payload.get("nodeExecutions") or [] if isinstance(item, dict)} upload = executions.get("upload-1") if not upload: raise SystemExit("expected upload-1 node execution in completed workflow") lineage = upload.get("inputLineage") or [] if not any( item.get("sourceNodeId") == "source-1" and item.get("targetNodeId") == "upload-1" and (item.get("artifactId") or (item.get("artifact") or {}).get("id")) for item in lineage ): raise SystemExit("expected source-1 to upload-1 inputLineage artifact") if not upload.get("inputArtifacts"): raise SystemExit("expected upload-1 inputArtifacts from source-1") gpu_limit = os.environ.get("WORKFLOW_ACCEPTANCE_GPU_LIMIT", "").strip() if gpu_limit: samples = [] for item in payload.get("resourceSeries") or []: if isinstance(item, dict): samples.append(item) for node in payload.get("nodeExecutions") or []: for item in node.get("resourceSeries") or []: if isinstance(item, dict): samples.append(item) if not any(int(sample.get("gpuMemoryTotalMiB") or 0) > 0 for sample in samples): raise SystemExit("expected GPU memory metrics in completed GPU workflow resourceSeries") PY printf '%s' "$artifacts_response" | python3 -c 'import json, sys; payload = json.load(sys.stdin); items = payload.get("items", []); if not items: raise SystemExit("expected at least one artifact")' fi