droboticflow/scripts/live-workflow-acceptance.sh
2026-06-28 19:47:06 +08:00

284 lines
12 KiB
Bash
Executable File

#!/usr/bin/env bash
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# shellcheck disable=SC1091
source "$SCRIPT_DIR/lib/service-endpoints.sh"
WORKFLOW_ACCEPTANCE_REQUIRE_EXPLICIT_ENDPOINT="${WORKFLOW_ACCEPTANCE_REQUIRE_EXPLICIT_ENDPOINT:-false}"
case "$(printf '%s' "$WORKFLOW_ACCEPTANCE_REQUIRE_EXPLICIT_ENDPOINT" | tr '[:upper:]' '[:lower:]')" in
1|true|yes|on)
WORKFLOW_ACCEPTANCE_REQUIRE_EXPLICIT_ENDPOINT="true"
;;
0|false|no|off)
WORKFLOW_ACCEPTANCE_REQUIRE_EXPLICIT_ENDPOINT="false"
;;
*)
echo "ERROR: WORKFLOW_ACCEPTANCE_REQUIRE_EXPLICIT_ENDPOINT must be a boolean." >&2
exit 1
;;
esac
if [[ $# -gt 0 ]]; then
API_BASE_URL="$(droboticflow_first_service_url "$1")"
WORKER_BASE_URL="$(droboticflow_first_service_url "${2:-}" "${WORKER_BASE_URL:-}")"
elif [[ "$WORKFLOW_ACCEPTANCE_REQUIRE_EXPLICIT_ENDPOINT" == "true" ]]; then
API_BASE_URL="$(droboticflow_first_service_url "${API_BASE_URL:-}" "${BASE_URL:-}")"
WORKER_BASE_URL="$(droboticflow_first_service_url "${WORKER_BASE_URL:-}" "${WORKER_API_BASE_URL:-}")"
if [[ -z "$API_BASE_URL" ]]; then
echo "ERROR: API_BASE_URL or BASE_URL is required when WORKFLOW_ACCEPTANCE_REQUIRE_EXPLICIT_ENDPOINT=true." >&2
exit 1
fi
else
API_BASE_URL="$(droboticflow_first_service_url "${API_BASE_URL:-}" "${BASE_URL:-}" "$(droboticflow_default_api_base_url)")"
WORKER_BASE_URL="$(droboticflow_first_service_url "${WORKER_BASE_URL:-}" "${WORKER_API_BASE_URL:-}" "$(droboticflow_default_worker_base_url)")"
fi
WORKFLOW_ACCEPTANCE_SOURCE_PATH="${WORKFLOW_ACCEPTANCE_SOURCE_PATH:-}"
WORKFLOW_ACCEPTANCE_RUN_TIMEOUT_SECONDS="${WORKFLOW_ACCEPTANCE_RUN_TIMEOUT_SECONDS:-180}"
WORKFLOW_ACCEPTANCE_EMAIL="${DROBOTICFLOW_ACCEPTANCE_EMAIL:-}"
WORKFLOW_ACCEPTANCE_PASSWORD="${DROBOTICFLOW_ACCEPTANCE_PASSWORD:-}"
WORKFLOW_ACCEPTANCE_GPU_LIMIT="${WORKFLOW_ACCEPTANCE_GPU_LIMIT:-}"
WORKFLOW_ACCEPTANCE_EXPECT_TERMINAL_STATUS="${WORKFLOW_ACCEPTANCE_EXPECT_TERMINAL_STATUS:-completed}"
WORKFLOW_ACCEPTANCE_EXPECT_LOG_SUBSTRING="${WORKFLOW_ACCEPTANCE_EXPECT_LOG_SUBSTRING:-}"
if [[ -z "$WORKFLOW_ACCEPTANCE_SOURCE_PATH" ]]; then
echo "ERROR: WORKFLOW_ACCEPTANCE_SOURCE_PATH is required for live workflow acceptance; provide a real path visible to operator-runner Jobs." >&2
exit 1
fi
if [[ -n "$WORKFLOW_ACCEPTANCE_EMAIL" && -z "$WORKFLOW_ACCEPTANCE_PASSWORD" ]] || [[ -z "$WORKFLOW_ACCEPTANCE_EMAIL" && -n "$WORKFLOW_ACCEPTANCE_PASSWORD" ]]; then
echo "ERROR: DROBOTICFLOW_ACCEPTANCE_EMAIL and DROBOTICFLOW_ACCEPTANCE_PASSWORD must be set together." >&2
exit 1
fi
if ! [[ "$WORKFLOW_ACCEPTANCE_RUN_TIMEOUT_SECONDS" =~ ^[0-9]+$ ]] || (( WORKFLOW_ACCEPTANCE_RUN_TIMEOUT_SECONDS < 1 )); then
echo "ERROR: WORKFLOW_ACCEPTANCE_RUN_TIMEOUT_SECONDS must be a positive integer." >&2
exit 1
fi
if [[ -n "$WORKFLOW_ACCEPTANCE_GPU_LIMIT" ]] && ! [[ "$WORKFLOW_ACCEPTANCE_GPU_LIMIT" =~ ^[1-9][0-9]*$ ]]; then
echo "ERROR: WORKFLOW_ACCEPTANCE_GPU_LIMIT must be a positive integer when set." >&2
exit 1
fi
case "$WORKFLOW_ACCEPTANCE_EXPECT_TERMINAL_STATUS" in
completed|failed|cancelled|waiting_for_audit)
;;
*)
echo "ERROR: WORKFLOW_ACCEPTANCE_EXPECT_TERMINAL_STATUS must be completed, failed, cancelled, or waiting_for_audit." >&2
exit 1
;;
esac
json_get() {
local expression="$1"
python3 -c "import json,sys; print(${expression})"
}
COOKIE_JAR="$(mktemp)"
cleanup() {
rm -f "$COOKIE_JAR"
}
trap cleanup EXIT
csrf_token() {
awk '$6 == "droboticflow_csrf" { value=$7 } END { print value }' "$COOKIE_JAR"
}
api_curl() {
local headers=()
if [[ -n "${DROBOTICFLOW_API_TOKEN:-}" ]]; then
headers+=(-H "Authorization: Bearer ${DROBOTICFLOW_API_TOKEN}")
fi
local curl_args=(--retry 6 --retry-delay 2 --retry-connrefused --retry-all-errors -fsS -b "$COOKIE_JAR" -c "$COOKIE_JAR")
if (( ${#headers[@]} > 0 )); then
curl_args+=("${headers[@]}")
fi
curl "${curl_args[@]}" "$@"
}
api_curl_write() {
local token
token="$(csrf_token)"
if [[ -n "$token" ]]; then
api_curl -H "X-Droboticflow-CSRF-Token: $token" "$@"
return
fi
api_curl "$@"
}
api_health="$(api_curl "$API_BASE_URL/api/health")"
printf 'api health: %s\n' "$api_health"
printf '%s' "$api_health" | grep -q '"status":"ok"'
if [[ -n "$WORKFLOW_ACCEPTANCE_EMAIL" && -n "$WORKFLOW_ACCEPTANCE_PASSWORD" ]]; then
login_payload="$(WORKFLOW_ACCEPTANCE_EMAIL="$WORKFLOW_ACCEPTANCE_EMAIL" WORKFLOW_ACCEPTANCE_PASSWORD="$WORKFLOW_ACCEPTANCE_PASSWORD" python3 - <<'PY'
import json
import os
print(json.dumps({
"email": os.environ["WORKFLOW_ACCEPTANCE_EMAIL"],
"password": os.environ["WORKFLOW_ACCEPTANCE_PASSWORD"],
}))
PY
)"
login_response="$(api_curl -X POST "$API_BASE_URL/api/auth/login" -H 'Content-Type: application/json' -d "$login_payload")"
printf 'logged in acceptance user: %s\n' "$login_response"
fi
project_payload="$(python3 - <<'PY'
import json
print(json.dumps({
"name": "Real Source Acceptance Project",
"description": "Created by scripts/live-workflow-acceptance.sh",
}))
PY
)"
project_response="$(api_curl_write -X POST "$API_BASE_URL/api/projects" -H 'Content-Type: application/json' -d "$project_payload")"
project_id="$(printf '%s' "$project_response" | json_get 'json.load(sys.stdin)["id"]')"
printf 'created project: %s\n' "$project_response"
workflow_payload="$(WORKFLOW_ACCEPTANCE_SOURCE_PATH="$WORKFLOW_ACCEPTANCE_SOURCE_PATH" WORKFLOW_ACCEPTANCE_GPU_LIMIT="$WORKFLOW_ACCEPTANCE_GPU_LIMIT" python3 - <<'PY'
import json
import os
source_data = {
"label": "真实数据源",
"category": "IO",
"description": "Read WORKFLOW_ACCEPTANCE_SOURCE_PATH to emit a real manifest artifact",
"inputs": [],
"outputs": [{"id": "out-1", "name": "manifest", "type": "json"}],
"hyperparams": [{"key": "SOURCE_PATH", "value": os.environ["WORKFLOW_ACCEPTANCE_SOURCE_PATH"]}],
}
gpu_limit = os.environ.get("WORKFLOW_ACCEPTANCE_GPU_LIMIT", "").strip()
if gpu_limit:
source_data["gpu"] = gpu_limit
print(json.dumps({
"name": "Real Source Acceptance Workflow",
"description": "End-to-end workflow acceptance with an explicit real source path",
"type": "batch",
"nodes": [
{
"id": "source-1",
"type": "operator",
"data": source_data,
"position": {"x": 0, "y": 0},
},
{
"id": "upload-1",
"type": "operator",
"data": {
"label": "结果上传",
"category": "IO",
"description": "Persist artifacts produced by upstream nodes",
"inputs": [{"id": "in-1", "name": "manifest", "type": "json"}],
"outputs": [],
},
"position": {"x": 240, "y": 0},
},
],
"edges": [
{
"id": "edge-source-upload",
"source": "source-1",
"sourceHandle": "out-1",
"target": "upload-1",
"targetHandle": "in-1",
}
],
}))
PY
)"
workflow_response="$(api_curl_write -X POST "$API_BASE_URL/api/projects/$project_id/workflows" -H 'Content-Type: application/json' -d "$workflow_payload")"
workflow_id="$(printf '%s' "$workflow_response" | json_get 'json.load(sys.stdin)["id"]')"
printf 'created workflow: %s\n' "$workflow_response"
queued_run="$(api_curl_write -X POST "$API_BASE_URL/api/workflows/$workflow_id/run" -H 'Content-Type: application/json' -d '{"trigger":"manual"}')"
run_id="$(printf '%s' "$queued_run" | json_get 'json.load(sys.stdin)["id"]')"
printf 'queued run: %s\n' "$queued_run"
maybe_process_once() {
if [[ -n "${WORKER_BASE_URL}" ]] && api_curl "$WORKER_BASE_URL/worker-api/health" >/dev/null 2>&1; then
api_curl -X POST "$WORKER_BASE_URL/worker-api/process-once" >/dev/null
fi
}
if [[ -n "${WORKER_BASE_URL}" ]] && api_curl "$WORKER_BASE_URL/worker-api/health" >/dev/null 2>&1; then
worker_health="$(api_curl "$WORKER_BASE_URL/worker-api/health")"
printf 'worker health: %s\n' "$worker_health"
fi
deadline=$((SECONDS + WORKFLOW_ACCEPTANCE_RUN_TIMEOUT_SECONDS))
final_run=""
final_status=""
while (( SECONDS < deadline )); do
maybe_process_once
final_run="$(api_curl "$API_BASE_URL/api/runs/$run_id")"
final_status="$(printf '%s' "$final_run" | json_get 'json.load(sys.stdin)["status"]')"
printf 'run status: %s\n' "$final_status"
case "$final_status" in
"$WORKFLOW_ACCEPTANCE_EXPECT_TERMINAL_STATUS")
case "$final_status" in
completed|failed|cancelled|waiting_for_audit)
break
;;
esac
;;
failed|cancelled|waiting_for_audit)
printf 'unexpected terminal status %s, expected %s: %s\n' "$final_status" "$WORKFLOW_ACCEPTANCE_EXPECT_TERMINAL_STATUS" "$final_run" >&2
exit 1
;;
esac
sleep 2
done
if [[ "$final_status" != "$WORKFLOW_ACCEPTANCE_EXPECT_TERMINAL_STATUS" ]]; then
printf 'run did not reach expected terminal status %s before timeout: %s\n' "$WORKFLOW_ACCEPTANCE_EXPECT_TERMINAL_STATUS" "$final_run" >&2
exit 1
fi
logs_response="$(api_curl "$API_BASE_URL/api/runs/$run_id/logs")"
artifacts_response="$(api_curl "$API_BASE_URL/api/runs/$run_id/artifacts")"
printf '%s' "$logs_response" | python3 -c 'import json, sys; payload = json.load(sys.stdin); items = payload.get("items", []); print("logs summary:", json.dumps({"total": payload.get("total", len(items)), "lastMessages": [item.get("message", "") for item in items[-3:]]}, ensure_ascii=False))'
printf '%s' "$artifacts_response" | python3 -c 'import json, sys; payload = json.load(sys.stdin); items = payload.get("items", []); print("artifacts summary:", json.dumps({"total": payload.get("total", len(items)), "items": [{"name": item.get("name"), "type": item.get("type"), "size": item.get("size")} for item in items[:10]]}, ensure_ascii=False))'
printf '%s' "$logs_response" | python3 -c 'import json, sys; payload = json.load(sys.stdin); items = payload.get("items", []);
if not items: raise SystemExit("expected at least one log entry")'
if [[ -n "$WORKFLOW_ACCEPTANCE_EXPECT_LOG_SUBSTRING" ]]; then
printf '%s' "$logs_response" | WORKFLOW_ACCEPTANCE_EXPECT_LOG_SUBSTRING="$WORKFLOW_ACCEPTANCE_EXPECT_LOG_SUBSTRING" python3 -c 'import json, os, sys; payload = json.load(sys.stdin); needle = os.environ["WORKFLOW_ACCEPTANCE_EXPECT_LOG_SUBSTRING"]; messages = "\n".join(str(item.get("message", "")) for item in payload.get("items", []));
if needle not in messages: raise SystemExit(f"expected log substring not found: {needle}")'
fi
if [[ "$WORKFLOW_ACCEPTANCE_EXPECT_TERMINAL_STATUS" == "completed" ]]; then
FINAL_RUN_JSON="$final_run" WORKFLOW_ACCEPTANCE_GPU_LIMIT="$WORKFLOW_ACCEPTANCE_GPU_LIMIT" python3 - <<'PY'
import json
import os
payload = json.loads(os.environ["FINAL_RUN_JSON"])
executions = {item.get("nodeId"): item for item in payload.get("nodeExecutions") or [] if isinstance(item, dict)}
upload = executions.get("upload-1")
if not upload:
raise SystemExit("expected upload-1 node execution in completed workflow")
lineage = upload.get("inputLineage") or []
if not any(item.get("sourceNodeId") == "source-1" and item.get("targetNodeId") == "upload-1" and item.get("artifactId") for item in lineage):
raise SystemExit("expected source-1 to upload-1 inputLineage artifact")
if not upload.get("inputArtifacts"):
raise SystemExit("expected upload-1 inputArtifacts from source-1")
gpu_limit = os.environ.get("WORKFLOW_ACCEPTANCE_GPU_LIMIT", "").strip()
if gpu_limit:
samples = []
for item in payload.get("resourceSeries") or []:
if isinstance(item, dict):
samples.append(item)
for node in payload.get("nodeExecutions") or []:
for item in node.get("resourceSeries") or []:
if isinstance(item, dict):
samples.append(item)
if not any(int(sample.get("gpuMemoryTotalMiB") or 0) > 0 for sample in samples):
raise SystemExit("expected GPU memory metrics in completed GPU workflow resourceSeries")
PY
printf '%s' "$artifacts_response" | python3 -c 'import json, sys; payload = json.load(sys.stdin); items = payload.get("items", []);
if not items: raise SystemExit("expected at least one artifact")'
fi