forked from longtao.wu/droboticflow
284 lines
12 KiB
Bash
Executable File
284 lines
12 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
# shellcheck disable=SC1091
|
|
source "$SCRIPT_DIR/lib/service-endpoints.sh"
|
|
|
|
WORKFLOW_ACCEPTANCE_REQUIRE_EXPLICIT_ENDPOINT="${WORKFLOW_ACCEPTANCE_REQUIRE_EXPLICIT_ENDPOINT:-false}"
|
|
case "$(printf '%s' "$WORKFLOW_ACCEPTANCE_REQUIRE_EXPLICIT_ENDPOINT" | tr '[:upper:]' '[:lower:]')" in
|
|
1|true|yes|on)
|
|
WORKFLOW_ACCEPTANCE_REQUIRE_EXPLICIT_ENDPOINT="true"
|
|
;;
|
|
0|false|no|off)
|
|
WORKFLOW_ACCEPTANCE_REQUIRE_EXPLICIT_ENDPOINT="false"
|
|
;;
|
|
*)
|
|
echo "ERROR: WORKFLOW_ACCEPTANCE_REQUIRE_EXPLICIT_ENDPOINT must be a boolean." >&2
|
|
exit 1
|
|
;;
|
|
esac
|
|
|
|
if [[ $# -gt 0 ]]; then
|
|
API_BASE_URL="$(droboticflow_first_service_url "$1")"
|
|
WORKER_BASE_URL="$(droboticflow_first_service_url "${2:-}" "${WORKER_BASE_URL:-}")"
|
|
elif [[ "$WORKFLOW_ACCEPTANCE_REQUIRE_EXPLICIT_ENDPOINT" == "true" ]]; then
|
|
API_BASE_URL="$(droboticflow_first_service_url "${API_BASE_URL:-}" "${BASE_URL:-}")"
|
|
WORKER_BASE_URL="$(droboticflow_first_service_url "${WORKER_BASE_URL:-}" "${WORKER_API_BASE_URL:-}")"
|
|
if [[ -z "$API_BASE_URL" ]]; then
|
|
echo "ERROR: API_BASE_URL or BASE_URL is required when WORKFLOW_ACCEPTANCE_REQUIRE_EXPLICIT_ENDPOINT=true." >&2
|
|
exit 1
|
|
fi
|
|
else
|
|
API_BASE_URL="$(droboticflow_first_service_url "${API_BASE_URL:-}" "${BASE_URL:-}" "$(droboticflow_default_api_base_url)")"
|
|
WORKER_BASE_URL="$(droboticflow_first_service_url "${WORKER_BASE_URL:-}" "${WORKER_API_BASE_URL:-}" "$(droboticflow_default_worker_base_url)")"
|
|
fi
|
|
|
|
WORKFLOW_ACCEPTANCE_SOURCE_PATH="${WORKFLOW_ACCEPTANCE_SOURCE_PATH:-}"
|
|
WORKFLOW_ACCEPTANCE_RUN_TIMEOUT_SECONDS="${WORKFLOW_ACCEPTANCE_RUN_TIMEOUT_SECONDS:-180}"
|
|
WORKFLOW_ACCEPTANCE_EMAIL="${DROBOTICFLOW_ACCEPTANCE_EMAIL:-}"
|
|
WORKFLOW_ACCEPTANCE_PASSWORD="${DROBOTICFLOW_ACCEPTANCE_PASSWORD:-}"
|
|
WORKFLOW_ACCEPTANCE_GPU_LIMIT="${WORKFLOW_ACCEPTANCE_GPU_LIMIT:-}"
|
|
WORKFLOW_ACCEPTANCE_EXPECT_TERMINAL_STATUS="${WORKFLOW_ACCEPTANCE_EXPECT_TERMINAL_STATUS:-completed}"
|
|
WORKFLOW_ACCEPTANCE_EXPECT_LOG_SUBSTRING="${WORKFLOW_ACCEPTANCE_EXPECT_LOG_SUBSTRING:-}"
|
|
|
|
if [[ -z "$WORKFLOW_ACCEPTANCE_SOURCE_PATH" ]]; then
|
|
echo "ERROR: WORKFLOW_ACCEPTANCE_SOURCE_PATH is required for live workflow acceptance; provide a real path visible to operator-runner Jobs." >&2
|
|
exit 1
|
|
fi
|
|
if [[ -n "$WORKFLOW_ACCEPTANCE_EMAIL" && -z "$WORKFLOW_ACCEPTANCE_PASSWORD" ]] || [[ -z "$WORKFLOW_ACCEPTANCE_EMAIL" && -n "$WORKFLOW_ACCEPTANCE_PASSWORD" ]]; then
|
|
echo "ERROR: DROBOTICFLOW_ACCEPTANCE_EMAIL and DROBOTICFLOW_ACCEPTANCE_PASSWORD must be set together." >&2
|
|
exit 1
|
|
fi
|
|
if ! [[ "$WORKFLOW_ACCEPTANCE_RUN_TIMEOUT_SECONDS" =~ ^[0-9]+$ ]] || (( WORKFLOW_ACCEPTANCE_RUN_TIMEOUT_SECONDS < 1 )); then
|
|
echo "ERROR: WORKFLOW_ACCEPTANCE_RUN_TIMEOUT_SECONDS must be a positive integer." >&2
|
|
exit 1
|
|
fi
|
|
if [[ -n "$WORKFLOW_ACCEPTANCE_GPU_LIMIT" ]] && ! [[ "$WORKFLOW_ACCEPTANCE_GPU_LIMIT" =~ ^[1-9][0-9]*$ ]]; then
|
|
echo "ERROR: WORKFLOW_ACCEPTANCE_GPU_LIMIT must be a positive integer when set." >&2
|
|
exit 1
|
|
fi
|
|
case "$WORKFLOW_ACCEPTANCE_EXPECT_TERMINAL_STATUS" in
|
|
completed|failed|cancelled|waiting_for_audit)
|
|
;;
|
|
*)
|
|
echo "ERROR: WORKFLOW_ACCEPTANCE_EXPECT_TERMINAL_STATUS must be completed, failed, cancelled, or waiting_for_audit." >&2
|
|
exit 1
|
|
;;
|
|
esac
|
|
|
|
json_get() {
|
|
local expression="$1"
|
|
python3 -c "import json,sys; print(${expression})"
|
|
}
|
|
|
|
COOKIE_JAR="$(mktemp)"
|
|
cleanup() {
|
|
rm -f "$COOKIE_JAR"
|
|
}
|
|
trap cleanup EXIT
|
|
|
|
csrf_token() {
|
|
awk '$6 == "droboticflow_csrf" { value=$7 } END { print value }' "$COOKIE_JAR"
|
|
}
|
|
|
|
api_curl() {
|
|
local headers=()
|
|
if [[ -n "${DROBOTICFLOW_API_TOKEN:-}" ]]; then
|
|
headers+=(-H "Authorization: Bearer ${DROBOTICFLOW_API_TOKEN}")
|
|
fi
|
|
local curl_args=(--retry 6 --retry-delay 2 --retry-connrefused --retry-all-errors -fsS -b "$COOKIE_JAR" -c "$COOKIE_JAR")
|
|
if (( ${#headers[@]} > 0 )); then
|
|
curl_args+=("${headers[@]}")
|
|
fi
|
|
curl "${curl_args[@]}" "$@"
|
|
}
|
|
|
|
api_curl_write() {
|
|
local token
|
|
token="$(csrf_token)"
|
|
if [[ -n "$token" ]]; then
|
|
api_curl -H "X-Droboticflow-CSRF-Token: $token" "$@"
|
|
return
|
|
fi
|
|
api_curl "$@"
|
|
}
|
|
|
|
api_health="$(api_curl "$API_BASE_URL/api/health")"
|
|
printf 'api health: %s\n' "$api_health"
|
|
printf '%s' "$api_health" | grep -q '"status":"ok"'
|
|
|
|
if [[ -n "$WORKFLOW_ACCEPTANCE_EMAIL" && -n "$WORKFLOW_ACCEPTANCE_PASSWORD" ]]; then
|
|
login_payload="$(WORKFLOW_ACCEPTANCE_EMAIL="$WORKFLOW_ACCEPTANCE_EMAIL" WORKFLOW_ACCEPTANCE_PASSWORD="$WORKFLOW_ACCEPTANCE_PASSWORD" python3 - <<'PY'
|
|
import json
|
|
import os
|
|
print(json.dumps({
|
|
"email": os.environ["WORKFLOW_ACCEPTANCE_EMAIL"],
|
|
"password": os.environ["WORKFLOW_ACCEPTANCE_PASSWORD"],
|
|
}))
|
|
PY
|
|
)"
|
|
login_response="$(api_curl -X POST "$API_BASE_URL/api/auth/login" -H 'Content-Type: application/json' -d "$login_payload")"
|
|
printf 'logged in acceptance user: %s\n' "$login_response"
|
|
fi
|
|
|
|
project_payload="$(python3 - <<'PY'
|
|
import json
|
|
print(json.dumps({
|
|
"name": "Real Source Acceptance Project",
|
|
"description": "Created by scripts/live-workflow-acceptance.sh",
|
|
}))
|
|
PY
|
|
)"
|
|
project_response="$(api_curl_write -X POST "$API_BASE_URL/api/projects" -H 'Content-Type: application/json' -d "$project_payload")"
|
|
project_id="$(printf '%s' "$project_response" | json_get 'json.load(sys.stdin)["id"]')"
|
|
printf 'created project: %s\n' "$project_response"
|
|
|
|
workflow_payload="$(WORKFLOW_ACCEPTANCE_SOURCE_PATH="$WORKFLOW_ACCEPTANCE_SOURCE_PATH" WORKFLOW_ACCEPTANCE_GPU_LIMIT="$WORKFLOW_ACCEPTANCE_GPU_LIMIT" python3 - <<'PY'
|
|
import json
|
|
import os
|
|
source_data = {
|
|
"label": "真实数据源",
|
|
"category": "IO",
|
|
"description": "Read WORKFLOW_ACCEPTANCE_SOURCE_PATH to emit a real manifest artifact",
|
|
"inputs": [],
|
|
"outputs": [{"id": "out-1", "name": "manifest", "type": "json"}],
|
|
"hyperparams": [{"key": "SOURCE_PATH", "value": os.environ["WORKFLOW_ACCEPTANCE_SOURCE_PATH"]}],
|
|
}
|
|
gpu_limit = os.environ.get("WORKFLOW_ACCEPTANCE_GPU_LIMIT", "").strip()
|
|
if gpu_limit:
|
|
source_data["gpu"] = gpu_limit
|
|
print(json.dumps({
|
|
"name": "Real Source Acceptance Workflow",
|
|
"description": "End-to-end workflow acceptance with an explicit real source path",
|
|
"type": "batch",
|
|
"nodes": [
|
|
{
|
|
"id": "source-1",
|
|
"type": "operator",
|
|
"data": source_data,
|
|
"position": {"x": 0, "y": 0},
|
|
},
|
|
{
|
|
"id": "upload-1",
|
|
"type": "operator",
|
|
"data": {
|
|
"label": "结果上传",
|
|
"category": "IO",
|
|
"description": "Persist artifacts produced by upstream nodes",
|
|
"inputs": [{"id": "in-1", "name": "manifest", "type": "json"}],
|
|
"outputs": [],
|
|
},
|
|
"position": {"x": 240, "y": 0},
|
|
},
|
|
],
|
|
"edges": [
|
|
{
|
|
"id": "edge-source-upload",
|
|
"source": "source-1",
|
|
"sourceHandle": "out-1",
|
|
"target": "upload-1",
|
|
"targetHandle": "in-1",
|
|
}
|
|
],
|
|
}))
|
|
PY
|
|
)"
|
|
workflow_response="$(api_curl_write -X POST "$API_BASE_URL/api/projects/$project_id/workflows" -H 'Content-Type: application/json' -d "$workflow_payload")"
|
|
workflow_id="$(printf '%s' "$workflow_response" | json_get 'json.load(sys.stdin)["id"]')"
|
|
printf 'created workflow: %s\n' "$workflow_response"
|
|
|
|
queued_run="$(api_curl_write -X POST "$API_BASE_URL/api/workflows/$workflow_id/run" -H 'Content-Type: application/json' -d '{"trigger":"manual"}')"
|
|
run_id="$(printf '%s' "$queued_run" | json_get 'json.load(sys.stdin)["id"]')"
|
|
printf 'queued run: %s\n' "$queued_run"
|
|
|
|
maybe_process_once() {
|
|
if [[ -n "${WORKER_BASE_URL}" ]] && api_curl "$WORKER_BASE_URL/worker-api/health" >/dev/null 2>&1; then
|
|
api_curl -X POST "$WORKER_BASE_URL/worker-api/process-once" >/dev/null
|
|
fi
|
|
}
|
|
|
|
if [[ -n "${WORKER_BASE_URL}" ]] && api_curl "$WORKER_BASE_URL/worker-api/health" >/dev/null 2>&1; then
|
|
worker_health="$(api_curl "$WORKER_BASE_URL/worker-api/health")"
|
|
printf 'worker health: %s\n' "$worker_health"
|
|
fi
|
|
|
|
deadline=$((SECONDS + WORKFLOW_ACCEPTANCE_RUN_TIMEOUT_SECONDS))
|
|
final_run=""
|
|
final_status=""
|
|
|
|
while (( SECONDS < deadline )); do
|
|
maybe_process_once
|
|
final_run="$(api_curl "$API_BASE_URL/api/runs/$run_id")"
|
|
final_status="$(printf '%s' "$final_run" | json_get 'json.load(sys.stdin)["status"]')"
|
|
printf 'run status: %s\n' "$final_status"
|
|
|
|
case "$final_status" in
|
|
"$WORKFLOW_ACCEPTANCE_EXPECT_TERMINAL_STATUS")
|
|
case "$final_status" in
|
|
completed|failed|cancelled|waiting_for_audit)
|
|
break
|
|
;;
|
|
esac
|
|
;;
|
|
failed|cancelled|waiting_for_audit)
|
|
printf 'unexpected terminal status %s, expected %s: %s\n' "$final_status" "$WORKFLOW_ACCEPTANCE_EXPECT_TERMINAL_STATUS" "$final_run" >&2
|
|
exit 1
|
|
;;
|
|
esac
|
|
|
|
sleep 2
|
|
done
|
|
|
|
if [[ "$final_status" != "$WORKFLOW_ACCEPTANCE_EXPECT_TERMINAL_STATUS" ]]; then
|
|
printf 'run did not reach expected terminal status %s before timeout: %s\n' "$WORKFLOW_ACCEPTANCE_EXPECT_TERMINAL_STATUS" "$final_run" >&2
|
|
exit 1
|
|
fi
|
|
|
|
logs_response="$(api_curl "$API_BASE_URL/api/runs/$run_id/logs")"
|
|
artifacts_response="$(api_curl "$API_BASE_URL/api/runs/$run_id/artifacts")"
|
|
printf '%s' "$logs_response" | python3 -c 'import json, sys; payload = json.load(sys.stdin); items = payload.get("items", []); print("logs summary:", json.dumps({"total": payload.get("total", len(items)), "lastMessages": [item.get("message", "") for item in items[-3:]]}, ensure_ascii=False))'
|
|
printf '%s' "$artifacts_response" | python3 -c 'import json, sys; payload = json.load(sys.stdin); items = payload.get("items", []); print("artifacts summary:", json.dumps({"total": payload.get("total", len(items)), "items": [{"name": item.get("name"), "type": item.get("type"), "size": item.get("size")} for item in items[:10]]}, ensure_ascii=False))'
|
|
|
|
printf '%s' "$logs_response" | python3 -c 'import json, sys; payload = json.load(sys.stdin); items = payload.get("items", []);
|
|
if not items: raise SystemExit("expected at least one log entry")'
|
|
|
|
if [[ -n "$WORKFLOW_ACCEPTANCE_EXPECT_LOG_SUBSTRING" ]]; then
|
|
printf '%s' "$logs_response" | WORKFLOW_ACCEPTANCE_EXPECT_LOG_SUBSTRING="$WORKFLOW_ACCEPTANCE_EXPECT_LOG_SUBSTRING" python3 -c 'import json, os, sys; payload = json.load(sys.stdin); needle = os.environ["WORKFLOW_ACCEPTANCE_EXPECT_LOG_SUBSTRING"]; messages = "\n".join(str(item.get("message", "")) for item in payload.get("items", []));
|
|
if needle not in messages: raise SystemExit(f"expected log substring not found: {needle}")'
|
|
fi
|
|
|
|
if [[ "$WORKFLOW_ACCEPTANCE_EXPECT_TERMINAL_STATUS" == "completed" ]]; then
|
|
FINAL_RUN_JSON="$final_run" WORKFLOW_ACCEPTANCE_GPU_LIMIT="$WORKFLOW_ACCEPTANCE_GPU_LIMIT" python3 - <<'PY'
|
|
import json
|
|
import os
|
|
|
|
payload = json.loads(os.environ["FINAL_RUN_JSON"])
|
|
executions = {item.get("nodeId"): item for item in payload.get("nodeExecutions") or [] if isinstance(item, dict)}
|
|
upload = executions.get("upload-1")
|
|
if not upload:
|
|
raise SystemExit("expected upload-1 node execution in completed workflow")
|
|
|
|
lineage = upload.get("inputLineage") or []
|
|
if not any(item.get("sourceNodeId") == "source-1" and item.get("targetNodeId") == "upload-1" and item.get("artifactId") for item in lineage):
|
|
raise SystemExit("expected source-1 to upload-1 inputLineage artifact")
|
|
if not upload.get("inputArtifacts"):
|
|
raise SystemExit("expected upload-1 inputArtifacts from source-1")
|
|
|
|
gpu_limit = os.environ.get("WORKFLOW_ACCEPTANCE_GPU_LIMIT", "").strip()
|
|
if gpu_limit:
|
|
samples = []
|
|
for item in payload.get("resourceSeries") or []:
|
|
if isinstance(item, dict):
|
|
samples.append(item)
|
|
for node in payload.get("nodeExecutions") or []:
|
|
for item in node.get("resourceSeries") or []:
|
|
if isinstance(item, dict):
|
|
samples.append(item)
|
|
if not any(int(sample.get("gpuMemoryTotalMiB") or 0) > 0 for sample in samples):
|
|
raise SystemExit("expected GPU memory metrics in completed GPU workflow resourceSeries")
|
|
PY
|
|
printf '%s' "$artifacts_response" | python3 -c 'import json, sys; payload = json.load(sys.stdin); items = payload.get("items", []);
|
|
if not items: raise SystemExit("expected at least one artifact")'
|
|
fi
|