forked from longtao.wu/droboticflow
587 lines
24 KiB
Bash
Executable File
587 lines
24 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
|
DEPLOY_ENV="${DEPLOY_ENV:-}"
|
|
export DEPLOY_ENV
|
|
# shellcheck source=scripts/load-deploy-env.sh
|
|
source "$ROOT_DIR/scripts/load-deploy-env.sh"
|
|
# shellcheck source=scripts/lib/service-endpoints.sh
|
|
source "$ROOT_DIR/scripts/lib/service-endpoints.sh"
|
|
|
|
K8S_NAMESPACE="${K8S_NAMESPACE:-droboticflow}"
|
|
API_PUBLIC_SERVICE_NAME="${API_PUBLIC_SERVICE_NAME:-droboticflow-api-public}"
|
|
WORKER_SERVICE_NAME="${POC_OPERATOR_WORKFLOW_WORKER_SERVICE_NAME:-droboticflow-worker}"
|
|
WORKER_SERVICE_PORT="${POC_OPERATOR_WORKFLOW_WORKER_SERVICE_PORT:-3002}"
|
|
API_TOKEN_SECRET_NAME="${POC_OPERATOR_WORKFLOW_API_TOKEN_SECRET_NAME:-droboticflow-auth-secret}"
|
|
API_TOKEN_SECRET_KEY="${POC_OPERATOR_WORKFLOW_API_TOKEN_SECRET_KEY:-DROBOTICFLOW_API_TOKEN}"
|
|
POC_OPERATOR_WORKFLOW_RUN_TIMEOUT_SECONDS="${POC_OPERATOR_WORKFLOW_RUN_TIMEOUT_SECONDS:-5400}"
|
|
POC_OPERATOR_WORKFLOW_FIXTURE_MANIFEST_PATH="${POC_OPERATOR_WORKFLOW_FIXTURE_MANIFEST_PATH:-/workspace/datasets/fixtures/poc-operator-acceptance/poc-operator-acceptance.json}"
|
|
POC_OPERATOR_WORKFLOW_MIN_OPERATOR_COUNT="${POC_OPERATOR_WORKFLOW_MIN_OPERATOR_COUNT:-$(python3 - "$ROOT_DIR/internal/api/builtin_operators_catalog.json" <<'PY'
|
|
import json
|
|
import os
|
|
import sys
|
|
print(len(json.load(open(sys.argv[1], encoding="utf-8"))))
|
|
PY
|
|
)}"
|
|
POC_OPERATOR_WORKFLOW_PROCESS_ONCE_TIMEOUT_SECONDS="${POC_OPERATOR_WORKFLOW_PROCESS_ONCE_TIMEOUT_SECONDS:-15}"
|
|
POC_OPERATOR_WORKFLOW_REQUIRE_AUDIT_RESOLUTION="${POC_OPERATOR_WORKFLOW_REQUIRE_AUDIT_RESOLUTION:-false}"
|
|
DROBOTICFLOW_ACCEPTANCE_EMAIL="${DROBOTICFLOW_ACCEPTANCE_EMAIL:-poc-acceptance-admin@droboticflow.local}"
|
|
DROBOTICFLOW_ACCEPTANCE_PASSWORD="${DROBOTICFLOW_ACCEPTANCE_PASSWORD:-youbixuanpoc}"
|
|
|
|
normalize_bool() {
|
|
local name="$1"
|
|
local value="$2"
|
|
local normalized
|
|
normalized="$(printf '%s' "$value" | tr '[:upper:]' '[:lower:]')"
|
|
case "$normalized" in
|
|
true|1|yes|y|on)
|
|
printf 'true\n'
|
|
;;
|
|
false|0|no|n|off)
|
|
printf 'false\n'
|
|
;;
|
|
*)
|
|
echo "ERROR: $name must be true or false." >&2
|
|
exit 1
|
|
;;
|
|
esac
|
|
}
|
|
|
|
if ! [[ "$POC_OPERATOR_WORKFLOW_RUN_TIMEOUT_SECONDS" =~ ^[0-9]+$ ]] || (( POC_OPERATOR_WORKFLOW_RUN_TIMEOUT_SECONDS < 1 )); then
|
|
echo "ERROR: POC_OPERATOR_WORKFLOW_RUN_TIMEOUT_SECONDS must be a positive integer." >&2
|
|
exit 1
|
|
fi
|
|
if [[ -z "$POC_OPERATOR_WORKFLOW_FIXTURE_MANIFEST_PATH" || "$POC_OPERATOR_WORKFLOW_FIXTURE_MANIFEST_PATH" != /* ]]; then
|
|
echo "ERROR: POC_OPERATOR_WORKFLOW_FIXTURE_MANIFEST_PATH must be an absolute workspace path." >&2
|
|
exit 1
|
|
fi
|
|
if ! [[ "$POC_OPERATOR_WORKFLOW_MIN_OPERATOR_COUNT" =~ ^[1-9][0-9]*$ ]]; then
|
|
echo "ERROR: POC_OPERATOR_WORKFLOW_MIN_OPERATOR_COUNT must be a positive integer." >&2
|
|
exit 1
|
|
fi
|
|
if ! [[ "$POC_OPERATOR_WORKFLOW_PROCESS_ONCE_TIMEOUT_SECONDS" =~ ^[1-9][0-9]*$ ]]; then
|
|
echo "ERROR: POC_OPERATOR_WORKFLOW_PROCESS_ONCE_TIMEOUT_SECONDS must be a positive integer." >&2
|
|
exit 1
|
|
fi
|
|
|
|
if [[ -z "${API_BASE_URL:-}" && -z "${BASE_URL:-}" ]]; then
|
|
public_host="$(
|
|
kubectl -n "$K8S_NAMESPACE" get svc "$API_PUBLIC_SERVICE_NAME" -o json | python3 -c '
|
|
import ipaddress
|
|
import json
|
|
import sys
|
|
payload = json.load(sys.stdin)
|
|
hosts = [item.get("ip") or item.get("hostname") for item in payload.get("status", {}).get("loadBalancer", {}).get("ingress", [])]
|
|
hosts = [item for item in hosts if item]
|
|
if not hosts:
|
|
raise SystemExit("no LoadBalancer ingress host is available")
|
|
for host in hosts:
|
|
try:
|
|
if not ipaddress.ip_address(host).is_private:
|
|
print(host)
|
|
raise SystemExit(0)
|
|
except ValueError:
|
|
print(host)
|
|
raise SystemExit(0)
|
|
print(hosts[0])
|
|
'
|
|
)"
|
|
export API_BASE_URL="http://${public_host}"
|
|
fi
|
|
API_BASE_URL="$(droboticflow_first_service_url "${API_BASE_URL:-}" "${BASE_URL:-}")"
|
|
|
|
if [[ -z "${DROBOTICFLOW_API_TOKEN:-}" ]]; then
|
|
api_token_secret_json="$(kubectl -n "$K8S_NAMESPACE" get secret "$API_TOKEN_SECRET_NAME" -o json 2>/dev/null || true)"
|
|
DROBOTICFLOW_API_TOKEN="$(
|
|
printf "%s" "$api_token_secret_json" | API_TOKEN_SECRET_KEY="$API_TOKEN_SECRET_KEY" python3 -c '
|
|
import base64
|
|
import json
|
|
import os
|
|
import sys
|
|
try:
|
|
payload = json.load(sys.stdin)
|
|
except json.JSONDecodeError:
|
|
raise SystemExit(0)
|
|
raw = payload.get("data", {}).get(os.environ["API_TOKEN_SECRET_KEY"])
|
|
if isinstance(raw, str) and raw:
|
|
print(base64.b64decode(raw, validate=True).decode("utf-8", errors="replace").strip(), end="")
|
|
'
|
|
)"
|
|
export DROBOTICFLOW_API_TOKEN
|
|
fi
|
|
if [[ -z "${DROBOTICFLOW_API_TOKEN:-}" ]]; then
|
|
echo "ERROR: DROBOTICFLOW_API_TOKEN or ${K8S_NAMESPACE}/${API_TOKEN_SECRET_NAME}.${API_TOKEN_SECRET_KEY} is required." >&2
|
|
exit 1
|
|
fi
|
|
|
|
json_get() {
|
|
local expression="$1"
|
|
python3 -c "import json,sys; print(${expression})"
|
|
}
|
|
|
|
api_curl() {
|
|
curl --retry 6 --retry-delay 2 --retry-connrefused --retry-all-errors -fsS \
|
|
-b "$COOKIE_JAR" -c "$COOKIE_JAR" -H "Authorization: Bearer ${DROBOTICFLOW_API_TOKEN}" "$@"
|
|
}
|
|
|
|
csrf_token() {
|
|
awk '$6 == "droboticflow_csrf" { value=$7 } END { print value }' "$COOKIE_JAR"
|
|
}
|
|
|
|
api_curl_write() {
|
|
local token
|
|
token="$(csrf_token)"
|
|
if [[ -n "$token" ]]; then
|
|
api_curl -H "X-Droboticflow-CSRF-Token: $token" "$@"
|
|
return
|
|
fi
|
|
api_curl "$@"
|
|
}
|
|
|
|
worker_curl() {
|
|
curl --retry 3 --retry-delay 1 --retry-connrefused --retry-all-errors -fsS "$@"
|
|
}
|
|
|
|
worker_process_once_curl() {
|
|
curl --max-time "$POC_OPERATOR_WORKFLOW_PROCESS_ONCE_TIMEOUT_SECONDS" -fsS "$@"
|
|
}
|
|
|
|
truncate_process_once_output() {
|
|
local value="$1"
|
|
local limit=1200
|
|
if (( ${#value} <= limit )); then
|
|
printf '%s' "$value"
|
|
return 0
|
|
fi
|
|
printf '%s... [truncated %d bytes]' "${value:0:limit}" "${#value}"
|
|
}
|
|
|
|
free_local_port() {
|
|
python3 - <<'PY'
|
|
import socket
|
|
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
|
|
sock.bind(("127.0.0.1", 0))
|
|
print(sock.getsockname()[1])
|
|
PY
|
|
}
|
|
|
|
PORT_FORWARD_PID=""
|
|
PORT_FORWARD_LOG="$(mktemp)"
|
|
RUNS_JSON_FILE="$(mktemp)"
|
|
COOKIE_JAR="$(mktemp)"
|
|
WORKER_LOCAL_BASE_URL=""
|
|
|
|
stop_worker_port_forward() {
|
|
if [[ -n "$PORT_FORWARD_PID" ]] && kill -0 "$PORT_FORWARD_PID" >/dev/null 2>&1; then
|
|
kill "$PORT_FORWARD_PID" >/dev/null 2>&1 || true
|
|
wait "$PORT_FORWARD_PID" >/dev/null 2>&1 || true
|
|
fi
|
|
PORT_FORWARD_PID=""
|
|
WORKER_LOCAL_BASE_URL=""
|
|
}
|
|
|
|
cleanup() {
|
|
stop_worker_port_forward
|
|
rm -f "$PORT_FORWARD_LOG" "$RUNS_JSON_FILE" "$COOKIE_JAR"
|
|
}
|
|
trap cleanup EXIT
|
|
|
|
start_worker_port_forward() {
|
|
local local_port
|
|
stop_worker_port_forward
|
|
: >"$PORT_FORWARD_LOG"
|
|
local_port="$(free_local_port)"
|
|
kubectl -n "$K8S_NAMESPACE" port-forward "service/${WORKER_SERVICE_NAME}" "${local_port}:${WORKER_SERVICE_PORT}" >"$PORT_FORWARD_LOG" 2>&1 &
|
|
PORT_FORWARD_PID="$!"
|
|
WORKER_LOCAL_BASE_URL="http://127.0.0.1:${local_port}"
|
|
local deadline=$((SECONDS + 20))
|
|
while (( SECONDS < deadline )); do
|
|
if ! kill -0 "$PORT_FORWARD_PID" >/dev/null 2>&1; then
|
|
echo "warning: worker port-forward exited early." >&2
|
|
tail -n 20 "$PORT_FORWARD_LOG" >&2 || true
|
|
stop_worker_port_forward
|
|
return 1
|
|
fi
|
|
if worker_curl "$WORKER_LOCAL_BASE_URL/worker-api/health" >/dev/null 2>&1; then
|
|
printf 'worker port-forward: %s -> service/%s:%s\n' "$WORKER_LOCAL_BASE_URL" "$WORKER_SERVICE_NAME" "$WORKER_SERVICE_PORT"
|
|
return 0
|
|
fi
|
|
sleep 0.25
|
|
done
|
|
echo "warning: timed out waiting for worker port-forward." >&2
|
|
tail -n 20 "$PORT_FORWARD_LOG" >&2 || true
|
|
stop_worker_port_forward
|
|
return 1
|
|
}
|
|
|
|
ensure_worker_port_forward() {
|
|
if [[ -n "$PORT_FORWARD_PID" && -n "$WORKER_LOCAL_BASE_URL" ]] && kill -0 "$PORT_FORWARD_PID" >/dev/null 2>&1; then
|
|
if worker_curl --max-time 5 "$WORKER_LOCAL_BASE_URL/worker-api/health" >/dev/null 2>&1; then
|
|
return 0
|
|
fi
|
|
printf 'warning: worker port-forward health check failed; restarting port-forward.\n' >&2
|
|
fi
|
|
start_worker_port_forward
|
|
}
|
|
|
|
trigger_worker_process_once() {
|
|
local output
|
|
if ! ensure_worker_port_forward; then
|
|
printf 'warning: worker port-forward is unavailable; continuing with run-state polling.\n' >&2
|
|
return 0
|
|
fi
|
|
if output="$(worker_process_once_curl -o /dev/null -X POST "$WORKER_LOCAL_BASE_URL/worker-api/process-once" 2>&1)"; then
|
|
return 0
|
|
fi
|
|
printf 'warning: worker process-once did not return cleanly; restarting port-forward and retrying once: %s\n' "$(truncate_process_once_output "$output")" >&2
|
|
if ! start_worker_port_forward; then
|
|
printf 'warning: worker port-forward retry is unavailable; continuing with run-state polling.\n' >&2
|
|
return 0
|
|
fi
|
|
if output="$(worker_process_once_curl -o /dev/null -X POST "$WORKER_LOCAL_BASE_URL/worker-api/process-once" 2>&1)"; then
|
|
return 0
|
|
fi
|
|
printf 'warning: worker process-once retry did not return cleanly; continuing with run-state polling: %s\n' "$(truncate_process_once_output "$output")" >&2
|
|
return 0
|
|
}
|
|
|
|
runs_need_process_once_file() {
|
|
python3 - "$1" <<'PY'
|
|
import datetime as dt
|
|
import json
|
|
import sys
|
|
|
|
path = sys.argv[1]
|
|
runs = json.loads(open(path, encoding="utf-8").read())
|
|
now = dt.datetime.now(dt.timezone.utc)
|
|
|
|
def parse_ts(value):
|
|
if not value:
|
|
return None
|
|
try:
|
|
return dt.datetime.fromisoformat(str(value).replace("Z", "+00:00"))
|
|
except ValueError:
|
|
return None
|
|
|
|
for run in runs:
|
|
if run.get("status") != "queued":
|
|
continue
|
|
active_node = False
|
|
for node in run.get("nodeExecutions") or []:
|
|
if node.get("status") in {"running", "waiting_for_audit", "queued"}:
|
|
active_node = True
|
|
break
|
|
if active_node:
|
|
continue
|
|
lease_owner = str(run.get("executionLeaseOwner") or "").strip()
|
|
lease_expires = parse_ts(run.get("executionLeaseExpiresAt"))
|
|
if lease_owner and lease_expires and lease_expires > now:
|
|
continue
|
|
raise SystemExit(0)
|
|
raise SystemExit(1)
|
|
PY
|
|
}
|
|
|
|
api_health="$(api_curl "$API_BASE_URL/api/health")"
|
|
printf 'api health: %s\n' "$api_health"
|
|
printf '%s' "$api_health" | grep -q '"status":"ok"'
|
|
|
|
login_payload="$(
|
|
DROBOTICFLOW_ACCEPTANCE_EMAIL="$DROBOTICFLOW_ACCEPTANCE_EMAIL" \
|
|
DROBOTICFLOW_ACCEPTANCE_PASSWORD="$DROBOTICFLOW_ACCEPTANCE_PASSWORD" python3 - <<'PY'
|
|
import json
|
|
import os
|
|
print(json.dumps({"email": os.environ["DROBOTICFLOW_ACCEPTANCE_EMAIL"], "password": os.environ["DROBOTICFLOW_ACCEPTANCE_PASSWORD"]}))
|
|
PY
|
|
)"
|
|
login_response="$(api_curl -H 'Content-Type: application/json' -X POST "$API_BASE_URL/api/auth/login" -d "$login_payload")"
|
|
LOGIN_RESPONSE="$login_response" DROBOTICFLOW_ACCEPTANCE_EMAIL="$DROBOTICFLOW_ACCEPTANCE_EMAIL" python3 - <<'PY'
|
|
import json
|
|
import os
|
|
payload = json.loads(os.environ["LOGIN_RESPONSE"])
|
|
user = payload.get("user") or payload
|
|
email = user.get("email") or payload.get("email")
|
|
if email != os.environ["DROBOTICFLOW_ACCEPTANCE_EMAIL"]:
|
|
raise SystemExit(f"acceptance login returned unexpected user: {email}")
|
|
print(f"logged in acceptance user: {email}")
|
|
PY
|
|
|
|
payload="$(
|
|
POC_OPERATOR_WORKFLOW_FIXTURE_MANIFEST_PATH="$POC_OPERATOR_WORKFLOW_FIXTURE_MANIFEST_PATH" python3 - <<'PY'
|
|
import json
|
|
import os
|
|
print(json.dumps({
|
|
"fixtureManifestPath": os.environ["POC_OPERATOR_WORKFLOW_FIXTURE_MANIFEST_PATH"],
|
|
"includeDcloud": False,
|
|
"includeShellCommand": True,
|
|
}))
|
|
PY
|
|
)"
|
|
response="$(api_curl_write -X POST "$API_BASE_URL/api/acceptance/poc-operator-workflows" -H 'Content-Type: application/json' -d "$payload")"
|
|
printf '%s' "$response" | python3 -c 'import json, sys; payload=json.load(sys.stdin); print("created workflow batches:", json.dumps({"operatorCount": payload.get("operatorCount"), "runs": [{"batch": item.get("batch"), "runId": item.get("runId"), "operators": len(item.get("operatorIds") or [])} for item in payload.get("runs", [])]}, ensure_ascii=False))'
|
|
RESPONSE="$response" POC_OPERATOR_WORKFLOW_MIN_OPERATOR_COUNT="$POC_OPERATOR_WORKFLOW_MIN_OPERATOR_COUNT" python3 - <<'PY'
|
|
import json
|
|
import os
|
|
import sys
|
|
|
|
payload = json.loads(os.environ["RESPONSE"])
|
|
summary = payload.get("summary") or {}
|
|
print("operator workflow contract:", json.dumps(summary, ensure_ascii=False))
|
|
operator_count = int(summary.get("operatorCount") or payload.get("operatorCount") or 0)
|
|
minimum_operator_count = int(os.environ.get("POC_OPERATOR_WORKFLOW_MIN_OPERATOR_COUNT", "96"))
|
|
coverage = int(summary.get("independentImageCoverage") or 0)
|
|
required = int(summary.get("independentImageRequired") or operator_count)
|
|
command_wrappers = int(summary.get("commandWrapperOnlyOperatorCount") or 0)
|
|
generated_fixtures = int(summary.get("generatedFixtureCount") or 0)
|
|
mock_fixtures = int(summary.get("mockFixtureCount") or 0)
|
|
if operator_count != minimum_operator_count:
|
|
raise SystemExit(f"operator workflow acceptance must cover the current catalog exactly: {operator_count}/{minimum_operator_count}")
|
|
if summary.get("contractVersion") != "poc-operator-workflows/v4":
|
|
raise SystemExit(f"operator workflow contract version must be poc-operator-workflows/v4, got {summary.get('contractVersion')}")
|
|
if int(summary.get("catalogCount") or 0) != operator_count or not summary.get("catalogHash"):
|
|
raise SystemExit("operator workflow acceptance catalog provenance is missing or inconsistent")
|
|
if summary.get("trainingMode") != "full" or summary.get("fullTrainingRequired") is not True:
|
|
raise SystemExit("operator workflow acceptance must require full training")
|
|
if summary.get("missingOperatorIds"):
|
|
raise SystemExit(f"operator workflow acceptance has missing operators: {summary.get('missingOperatorIds')}")
|
|
if coverage != required:
|
|
raise SystemExit(f"independent image coverage is incomplete: {coverage}/{required}")
|
|
if command_wrappers != 0:
|
|
raise SystemExit(f"non-shell command-wrapper operators are not allowed: {command_wrappers}")
|
|
if generated_fixtures != 0 or mock_fixtures != 0:
|
|
raise SystemExit(f"generated/mock fixtures are not allowed: generated={generated_fixtures} mock={mock_fixtures}")
|
|
for run in payload.get("runs") or []:
|
|
evidence = run.get("imageEvidence") or []
|
|
if len(evidence) != len(run.get("operatorIds") or []):
|
|
raise SystemExit(f"run {run.get('runId')} image evidence count does not match operators")
|
|
for item in evidence:
|
|
image = str(item.get("image") or "")
|
|
image_id = str(item.get("imageId") or "")
|
|
if "droboticflow-operator-runner" in image or "droboticflow-operator-" not in image:
|
|
raise SystemExit(f"operator {item.get('operatorId')} does not use an independent operator image")
|
|
if "@sha256:" not in image_id:
|
|
raise SystemExit(f"operator {item.get('operatorId')} imageId is not digest-pinned")
|
|
PY
|
|
run_ids="$(printf '%s' "$response" | python3 -c 'import json, sys; [print(item["runId"]) for item in json.load(sys.stdin).get("runs", [])]')"
|
|
if [[ -z "$run_ids" ]]; then
|
|
echo "ERROR: no runs were created." >&2
|
|
exit 1
|
|
fi
|
|
|
|
start_worker_port_forward || printf 'warning: initial worker port-forward is unavailable; relying on background worker polling.\n' >&2
|
|
|
|
fetch_runs_json() {
|
|
printf '[' >"$RUNS_JSON_FILE"
|
|
local first=true
|
|
while IFS= read -r run_id; do
|
|
run_id="$(printf '%s' "$run_id" | tr -d '\r' | xargs)"
|
|
[[ -z "$run_id" ]] && continue
|
|
if [[ "$first" == "true" ]]; then
|
|
first=false
|
|
else
|
|
printf ',' >>"$RUNS_JSON_FILE"
|
|
fi
|
|
api_curl "$API_BASE_URL/api/runs/$run_id" >>"$RUNS_JSON_FILE"
|
|
done <<<"$run_ids"
|
|
printf ']' >>"$RUNS_JSON_FILE"
|
|
}
|
|
|
|
deadline=$((SECONDS + POC_OPERATOR_WORKFLOW_RUN_TIMEOUT_SECONDS))
|
|
last_process_once_at=0
|
|
while (( SECONDS < deadline )); do
|
|
fetch_runs_json
|
|
if (( SECONDS - last_process_once_at >= 3 )) && runs_need_process_once_file "$RUNS_JSON_FILE"; then
|
|
trigger_worker_process_once
|
|
last_process_once_at=$SECONDS
|
|
fetch_runs_json
|
|
fi
|
|
summary="$(
|
|
python3 - "$RUNS_JSON_FILE" <<'PY'
|
|
import json
|
|
import os
|
|
import sys
|
|
runs = json.loads(open(sys.argv[1], encoding="utf-8").read())
|
|
print(json.dumps([{"id": run.get("id"), "status": run.get("status"), "nodes": len(run.get("nodeExecutions") or [])} for run in runs], ensure_ascii=False))
|
|
PY
|
|
)"
|
|
printf 'run statuses: %s\n' "$summary"
|
|
if SUMMARY="$summary" python3 - <<'PY'
|
|
import json
|
|
import os
|
|
items = json.loads(os.environ["SUMMARY"])
|
|
terminal = {"completed", "failed", "cancelled", "waiting_for_audit"}
|
|
raise SystemExit(0 if items and all(item.get("status") in terminal for item in items) else 1)
|
|
PY
|
|
then
|
|
break
|
|
fi
|
|
sleep 5
|
|
done
|
|
|
|
fetch_runs_json
|
|
python3 - "$RUNS_JSON_FILE" <<'PY'
|
|
import json
|
|
import os
|
|
import sys
|
|
|
|
runs = json.loads(open(sys.argv[1], encoding="utf-8").read())
|
|
|
|
covered = set()
|
|
failed = []
|
|
waiting = []
|
|
lineage_failures = []
|
|
production_failures = []
|
|
minimum_train_samples = int(__import__("os").environ.get("POC_OPERATOR_WORKFLOW_MIN_TRAIN_SAMPLE_COUNT", "128"))
|
|
require_audit_resolution = os.environ.get("POC_OPERATOR_WORKFLOW_REQUIRE_AUDIT_RESOLUTION", "false").lower() in {"1", "true", "yes", "on"}
|
|
|
|
expected_edges_by_suffix = {
|
|
"外部存储与审计链路": [
|
|
("op-shell-command", "op-bos-incremental-scan"),
|
|
("op-bos-incremental-scan", "op-mongodb-write"),
|
|
],
|
|
"视频检测裁剪与成品链路": [
|
|
("op-xplatform-dataset-index", "op-xplatform-dataset-validate"),
|
|
("op-xplatform-dataset-validate", "op-video-quality"),
|
|
("op-video-quality", "op-video-filter"),
|
|
("op-video-filter", "op-video-slice"),
|
|
("op-video-slice", "op-video-target-clip"),
|
|
("op-video-target-clip", "op-yolo-video-target-clip"),
|
|
("op-yolo-video-target-clip", "op-video-annotate"),
|
|
("op-video-annotate", "op-package"),
|
|
],
|
|
}
|
|
|
|
def workflow_suffix(run):
|
|
name = str(run.get("workflowName") or "")
|
|
marker = "POC Operator Workflow Acceptance - "
|
|
if marker in name:
|
|
return name.split(marker, 1)[1].strip()
|
|
return name.rsplit(" - ", 1)[-1].strip()
|
|
|
|
def validate_workflow_lineage(run):
|
|
suffix = workflow_suffix(run)
|
|
expected_edges = expected_edges_by_suffix.get(suffix, [])
|
|
if not expected_edges:
|
|
return
|
|
executions = {node.get("nodeId"): node for node in run.get("nodeExecutions") or []}
|
|
for source, target in expected_edges:
|
|
target_execution = executions.get(target)
|
|
if not target_execution:
|
|
lineage_failures.append({"runId": run.get("id"), "workflow": run.get("workflowName"), "edge": f"{source}->{target}", "error": "target execution missing"})
|
|
continue
|
|
lineage = target_execution.get("inputLineage") or []
|
|
artifacts = target_execution.get("inputArtifacts") or []
|
|
matched = [
|
|
item for item in lineage
|
|
if item.get("sourceNodeId") == source
|
|
and item.get("targetNodeId") == target
|
|
and (item.get("artifactId") or (item.get("artifact") or {}).get("id"))
|
|
]
|
|
if not matched:
|
|
lineage_failures.append({"runId": run.get("id"), "workflow": run.get("workflowName"), "edge": f"{source}->{target}", "error": "missing inputLineage artifact"})
|
|
if not artifacts:
|
|
lineage_failures.append({"runId": run.get("id"), "workflow": run.get("workflowName"), "edge": f"{source}->{target}", "error": "target inputArtifacts empty"})
|
|
|
|
for run in runs:
|
|
validate_workflow_lineage(run)
|
|
for node in run.get("nodeExecutions") or []:
|
|
operator_id = node.get("operatorId") or node.get("nodeId")
|
|
status = node.get("status")
|
|
artifacts = node.get("artifacts") or []
|
|
if operator_id in {
|
|
"operator-builtin-act-policy-train",
|
|
"operator-builtin-vla-model-train",
|
|
"operator-builtin-diffusion-policy-train",
|
|
"operator-builtin-openvla-finetune",
|
|
"operator-builtin-yolo-embodied-detection-train",
|
|
} and status == "completed":
|
|
artifact_names = {str(item.get("name") or "") for item in artifacts}
|
|
has_checkpoint = any(
|
|
name.endswith((".pt", ".pth", ".safetensors", ".onnx"))
|
|
or "checkpoint" in name
|
|
or (operator_id == "operator-builtin-openvla-finetune" and name.endswith(".zip"))
|
|
or (operator_id == "operator-builtin-vla-model-train" and name.endswith(".json"))
|
|
for name in artifact_names
|
|
)
|
|
has_metrics = any("metric" in name.lower() for name in artifact_names)
|
|
if not has_checkpoint:
|
|
production_failures.append({"runId": run.get("id"), "operatorId": operator_id, "error": "missing checkpoint artifact"})
|
|
if not has_metrics:
|
|
production_failures.append({"runId": run.get("id"), "operatorId": operator_id, "error": "missing metrics artifact"})
|
|
for artifact in artifacts:
|
|
data = artifact.get("data") if isinstance(artifact.get("data"), dict) else {}
|
|
if data.get("requiresDNNModelPathForProduction") is True:
|
|
production_failures.append({
|
|
"runId": run.get("id"),
|
|
"operatorId": operator_id,
|
|
"artifact": artifact.get("name"),
|
|
"error": "YOLO/video production validation requires a real DNN_MODEL_PATH",
|
|
})
|
|
sample_count = data.get("sampleCount")
|
|
model_family = str(data.get("modelFamily") or data.get("type") or "")
|
|
if operator_id in {
|
|
"operator-builtin-act-policy-train",
|
|
"operator-builtin-vla-model-train",
|
|
"operator-builtin-diffusion-policy-train",
|
|
"operator-builtin-openvla-finetune",
|
|
"operator-builtin-yolo-embodied-detection-train",
|
|
} and sample_count is not None:
|
|
try:
|
|
parsed_sample_count = int(sample_count)
|
|
except (TypeError, ValueError):
|
|
parsed_sample_count = 0
|
|
if parsed_sample_count < minimum_train_samples:
|
|
production_failures.append({
|
|
"runId": run.get("id"),
|
|
"operatorId": operator_id,
|
|
"artifact": artifact.get("name"),
|
|
"error": f"training sampleCount {parsed_sample_count} is below production minimum {minimum_train_samples}",
|
|
})
|
|
if "vla_linear_policy" in model_family and operator_id != "operator-builtin-vla-model-train":
|
|
production_failures.append({
|
|
"runId": run.get("id"),
|
|
"operatorId": operator_id,
|
|
"artifact": artifact.get("name"),
|
|
"error": "linear VLA policy is not accepted as production VLA training evidence",
|
|
})
|
|
if status == "completed" or (status == "waiting_for_audit" and not require_audit_resolution):
|
|
covered.add(operator_id)
|
|
elif status == "failed":
|
|
failed.append({
|
|
"runId": run.get("id"),
|
|
"workflow": run.get("workflowName"),
|
|
"operatorId": operator_id,
|
|
"label": node.get("nodeLabel"),
|
|
"error": node.get("error"),
|
|
})
|
|
if run.get("status") == "waiting_for_audit":
|
|
waiting.append(run.get("id"))
|
|
|
|
summary = {
|
|
"runCount": len(runs),
|
|
"terminalStatuses": {status: sum(1 for run in runs if run.get("status") == status) for status in sorted({run.get("status") for run in runs})},
|
|
"coveredExecutionCount": len(covered),
|
|
"failedExecutionCount": len(failed),
|
|
"lineageFailureCount": len(lineage_failures),
|
|
"productionFailureCount": len(production_failures),
|
|
"waitingForAuditRuns": waiting,
|
|
"failed": failed[:50],
|
|
"lineageFailures": lineage_failures[:50],
|
|
"productionFailures": production_failures[:50],
|
|
}
|
|
print("operator workflow acceptance summary:", json.dumps(summary, ensure_ascii=False))
|
|
if failed:
|
|
sys.exit(1)
|
|
if waiting and require_audit_resolution:
|
|
raise SystemExit(f"human review runs are still waiting for audit resolution: {waiting}")
|
|
if lineage_failures:
|
|
raise SystemExit("workflow data-transfer lineage validation failed")
|
|
if production_failures:
|
|
raise SystemExit("production-grade artifact validation failed")
|
|
allowed_terminal = {"completed"} if require_audit_resolution else {"completed", "waiting_for_audit"}
|
|
non_terminal = [run.get("id") for run in runs if run.get("status") not in allowed_terminal]
|
|
if non_terminal:
|
|
raise SystemExit(f"non-terminal or unsuccessful runs: {non_terminal}")
|
|
PY
|