#!/usr/bin/env bash : "${DROBOTICFLOW_ACCEPTANCE_PASSWORD:?Set DROBOTICFLOW_ACCEPTANCE_PASSWORD before running acceptance}" set -euo pipefail ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" DEPLOY_ENV="${DEPLOY_ENV:-}" export DEPLOY_ENV # shellcheck source=scripts/load-deploy-env.sh source "$ROOT_DIR/scripts/load-deploy-env.sh" # shellcheck source=scripts/lib/service-endpoints.sh source "$ROOT_DIR/scripts/lib/service-endpoints.sh" K8S_NAMESPACE="${K8S_NAMESPACE:-droboticflow}" API_PUBLIC_SERVICE_NAME="${API_PUBLIC_SERVICE_NAME:-droboticflow-api-public}" WORKER_SERVICE_NAME="${POC_OPERATOR_WORKFLOW_WORKER_SERVICE_NAME:-droboticflow-worker}" WORKER_SERVICE_PORT="${POC_OPERATOR_WORKFLOW_WORKER_SERVICE_PORT:-3002}" API_TOKEN_SECRET_NAME="${POC_OPERATOR_WORKFLOW_API_TOKEN_SECRET_NAME:-droboticflow-auth-secret}" API_TOKEN_SECRET_KEY="${POC_OPERATOR_WORKFLOW_API_TOKEN_SECRET_KEY:-DROBOTICFLOW_API_TOKEN}" POC_OPERATOR_WORKFLOW_RUN_TIMEOUT_SECONDS="${POC_OPERATOR_WORKFLOW_RUN_TIMEOUT_SECONDS:-5400}" POC_OPERATOR_WORKFLOW_FIXTURE_MANIFEST_PATH="${POC_OPERATOR_WORKFLOW_FIXTURE_MANIFEST_PATH:-/workspace/datasets/fixtures/poc-operator-acceptance/poc-operator-acceptance.json}" POC_OPERATOR_WORKFLOW_MIN_OPERATOR_COUNT="${POC_OPERATOR_WORKFLOW_MIN_OPERATOR_COUNT:-$(python3 - "$ROOT_DIR/internal/api/builtin_operators_catalog.json" <<'PY' import json import os import sys print(len(json.load(open(sys.argv[1], encoding="utf-8")))) PY )}" POC_OPERATOR_WORKFLOW_PROCESS_ONCE_TIMEOUT_SECONDS="${POC_OPERATOR_WORKFLOW_PROCESS_ONCE_TIMEOUT_SECONDS:-15}" POC_OPERATOR_WORKFLOW_REQUIRE_AUDIT_RESOLUTION="${POC_OPERATOR_WORKFLOW_REQUIRE_AUDIT_RESOLUTION:-false}" DROBOTICFLOW_ACCEPTANCE_EMAIL="${DROBOTICFLOW_ACCEPTANCE_EMAIL:-poc-acceptance-admin@droboticflow.local}" normalize_bool() { local name="$1" local value="$2" local normalized normalized="$(printf '%s' "$value" | tr '[:upper:]' '[:lower:]')" case "$normalized" in true|1|yes|y|on) printf 'true\n' ;; false|0|no|n|off) printf 'false\n' ;; *) echo "ERROR: $name must be true or false." >&2 exit 1 ;; esac } if ! [[ "$POC_OPERATOR_WORKFLOW_RUN_TIMEOUT_SECONDS" =~ ^[0-9]+$ ]] || (( POC_OPERATOR_WORKFLOW_RUN_TIMEOUT_SECONDS < 1 )); then echo "ERROR: POC_OPERATOR_WORKFLOW_RUN_TIMEOUT_SECONDS must be a positive integer." >&2 exit 1 fi if [[ -z "$POC_OPERATOR_WORKFLOW_FIXTURE_MANIFEST_PATH" || "$POC_OPERATOR_WORKFLOW_FIXTURE_MANIFEST_PATH" != /* ]]; then echo "ERROR: POC_OPERATOR_WORKFLOW_FIXTURE_MANIFEST_PATH must be an absolute workspace path." >&2 exit 1 fi if ! [[ "$POC_OPERATOR_WORKFLOW_MIN_OPERATOR_COUNT" =~ ^[1-9][0-9]*$ ]]; then echo "ERROR: POC_OPERATOR_WORKFLOW_MIN_OPERATOR_COUNT must be a positive integer." >&2 exit 1 fi if ! [[ "$POC_OPERATOR_WORKFLOW_PROCESS_ONCE_TIMEOUT_SECONDS" =~ ^[1-9][0-9]*$ ]]; then echo "ERROR: POC_OPERATOR_WORKFLOW_PROCESS_ONCE_TIMEOUT_SECONDS must be a positive integer." >&2 exit 1 fi if [[ -z "${API_BASE_URL:-}" && -z "${BASE_URL:-}" ]]; then public_host="$( kubectl -n "$K8S_NAMESPACE" get svc "$API_PUBLIC_SERVICE_NAME" -o json | python3 -c ' import ipaddress import json import sys payload = json.load(sys.stdin) hosts = [item.get("ip") or item.get("hostname") for item in payload.get("status", {}).get("loadBalancer", {}).get("ingress", [])] hosts = [item for item in hosts if item] if not hosts: raise SystemExit("no LoadBalancer ingress host is available") for host in hosts: try: if not ipaddress.ip_address(host).is_private: print(host) raise SystemExit(0) except ValueError: print(host) raise SystemExit(0) print(hosts[0]) ' )" export API_BASE_URL="http://${public_host}" fi API_BASE_URL="$(droboticflow_first_service_url "${API_BASE_URL:-}" "${BASE_URL:-}")" if [[ -z "${DROBOTICFLOW_API_TOKEN:-}" ]]; then api_token_secret_json="$(kubectl -n "$K8S_NAMESPACE" get secret "$API_TOKEN_SECRET_NAME" -o json 2>/dev/null || true)" DROBOTICFLOW_API_TOKEN="$( printf "%s" "$api_token_secret_json" | API_TOKEN_SECRET_KEY="$API_TOKEN_SECRET_KEY" python3 -c ' import base64 import json import os import sys try: payload = json.load(sys.stdin) except json.JSONDecodeError: raise SystemExit(0) raw = payload.get("data", {}).get(os.environ["API_TOKEN_SECRET_KEY"]) if isinstance(raw, str) and raw: print(base64.b64decode(raw, validate=True).decode("utf-8", errors="replace").strip(), end="") ' )" export DROBOTICFLOW_API_TOKEN fi if [[ -z "${DROBOTICFLOW_API_TOKEN:-}" ]]; then echo "ERROR: DROBOTICFLOW_API_TOKEN or ${K8S_NAMESPACE}/${API_TOKEN_SECRET_NAME}.${API_TOKEN_SECRET_KEY} is required." >&2 exit 1 fi json_get() { local expression="$1" python3 -c "import json,sys; print(${expression})" } api_curl() { curl --retry 6 --retry-delay 2 --retry-connrefused --retry-all-errors -fsS \ -b "$COOKIE_JAR" -c "$COOKIE_JAR" -H "Authorization: Bearer ${DROBOTICFLOW_API_TOKEN}" "$@" } csrf_token() { awk '$6 == "droboticflow_csrf" { value=$7 } END { print value }' "$COOKIE_JAR" } api_curl_write() { local token token="$(csrf_token)" if [[ -n "$token" ]]; then api_curl -H "X-Droboticflow-CSRF-Token: $token" "$@" return fi api_curl "$@" } worker_curl() { curl --retry 3 --retry-delay 1 --retry-connrefused --retry-all-errors -fsS "$@" } worker_process_once_curl() { curl --max-time "$POC_OPERATOR_WORKFLOW_PROCESS_ONCE_TIMEOUT_SECONDS" -fsS "$@" } truncate_process_once_output() { local value="$1" local limit=1200 if (( ${#value} <= limit )); then printf '%s' "$value" return 0 fi printf '%s... [truncated %d bytes]' "${value:0:limit}" "${#value}" } free_local_port() { python3 - <<'PY' import socket with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: sock.bind(("127.0.0.1", 0)) print(sock.getsockname()[1]) PY } PORT_FORWARD_PID="" PORT_FORWARD_LOG="$(mktemp)" RUNS_JSON_FILE="$(mktemp)" COOKIE_JAR="$(mktemp)" WORKER_LOCAL_BASE_URL="" stop_worker_port_forward() { if [[ -n "$PORT_FORWARD_PID" ]] && kill -0 "$PORT_FORWARD_PID" >/dev/null 2>&1; then kill "$PORT_FORWARD_PID" >/dev/null 2>&1 || true wait "$PORT_FORWARD_PID" >/dev/null 2>&1 || true fi PORT_FORWARD_PID="" WORKER_LOCAL_BASE_URL="" } cleanup() { stop_worker_port_forward rm -f "$PORT_FORWARD_LOG" "$RUNS_JSON_FILE" "$COOKIE_JAR" } trap cleanup EXIT start_worker_port_forward() { local local_port stop_worker_port_forward : >"$PORT_FORWARD_LOG" local_port="$(free_local_port)" kubectl -n "$K8S_NAMESPACE" port-forward "service/${WORKER_SERVICE_NAME}" "${local_port}:${WORKER_SERVICE_PORT}" >"$PORT_FORWARD_LOG" 2>&1 & PORT_FORWARD_PID="$!" WORKER_LOCAL_BASE_URL="http://127.0.0.1:${local_port}" local deadline=$((SECONDS + 20)) while (( SECONDS < deadline )); do if ! kill -0 "$PORT_FORWARD_PID" >/dev/null 2>&1; then echo "warning: worker port-forward exited early." >&2 tail -n 20 "$PORT_FORWARD_LOG" >&2 || true stop_worker_port_forward return 1 fi if worker_curl "$WORKER_LOCAL_BASE_URL/worker-api/health" >/dev/null 2>&1; then printf 'worker port-forward: %s -> service/%s:%s\n' "$WORKER_LOCAL_BASE_URL" "$WORKER_SERVICE_NAME" "$WORKER_SERVICE_PORT" return 0 fi sleep 0.25 done echo "warning: timed out waiting for worker port-forward." >&2 tail -n 20 "$PORT_FORWARD_LOG" >&2 || true stop_worker_port_forward return 1 } ensure_worker_port_forward() { if [[ -n "$PORT_FORWARD_PID" && -n "$WORKER_LOCAL_BASE_URL" ]] && kill -0 "$PORT_FORWARD_PID" >/dev/null 2>&1; then if worker_curl --max-time 5 "$WORKER_LOCAL_BASE_URL/worker-api/health" >/dev/null 2>&1; then return 0 fi printf 'warning: worker port-forward health check failed; restarting port-forward.\n' >&2 fi start_worker_port_forward } trigger_worker_process_once() { local output if ! ensure_worker_port_forward; then printf 'warning: worker port-forward is unavailable; continuing with run-state polling.\n' >&2 return 0 fi if output="$(worker_process_once_curl -o /dev/null -X POST "$WORKER_LOCAL_BASE_URL/worker-api/process-once" 2>&1)"; then return 0 fi printf 'warning: worker process-once did not return cleanly; restarting port-forward and retrying once: %s\n' "$(truncate_process_once_output "$output")" >&2 if ! start_worker_port_forward; then printf 'warning: worker port-forward retry is unavailable; continuing with run-state polling.\n' >&2 return 0 fi if output="$(worker_process_once_curl -o /dev/null -X POST "$WORKER_LOCAL_BASE_URL/worker-api/process-once" 2>&1)"; then return 0 fi printf 'warning: worker process-once retry did not return cleanly; continuing with run-state polling: %s\n' "$(truncate_process_once_output "$output")" >&2 return 0 } runs_need_process_once_file() { python3 - "$1" <<'PY' import datetime as dt import json import sys path = sys.argv[1] runs = json.loads(open(path, encoding="utf-8").read()) now = dt.datetime.now(dt.timezone.utc) def parse_ts(value): if not value: return None try: return dt.datetime.fromisoformat(str(value).replace("Z", "+00:00")) except ValueError: return None for run in runs: if run.get("status") != "queued": continue active_node = False for node in run.get("nodeExecutions") or []: if node.get("status") in {"running", "waiting_for_audit", "queued"}: active_node = True break if active_node: continue lease_owner = str(run.get("executionLeaseOwner") or "").strip() lease_expires = parse_ts(run.get("executionLeaseExpiresAt")) if lease_owner and lease_expires and lease_expires > now: continue raise SystemExit(0) raise SystemExit(1) PY } api_health="$(api_curl "$API_BASE_URL/api/health")" printf 'api health: %s\n' "$api_health" printf '%s' "$api_health" | grep -q '"status":"ok"' login_payload="$( DROBOTICFLOW_ACCEPTANCE_EMAIL="$DROBOTICFLOW_ACCEPTANCE_EMAIL" \ DROBOTICFLOW_ACCEPTANCE_PASSWORD="$DROBOTICFLOW_ACCEPTANCE_PASSWORD" python3 - <<'PY' import json import os print(json.dumps({"email": os.environ["DROBOTICFLOW_ACCEPTANCE_EMAIL"], "password": os.environ["DROBOTICFLOW_ACCEPTANCE_PASSWORD"]})) PY )" login_response="$(api_curl -H 'Content-Type: application/json' -X POST "$API_BASE_URL/api/auth/login" -d "$login_payload")" LOGIN_RESPONSE="$login_response" DROBOTICFLOW_ACCEPTANCE_EMAIL="$DROBOTICFLOW_ACCEPTANCE_EMAIL" python3 - <<'PY' import json import os payload = json.loads(os.environ["LOGIN_RESPONSE"]) user = payload.get("user") or payload email = user.get("email") or payload.get("email") if email != os.environ["DROBOTICFLOW_ACCEPTANCE_EMAIL"]: raise SystemExit(f"acceptance login returned unexpected user: {email}") print(f"logged in acceptance user: {email}") PY payload="$( POC_OPERATOR_WORKFLOW_FIXTURE_MANIFEST_PATH="$POC_OPERATOR_WORKFLOW_FIXTURE_MANIFEST_PATH" python3 - <<'PY' import json import os print(json.dumps({ "fixtureManifestPath": os.environ["POC_OPERATOR_WORKFLOW_FIXTURE_MANIFEST_PATH"], "includeDcloud": False, "includeShellCommand": True, })) PY )" response="$(api_curl_write -X POST "$API_BASE_URL/api/acceptance/poc-operator-workflows" -H 'Content-Type: application/json' -d "$payload")" printf '%s' "$response" | python3 -c 'import json, sys; payload=json.load(sys.stdin); print("created workflow batches:", json.dumps({"operatorCount": payload.get("operatorCount"), "runs": [{"batch": item.get("batch"), "runId": item.get("runId"), "operators": len(item.get("operatorIds") or [])} for item in payload.get("runs", [])]}, ensure_ascii=False))' RESPONSE="$response" POC_OPERATOR_WORKFLOW_MIN_OPERATOR_COUNT="$POC_OPERATOR_WORKFLOW_MIN_OPERATOR_COUNT" python3 - <<'PY' import json import os import sys payload = json.loads(os.environ["RESPONSE"]) summary = payload.get("summary") or {} print("operator workflow contract:", json.dumps(summary, ensure_ascii=False)) operator_count = int(summary.get("operatorCount") or payload.get("operatorCount") or 0) minimum_operator_count = int(os.environ.get("POC_OPERATOR_WORKFLOW_MIN_OPERATOR_COUNT", "96")) coverage = int(summary.get("independentImageCoverage") or 0) required = int(summary.get("independentImageRequired") or operator_count) command_wrappers = int(summary.get("commandWrapperOnlyOperatorCount") or 0) generated_fixtures = int(summary.get("generatedFixtureCount") or 0) mock_fixtures = int(summary.get("mockFixtureCount") or 0) if operator_count != minimum_operator_count: raise SystemExit(f"operator workflow acceptance must cover the current catalog exactly: {operator_count}/{minimum_operator_count}") if summary.get("contractVersion") != "poc-operator-workflows/v4": raise SystemExit(f"operator workflow contract version must be poc-operator-workflows/v4, got {summary.get('contractVersion')}") if int(summary.get("catalogCount") or 0) != operator_count or not summary.get("catalogHash"): raise SystemExit("operator workflow acceptance catalog provenance is missing or inconsistent") if summary.get("trainingMode") != "full" or summary.get("fullTrainingRequired") is not True: raise SystemExit("operator workflow acceptance must require full training") if summary.get("missingOperatorIds"): raise SystemExit(f"operator workflow acceptance has missing operators: {summary.get('missingOperatorIds')}") if coverage != required: raise SystemExit(f"independent image coverage is incomplete: {coverage}/{required}") if command_wrappers != 0: raise SystemExit(f"non-shell command-wrapper operators are not allowed: {command_wrappers}") if generated_fixtures != 0 or mock_fixtures != 0: raise SystemExit(f"generated/mock fixtures are not allowed: generated={generated_fixtures} mock={mock_fixtures}") for run in payload.get("runs") or []: evidence = run.get("imageEvidence") or [] if len(evidence) != len(run.get("operatorIds") or []): raise SystemExit(f"run {run.get('runId')} image evidence count does not match operators") for item in evidence: image = str(item.get("image") or "") image_id = str(item.get("imageId") or "") if "droboticflow-operator-runner" in image or "droboticflow-operator-" not in image: raise SystemExit(f"operator {item.get('operatorId')} does not use an independent operator image") if "@sha256:" not in image_id: raise SystemExit(f"operator {item.get('operatorId')} imageId is not digest-pinned") PY run_ids="$(printf '%s' "$response" | python3 -c 'import json, sys; [print(item["runId"]) for item in json.load(sys.stdin).get("runs", [])]')" if [[ -z "$run_ids" ]]; then echo "ERROR: no runs were created." >&2 exit 1 fi start_worker_port_forward || printf 'warning: initial worker port-forward is unavailable; relying on background worker polling.\n' >&2 fetch_runs_json() { printf '[' >"$RUNS_JSON_FILE" local first=true while IFS= read -r run_id; do run_id="$(printf '%s' "$run_id" | tr -d '\r' | xargs)" [[ -z "$run_id" ]] && continue if [[ "$first" == "true" ]]; then first=false else printf ',' >>"$RUNS_JSON_FILE" fi api_curl "$API_BASE_URL/api/runs/$run_id" >>"$RUNS_JSON_FILE" done <<<"$run_ids" printf ']' >>"$RUNS_JSON_FILE" } deadline=$((SECONDS + POC_OPERATOR_WORKFLOW_RUN_TIMEOUT_SECONDS)) last_process_once_at=0 while (( SECONDS < deadline )); do fetch_runs_json if (( SECONDS - last_process_once_at >= 3 )) && runs_need_process_once_file "$RUNS_JSON_FILE"; then trigger_worker_process_once last_process_once_at=$SECONDS fetch_runs_json fi summary="$( python3 - "$RUNS_JSON_FILE" <<'PY' import json import os import sys runs = json.loads(open(sys.argv[1], encoding="utf-8").read()) print(json.dumps([{"id": run.get("id"), "status": run.get("status"), "nodes": len(run.get("nodeExecutions") or [])} for run in runs], ensure_ascii=False)) PY )" printf 'run statuses: %s\n' "$summary" if SUMMARY="$summary" python3 - <<'PY' import json import os items = json.loads(os.environ["SUMMARY"]) terminal = {"completed", "failed", "cancelled", "waiting_for_audit"} raise SystemExit(0 if items and all(item.get("status") in terminal for item in items) else 1) PY then break fi sleep 5 done fetch_runs_json python3 - "$RUNS_JSON_FILE" <<'PY' import json import os import sys runs = json.loads(open(sys.argv[1], encoding="utf-8").read()) covered = set() failed = [] waiting = [] lineage_failures = [] production_failures = [] minimum_train_samples = int(__import__("os").environ.get("POC_OPERATOR_WORKFLOW_MIN_TRAIN_SAMPLE_COUNT", "128")) require_audit_resolution = os.environ.get("POC_OPERATOR_WORKFLOW_REQUIRE_AUDIT_RESOLUTION", "false").lower() in {"1", "true", "yes", "on"} expected_edges_by_suffix = { "外部存储与审计链路": [ ("op-shell-command", "op-bos-incremental-scan"), ("op-bos-incremental-scan", "op-mongodb-write"), ], "视频检测裁剪与成品链路": [ ("op-xplatform-dataset-index", "op-xplatform-dataset-validate"), ("op-xplatform-dataset-validate", "op-video-quality"), ("op-video-quality", "op-video-filter"), ("op-video-filter", "op-video-slice"), ("op-video-slice", "op-video-target-clip"), ("op-video-target-clip", "op-yolo-video-target-clip"), ("op-yolo-video-target-clip", "op-video-annotate"), ("op-video-annotate", "op-package"), ], } def workflow_suffix(run): name = str(run.get("workflowName") or "") marker = "POC Operator Workflow Acceptance - " if marker in name: return name.split(marker, 1)[1].strip() return name.rsplit(" - ", 1)[-1].strip() def validate_workflow_lineage(run): suffix = workflow_suffix(run) expected_edges = expected_edges_by_suffix.get(suffix, []) if not expected_edges: return executions = {node.get("nodeId"): node for node in run.get("nodeExecutions") or []} for source, target in expected_edges: target_execution = executions.get(target) if not target_execution: lineage_failures.append({"runId": run.get("id"), "workflow": run.get("workflowName"), "edge": f"{source}->{target}", "error": "target execution missing"}) continue lineage = target_execution.get("inputLineage") or [] artifacts = target_execution.get("inputArtifacts") or [] matched = [ item for item in lineage if item.get("sourceNodeId") == source and item.get("targetNodeId") == target and (item.get("artifactId") or (item.get("artifact") or {}).get("id")) ] if not matched: lineage_failures.append({"runId": run.get("id"), "workflow": run.get("workflowName"), "edge": f"{source}->{target}", "error": "missing inputLineage artifact"}) if not artifacts: lineage_failures.append({"runId": run.get("id"), "workflow": run.get("workflowName"), "edge": f"{source}->{target}", "error": "target inputArtifacts empty"}) for run in runs: validate_workflow_lineage(run) for node in run.get("nodeExecutions") or []: operator_id = node.get("operatorId") or node.get("nodeId") status = node.get("status") artifacts = node.get("artifacts") or [] if operator_id in { "operator-builtin-act-policy-train", "operator-builtin-vla-model-train", "operator-builtin-diffusion-policy-train", "operator-builtin-openvla-finetune", "operator-builtin-yolo-embodied-detection-train", } and status == "completed": artifact_names = {str(item.get("name") or "") for item in artifacts} has_checkpoint = any( name.endswith((".pt", ".pth", ".safetensors", ".onnx")) or "checkpoint" in name or (operator_id == "operator-builtin-openvla-finetune" and name.endswith(".zip")) or (operator_id == "operator-builtin-vla-model-train" and name.endswith(".json")) for name in artifact_names ) has_metrics = any("metric" in name.lower() for name in artifact_names) if not has_checkpoint: production_failures.append({"runId": run.get("id"), "operatorId": operator_id, "error": "missing checkpoint artifact"}) if not has_metrics: production_failures.append({"runId": run.get("id"), "operatorId": operator_id, "error": "missing metrics artifact"}) for artifact in artifacts: data = artifact.get("data") if isinstance(artifact.get("data"), dict) else {} if data.get("requiresDNNModelPathForProduction") is True: production_failures.append({ "runId": run.get("id"), "operatorId": operator_id, "artifact": artifact.get("name"), "error": "YOLO/video production validation requires a real DNN_MODEL_PATH", }) sample_count = data.get("sampleCount") model_family = str(data.get("modelFamily") or data.get("type") or "") if operator_id in { "operator-builtin-act-policy-train", "operator-builtin-vla-model-train", "operator-builtin-diffusion-policy-train", "operator-builtin-openvla-finetune", "operator-builtin-yolo-embodied-detection-train", } and sample_count is not None: try: parsed_sample_count = int(sample_count) except (TypeError, ValueError): parsed_sample_count = 0 if parsed_sample_count < minimum_train_samples: production_failures.append({ "runId": run.get("id"), "operatorId": operator_id, "artifact": artifact.get("name"), "error": f"training sampleCount {parsed_sample_count} is below production minimum {minimum_train_samples}", }) if "vla_linear_policy" in model_family and operator_id != "operator-builtin-vla-model-train": production_failures.append({ "runId": run.get("id"), "operatorId": operator_id, "artifact": artifact.get("name"), "error": "linear VLA policy is not accepted as production VLA training evidence", }) if status == "completed" or (status == "waiting_for_audit" and not require_audit_resolution): covered.add(operator_id) elif status == "failed": failed.append({ "runId": run.get("id"), "workflow": run.get("workflowName"), "operatorId": operator_id, "label": node.get("nodeLabel"), "error": node.get("error"), }) if run.get("status") == "waiting_for_audit": waiting.append(run.get("id")) summary = { "runCount": len(runs), "terminalStatuses": {status: sum(1 for run in runs if run.get("status") == status) for status in sorted({run.get("status") for run in runs})}, "coveredExecutionCount": len(covered), "failedExecutionCount": len(failed), "lineageFailureCount": len(lineage_failures), "productionFailureCount": len(production_failures), "waitingForAuditRuns": waiting, "failed": failed[:50], "lineageFailures": lineage_failures[:50], "productionFailures": production_failures[:50], } print("operator workflow acceptance summary:", json.dumps(summary, ensure_ascii=False)) if failed: sys.exit(1) if waiting and require_audit_resolution: raise SystemExit(f"human review runs are still waiting for audit resolution: {waiting}") if lineage_failures: raise SystemExit("workflow data-transfer lineage validation failed") if production_failures: raise SystemExit("production-grade artifact validation failed") allowed_terminal = {"completed"} if require_audit_resolution else {"completed", "waiting_for_audit"} non_terminal = [run.get("id") for run in runs if run.get("status") not in allowed_terminal] if non_terminal: raise SystemExit(f"non-terminal or unsuccessful runs: {non_terminal}") PY