droboticflow-new/scripts/poc-operator-workflow-acceptance.sh

587 lines
24 KiB
Bash
Executable File

#!/usr/bin/env bash
: "${DROBOTICFLOW_ACCEPTANCE_PASSWORD:?Set DROBOTICFLOW_ACCEPTANCE_PASSWORD before running acceptance}"
set -euo pipefail
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
DEPLOY_ENV="${DEPLOY_ENV:-}"
export DEPLOY_ENV
# shellcheck source=scripts/load-deploy-env.sh
source "$ROOT_DIR/scripts/load-deploy-env.sh"
# shellcheck source=scripts/lib/service-endpoints.sh
source "$ROOT_DIR/scripts/lib/service-endpoints.sh"
K8S_NAMESPACE="${K8S_NAMESPACE:-droboticflow}"
API_PUBLIC_SERVICE_NAME="${API_PUBLIC_SERVICE_NAME:-droboticflow-api-public}"
WORKER_SERVICE_NAME="${POC_OPERATOR_WORKFLOW_WORKER_SERVICE_NAME:-droboticflow-worker}"
WORKER_SERVICE_PORT="${POC_OPERATOR_WORKFLOW_WORKER_SERVICE_PORT:-3002}"
API_TOKEN_SECRET_NAME="${POC_OPERATOR_WORKFLOW_API_TOKEN_SECRET_NAME:-droboticflow-auth-secret}"
API_TOKEN_SECRET_KEY="${POC_OPERATOR_WORKFLOW_API_TOKEN_SECRET_KEY:-DROBOTICFLOW_API_TOKEN}"
POC_OPERATOR_WORKFLOW_RUN_TIMEOUT_SECONDS="${POC_OPERATOR_WORKFLOW_RUN_TIMEOUT_SECONDS:-5400}"
POC_OPERATOR_WORKFLOW_FIXTURE_MANIFEST_PATH="${POC_OPERATOR_WORKFLOW_FIXTURE_MANIFEST_PATH:-/workspace/datasets/fixtures/poc-operator-acceptance/poc-operator-acceptance.json}"
POC_OPERATOR_WORKFLOW_MIN_OPERATOR_COUNT="${POC_OPERATOR_WORKFLOW_MIN_OPERATOR_COUNT:-$(python3 - "$ROOT_DIR/internal/api/builtin_operators_catalog.json" <<'PY'
import json
import os
import sys
print(len(json.load(open(sys.argv[1], encoding="utf-8"))))
PY
)}"
POC_OPERATOR_WORKFLOW_PROCESS_ONCE_TIMEOUT_SECONDS="${POC_OPERATOR_WORKFLOW_PROCESS_ONCE_TIMEOUT_SECONDS:-15}"
POC_OPERATOR_WORKFLOW_REQUIRE_AUDIT_RESOLUTION="${POC_OPERATOR_WORKFLOW_REQUIRE_AUDIT_RESOLUTION:-false}"
DROBOTICFLOW_ACCEPTANCE_EMAIL="${DROBOTICFLOW_ACCEPTANCE_EMAIL:-poc-acceptance-admin@droboticflow.local}"
normalize_bool() {
local name="$1"
local value="$2"
local normalized
normalized="$(printf '%s' "$value" | tr '[:upper:]' '[:lower:]')"
case "$normalized" in
true|1|yes|y|on)
printf 'true\n'
;;
false|0|no|n|off)
printf 'false\n'
;;
*)
echo "ERROR: $name must be true or false." >&2
exit 1
;;
esac
}
if ! [[ "$POC_OPERATOR_WORKFLOW_RUN_TIMEOUT_SECONDS" =~ ^[0-9]+$ ]] || (( POC_OPERATOR_WORKFLOW_RUN_TIMEOUT_SECONDS < 1 )); then
echo "ERROR: POC_OPERATOR_WORKFLOW_RUN_TIMEOUT_SECONDS must be a positive integer." >&2
exit 1
fi
if [[ -z "$POC_OPERATOR_WORKFLOW_FIXTURE_MANIFEST_PATH" || "$POC_OPERATOR_WORKFLOW_FIXTURE_MANIFEST_PATH" != /* ]]; then
echo "ERROR: POC_OPERATOR_WORKFLOW_FIXTURE_MANIFEST_PATH must be an absolute workspace path." >&2
exit 1
fi
if ! [[ "$POC_OPERATOR_WORKFLOW_MIN_OPERATOR_COUNT" =~ ^[1-9][0-9]*$ ]]; then
echo "ERROR: POC_OPERATOR_WORKFLOW_MIN_OPERATOR_COUNT must be a positive integer." >&2
exit 1
fi
if ! [[ "$POC_OPERATOR_WORKFLOW_PROCESS_ONCE_TIMEOUT_SECONDS" =~ ^[1-9][0-9]*$ ]]; then
echo "ERROR: POC_OPERATOR_WORKFLOW_PROCESS_ONCE_TIMEOUT_SECONDS must be a positive integer." >&2
exit 1
fi
if [[ -z "${API_BASE_URL:-}" && -z "${BASE_URL:-}" ]]; then
public_host="$(
kubectl -n "$K8S_NAMESPACE" get svc "$API_PUBLIC_SERVICE_NAME" -o json | python3 -c '
import ipaddress
import json
import sys
payload = json.load(sys.stdin)
hosts = [item.get("ip") or item.get("hostname") for item in payload.get("status", {}).get("loadBalancer", {}).get("ingress", [])]
hosts = [item for item in hosts if item]
if not hosts:
raise SystemExit("no LoadBalancer ingress host is available")
for host in hosts:
try:
if not ipaddress.ip_address(host).is_private:
print(host)
raise SystemExit(0)
except ValueError:
print(host)
raise SystemExit(0)
print(hosts[0])
'
)"
export API_BASE_URL="http://${public_host}"
fi
API_BASE_URL="$(droboticflow_first_service_url "${API_BASE_URL:-}" "${BASE_URL:-}")"
if [[ -z "${DROBOTICFLOW_API_TOKEN:-}" ]]; then
api_token_secret_json="$(kubectl -n "$K8S_NAMESPACE" get secret "$API_TOKEN_SECRET_NAME" -o json 2>/dev/null || true)"
DROBOTICFLOW_API_TOKEN="$(
printf "%s" "$api_token_secret_json" | API_TOKEN_SECRET_KEY="$API_TOKEN_SECRET_KEY" python3 -c '
import base64
import json
import os
import sys
try:
payload = json.load(sys.stdin)
except json.JSONDecodeError:
raise SystemExit(0)
raw = payload.get("data", {}).get(os.environ["API_TOKEN_SECRET_KEY"])
if isinstance(raw, str) and raw:
print(base64.b64decode(raw, validate=True).decode("utf-8", errors="replace").strip(), end="")
'
)"
export DROBOTICFLOW_API_TOKEN
fi
if [[ -z "${DROBOTICFLOW_API_TOKEN:-}" ]]; then
echo "ERROR: DROBOTICFLOW_API_TOKEN or ${K8S_NAMESPACE}/${API_TOKEN_SECRET_NAME}.${API_TOKEN_SECRET_KEY} is required." >&2
exit 1
fi
json_get() {
local expression="$1"
python3 -c "import json,sys; print(${expression})"
}
api_curl() {
curl --retry 6 --retry-delay 2 --retry-connrefused --retry-all-errors -fsS \
-b "$COOKIE_JAR" -c "$COOKIE_JAR" -H "Authorization: Bearer ${DROBOTICFLOW_API_TOKEN}" "$@"
}
csrf_token() {
awk '$6 == "droboticflow_csrf" { value=$7 } END { print value }' "$COOKIE_JAR"
}
api_curl_write() {
local token
token="$(csrf_token)"
if [[ -n "$token" ]]; then
api_curl -H "X-Droboticflow-CSRF-Token: $token" "$@"
return
fi
api_curl "$@"
}
worker_curl() {
curl --retry 3 --retry-delay 1 --retry-connrefused --retry-all-errors -fsS "$@"
}
worker_process_once_curl() {
curl --max-time "$POC_OPERATOR_WORKFLOW_PROCESS_ONCE_TIMEOUT_SECONDS" -fsS "$@"
}
truncate_process_once_output() {
local value="$1"
local limit=1200
if (( ${#value} <= limit )); then
printf '%s' "$value"
return 0
fi
printf '%s... [truncated %d bytes]' "${value:0:limit}" "${#value}"
}
free_local_port() {
python3 - <<'PY'
import socket
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
sock.bind(("127.0.0.1", 0))
print(sock.getsockname()[1])
PY
}
PORT_FORWARD_PID=""
PORT_FORWARD_LOG="$(mktemp)"
RUNS_JSON_FILE="$(mktemp)"
COOKIE_JAR="$(mktemp)"
WORKER_LOCAL_BASE_URL=""
stop_worker_port_forward() {
if [[ -n "$PORT_FORWARD_PID" ]] && kill -0 "$PORT_FORWARD_PID" >/dev/null 2>&1; then
kill "$PORT_FORWARD_PID" >/dev/null 2>&1 || true
wait "$PORT_FORWARD_PID" >/dev/null 2>&1 || true
fi
PORT_FORWARD_PID=""
WORKER_LOCAL_BASE_URL=""
}
cleanup() {
stop_worker_port_forward
rm -f "$PORT_FORWARD_LOG" "$RUNS_JSON_FILE" "$COOKIE_JAR"
}
trap cleanup EXIT
start_worker_port_forward() {
local local_port
stop_worker_port_forward
: >"$PORT_FORWARD_LOG"
local_port="$(free_local_port)"
kubectl -n "$K8S_NAMESPACE" port-forward "service/${WORKER_SERVICE_NAME}" "${local_port}:${WORKER_SERVICE_PORT}" >"$PORT_FORWARD_LOG" 2>&1 &
PORT_FORWARD_PID="$!"
WORKER_LOCAL_BASE_URL="http://127.0.0.1:${local_port}"
local deadline=$((SECONDS + 20))
while (( SECONDS < deadline )); do
if ! kill -0 "$PORT_FORWARD_PID" >/dev/null 2>&1; then
echo "warning: worker port-forward exited early." >&2
tail -n 20 "$PORT_FORWARD_LOG" >&2 || true
stop_worker_port_forward
return 1
fi
if worker_curl "$WORKER_LOCAL_BASE_URL/worker-api/health" >/dev/null 2>&1; then
printf 'worker port-forward: %s -> service/%s:%s\n' "$WORKER_LOCAL_BASE_URL" "$WORKER_SERVICE_NAME" "$WORKER_SERVICE_PORT"
return 0
fi
sleep 0.25
done
echo "warning: timed out waiting for worker port-forward." >&2
tail -n 20 "$PORT_FORWARD_LOG" >&2 || true
stop_worker_port_forward
return 1
}
ensure_worker_port_forward() {
if [[ -n "$PORT_FORWARD_PID" && -n "$WORKER_LOCAL_BASE_URL" ]] && kill -0 "$PORT_FORWARD_PID" >/dev/null 2>&1; then
if worker_curl --max-time 5 "$WORKER_LOCAL_BASE_URL/worker-api/health" >/dev/null 2>&1; then
return 0
fi
printf 'warning: worker port-forward health check failed; restarting port-forward.\n' >&2
fi
start_worker_port_forward
}
trigger_worker_process_once() {
local output
if ! ensure_worker_port_forward; then
printf 'warning: worker port-forward is unavailable; continuing with run-state polling.\n' >&2
return 0
fi
if output="$(worker_process_once_curl -o /dev/null -X POST "$WORKER_LOCAL_BASE_URL/worker-api/process-once" 2>&1)"; then
return 0
fi
printf 'warning: worker process-once did not return cleanly; restarting port-forward and retrying once: %s\n' "$(truncate_process_once_output "$output")" >&2
if ! start_worker_port_forward; then
printf 'warning: worker port-forward retry is unavailable; continuing with run-state polling.\n' >&2
return 0
fi
if output="$(worker_process_once_curl -o /dev/null -X POST "$WORKER_LOCAL_BASE_URL/worker-api/process-once" 2>&1)"; then
return 0
fi
printf 'warning: worker process-once retry did not return cleanly; continuing with run-state polling: %s\n' "$(truncate_process_once_output "$output")" >&2
return 0
}
runs_need_process_once_file() {
python3 - "$1" <<'PY'
import datetime as dt
import json
import sys
path = sys.argv[1]
runs = json.loads(open(path, encoding="utf-8").read())
now = dt.datetime.now(dt.timezone.utc)
def parse_ts(value):
if not value:
return None
try:
return dt.datetime.fromisoformat(str(value).replace("Z", "+00:00"))
except ValueError:
return None
for run in runs:
if run.get("status") != "queued":
continue
active_node = False
for node in run.get("nodeExecutions") or []:
if node.get("status") in {"running", "waiting_for_audit", "queued"}:
active_node = True
break
if active_node:
continue
lease_owner = str(run.get("executionLeaseOwner") or "").strip()
lease_expires = parse_ts(run.get("executionLeaseExpiresAt"))
if lease_owner and lease_expires and lease_expires > now:
continue
raise SystemExit(0)
raise SystemExit(1)
PY
}
api_health="$(api_curl "$API_BASE_URL/api/health")"
printf 'api health: %s\n' "$api_health"
printf '%s' "$api_health" | grep -q '"status":"ok"'
login_payload="$(
DROBOTICFLOW_ACCEPTANCE_EMAIL="$DROBOTICFLOW_ACCEPTANCE_EMAIL" \
DROBOTICFLOW_ACCEPTANCE_PASSWORD="$DROBOTICFLOW_ACCEPTANCE_PASSWORD" python3 - <<'PY'
import json
import os
print(json.dumps({"email": os.environ["DROBOTICFLOW_ACCEPTANCE_EMAIL"], "password": os.environ["DROBOTICFLOW_ACCEPTANCE_PASSWORD"]}))
PY
)"
login_response="$(api_curl -H 'Content-Type: application/json' -X POST "$API_BASE_URL/api/auth/login" -d "$login_payload")"
LOGIN_RESPONSE="$login_response" DROBOTICFLOW_ACCEPTANCE_EMAIL="$DROBOTICFLOW_ACCEPTANCE_EMAIL" python3 - <<'PY'
import json
import os
payload = json.loads(os.environ["LOGIN_RESPONSE"])
user = payload.get("user") or payload
email = user.get("email") or payload.get("email")
if email != os.environ["DROBOTICFLOW_ACCEPTANCE_EMAIL"]:
raise SystemExit(f"acceptance login returned unexpected user: {email}")
print(f"logged in acceptance user: {email}")
PY
payload="$(
POC_OPERATOR_WORKFLOW_FIXTURE_MANIFEST_PATH="$POC_OPERATOR_WORKFLOW_FIXTURE_MANIFEST_PATH" python3 - <<'PY'
import json
import os
print(json.dumps({
"fixtureManifestPath": os.environ["POC_OPERATOR_WORKFLOW_FIXTURE_MANIFEST_PATH"],
"includeDcloud": False,
"includeShellCommand": True,
}))
PY
)"
response="$(api_curl_write -X POST "$API_BASE_URL/api/acceptance/poc-operator-workflows" -H 'Content-Type: application/json' -d "$payload")"
printf '%s' "$response" | python3 -c 'import json, sys; payload=json.load(sys.stdin); print("created workflow batches:", json.dumps({"operatorCount": payload.get("operatorCount"), "runs": [{"batch": item.get("batch"), "runId": item.get("runId"), "operators": len(item.get("operatorIds") or [])} for item in payload.get("runs", [])]}, ensure_ascii=False))'
RESPONSE="$response" POC_OPERATOR_WORKFLOW_MIN_OPERATOR_COUNT="$POC_OPERATOR_WORKFLOW_MIN_OPERATOR_COUNT" python3 - <<'PY'
import json
import os
import sys
payload = json.loads(os.environ["RESPONSE"])
summary = payload.get("summary") or {}
print("operator workflow contract:", json.dumps(summary, ensure_ascii=False))
operator_count = int(summary.get("operatorCount") or payload.get("operatorCount") or 0)
minimum_operator_count = int(os.environ.get("POC_OPERATOR_WORKFLOW_MIN_OPERATOR_COUNT", "96"))
coverage = int(summary.get("independentImageCoverage") or 0)
required = int(summary.get("independentImageRequired") or operator_count)
command_wrappers = int(summary.get("commandWrapperOnlyOperatorCount") or 0)
generated_fixtures = int(summary.get("generatedFixtureCount") or 0)
mock_fixtures = int(summary.get("mockFixtureCount") or 0)
if operator_count != minimum_operator_count:
raise SystemExit(f"operator workflow acceptance must cover the current catalog exactly: {operator_count}/{minimum_operator_count}")
if summary.get("contractVersion") != "poc-operator-workflows/v4":
raise SystemExit(f"operator workflow contract version must be poc-operator-workflows/v4, got {summary.get('contractVersion')}")
if int(summary.get("catalogCount") or 0) != operator_count or not summary.get("catalogHash"):
raise SystemExit("operator workflow acceptance catalog provenance is missing or inconsistent")
if summary.get("trainingMode") != "full" or summary.get("fullTrainingRequired") is not True:
raise SystemExit("operator workflow acceptance must require full training")
if summary.get("missingOperatorIds"):
raise SystemExit(f"operator workflow acceptance has missing operators: {summary.get('missingOperatorIds')}")
if coverage != required:
raise SystemExit(f"independent image coverage is incomplete: {coverage}/{required}")
if command_wrappers != 0:
raise SystemExit(f"non-shell command-wrapper operators are not allowed: {command_wrappers}")
if generated_fixtures != 0 or mock_fixtures != 0:
raise SystemExit(f"generated/mock fixtures are not allowed: generated={generated_fixtures} mock={mock_fixtures}")
for run in payload.get("runs") or []:
evidence = run.get("imageEvidence") or []
if len(evidence) != len(run.get("operatorIds") or []):
raise SystemExit(f"run {run.get('runId')} image evidence count does not match operators")
for item in evidence:
image = str(item.get("image") or "")
image_id = str(item.get("imageId") or "")
if "droboticflow-operator-runner" in image or "droboticflow-operator-" not in image:
raise SystemExit(f"operator {item.get('operatorId')} does not use an independent operator image")
if "@sha256:" not in image_id:
raise SystemExit(f"operator {item.get('operatorId')} imageId is not digest-pinned")
PY
run_ids="$(printf '%s' "$response" | python3 -c 'import json, sys; [print(item["runId"]) for item in json.load(sys.stdin).get("runs", [])]')"
if [[ -z "$run_ids" ]]; then
echo "ERROR: no runs were created." >&2
exit 1
fi
start_worker_port_forward || printf 'warning: initial worker port-forward is unavailable; relying on background worker polling.\n' >&2
fetch_runs_json() {
printf '[' >"$RUNS_JSON_FILE"
local first=true
while IFS= read -r run_id; do
run_id="$(printf '%s' "$run_id" | tr -d '\r' | xargs)"
[[ -z "$run_id" ]] && continue
if [[ "$first" == "true" ]]; then
first=false
else
printf ',' >>"$RUNS_JSON_FILE"
fi
api_curl "$API_BASE_URL/api/runs/$run_id" >>"$RUNS_JSON_FILE"
done <<<"$run_ids"
printf ']' >>"$RUNS_JSON_FILE"
}
deadline=$((SECONDS + POC_OPERATOR_WORKFLOW_RUN_TIMEOUT_SECONDS))
last_process_once_at=0
while (( SECONDS < deadline )); do
fetch_runs_json
if (( SECONDS - last_process_once_at >= 3 )) && runs_need_process_once_file "$RUNS_JSON_FILE"; then
trigger_worker_process_once
last_process_once_at=$SECONDS
fetch_runs_json
fi
summary="$(
python3 - "$RUNS_JSON_FILE" <<'PY'
import json
import os
import sys
runs = json.loads(open(sys.argv[1], encoding="utf-8").read())
print(json.dumps([{"id": run.get("id"), "status": run.get("status"), "nodes": len(run.get("nodeExecutions") or [])} for run in runs], ensure_ascii=False))
PY
)"
printf 'run statuses: %s\n' "$summary"
if SUMMARY="$summary" python3 - <<'PY'
import json
import os
items = json.loads(os.environ["SUMMARY"])
terminal = {"completed", "failed", "cancelled", "waiting_for_audit"}
raise SystemExit(0 if items and all(item.get("status") in terminal for item in items) else 1)
PY
then
break
fi
sleep 5
done
fetch_runs_json
python3 - "$RUNS_JSON_FILE" <<'PY'
import json
import os
import sys
runs = json.loads(open(sys.argv[1], encoding="utf-8").read())
covered = set()
failed = []
waiting = []
lineage_failures = []
production_failures = []
minimum_train_samples = int(__import__("os").environ.get("POC_OPERATOR_WORKFLOW_MIN_TRAIN_SAMPLE_COUNT", "128"))
require_audit_resolution = os.environ.get("POC_OPERATOR_WORKFLOW_REQUIRE_AUDIT_RESOLUTION", "false").lower() in {"1", "true", "yes", "on"}
expected_edges_by_suffix = {
"外部存储与审计链路": [
("op-shell-command", "op-bos-incremental-scan"),
("op-bos-incremental-scan", "op-mongodb-write"),
],
"视频检测裁剪与成品链路": [
("op-xplatform-dataset-index", "op-xplatform-dataset-validate"),
("op-xplatform-dataset-validate", "op-video-quality"),
("op-video-quality", "op-video-filter"),
("op-video-filter", "op-video-slice"),
("op-video-slice", "op-video-target-clip"),
("op-video-target-clip", "op-yolo-video-target-clip"),
("op-yolo-video-target-clip", "op-video-annotate"),
("op-video-annotate", "op-package"),
],
}
def workflow_suffix(run):
name = str(run.get("workflowName") or "")
marker = "POC Operator Workflow Acceptance - "
if marker in name:
return name.split(marker, 1)[1].strip()
return name.rsplit(" - ", 1)[-1].strip()
def validate_workflow_lineage(run):
suffix = workflow_suffix(run)
expected_edges = expected_edges_by_suffix.get(suffix, [])
if not expected_edges:
return
executions = {node.get("nodeId"): node for node in run.get("nodeExecutions") or []}
for source, target in expected_edges:
target_execution = executions.get(target)
if not target_execution:
lineage_failures.append({"runId": run.get("id"), "workflow": run.get("workflowName"), "edge": f"{source}->{target}", "error": "target execution missing"})
continue
lineage = target_execution.get("inputLineage") or []
artifacts = target_execution.get("inputArtifacts") or []
matched = [
item for item in lineage
if item.get("sourceNodeId") == source
and item.get("targetNodeId") == target
and (item.get("artifactId") or (item.get("artifact") or {}).get("id"))
]
if not matched:
lineage_failures.append({"runId": run.get("id"), "workflow": run.get("workflowName"), "edge": f"{source}->{target}", "error": "missing inputLineage artifact"})
if not artifacts:
lineage_failures.append({"runId": run.get("id"), "workflow": run.get("workflowName"), "edge": f"{source}->{target}", "error": "target inputArtifacts empty"})
for run in runs:
validate_workflow_lineage(run)
for node in run.get("nodeExecutions") or []:
operator_id = node.get("operatorId") or node.get("nodeId")
status = node.get("status")
artifacts = node.get("artifacts") or []
if operator_id in {
"operator-builtin-act-policy-train",
"operator-builtin-vla-model-train",
"operator-builtin-diffusion-policy-train",
"operator-builtin-openvla-finetune",
"operator-builtin-yolo-embodied-detection-train",
} and status == "completed":
artifact_names = {str(item.get("name") or "") for item in artifacts}
has_checkpoint = any(
name.endswith((".pt", ".pth", ".safetensors", ".onnx"))
or "checkpoint" in name
or (operator_id == "operator-builtin-openvla-finetune" and name.endswith(".zip"))
or (operator_id == "operator-builtin-vla-model-train" and name.endswith(".json"))
for name in artifact_names
)
has_metrics = any("metric" in name.lower() for name in artifact_names)
if not has_checkpoint:
production_failures.append({"runId": run.get("id"), "operatorId": operator_id, "error": "missing checkpoint artifact"})
if not has_metrics:
production_failures.append({"runId": run.get("id"), "operatorId": operator_id, "error": "missing metrics artifact"})
for artifact in artifacts:
data = artifact.get("data") if isinstance(artifact.get("data"), dict) else {}
if data.get("requiresDNNModelPathForProduction") is True:
production_failures.append({
"runId": run.get("id"),
"operatorId": operator_id,
"artifact": artifact.get("name"),
"error": "YOLO/video production validation requires a real DNN_MODEL_PATH",
})
sample_count = data.get("sampleCount")
model_family = str(data.get("modelFamily") or data.get("type") or "")
if operator_id in {
"operator-builtin-act-policy-train",
"operator-builtin-vla-model-train",
"operator-builtin-diffusion-policy-train",
"operator-builtin-openvla-finetune",
"operator-builtin-yolo-embodied-detection-train",
} and sample_count is not None:
try:
parsed_sample_count = int(sample_count)
except (TypeError, ValueError):
parsed_sample_count = 0
if parsed_sample_count < minimum_train_samples:
production_failures.append({
"runId": run.get("id"),
"operatorId": operator_id,
"artifact": artifact.get("name"),
"error": f"training sampleCount {parsed_sample_count} is below production minimum {minimum_train_samples}",
})
if "vla_linear_policy" in model_family and operator_id != "operator-builtin-vla-model-train":
production_failures.append({
"runId": run.get("id"),
"operatorId": operator_id,
"artifact": artifact.get("name"),
"error": "linear VLA policy is not accepted as production VLA training evidence",
})
if status == "completed" or (status == "waiting_for_audit" and not require_audit_resolution):
covered.add(operator_id)
elif status == "failed":
failed.append({
"runId": run.get("id"),
"workflow": run.get("workflowName"),
"operatorId": operator_id,
"label": node.get("nodeLabel"),
"error": node.get("error"),
})
if run.get("status") == "waiting_for_audit":
waiting.append(run.get("id"))
summary = {
"runCount": len(runs),
"terminalStatuses": {status: sum(1 for run in runs if run.get("status") == status) for status in sorted({run.get("status") for run in runs})},
"coveredExecutionCount": len(covered),
"failedExecutionCount": len(failed),
"lineageFailureCount": len(lineage_failures),
"productionFailureCount": len(production_failures),
"waitingForAuditRuns": waiting,
"failed": failed[:50],
"lineageFailures": lineage_failures[:50],
"productionFailures": production_failures[:50],
}
print("operator workflow acceptance summary:", json.dumps(summary, ensure_ascii=False))
if failed:
sys.exit(1)
if waiting and require_audit_resolution:
raise SystemExit(f"human review runs are still waiting for audit resolution: {waiting}")
if lineage_failures:
raise SystemExit("workflow data-transfer lineage validation failed")
if production_failures:
raise SystemExit("production-grade artifact validation failed")
allowed_terminal = {"completed"} if require_audit_resolution else {"completed", "waiting_for_audit"}
non_terminal = [run.get("id") for run in runs if run.get("status") not in allowed_terminal]
if non_terminal:
raise SystemExit(f"non-terminal or unsuccessful runs: {non_terminal}")
PY