droboticflow/scripts/poc-operator-full-acceptance.sh

906 lines
34 KiB
Bash
Executable File

#!/usr/bin/env bash
set -euo pipefail
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
DEPLOY_ENV="${DEPLOY_ENV:-}"
export DEPLOY_ENV
# shellcheck source=scripts/load-deploy-env.sh
source "$ROOT_DIR/scripts/load-deploy-env.sh"
K8S_NAMESPACE="${K8S_NAMESPACE:-droboticflow}"
POC_OPERATOR_ACCEPTANCE_JOB_NAME="${POC_OPERATOR_ACCEPTANCE_JOB_NAME:-droboticflow-operator-acceptance-$(TZ=Asia/Shanghai date +%Y%m%d%H%M%S)}"
POC_OPERATOR_ACCEPTANCE_WAIT_TIMEOUT="${POC_OPERATOR_ACCEPTANCE_WAIT_TIMEOUT:-1800s}"
POC_ACCEPTANCE_REQUIRED_EXTERNALS="${POC_ACCEPTANCE_REQUIRED_EXTERNALS:-bos,mongo}"
POC_OPERATOR_ACCEPTANCE_CLEANUP_JOB="${POC_OPERATOR_ACCEPTANCE_CLEANUP_JOB:-false}"
POC_OPERATOR_ACCEPTANCE_DRY_RUN="${POC_OPERATOR_ACCEPTANCE_DRY_RUN:-false}"
POC_ACCEPTANCE_ALLOW_TMP_FALLBACK="${POC_ACCEPTANCE_ALLOW_TMP_FALLBACK:-false}"
POC_ACCEPTANCE_ALLOW_GENERATED_FIXTURES="${POC_ACCEPTANCE_ALLOW_GENERATED_FIXTURES:-false}"
POC_ACCEPTANCE_FIXTURE_MANIFEST_PATH="${POC_ACCEPTANCE_FIXTURE_MANIFEST_PATH:-}"
POC_ACCEPTANCE_DISCOVERY_ROOTS="${POC_ACCEPTANCE_DISCOVERY_ROOTS:-}"
POC_ACCEPTANCE_DISCOVERY_EXCLUDE_ROOTS="${POC_ACCEPTANCE_DISCOVERY_EXCLUDE_ROOTS:-}"
POC_ACCEPTANCE_DISCOVERY_MAX_RESULTS="${POC_ACCEPTANCE_DISCOVERY_MAX_RESULTS:-10}"
POC_ACCEPTANCE_DISCOVERY_INCLUDE_ACCEPTANCE_OUTPUT="${POC_ACCEPTANCE_DISCOVERY_INCLUDE_ACCEPTANCE_OUTPUT:-false}"
POC_ACCEPTANCE_IMPORT_RESULT="${POC_ACCEPTANCE_IMPORT_RESULT:-true}"
POC_ACCEPTANCE_IMPORT_API_BASE_URL="${POC_ACCEPTANCE_IMPORT_API_BASE_URL:-${API_BASE_URL:-${BASE_URL:-}}}"
SCRIPT_PATH="${POC_OPERATOR_ACCEPTANCE_SCRIPT_PATH:-$ROOT_DIR/scripts/poc_operator_full_acceptance_job.py}"
POC_ACCEPTANCE_SOURCE_COMMIT="${POC_ACCEPTANCE_SOURCE_COMMIT:-${DROBOTICFLOW_BUILD_COMMIT:-}}"
if [[ -z "$POC_ACCEPTANCE_SOURCE_COMMIT" ]]; then
POC_ACCEPTANCE_SOURCE_COMMIT="$(git -C "$ROOT_DIR" rev-parse HEAD 2>/dev/null || true)"
fi
POC_ACCEPTANCE_SOURCE_COMMIT="${POC_ACCEPTANCE_SOURCE_COMMIT:-unknown}"
catalog_metadata="$({ python3 - "$ROOT_DIR/internal/api/builtin_operators_catalog.json" <<'PY'
from __future__ import annotations
import hashlib
import json
import pathlib
import sys
catalog_path = pathlib.Path(sys.argv[1])
catalog = json.loads(catalog_path.read_text(encoding="utf-8"))
if not isinstance(catalog, list) or not catalog:
raise SystemExit("builtin operator catalog must be a non-empty JSON list")
canonical_text = json.dumps(catalog, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
for raw, escaped in (
("&", "\\u0026"),
("<", "\\u003c"),
(">", "\\u003e"),
("\u2028", "\\u2028"),
("\u2029", "\\u2029"),
):
canonical_text = canonical_text.replace(raw, escaped)
canonical = canonical_text.encode("utf-8")
print(f"{hashlib.sha256(canonical).hexdigest()}\t{len(catalog)}")
PY
} 2>&1)" || {
echo "ERROR: failed to compute builtin operator catalog provenance: $catalog_metadata" >&2
exit 1
}
IFS=$'\t' read -r POC_ACCEPTANCE_CATALOG_HASH POC_ACCEPTANCE_CATALOG_COUNT <<< "$catalog_metadata"
export POC_ACCEPTANCE_SOURCE_COMMIT
export POC_ACCEPTANCE_CATALOG_HASH
export POC_ACCEPTANCE_CATALOG_COUNT
validate_dns_label() {
local name="$1"
local value="$2"
if [[ -z "$value" ]] || [[ ${#value} -gt 63 ]] || ! [[ "$value" =~ ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ ]]; then
echo "ERROR: $name must be a lowercase DNS label" >&2
exit 1
fi
}
validate_image_reference() {
local name="$1"
local value="$2"
if [[ -z "$value" ]] || [[ "$value" =~ [[:space:]] ]]; then
echo "ERROR: $name must be a non-empty container image reference without whitespace" >&2
exit 1
fi
}
validate_go_duration() {
local name="$1"
local value="$2"
if ! [[ "$value" =~ ^([1-9][0-9]*(ns|us|ms|s|m|h))+$ ]]; then
echo "ERROR: $name must be a positive Go-style duration such as 300s, 15m, or 1h30m" >&2
exit 1
fi
}
validate_bool() {
local name="$1"
local value="$2"
local normalized
normalized="$(printf '%s' "$value" | tr '[:upper:]' '[:lower:]')"
case "$normalized" in
true|false|1|0|yes|no|on|off) ;;
*)
echo "ERROR: $name must be a boolean value" >&2
exit 1
;;
esac
}
validate_positive_int() {
local name="$1"
local value="$2"
if ! [[ "$value" =~ ^[1-9][0-9]*$ ]]; then
echo "ERROR: $name must be a positive integer" >&2
exit 1
fi
}
duration_seconds() {
python3 - "$1" <<'PY'
import re
import sys
value = sys.argv[1]
units = {"ns": 0, "us": 0, "ms": 0, "s": 1, "m": 60, "h": 3600}
total = 0
for number, unit in re.findall(r"([1-9][0-9]*)(ns|us|ms|s|m|h)", value):
total += int(number) * units[unit]
print(max(total, 1))
PY
}
json_string() {
python3 - "$1" <<'PY'
import json
import sys
print(json.dumps(sys.argv[1]))
PY
}
required_external_enabled() {
local target="$1"
local raw="$POC_ACCEPTANCE_REQUIRED_EXTERNALS"
local item
local items
IFS=',' read -r -a items <<< "$raw"
for item in "${items[@]}"; do
item="$(printf '%s' "$item" | tr '[:upper:]' '[:lower:]')"
item="${item#"${item%%[![:space:]]*}"}"
item="${item%"${item##*[![:space:]]}"}"
if [[ "$item" == "$target" ]]; then
return 0
fi
done
return 1
}
bool_is_true() {
local value
value="$(printf '%s' "${1:-}" | tr '[:upper:]' '[:lower:]')"
case "$value" in
true|1|yes|on)
return 0
;;
*)
return 1
;;
esac
}
verify_workspace_path_error() {
local name="$1"
local value="$2"
local kind="$3"
local description="$4"
if [[ -z "$value" ]]; then
return 0
fi
if [[ -z "$WORKSPACE_MOUNT_PATH" ]]; then
printf 'cannot validate %s because WORKSPACE_MOUNT_PATH is empty' "$description"
return 0
fi
case "$value" in
"$WORKSPACE_MOUNT_PATH"|"$WORKSPACE_MOUNT_PATH"/*) ;;
*)
printf '%s must point under WORKSPACE_MOUNT_PATH %s: %s=%s' "$description" "$WORKSPACE_MOUNT_PATH" "$name" "$value"
return 0
;;
esac
local api_pod
api_pod="$(kubectl -n "$K8S_NAMESPACE" get pod -l app=droboticflow-api -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)"
if [[ -z "$api_pod" ]]; then
printf 'cannot validate %s because no droboticflow-api pod is available in namespace %s' "$description" "$K8S_NAMESPACE"
return 0
fi
local test_script
case "$kind" in
file)
test_script='test -f "$1" && test -r "$1" && test -s "$1"'
;;
dir)
test_script='test -d "$1" && test -r "$1"'
;;
any)
test_script='test -e "$1" && test -r "$1"'
;;
*)
printf 'internal error: unsupported workspace path kind %s for %s' "$kind" "$description"
return 0
;;
esac
if ! kubectl -n "$K8S_NAMESPACE" exec "$api_pod" -- sh -c "$test_script" -- "$value" >/dev/null 2>&1; then
if [[ "$kind" == "file" ]]; then
printf '%s is not a readable non-empty file inside the POC workspace mount: %s=%s' "$description" "$name" "$value"
else
printf '%s is not a readable %s inside the POC workspace mount: %s=%s' "$description" "$kind" "$name" "$value"
fi
fi
}
fixture_readiness_error() {
if bool_is_true "$POC_ACCEPTANCE_ALLOW_GENERATED_FIXTURES"; then
return 0
fi
if [[ -n "$POC_ACCEPTANCE_FIXTURE_MANIFEST_PATH" ]]; then
verify_workspace_path_error \
POC_ACCEPTANCE_FIXTURE_MANIFEST_PATH \
"$POC_ACCEPTANCE_FIXTURE_MANIFEST_PATH" \
file \
"POC acceptance fixture manifest"
return 0
fi
local missing=()
local key
for key in \
POC_ACCEPTANCE_XPLATFORM_SOURCE_PATH \
POC_ACCEPTANCE_CSV_INPUT_PATH \
POC_ACCEPTANCE_JSON_INPUT_PATH \
POC_ACCEPTANCE_LEROBOT_SOURCE_PATH \
POC_ACCEPTANCE_ROSBAG_SOURCE_PATH \
POC_ACCEPTANCE_MODEL_JSON_PATH \
POC_ACCEPTANCE_SHELL_COMMAND
do
if [[ -z "${!key:-}" ]]; then
missing+=("$key")
fi
done
if [[ "${#missing[@]}" -gt 0 ]]; then
printf 'POC acceptance real fixtures are incomplete: set POC_ACCEPTANCE_FIXTURE_MANIFEST_PATH or provide %s; generated fixtures are disabled' "$(IFS=', '; printf '%s' "${missing[*]}")"
return 0
fi
local path_errors=()
local path_error
for key in \
POC_ACCEPTANCE_XPLATFORM_SOURCE_PATH:dir:X-platform_acceptance_dataset \
POC_ACCEPTANCE_CSV_INPUT_PATH:file:CSV_acceptance_input \
POC_ACCEPTANCE_JSON_INPUT_PATH:file:JSON_acceptance_input \
POC_ACCEPTANCE_LEROBOT_SOURCE_PATH:dir:LeRobot_acceptance_dataset \
POC_ACCEPTANCE_ROSBAG_SOURCE_PATH:dir:ROSBag_acceptance_dataset \
POC_ACCEPTANCE_MODEL_JSON_PATH:file:Model_acceptance_JSON
do
IFS=':' read -r env_name kind description <<< "$key"
path_error="$(verify_workspace_path_error "$env_name" "${!env_name:-}" "$kind" "${description//_/ }")"
if [[ -n "$path_error" ]]; then
path_errors+=("$path_error")
fi
done
if [[ "${#path_errors[@]}" -gt 0 ]]; then
printf 'POC acceptance real fixture paths are not ready: %s' "$(IFS='; '; printf '%s' "${path_errors[*]}")"
fi
}
fixture_discovery_evidence_json() {
local discovery_script="$ROOT_DIR/scripts/discover_poc_acceptance_fixtures.py"
if [[ ! -f "$discovery_script" ]]; then
printf '{"status":"unavailable","message":"fixture discovery script is not present"}\n'
return 0
fi
local discovery_manifest_path="$POC_ACCEPTANCE_FIXTURE_MANIFEST_PATH"
if [[ -n "$discovery_manifest_path" ]]; then
case "$discovery_manifest_path" in
"$WORKSPACE_MOUNT_PATH"|"$WORKSPACE_MOUNT_PATH"/*) ;;
*)
discovery_manifest_path=""
;;
esac
fi
local output
local status=0
set +e
output="$(
K8S_NAMESPACE="$K8S_NAMESPACE" \
WORKSPACE_MOUNT_PATH="$WORKSPACE_MOUNT_PATH" \
POC_ACCEPTANCE_FIXTURE_MANIFEST_PATH="$discovery_manifest_path" \
POC_ACCEPTANCE_DISCOVERY_ROOTS="$POC_ACCEPTANCE_DISCOVERY_ROOTS" \
POC_ACCEPTANCE_DISCOVERY_EXCLUDE_ROOTS="$POC_ACCEPTANCE_DISCOVERY_EXCLUDE_ROOTS" \
POC_ACCEPTANCE_DISCOVERY_MAX_RESULTS="$POC_ACCEPTANCE_DISCOVERY_MAX_RESULTS" \
POC_ACCEPTANCE_DISCOVERY_INCLUDE_ACCEPTANCE_OUTPUT="$POC_ACCEPTANCE_DISCOVERY_INCLUDE_ACCEPTANCE_OUTPUT" \
POC_ACCEPTANCE_SHELL_COMMAND="${POC_ACCEPTANCE_SHELL_COMMAND:-}" \
python3 "$discovery_script" --namespace "$K8S_NAMESPACE" 2>/dev/null
)"
status=$?
set -e
DISCOVERY_OUTPUT="$output" DISCOVERY_STATUS="$status" python3 - <<'PY'
from __future__ import annotations
import json
import os
raw = os.environ.get("DISCOVERY_OUTPUT", "")
status = int(os.environ.get("DISCOVERY_STATUS", "0") or "0")
try:
payload = json.loads(raw) if raw.strip() else {}
except json.JSONDecodeError:
payload = {"status": "error", "message": "fixture discovery returned non-JSON output"}
if status and payload.get("status") != "error":
payload = {
"status": "error",
"message": f"fixture discovery command exited with status {status}",
"partialStatus": payload.get("status"),
}
print(json.dumps(payload, ensure_ascii=False, sort_keys=True))
PY
}
write_preflight_acceptance_failure_result() {
local result_file="$1"
local fixture_discovery_json="$2"
local issue_count="$3"
shift 3
python3 - "$ROOT_DIR" "$POC_OPERATOR_ACCEPTANCE_JOB_NAME" "$WORKSPACE_MOUNT_PATH" "$POC_ACCEPTANCE_REQUIRED_EXTERNALS" "$POC_ACCEPTANCE_FIXTURE_MANIFEST_PATH" "$POC_ACCEPTANCE_ALLOW_GENERATED_FIXTURES" "$fixture_discovery_json" "$issue_count" "$@" >"$result_file" <<'PY'
from __future__ import annotations
import datetime
import json
import os
import pathlib
import sys
from typing import Any
root = pathlib.Path(sys.argv[1])
run_id = sys.argv[2]
workspace_mount = sys.argv[3].rstrip("/") or "/workspace"
required_raw = sys.argv[4]
fixture_manifest_path = sys.argv[5]
allow_generated_fixtures = sys.argv[6]
fixture_discovery_raw = sys.argv[7]
issue_count = int(sys.argv[8])
issue_values = sys.argv[9:]
stages = issue_values[:issue_count]
reasons = issue_values[issue_count : issue_count * 2]
if len(stages) != issue_count or len(reasons) != issue_count:
raise SystemExit("preflight issue argument mismatch")
required = sorted({item.strip().lower() for item in required_raw.split(",") if item.strip()})
scope = "full" if {"bos", "mongo"}.issubset(set(required)) else "dependency-isolation"
operator_slugs: set[str] = set()
for manifest_path in sorted((root / "backend" / "operators" / "builtin").glob("*/repository.json")):
try:
manifest: dict[str, Any] = json.loads(manifest_path.read_text(encoding="utf-8"))
except Exception:
continue
if manifest.get("type") != "builtin":
continue
slug = str(manifest.get("operatorSlug") or manifest_path.parent.name).strip()
if not slug:
continue
operator_slugs.add(slug)
def compact_fixture_discovery(raw: str) -> dict[str, Any]:
try:
payload = json.loads(raw) if raw.strip() else {}
except json.JSONDecodeError:
return {"status": "error", "message": "fixture discovery returned non-JSON output"}
if not isinstance(payload, dict) or not payload:
return {}
compact: dict[str, Any] = {}
for key in (
"status",
"namespace",
"apiPod",
"workspaceMountPath",
"roots",
"excludedRoots",
"acceptanceOutputIncluded",
"manifestPath",
"missing",
"nextActions",
"message",
"writeResult",
):
value = payload.get(key)
if value not in (None, "", [], {}):
compact[key] = value
candidates = payload.get("candidates")
if isinstance(candidates, dict):
compact["candidateCounts"] = {
str(name): len(values) if isinstance(values, list) else 0
for name, values in sorted(candidates.items())
}
selected = payload.get("selected")
if isinstance(selected, dict):
selected_paths: dict[str, str] = {}
for name, value in selected.items():
if isinstance(value, dict) and isinstance(value.get("path"), str):
selected_paths[str(name)] = value["path"]
if selected_paths:
compact["selectedPaths"] = selected_paths
return compact
fixture_discovery = compact_fixture_discovery(fixture_discovery_raw)
now = datetime.datetime.now(datetime.timezone.utc).isoformat().replace("+00:00", "Z")
records: list[dict[str, Any]] = []
missing_coverage_set: set[str] = set()
for stage, reason in zip(stages, reasons, strict=True):
if stage == "fixture-readiness":
record_missing_coverage = sorted(operator_slugs)
handler = "fixture_readiness"
evidence = {
"fixtureManifestPath": fixture_manifest_path,
"allowGeneratedFixtures": allow_generated_fixtures,
"missingOperatorCoverage": record_missing_coverage,
}
if fixture_discovery:
evidence["fixtureDiscovery"] = fixture_discovery
else:
record_missing_coverage = sorted(operator_slugs)
handler = "preflight_readiness"
evidence = {"missingOperatorCoverage": record_missing_coverage}
missing_coverage_set.update(record_missing_coverage)
records.append(
{
"operator": "poc_acceptance_preflight",
"handler": handler,
"status": "failed",
"error": reason,
"evidence": evidence,
"preflightStage": stage,
}
)
unique_stages = list(dict.fromkeys(stages))
stage = unique_stages[0] if len(unique_stages) == 1 else "preflight-readiness"
missing_coverage = sorted(missing_coverage_set)
required_external_systems = sorted({"bos", "mongo"}.intersection(set(required)))
payload = {
"runId": run_id,
"workspace": f"{workspace_mount}/acceptance/{run_id}",
"preflightFailure": True,
"preflightStage": stage,
"preflightStages": unique_stages,
"acceptanceScope": scope,
"fullExternalAcceptance": False,
"operatorCount": len(operator_slugs),
"operatorPassedCount": 0,
"coveredOperatorCount": 0,
"allOperatorCoverageComplete": False,
"blockingOperatorCoverageComplete": False,
"missingCoverage": missing_coverage,
"blockingMissingCoverage": missing_coverage,
"requiredExternals": required,
"statusCounts": {"failed": len(records)},
"passedCount": 0,
"externalVerificationPassed": [],
"externalVerificationMissing": required_external_systems,
"externalVerificationPassedCount": 0,
"externalVerificationRequiredCount": len(required_external_systems),
"failedCount": len(records),
"skippedCount": 0,
"requiredSkippedCount": 0,
"optionalSkippedCount": 0,
"records": records,
"failures": reasons,
"status": "failed",
"startedAt": now,
"endedAt": now,
"sourceCommit": os.environ["POC_ACCEPTANCE_SOURCE_COMMIT"],
"catalogHash": os.environ["POC_ACCEPTANCE_CATALOG_HASH"],
"catalogCount": int(os.environ["POC_ACCEPTANCE_CATALOG_COUNT"]),
"completedAt": now,
}
print("DROBOTICFLOW_POC_OPERATOR_ACCEPTANCE=" + json.dumps(payload, ensure_ascii=False, sort_keys=True))
PY
}
import_preflight_acceptance_failure() {
local fixture_discovery_json="$1"
local issue_count="$2"
shift 2
local result_file
result_file="$(mktemp)"
write_preflight_acceptance_failure_result "$result_file" "$fixture_discovery_json" "$issue_count" "$@"
cat "$result_file"
local import_result_normalized
import_result_normalized="$(printf '%s' "$POC_ACCEPTANCE_IMPORT_RESULT" | tr '[:upper:]' '[:lower:]')"
if [[ "$import_result_normalized" == "true" || "$import_result_normalized" == "1" || "$import_result_normalized" == "yes" || "$import_result_normalized" == "on" ]]; then
if [[ -n "$POC_ACCEPTANCE_IMPORT_API_BASE_URL" ]]; then
API_BASE_URL="$POC_ACCEPTANCE_IMPORT_API_BASE_URL" \
K8S_NAMESPACE="$K8S_NAMESPACE" \
OPERATOR_RUNNER_IMAGE="$OPERATOR_RUNNER_IMAGE" \
POC_OPERATOR_ACCEPTANCE_JOB_NAME="$POC_OPERATOR_ACCEPTANCE_JOB_NAME" \
python3 "$ROOT_DIR/scripts/import_poc_acceptance_result.py" \
--api-base-url "$POC_ACCEPTANCE_IMPORT_API_BASE_URL" \
--result-file "$result_file" \
--environment "${DEPLOY_ENV:-poc}" \
--namespace "$K8S_NAMESPACE" \
--job-name "$POC_OPERATOR_ACCEPTANCE_JOB_NAME" \
--runner-image "$OPERATOR_RUNNER_IMAGE"
else
echo "POC acceptance platform import skipped: set POC_ACCEPTANCE_IMPORT_API_BASE_URL, API_BASE_URL, or BASE_URL to enable it." >&2
fi
fi
rm -f "$result_file"
}
append_preflight_issue() {
local stage="$1"
local reason="$2"
preflight_stages+=("$stage")
preflight_reasons+=("$reason")
}
has_preflight_stage() {
local target="$1"
local stage
for stage in "${preflight_stages[@]:-}"; do
if [[ "$stage" == "$target" ]]; then
return 0
fi
done
return 1
}
config_value() {
local key="$1"
kubectl -n "$K8S_NAMESPACE" get configmap droboticflow-config -o "jsonpath={.data.$key}" 2>/dev/null || true
}
infer_workspace_bucket() {
if [[ -n "${POC_ACCEPTANCE_BOS_BUCKET:-}" ]]; then
printf '%s\n' "$POC_ACCEPTANCE_BOS_BUCKET"
return 0
fi
if [[ -n "${WORKSPACE_BOS_BUCKET:-}" ]]; then
printf '%s\n' "$WORKSPACE_BOS_BUCKET"
return 0
fi
local pv_name
pv_name="$(kubectl -n "$K8S_NAMESPACE" get pvc "$WORKSPACE_PVC_NAME" -o jsonpath='{.spec.volumeName}' 2>/dev/null || true)"
if [[ -z "$pv_name" ]]; then
return 0
fi
kubectl get pv "$pv_name" -o jsonpath='{.spec.csi.volumeHandle}' 2>/dev/null || true
}
if [[ ! -f "$SCRIPT_PATH" ]]; then
echo "ERROR: acceptance job script not found: $SCRIPT_PATH" >&2
exit 1
fi
validate_dns_label K8S_NAMESPACE "$K8S_NAMESPACE"
validate_dns_label POC_OPERATOR_ACCEPTANCE_JOB_NAME "$POC_OPERATOR_ACCEPTANCE_JOB_NAME"
validate_go_duration POC_OPERATOR_ACCEPTANCE_WAIT_TIMEOUT "$POC_OPERATOR_ACCEPTANCE_WAIT_TIMEOUT"
validate_bool POC_ACCEPTANCE_ALLOW_TMP_FALLBACK "$POC_ACCEPTANCE_ALLOW_TMP_FALLBACK"
validate_bool POC_ACCEPTANCE_ALLOW_GENERATED_FIXTURES "$POC_ACCEPTANCE_ALLOW_GENERATED_FIXTURES"
validate_bool POC_ACCEPTANCE_IMPORT_RESULT "$POC_ACCEPTANCE_IMPORT_RESULT"
validate_bool POC_ACCEPTANCE_DISCOVERY_INCLUDE_ACCEPTANCE_OUTPUT "$POC_ACCEPTANCE_DISCOVERY_INCLUDE_ACCEPTANCE_OUTPUT"
validate_positive_int POC_ACCEPTANCE_DISCOVERY_MAX_RESULTS "$POC_ACCEPTANCE_DISCOVERY_MAX_RESULTS"
OPERATOR_RUNNER_IMAGE="${OPERATOR_RUNNER_IMAGE:-$(config_value OPERATOR_RUNNER_IMAGE)}"
WORKSPACE_PVC_NAME="${WORKSPACE_PVC_NAME:-$(config_value WORKSPACE_PVC_NAME)}"
WORKSPACE_MOUNT_PATH="${WORKSPACE_MOUNT_PATH:-$(config_value WORKSPACE_MOUNT_PATH)}"
K8S_IMAGE_PULL_SECRET_NAME="${K8S_IMAGE_PULL_SECRET_NAME:-$(config_value K8S_IMAGE_PULL_SECRET_NAME)}"
BOS_ENDPOINT="${POC_ACCEPTANCE_BOS_ENDPOINT:-${BOS_ENDPOINT:-$(config_value BOS_ENDPOINT)}}"
POC_ACCEPTANCE_BOS_BUCKET="${POC_ACCEPTANCE_BOS_BUCKET:-$(infer_workspace_bucket)}"
POC_ACCEPTANCE_FIXTURE_MANIFEST_PATH="${POC_ACCEPTANCE_FIXTURE_MANIFEST_PATH:-${WORKSPACE_MOUNT_PATH%/}/datasets/fixtures/poc-operator-acceptance/poc-operator-acceptance.json}"
validate_image_reference OPERATOR_RUNNER_IMAGE "$OPERATOR_RUNNER_IMAGE"
validate_dns_label WORKSPACE_PVC_NAME "$WORKSPACE_PVC_NAME"
if [[ -z "$WORKSPACE_MOUNT_PATH" ]] || ! [[ "$WORKSPACE_MOUNT_PATH" =~ ^/[-A-Za-z0-9._/]*$ ]]; then
echo "ERROR: WORKSPACE_MOUNT_PATH must be an absolute container path without spaces" >&2
exit 1
fi
if [[ -n "$K8S_IMAGE_PULL_SECRET_NAME" ]]; then
validate_dns_label K8S_IMAGE_PULL_SECRET_NAME "$K8S_IMAGE_PULL_SECRET_NAME"
fi
preflight_stages=()
preflight_reasons=()
if [[ "$POC_OPERATOR_ACCEPTANCE_DRY_RUN" != "true" ]]; then
fixture_readiness_error="$(fixture_readiness_error)"
if [[ -n "$fixture_readiness_error" ]]; then
echo "ERROR: $fixture_readiness_error." >&2
append_preflight_issue "fixture-readiness" "$fixture_readiness_error"
fi
fi
if [[ "${#preflight_stages[@]}" -gt 0 ]]; then
fixture_discovery_json="{}"
if has_preflight_stage "fixture-readiness"; then
echo "Provide a real fixture manifest under the shared workspace PVC or set all explicit POC_ACCEPTANCE_* fixture paths before final full acceptance." >&2
fixture_discovery_json="$(fixture_discovery_evidence_json)"
fi
import_preflight_acceptance_failure "$fixture_discovery_json" "${#preflight_stages[@]}" "${preflight_stages[@]}" "${preflight_reasons[@]}"
exit 1
fi
job_name_json="$(json_string "$POC_OPERATOR_ACCEPTANCE_JOB_NAME")"
namespace_json="$(json_string "$K8S_NAMESPACE")"
image_json="$(json_string "$OPERATOR_RUNNER_IMAGE")"
workspace_mount_json="$(json_string "$WORKSPACE_MOUNT_PATH")"
workspace_pvc_json="$(json_string "$WORKSPACE_PVC_NAME")"
required_externals_json="$(json_string "$POC_ACCEPTANCE_REQUIRED_EXTERNALS")"
allow_tmp_fallback_json="$(json_string "$POC_ACCEPTANCE_ALLOW_TMP_FALLBACK")"
allow_generated_fixtures_json="$(json_string "$POC_ACCEPTANCE_ALLOW_GENERATED_FIXTURES")"
fixture_manifest_path_json="$(json_string "$POC_ACCEPTANCE_FIXTURE_MANIFEST_PATH")"
bos_endpoint_json="$(json_string "$BOS_ENDPOINT")"
bos_bucket_json="$(json_string "$POC_ACCEPTANCE_BOS_BUCKET")"
xplatform_source_json="$(json_string "${POC_ACCEPTANCE_XPLATFORM_SOURCE_PATH:-}")"
csv_input_json="$(json_string "${POC_ACCEPTANCE_CSV_INPUT_PATH:-}")"
json_input_json="$(json_string "${POC_ACCEPTANCE_JSON_INPUT_PATH:-}")"
lerobot_source_json="$(json_string "${POC_ACCEPTANCE_LEROBOT_SOURCE_PATH:-}")"
rosbag_source_json="$(json_string "${POC_ACCEPTANCE_ROSBAG_SOURCE_PATH:-}")"
model_json_path_json="$(json_string "${POC_ACCEPTANCE_MODEL_JSON_PATH:-}")"
json_filter_field_json="$(json_string "${POC_ACCEPTANCE_JSON_FILTER_FIELD:-}")"
json_filter_expected_json="$(json_string "${POC_ACCEPTANCE_JSON_FILTER_EXPECTED:-}")"
json_extract_field_json="$(json_string "${POC_ACCEPTANCE_JSON_EXTRACT_FIELD:-}")"
json_field_map_json="$(json_string "${POC_ACCEPTANCE_JSON_FIELD_MAP_JSON:-}")"
template_text_json="$(json_string "${POC_ACCEPTANCE_TEMPLATE_TEXT:-}")"
min_clarity_json="$(json_string "${POC_ACCEPTANCE_MIN_CLARITY:-}")"
max_shake_json="$(json_string "${POC_ACCEPTANCE_MAX_SHAKE:-}")"
max_hand_occlusion_json="$(json_string "${POC_ACCEPTANCE_MAX_HAND_OCCLUSION:-}")"
clarity_scale_json="$(json_string "${POC_ACCEPTANCE_CLARITY_SCALE:-}")"
xplatform_date_start_json="$(json_string "${POC_ACCEPTANCE_XPLATFORM_DATE_START:-}")"
xplatform_date_end_json="$(json_string "${POC_ACCEPTANCE_XPLATFORM_DATE_END:-}")"
xplatform_place_json="$(json_string "${POC_ACCEPTANCE_XPLATFORM_PLACE:-}")"
xplatform_type_json="$(json_string "${POC_ACCEPTANCE_XPLATFORM_TYPE:-}")"
segment_duration_json="$(json_string "${POC_ACCEPTANCE_SEGMENT_DURATION_SECONDS:-}")"
frame_interval_json="$(json_string "${POC_ACCEPTANCE_FRAME_INTERVAL_SECONDS:-}")"
audit_rules_json="$(json_string "${POC_ACCEPTANCE_AUDIT_RULES_JSON:-}")"
shell_command_json="$(json_string "${POC_ACCEPTANCE_SHELL_COMMAND:-}")"
source_commit_json="$(json_string "$POC_ACCEPTANCE_SOURCE_COMMIT")"
catalog_hash_json="$(json_string "$POC_ACCEPTANCE_CATALOG_HASH")"
catalog_count_json="$(json_string "$POC_ACCEPTANCE_CATALOG_COUNT")"
image_pull_secrets_yaml=""
if [[ -n "$K8S_IMAGE_PULL_SECRET_NAME" ]]; then
image_pull_secret_json="$(json_string "$K8S_IMAGE_PULL_SECRET_NAME")"
image_pull_secrets_yaml=" imagePullSecrets:
- name: $image_pull_secret_json"
fi
manifest="$(cat <<YAML
apiVersion: batch/v1
kind: Job
metadata:
name: $job_name_json
namespace: $namespace_json
labels:
app: droboticflow
component: operator-full-acceptance
spec:
backoffLimit: 0
ttlSecondsAfterFinished: 86400
template:
metadata:
labels:
app: droboticflow
component: operator-full-acceptance
spec:
restartPolicy: Never
$image_pull_secrets_yaml
securityContext:
runAsNonRoot: true
runAsUser: 10001
runAsGroup: 10001
fsGroup: 10001
seccompProfile:
type: RuntimeDefault
containers:
- name: acceptance
image: $image_json
imagePullPolicy: Always
command: ["python", "/acceptance/poc_operator_full_acceptance_job.py"]
env:
- name: POC_ACCEPTANCE_RUN_ID
value: $job_name_json
- name: POC_ACCEPTANCE_SOURCE_COMMIT
value: $source_commit_json
- name: POC_ACCEPTANCE_CATALOG_HASH
value: $catalog_hash_json
- name: POC_ACCEPTANCE_CATALOG_COUNT
value: $catalog_count_json
- name: WORKSPACE_MOUNT_PATH
value: $workspace_mount_json
- name: POC_ACCEPTANCE_REQUIRED_EXTERNALS
value: $required_externals_json
- name: POC_ACCEPTANCE_ALLOW_TMP_FALLBACK
value: $allow_tmp_fallback_json
- name: POC_ACCEPTANCE_ALLOW_GENERATED_FIXTURES
value: $allow_generated_fixtures_json
- name: POC_ACCEPTANCE_FIXTURE_MANIFEST_PATH
value: $fixture_manifest_path_json
- name: POC_ACCEPTANCE_XPLATFORM_SOURCE_PATH
value: $xplatform_source_json
- name: POC_ACCEPTANCE_CSV_INPUT_PATH
value: $csv_input_json
- name: POC_ACCEPTANCE_JSON_INPUT_PATH
value: $json_input_json
- name: POC_ACCEPTANCE_LEROBOT_SOURCE_PATH
value: $lerobot_source_json
- name: POC_ACCEPTANCE_ROSBAG_SOURCE_PATH
value: $rosbag_source_json
- name: POC_ACCEPTANCE_MODEL_JSON_PATH
value: $model_json_path_json
- name: POC_ACCEPTANCE_JSON_FILTER_FIELD
value: $json_filter_field_json
- name: POC_ACCEPTANCE_JSON_FILTER_EXPECTED
value: $json_filter_expected_json
- name: POC_ACCEPTANCE_JSON_EXTRACT_FIELD
value: $json_extract_field_json
- name: POC_ACCEPTANCE_JSON_FIELD_MAP_JSON
value: $json_field_map_json
- name: POC_ACCEPTANCE_TEMPLATE_TEXT
value: $template_text_json
- name: POC_ACCEPTANCE_MIN_CLARITY
value: $min_clarity_json
- name: POC_ACCEPTANCE_MAX_SHAKE
value: $max_shake_json
- name: POC_ACCEPTANCE_MAX_HAND_OCCLUSION
value: $max_hand_occlusion_json
- name: POC_ACCEPTANCE_CLARITY_SCALE
value: $clarity_scale_json
- name: POC_ACCEPTANCE_XPLATFORM_DATE_START
value: $xplatform_date_start_json
- name: POC_ACCEPTANCE_XPLATFORM_DATE_END
value: $xplatform_date_end_json
- name: POC_ACCEPTANCE_XPLATFORM_PLACE
value: $xplatform_place_json
- name: POC_ACCEPTANCE_XPLATFORM_TYPE
value: $xplatform_type_json
- name: POC_ACCEPTANCE_SEGMENT_DURATION_SECONDS
value: $segment_duration_json
- name: POC_ACCEPTANCE_FRAME_INTERVAL_SECONDS
value: $frame_interval_json
- name: POC_ACCEPTANCE_AUDIT_RULES_JSON
value: $audit_rules_json
- name: POC_ACCEPTANCE_SHELL_COMMAND
value: $shell_command_json
- name: POC_ACCEPTANCE_BOS_BUCKET
value: $bos_bucket_json
- name: POC_ACCEPTANCE_BOS_ENDPOINT
value: $bos_endpoint_json
- name: BOS_ENDPOINT
value: $bos_endpoint_json
- name: BOS_ACCESS_KEY_ID
valueFrom:
secretKeyRef:
name: droboticflow-bos-workspace
key: ak
optional: true
- name: BOS_SECRET_ACCESS_KEY
valueFrom:
secretKeyRef:
name: droboticflow-bos-workspace
key: sk
optional: true
- name: MONGO_URI
valueFrom:
secretKeyRef:
name: droboticflow-mongo-secret
key: MONGO_URI
optional: true
- name: MONGO_DB
valueFrom:
configMapKeyRef:
name: droboticflow-config
key: MONGO_DB
optional: true
volumeMounts:
- name: acceptance-script
mountPath: /acceptance
- name: workspace
mountPath: $workspace_mount_json
resources:
requests:
cpu: 500m
memory: 512Mi
limits:
cpu: "2"
memory: 2Gi
volumes:
- name: acceptance-script
configMap:
name: $job_name_json
defaultMode: 0555
- name: workspace
persistentVolumeClaim:
claimName: $workspace_pvc_json
YAML
)"
if [[ "$POC_OPERATOR_ACCEPTANCE_DRY_RUN" == "true" ]]; then
printf '%s\n' "$manifest"
exit 0
fi
kubectl -n "$K8S_NAMESPACE" create configmap "$POC_OPERATOR_ACCEPTANCE_JOB_NAME" \
--from-file=poc_operator_full_acceptance_job.py="$SCRIPT_PATH" \
--dry-run=client \
-o yaml | kubectl apply -f -
printf '%s\n' "$manifest" | kubectl apply -f -
timeout_seconds="$(duration_seconds "$POC_OPERATOR_ACCEPTANCE_WAIT_TIMEOUT")"
deadline=$(( $(date +%s) + timeout_seconds ))
wait_status=1
while true; do
job_status_json="$(kubectl -n "$K8S_NAMESPACE" get job "$POC_OPERATOR_ACCEPTANCE_JOB_NAME" -o json)"
job_state="$(
JOB_STATUS_JSON="$job_status_json" python3 - <<'PY'
import json
import os
payload = json.loads(os.environ["JOB_STATUS_JSON"])
status = payload.get("status", {})
if int(status.get("succeeded") or 0) > 0:
print("complete")
elif int(status.get("failed") or 0) > 0:
print("failed")
else:
print("running")
PY
)"
if [[ "$job_state" == "complete" ]]; then
wait_status=0
break
fi
if [[ "$job_state" == "failed" ]]; then
wait_status=1
break
fi
if (( $(date +%s) >= deadline )); then
wait_status=1
break
fi
sleep 5
done
logs_file="$(mktemp)"
result_file="$(mktemp)"
cleanup_acceptance_import_files() {
rm -f "$logs_file" "$result_file"
}
trap cleanup_acceptance_import_files EXIT
kubectl -n "$K8S_NAMESPACE" logs "job/$POC_OPERATOR_ACCEPTANCE_JOB_NAME" | tee "$logs_file"
kubectl -n "$K8S_NAMESPACE" delete configmap "$POC_OPERATOR_ACCEPTANCE_JOB_NAME" --ignore-not-found=true >/dev/null
if [[ "$POC_OPERATOR_ACCEPTANCE_CLEANUP_JOB" == "true" ]]; then
kubectl -n "$K8S_NAMESPACE" delete job "$POC_OPERATOR_ACCEPTANCE_JOB_NAME" --ignore-not-found=true >/dev/null
fi
result_marker_found=false
if grep '^DROBOTICFLOW_POC_OPERATOR_ACCEPTANCE=' "$logs_file" | tail -n 1 >"$result_file" && [[ -s "$result_file" ]]; then
result_marker_found=true
fi
import_result_normalized="$(printf '%s' "$POC_ACCEPTANCE_IMPORT_RESULT" | tr '[:upper:]' '[:lower:]')"
if [[ "$result_marker_found" == "true" ]] && [[ "$import_result_normalized" == "true" || "$import_result_normalized" == "1" || "$import_result_normalized" == "yes" || "$import_result_normalized" == "on" ]]; then
if [[ -n "$POC_ACCEPTANCE_IMPORT_API_BASE_URL" ]]; then
runner_image_id="$(kubectl -n "$K8S_NAMESPACE" get pod -l "job-name=$POC_OPERATOR_ACCEPTANCE_JOB_NAME" -o jsonpath='{range .items[*]}{range .status.containerStatuses[*]}{.imageID}{end}{end}' 2>/dev/null || true)"
POC_ACCEPTANCE_RUNNER_IMAGE_ID="$runner_image_id" \
API_BASE_URL="$POC_ACCEPTANCE_IMPORT_API_BASE_URL" \
K8S_NAMESPACE="$K8S_NAMESPACE" \
OPERATOR_RUNNER_IMAGE="$OPERATOR_RUNNER_IMAGE" \
POC_OPERATOR_ACCEPTANCE_JOB_NAME="$POC_OPERATOR_ACCEPTANCE_JOB_NAME" \
python3 "$ROOT_DIR/scripts/import_poc_acceptance_result.py" \
--api-base-url "$POC_ACCEPTANCE_IMPORT_API_BASE_URL" \
--result-file "$result_file" \
--environment "${DEPLOY_ENV:-poc}" \
--namespace "$K8S_NAMESPACE" \
--job-name "$POC_OPERATOR_ACCEPTANCE_JOB_NAME" \
--runner-image "$OPERATOR_RUNNER_IMAGE" \
--runner-image-id "$runner_image_id"
else
echo "POC acceptance platform import skipped: set POC_ACCEPTANCE_IMPORT_API_BASE_URL, API_BASE_URL, or BASE_URL to enable it." >&2
fi
fi
if [[ $wait_status -ne 0 ]]; then
echo "ERROR: POC operator full acceptance Job did not complete successfully." >&2
exit "$wait_status"
fi
if [[ "$result_marker_found" != "true" ]]; then
echo "ERROR: POC operator full acceptance result marker was not found in Job logs." >&2
exit 1
fi