#!/usr/bin/env python3 from __future__ import annotations import argparse import json import os import pathlib import re import subprocess import sys import time from typing import Any RESULT_PREFIX = "DROBOTICFLOW_POC_OPERATOR_ACCEPTANCE=" DEFAULT_MONGO_IMAGE = "mongo:7" K8S_NAME_PATTERN = re.compile(r"^[a-z0-9]([-a-z0-9]*[a-z0-9])?$") EXTERNAL_NAME_PATTERN = re.compile(r"^[a-z][a-z0-9_-]*$") IMAGE_DIGEST_PATTERN = r"^.+@sha256:[0-9a-f]{64}$" def sanitize_acceptance_id_segment(value: str) -> str: cleaned = re.sub(r"[^a-zA-Z0-9_-]+", "-", value.strip()).strip("-_") return cleaned or "unknown" def platform_run_id_from_acceptance_run_id(run_id: str) -> str: return "run-poc-operator-acceptance-" + sanitize_acceptance_id_segment(run_id) def load_acceptance_result(path: pathlib.Path) -> dict[str, Any]: text = path.read_text(encoding="utf-8").strip() if not text: raise SystemExit(f"{path} is empty") if text.startswith(RESULT_PREFIX): text = text[len(RESULT_PREFIX) :].strip() payload = json.loads(text) if not isinstance(payload, dict): raise SystemExit("acceptance result must be a JSON object") return payload def validate_k8s_name(name: str, label: str) -> None: if not name or len(name) > 63 or not K8S_NAME_PATTERN.match(name): raise SystemExit(f"{label} must be a lowercase Kubernetes DNS label") def validate_image_reference(value: str, label: str) -> None: if not value or any(char.isspace() for char in value): raise SystemExit(f"{label} must be a non-empty container image reference without whitespace") def parse_expected_externals(values: list[str]) -> list[str]: externals: list[str] = [] seen: set[str] = set() for value in values: for item in value.split(","): external = item.strip().lower() if not external: continue if not EXTERNAL_NAME_PATTERN.match(external): raise SystemExit("--expected-external values must be lowercase external system names") if external not in seen: externals.append(external) seen.add(external) return externals def parse_expected_preflight_stages(values: list[str]) -> list[str]: stages: list[str] = [] seen: set[str] = set() for value in values: for item in value.split(","): stage = item.strip().lower() if not stage: continue if not K8S_NAME_PATTERN.match(stage): raise SystemExit("--expected-preflight-stage values must be lowercase stage names") if stage not in seen: stages.append(stage) seen.add(stage) return stages def resolve_run_id(args: argparse.Namespace) -> str: if args.run_id and args.result_file: raise SystemExit("set only one of --run-id or --result-file") if args.run_id: return args.run_id.strip() if args.result_file: result = load_acceptance_result(pathlib.Path(args.result_file)) acceptance_run_id = str(result.get("runId", "")).strip() if not acceptance_run_id: raise SystemExit("acceptance result is missing runId") return platform_run_id_from_acceptance_run_id(acceptance_run_id) env_run_id = os.getenv("POC_ACCEPTANCE_IMPORTED_RUN_ID", "").strip() if env_run_id: return env_run_id raise SystemExit("--run-id, --result-file, or POC_ACCEPTANCE_IMPORTED_RUN_ID is required") def mongo_verify_js() -> str: return f""" const runId = process.env.RUN_ID; const expectedImage = process.env.EXPECTED_IMAGE || ""; const expectedScope = process.env.EXPECTED_SCOPE || ""; const expectedAcceptanceStatus = process.env.EXPECTED_ACCEPTANCE_STATUS || "passed"; const expectedRunStatus = process.env.EXPECTED_RUN_STATUS || "completed"; const requireImageDigest = process.env.REQUIRE_IMAGE_DIGEST !== "false"; const requireRepositoryPolicyClean = process.env.REQUIRE_REPOSITORY_POLICY_CLEAN !== "false"; const requireRealFixtures = process.env.REQUIRE_REAL_FIXTURES !== "false"; const requireKubernetesSource = process.env.REQUIRE_KUBERNETES_SOURCE !== "false"; const expectedHumanReviewTaskCount = Number(process.env.EXPECTED_HUMAN_REVIEW_TASK_COUNT || "0"); let expectedExternals = []; let expectedPreflightStages = []; try {{ expectedExternals = JSON.parse(process.env.EXPECTED_EXTERNALS || "[]"); expectedPreflightStages = JSON.parse(process.env.EXPECTED_PREFLIGHT_STAGES || "[]"); }} catch (error) {{ throw new Error("EXPECTED_EXTERNALS and EXPECTED_PREFLIGHT_STAGES must be JSON string arrays"); }} if (!Array.isArray(expectedExternals) || expectedExternals.some((item) => typeof item !== "string" || item.length === 0)) {{ throw new Error("EXPECTED_EXTERNALS must be a JSON string array"); }} if (!Array.isArray(expectedPreflightStages) || expectedPreflightStages.some((item) => typeof item !== "string" || item.length === 0)) {{ throw new Error("EXPECTED_PREFLIGHT_STAGES must be a JSON string array"); }} const database = db.getSiblingDB(process.env.MONGO_DB || "poc"); const run = database.runs.findOne({{ id: runId }}); if (!run) throw new Error("missing imported run " + runId); const nestedArtifacts = []; for (const execution of (run.nodeExecutions || [])) {{ for (const artifact of (execution.artifacts || [])) nestedArtifacts.push(artifact); }} const artifact = nestedArtifacts.find((item) => item.name === "poc-operator-full-acceptance-result.json"); if (!artifact || !artifact.data) throw new Error("missing imported acceptance artifact data"); const data = artifact.data; const source = data.importSource || {{}}; const human = (data.records || []).find((item) => item.operator === "human_review"); const fixtureItems = data.fixtureSources && data.fixtureSources.items && typeof data.fixtureSources.items === "object" ? data.fixtureSources.items : {{}}; const fixtureNames = Object.keys(fixtureItems); const generatedFixtureSources = fixtureNames.filter((name) => fixtureItems[name] && fixtureItems[name].generated !== false); const fixtureSourcesWithInvalidPath = fixtureNames.filter((name) => {{ const item = fixtureItems[name] || {{}}; return typeof item.path !== "string" || !item.path.startsWith("/"); }}); const externalVerificationPassed = Array.isArray(data.externalVerificationPassed) ? data.externalVerificationPassed : []; const externalVerificationMissing = Array.isArray(data.externalVerificationMissing) ? data.externalVerificationMissing : []; const expectedExternalVerificationPassed = expectedExternals.every((external) => externalVerificationPassed.includes(external) && !externalVerificationMissing.includes(external) ); const preflightStages = Array.isArray(data.preflightStages) ? data.preflightStages : data.preflightStage ? [data.preflightStage] : []; const expectedPreflightStagesPresent = expectedPreflightStages.every((stage) => preflightStages.includes(stage)); const preflightRecords = (data.records || []).filter((item) => item && item.operator === "poc_acceptance_preflight" && item.status === "failed" ); const expectedPreflightRecordsPresent = expectedPreflightStages.every((stage) => preflightRecords.some((item) => item.preflightStage === stage) ); const repositoryManifestErrors = data.static && Array.isArray(data.static.repositoryManifestErrors) ? data.static.repositoryManifestErrors : []; const summary = {{ runId: run.id, acceptanceRunId: data.runId, status: run.status, artifactName: artifact.name, acceptanceStatus: data.status, acceptanceScope: data.acceptanceScope, fullExternalAcceptance: data.fullExternalAcceptance, operatorCount: data.operatorCount, operatorPassedCount: data.operatorPassedCount, allFixturesReal: data.allFixturesReal, generatedFixtureCount: data.generatedFixtureCount, realFixtureCount: data.realFixtureCount, fixtureSourceCount: fixtureNames.length, generatedFixtureSources, fixtureSourcesWithInvalidPath, externalVerificationPassed, externalVerificationMissing, externalVerificationPassedCount: data.externalVerificationPassedCount, externalVerificationRequiredCount: data.externalVerificationRequiredCount, expectedExternals, expectedExternalVerificationPassed, preflightFailure: data.preflightFailure, preflightStage: data.preflightStage, preflightStages, expectedPreflightStages, expectedPreflightStagesPresent, expectedPreflightRecordsPresent, preflightRecordCount: preflightRecords.length, blockingMissingCoverage: Array.isArray(data.blockingMissingCoverage) ? data.blockingMissingCoverage : [], failures: Array.isArray(data.failures) ? data.failures : [], operatorRepositoryPolicyClean: data.operatorRepositoryPolicyClean, repositoryManifestErrors, importEnvironment: source.environment, importNamespace: source.namespace, importJobName: source.jobName, importImage: source.image, importSourceComplete: Boolean(source.namespace && source.jobName && source.image && source.imageId), runIdMatchesJobName: source.jobName ? data.runId === source.jobName : null, importImageMatches: expectedImage ? source.image === expectedImage : null, imageIdHasSha256: /{IMAGE_DIGEST_PATTERN}/.test(source.imageId || ""), humanReview: human ? {{ handler: human.handler, status: human.status, taskCount: human.evidence && human.evidence.taskCount, }} : null, }}; if (summary.status !== expectedRunStatus) throw new Error(JSON.stringify({{ reason: "run status mismatch", summary }})); if (summary.acceptanceStatus !== expectedAcceptanceStatus) throw new Error(JSON.stringify({{ reason: "acceptance status mismatch", summary }})); if (summary.runIdMatchesJobName === false) throw new Error(JSON.stringify({{ reason: "acceptance runId does not match import jobName", summary }})); if (expectedScope && summary.acceptanceScope !== expectedScope) throw new Error(JSON.stringify({{ reason: "acceptance scope mismatch", summary }})); if (expectedImage && !summary.importImageMatches) throw new Error(JSON.stringify({{ reason: "runner image mismatch", summary }})); if (requireKubernetesSource && !summary.importSourceComplete) throw new Error(JSON.stringify({{ reason: "missing kubernetes import source", summary }})); if (requireImageDigest && !summary.imageIdHasSha256) throw new Error(JSON.stringify({{ reason: "missing sha256 image id", summary }})); if (requireRealFixtures) {{ if (data.fixtureSources && data.fixtureSources.allowGeneratedFixtures === true) throw new Error(JSON.stringify({{ reason: "generated fixtures allowed", summary }})); if (summary.allFixturesReal !== true) throw new Error(JSON.stringify({{ reason: "fixtures not marked real", summary }})); if (summary.generatedFixtureCount !== 0) throw new Error(JSON.stringify({{ reason: "generated fixture count is not zero", summary }})); if (!Number.isFinite(summary.realFixtureCount) || summary.realFixtureCount <= 0) throw new Error(JSON.stringify({{ reason: "real fixture count missing", summary }})); if (summary.fixtureSourceCount <= 0) throw new Error(JSON.stringify({{ reason: "fixture source evidence missing", summary }})); if (summary.generatedFixtureSources.length > 0) throw new Error(JSON.stringify({{ reason: "generated fixture source evidence", summary }})); if (summary.fixtureSourcesWithInvalidPath.length > 0) throw new Error(JSON.stringify({{ reason: "fixture source path invalid", summary }})); }} if (expectedExternals.length > 0) {{ if (!summary.expectedExternalVerificationPassed) throw new Error(JSON.stringify({{ reason: "expected external verification missing", summary }})); if (!Number.isFinite(summary.externalVerificationPassedCount) || summary.externalVerificationPassedCount < expectedExternals.length) {{ throw new Error(JSON.stringify({{ reason: "external verification count too low", summary }})); }} }} if (expectedPreflightStages.length > 0) {{ if (summary.preflightFailure !== true) throw new Error(JSON.stringify({{ reason: "preflight failure flag missing", summary }})); if (!summary.expectedPreflightStagesPresent) throw new Error(JSON.stringify({{ reason: "expected preflight stage missing", summary }})); if (!summary.expectedPreflightRecordsPresent) throw new Error(JSON.stringify({{ reason: "expected preflight record missing", summary }})); if (summary.preflightRecordCount <= 0) throw new Error(JSON.stringify({{ reason: "preflight records missing", summary }})); if (summary.blockingMissingCoverage.length <= 0) throw new Error(JSON.stringify({{ reason: "preflight blocking coverage missing", summary }})); if (summary.failures.length <= 0) throw new Error(JSON.stringify({{ reason: "preflight failure messages missing", summary }})); }} if (requireRepositoryPolicyClean && !summary.operatorRepositoryPolicyClean) throw new Error(JSON.stringify({{ reason: "repository policy not clean", summary }})); if (repositoryManifestErrors.length !== 0) throw new Error(JSON.stringify({{ reason: "repository manifest errors", summary }})); if (expectedHumanReviewTaskCount > 0) {{ if (!summary.humanReview || summary.humanReview.handler !== "human_review" || summary.humanReview.status !== "passed" || summary.humanReview.taskCount !== expectedHumanReviewTaskCount) {{ throw new Error(JSON.stringify({{ reason: "human review evidence invalid", summary }})); }} }} print(JSON.stringify(summary, null, 2)); """.strip() def build_pod_manifest(args: argparse.Namespace, run_id: str) -> dict[str, Any]: return { "apiVersion": "v1", "kind": "Pod", "metadata": {"name": args.pod_name}, "spec": { "restartPolicy": "Never", "containers": [ { "name": "mongo-verify", "image": args.mongo_image, "command": ["sh", "-lc"], "args": ['mongosh "$MONGO_URI" --quiet --eval "$MONGO_VERIFY_JS"'], "env": [ { "name": "MONGO_URI", "valueFrom": { "secretKeyRef": { "name": args.mongo_secret, "key": args.mongo_uri_key, } }, }, { "name": "MONGO_DB", "valueFrom": { "configMapKeyRef": { "name": args.configmap, "key": args.mongo_db_key, } }, }, {"name": "RUN_ID", "value": run_id}, {"name": "EXPECTED_IMAGE", "value": args.expected_runner_image}, {"name": "EXPECTED_SCOPE", "value": args.expected_scope}, {"name": "EXPECTED_ACCEPTANCE_STATUS", "value": args.expected_acceptance_status}, {"name": "EXPECTED_RUN_STATUS", "value": args.expected_run_status}, {"name": "REQUIRE_IMAGE_DIGEST", "value": str(args.require_image_digest).lower()}, { "name": "REQUIRE_REPOSITORY_POLICY_CLEAN", "value": str(args.require_repository_policy_clean).lower(), }, { "name": "EXPECTED_HUMAN_REVIEW_TASK_COUNT", "value": str(args.expected_human_review_task_count), }, {"name": "EXPECTED_EXTERNALS", "value": json.dumps(args.expected_externals)}, {"name": "EXPECTED_PREFLIGHT_STAGES", "value": json.dumps(args.expected_preflight_stages)}, {"name": "REQUIRE_REAL_FIXTURES", "value": str(args.require_real_fixtures).lower()}, {"name": "REQUIRE_KUBERNETES_SOURCE", "value": str(args.require_kubernetes_source).lower()}, {"name": "MONGO_VERIFY_JS", "value": mongo_verify_js()}, ], } ], }, } def kubectl_base(args: argparse.Namespace) -> list[str]: command = [args.kubectl] if args.kubeconfig: command.append(f"--kubeconfig={args.kubeconfig}") command.extend(["-n", args.namespace]) return command def run_command(args: argparse.Namespace, command: list[str], *, input_text: str | None = None, check: bool = True) -> subprocess.CompletedProcess[str]: result = subprocess.run( command, input=input_text, text=True, capture_output=True, check=False, ) if check and result.returncode != 0: sys.stderr.write(result.stderr) sys.stderr.write(result.stdout) raise SystemExit(result.returncode) return result def delete_pod(args: argparse.Namespace) -> None: command = kubectl_base(args) + ["delete", "pod", args.pod_name, "--ignore-not-found=true"] run_command(args, command, check=False) def verify_import(args: argparse.Namespace, run_id: str) -> int: manifest = build_pod_manifest(args, run_id) delete_pod(args) create = run_command(args, kubectl_base(args) + ["create", "-f", "-"], input_text=json.dumps(manifest)) if create.stdout.strip(): print(create.stdout.strip(), file=sys.stderr) try: wait_command = kubectl_base(args) + [ "wait", f"--for=jsonpath={{.status.phase}}=Succeeded", f"pod/{args.pod_name}", f"--timeout={args.timeout}", ] wait = run_command(args, wait_command, check=False) if wait.returncode != 0: logs = run_command(args, kubectl_base(args) + ["logs", f"pod/{args.pod_name}"], check=False) if logs.stdout: sys.stderr.write(logs.stdout) if logs.stderr: sys.stderr.write(logs.stderr) describe = run_command(args, kubectl_base(args) + ["describe", "pod", args.pod_name], check=False) if describe.stdout: sys.stderr.write(describe.stdout) if describe.stderr: sys.stderr.write(describe.stderr) return wait.returncode or 1 logs = run_command(args, kubectl_base(args) + ["logs", f"pod/{args.pod_name}"], check=True) print(logs.stdout.strip()) return 0 finally: if args.cleanup: delete_pod(args) def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser(description="Verify an imported POC operator acceptance run in the platform Mongo store") parser.add_argument("--run-id", default="") parser.add_argument("--result-file", default="") parser.add_argument("--namespace", default=os.getenv("K8S_NAMESPACE") or "droboticflow") parser.add_argument("--kubeconfig", default=os.getenv("KUBECONFIG") or "") parser.add_argument("--kubectl", default=os.getenv("KUBECTL") or "kubectl") parser.add_argument("--mongo-secret", default=os.getenv("POC_ACCEPTANCE_MONGO_SECRET") or "droboticflow-mongo-secret") parser.add_argument("--mongo-uri-key", default=os.getenv("POC_ACCEPTANCE_MONGO_URI_KEY") or "MONGO_URI") parser.add_argument("--configmap", default=os.getenv("POC_ACCEPTANCE_CONFIGMAP") or "droboticflow-config") parser.add_argument("--mongo-db-key", default=os.getenv("POC_ACCEPTANCE_MONGO_DB_KEY") or "MONGO_DB") parser.add_argument("--mongo-image", default=os.getenv("POC_ACCEPTANCE_MONGO_VERIFY_IMAGE") or DEFAULT_MONGO_IMAGE) parser.add_argument("--pod-name", default=os.getenv("POC_ACCEPTANCE_VERIFY_POD_NAME") or f"droboticflow-poc-import-verify-{int(time.time())}") parser.add_argument("--timeout", default=os.getenv("POC_ACCEPTANCE_IMPORT_VERIFY_TIMEOUT") or "180s") parser.add_argument("--expected-runner-image", default=os.getenv("OPERATOR_RUNNER_IMAGE") or "") parser.add_argument("--expected-scope", default=os.getenv("POC_ACCEPTANCE_EXPECT_SCOPE") or "") parser.add_argument("--expected-external", action="append", default=[]) parser.add_argument("--expected-preflight-stage", action="append", default=[]) parser.add_argument("--expected-acceptance-status", default=os.getenv("POC_ACCEPTANCE_EXPECT_STATUS") or "passed") parser.add_argument("--expected-run-status", default=os.getenv("POC_ACCEPTANCE_EXPECT_RUN_STATUS") or "completed") parser.add_argument( "--expected-human-review-task-count", type=int, default=int(os.getenv("POC_ACCEPTANCE_EXPECT_HUMAN_REVIEW_TASK_COUNT") or "0"), ) parser.add_argument("--skip-image-digest", dest="require_image_digest", action="store_false") parser.add_argument("--skip-real-fixtures", dest="require_real_fixtures", action="store_false") parser.add_argument("--skip-kubernetes-source", dest="require_kubernetes_source", action="store_false") parser.add_argument("--skip-repository-policy-clean", dest="require_repository_policy_clean", action="store_false") parser.add_argument("--no-cleanup", dest="cleanup", action="store_false") parser.set_defaults( require_image_digest=True, require_real_fixtures=True, require_kubernetes_source=True, require_repository_policy_clean=True, cleanup=True, ) args = parser.parse_args() args.expected_externals = parse_expected_externals( [os.getenv("POC_ACCEPTANCE_EXPECT_EXTERNALS") or "", *args.expected_external] ) args.expected_preflight_stages = parse_expected_preflight_stages( [os.getenv("POC_ACCEPTANCE_EXPECT_PREFLIGHT_STAGES") or "", *args.expected_preflight_stage] ) if args.expected_preflight_stages: if args.expected_acceptance_status == "passed": args.expected_acceptance_status = "failed" if args.expected_run_status == "completed": args.expected_run_status = "failed" args.require_image_digest = False args.require_real_fixtures = False args.require_kubernetes_source = False validate_k8s_name(args.namespace, "--namespace") validate_k8s_name(args.mongo_secret, "--mongo-secret") validate_k8s_name(args.configmap, "--configmap") validate_k8s_name(args.pod_name, "--pod-name") validate_image_reference(args.mongo_image, "--mongo-image") if args.expected_runner_image: validate_image_reference(args.expected_runner_image, "--expected-runner-image") if args.expected_human_review_task_count < 0: raise SystemExit("--expected-human-review-task-count must be zero or positive") return args def main() -> int: args = parse_args() run_id = resolve_run_id(args) return verify_import(args, run_id) if __name__ == "__main__": sys.exit(main())