droboticflow-new/scripts/verify_poc_acceptance_import.py
2026-06-23 23:14:40 +08:00

450 lines
22 KiB
Python

#!/usr/bin/env python3
from __future__ import annotations
import argparse
import json
import os
import pathlib
import re
import subprocess
import sys
import time
from typing import Any
RESULT_PREFIX = "DROBOTICFLOW_POC_OPERATOR_ACCEPTANCE="
DEFAULT_MONGO_IMAGE = "mongo:7"
K8S_NAME_PATTERN = re.compile(r"^[a-z0-9]([-a-z0-9]*[a-z0-9])?$")
EXTERNAL_NAME_PATTERN = re.compile(r"^[a-z][a-z0-9_-]*$")
IMAGE_DIGEST_PATTERN = r"^.+@sha256:[0-9a-f]{64}$"
def sanitize_acceptance_id_segment(value: str) -> str:
cleaned = re.sub(r"[^a-zA-Z0-9_-]+", "-", value.strip()).strip("-_")
return cleaned or "unknown"
def platform_run_id_from_acceptance_run_id(run_id: str) -> str:
return "run-poc-operator-acceptance-" + sanitize_acceptance_id_segment(run_id)
def load_acceptance_result(path: pathlib.Path) -> dict[str, Any]:
text = path.read_text(encoding="utf-8").strip()
if not text:
raise SystemExit(f"{path} is empty")
if text.startswith(RESULT_PREFIX):
text = text[len(RESULT_PREFIX) :].strip()
payload = json.loads(text)
if not isinstance(payload, dict):
raise SystemExit("acceptance result must be a JSON object")
return payload
def validate_k8s_name(name: str, label: str) -> None:
if not name or len(name) > 63 or not K8S_NAME_PATTERN.match(name):
raise SystemExit(f"{label} must be a lowercase Kubernetes DNS label")
def validate_image_reference(value: str, label: str) -> None:
if not value or any(char.isspace() for char in value):
raise SystemExit(f"{label} must be a non-empty container image reference without whitespace")
def parse_expected_externals(values: list[str]) -> list[str]:
externals: list[str] = []
seen: set[str] = set()
for value in values:
for item in value.split(","):
external = item.strip().lower()
if not external:
continue
if not EXTERNAL_NAME_PATTERN.match(external):
raise SystemExit("--expected-external values must be lowercase external system names")
if external not in seen:
externals.append(external)
seen.add(external)
return externals
def parse_expected_preflight_stages(values: list[str]) -> list[str]:
stages: list[str] = []
seen: set[str] = set()
for value in values:
for item in value.split(","):
stage = item.strip().lower()
if not stage:
continue
if not K8S_NAME_PATTERN.match(stage):
raise SystemExit("--expected-preflight-stage values must be lowercase stage names")
if stage not in seen:
stages.append(stage)
seen.add(stage)
return stages
def resolve_run_id(args: argparse.Namespace) -> str:
if args.run_id and args.result_file:
raise SystemExit("set only one of --run-id or --result-file")
if args.run_id:
return args.run_id.strip()
if args.result_file:
result = load_acceptance_result(pathlib.Path(args.result_file))
acceptance_run_id = str(result.get("runId", "")).strip()
if not acceptance_run_id:
raise SystemExit("acceptance result is missing runId")
return platform_run_id_from_acceptance_run_id(acceptance_run_id)
env_run_id = os.getenv("POC_ACCEPTANCE_IMPORTED_RUN_ID", "").strip()
if env_run_id:
return env_run_id
raise SystemExit("--run-id, --result-file, or POC_ACCEPTANCE_IMPORTED_RUN_ID is required")
def mongo_verify_js() -> str:
return f"""
const runId = process.env.RUN_ID;
const expectedImage = process.env.EXPECTED_IMAGE || "";
const expectedScope = process.env.EXPECTED_SCOPE || "";
const expectedAcceptanceStatus = process.env.EXPECTED_ACCEPTANCE_STATUS || "passed";
const expectedRunStatus = process.env.EXPECTED_RUN_STATUS || "completed";
const requireImageDigest = process.env.REQUIRE_IMAGE_DIGEST !== "false";
const requireRepositoryPolicyClean = process.env.REQUIRE_REPOSITORY_POLICY_CLEAN !== "false";
const requireRealFixtures = process.env.REQUIRE_REAL_FIXTURES !== "false";
const requireKubernetesSource = process.env.REQUIRE_KUBERNETES_SOURCE !== "false";
const expectedHumanReviewTaskCount = Number(process.env.EXPECTED_HUMAN_REVIEW_TASK_COUNT || "0");
let expectedExternals = [];
let expectedPreflightStages = [];
try {{
expectedExternals = JSON.parse(process.env.EXPECTED_EXTERNALS || "[]");
expectedPreflightStages = JSON.parse(process.env.EXPECTED_PREFLIGHT_STAGES || "[]");
}} catch (error) {{
throw new Error("EXPECTED_EXTERNALS and EXPECTED_PREFLIGHT_STAGES must be JSON string arrays");
}}
if (!Array.isArray(expectedExternals) || expectedExternals.some((item) => typeof item !== "string" || item.length === 0)) {{
throw new Error("EXPECTED_EXTERNALS must be a JSON string array");
}}
if (!Array.isArray(expectedPreflightStages) || expectedPreflightStages.some((item) => typeof item !== "string" || item.length === 0)) {{
throw new Error("EXPECTED_PREFLIGHT_STAGES must be a JSON string array");
}}
const database = db.getSiblingDB(process.env.MONGO_DB || "poc");
const run = database.runs.findOne({{ id: runId }});
if (!run) throw new Error("missing imported run " + runId);
const nestedArtifacts = [];
for (const execution of (run.nodeExecutions || [])) {{
for (const artifact of (execution.artifacts || [])) nestedArtifacts.push(artifact);
}}
const artifact = nestedArtifacts.find((item) => item.name === "poc-operator-full-acceptance-result.json");
if (!artifact || !artifact.data) throw new Error("missing imported acceptance artifact data");
const data = artifact.data;
const source = data.importSource || {{}};
const human = (data.records || []).find((item) => item.operator === "human_review");
const fixtureItems = data.fixtureSources && data.fixtureSources.items && typeof data.fixtureSources.items === "object"
? data.fixtureSources.items
: {{}};
const fixtureNames = Object.keys(fixtureItems);
const generatedFixtureSources = fixtureNames.filter((name) => fixtureItems[name] && fixtureItems[name].generated !== false);
const fixtureSourcesWithInvalidPath = fixtureNames.filter((name) => {{
const item = fixtureItems[name] || {{}};
return typeof item.path !== "string" || !item.path.startsWith("/");
}});
const externalVerificationPassed = Array.isArray(data.externalVerificationPassed) ? data.externalVerificationPassed : [];
const externalVerificationMissing = Array.isArray(data.externalVerificationMissing) ? data.externalVerificationMissing : [];
const expectedExternalVerificationPassed = expectedExternals.every((external) =>
externalVerificationPassed.includes(external) && !externalVerificationMissing.includes(external)
);
const preflightStages = Array.isArray(data.preflightStages)
? data.preflightStages
: data.preflightStage
? [data.preflightStage]
: [];
const expectedPreflightStagesPresent = expectedPreflightStages.every((stage) => preflightStages.includes(stage));
const preflightRecords = (data.records || []).filter((item) =>
item && item.operator === "poc_acceptance_preflight" && item.status === "failed"
);
const expectedPreflightRecordsPresent = expectedPreflightStages.every((stage) =>
preflightRecords.some((item) => item.preflightStage === stage)
);
const repositoryManifestErrors = data.static && Array.isArray(data.static.repositoryManifestErrors)
? data.static.repositoryManifestErrors
: [];
const summary = {{
runId: run.id,
acceptanceRunId: data.runId,
status: run.status,
artifactName: artifact.name,
acceptanceStatus: data.status,
acceptanceScope: data.acceptanceScope,
fullExternalAcceptance: data.fullExternalAcceptance,
operatorCount: data.operatorCount,
operatorPassedCount: data.operatorPassedCount,
allFixturesReal: data.allFixturesReal,
generatedFixtureCount: data.generatedFixtureCount,
realFixtureCount: data.realFixtureCount,
fixtureSourceCount: fixtureNames.length,
generatedFixtureSources,
fixtureSourcesWithInvalidPath,
externalVerificationPassed,
externalVerificationMissing,
externalVerificationPassedCount: data.externalVerificationPassedCount,
externalVerificationRequiredCount: data.externalVerificationRequiredCount,
expectedExternals,
expectedExternalVerificationPassed,
preflightFailure: data.preflightFailure,
preflightStage: data.preflightStage,
preflightStages,
expectedPreflightStages,
expectedPreflightStagesPresent,
expectedPreflightRecordsPresent,
preflightRecordCount: preflightRecords.length,
blockingMissingCoverage: Array.isArray(data.blockingMissingCoverage) ? data.blockingMissingCoverage : [],
failures: Array.isArray(data.failures) ? data.failures : [],
operatorRepositoryPolicyClean: data.operatorRepositoryPolicyClean,
repositoryManifestErrors,
importEnvironment: source.environment,
importNamespace: source.namespace,
importJobName: source.jobName,
importImage: source.image,
importSourceComplete: Boolean(source.namespace && source.jobName && source.image && source.imageId),
runIdMatchesJobName: source.jobName ? data.runId === source.jobName : null,
importImageMatches: expectedImage ? source.image === expectedImage : null,
imageIdHasSha256: /{IMAGE_DIGEST_PATTERN}/.test(source.imageId || ""),
humanReview: human ? {{
handler: human.handler,
status: human.status,
taskCount: human.evidence && human.evidence.taskCount,
}} : null,
}};
if (summary.status !== expectedRunStatus) throw new Error(JSON.stringify({{ reason: "run status mismatch", summary }}));
if (summary.acceptanceStatus !== expectedAcceptanceStatus) throw new Error(JSON.stringify({{ reason: "acceptance status mismatch", summary }}));
if (summary.runIdMatchesJobName === false) throw new Error(JSON.stringify({{ reason: "acceptance runId does not match import jobName", summary }}));
if (expectedScope && summary.acceptanceScope !== expectedScope) throw new Error(JSON.stringify({{ reason: "acceptance scope mismatch", summary }}));
if (expectedImage && !summary.importImageMatches) throw new Error(JSON.stringify({{ reason: "runner image mismatch", summary }}));
if (requireKubernetesSource && !summary.importSourceComplete) throw new Error(JSON.stringify({{ reason: "missing kubernetes import source", summary }}));
if (requireImageDigest && !summary.imageIdHasSha256) throw new Error(JSON.stringify({{ reason: "missing sha256 image id", summary }}));
if (requireRealFixtures) {{
if (data.fixtureSources && data.fixtureSources.allowGeneratedFixtures === true) throw new Error(JSON.stringify({{ reason: "generated fixtures allowed", summary }}));
if (summary.allFixturesReal !== true) throw new Error(JSON.stringify({{ reason: "fixtures not marked real", summary }}));
if (summary.generatedFixtureCount !== 0) throw new Error(JSON.stringify({{ reason: "generated fixture count is not zero", summary }}));
if (!Number.isFinite(summary.realFixtureCount) || summary.realFixtureCount <= 0) throw new Error(JSON.stringify({{ reason: "real fixture count missing", summary }}));
if (summary.fixtureSourceCount <= 0) throw new Error(JSON.stringify({{ reason: "fixture source evidence missing", summary }}));
if (summary.generatedFixtureSources.length > 0) throw new Error(JSON.stringify({{ reason: "generated fixture source evidence", summary }}));
if (summary.fixtureSourcesWithInvalidPath.length > 0) throw new Error(JSON.stringify({{ reason: "fixture source path invalid", summary }}));
}}
if (expectedExternals.length > 0) {{
if (!summary.expectedExternalVerificationPassed) throw new Error(JSON.stringify({{ reason: "expected external verification missing", summary }}));
if (!Number.isFinite(summary.externalVerificationPassedCount) || summary.externalVerificationPassedCount < expectedExternals.length) {{
throw new Error(JSON.stringify({{ reason: "external verification count too low", summary }}));
}}
}}
if (expectedPreflightStages.length > 0) {{
if (summary.preflightFailure !== true) throw new Error(JSON.stringify({{ reason: "preflight failure flag missing", summary }}));
if (!summary.expectedPreflightStagesPresent) throw new Error(JSON.stringify({{ reason: "expected preflight stage missing", summary }}));
if (!summary.expectedPreflightRecordsPresent) throw new Error(JSON.stringify({{ reason: "expected preflight record missing", summary }}));
if (summary.preflightRecordCount <= 0) throw new Error(JSON.stringify({{ reason: "preflight records missing", summary }}));
if (summary.blockingMissingCoverage.length <= 0) throw new Error(JSON.stringify({{ reason: "preflight blocking coverage missing", summary }}));
if (summary.failures.length <= 0) throw new Error(JSON.stringify({{ reason: "preflight failure messages missing", summary }}));
}}
if (requireRepositoryPolicyClean && !summary.operatorRepositoryPolicyClean) throw new Error(JSON.stringify({{ reason: "repository policy not clean", summary }}));
if (repositoryManifestErrors.length !== 0) throw new Error(JSON.stringify({{ reason: "repository manifest errors", summary }}));
if (expectedHumanReviewTaskCount > 0) {{
if (!summary.humanReview || summary.humanReview.handler !== "human_review" || summary.humanReview.status !== "passed" || summary.humanReview.taskCount !== expectedHumanReviewTaskCount) {{
throw new Error(JSON.stringify({{ reason: "human review evidence invalid", summary }}));
}}
}}
print(JSON.stringify(summary, null, 2));
""".strip()
def build_pod_manifest(args: argparse.Namespace, run_id: str) -> dict[str, Any]:
return {
"apiVersion": "v1",
"kind": "Pod",
"metadata": {"name": args.pod_name},
"spec": {
"restartPolicy": "Never",
"containers": [
{
"name": "mongo-verify",
"image": args.mongo_image,
"command": ["sh", "-lc"],
"args": ['mongosh "$MONGO_URI" --quiet --eval "$MONGO_VERIFY_JS"'],
"env": [
{
"name": "MONGO_URI",
"valueFrom": {
"secretKeyRef": {
"name": args.mongo_secret,
"key": args.mongo_uri_key,
}
},
},
{
"name": "MONGO_DB",
"valueFrom": {
"configMapKeyRef": {
"name": args.configmap,
"key": args.mongo_db_key,
}
},
},
{"name": "RUN_ID", "value": run_id},
{"name": "EXPECTED_IMAGE", "value": args.expected_runner_image},
{"name": "EXPECTED_SCOPE", "value": args.expected_scope},
{"name": "EXPECTED_ACCEPTANCE_STATUS", "value": args.expected_acceptance_status},
{"name": "EXPECTED_RUN_STATUS", "value": args.expected_run_status},
{"name": "REQUIRE_IMAGE_DIGEST", "value": str(args.require_image_digest).lower()},
{
"name": "REQUIRE_REPOSITORY_POLICY_CLEAN",
"value": str(args.require_repository_policy_clean).lower(),
},
{
"name": "EXPECTED_HUMAN_REVIEW_TASK_COUNT",
"value": str(args.expected_human_review_task_count),
},
{"name": "EXPECTED_EXTERNALS", "value": json.dumps(args.expected_externals)},
{"name": "EXPECTED_PREFLIGHT_STAGES", "value": json.dumps(args.expected_preflight_stages)},
{"name": "REQUIRE_REAL_FIXTURES", "value": str(args.require_real_fixtures).lower()},
{"name": "REQUIRE_KUBERNETES_SOURCE", "value": str(args.require_kubernetes_source).lower()},
{"name": "MONGO_VERIFY_JS", "value": mongo_verify_js()},
],
}
],
},
}
def kubectl_base(args: argparse.Namespace) -> list[str]:
command = [args.kubectl]
if args.kubeconfig:
command.append(f"--kubeconfig={args.kubeconfig}")
command.extend(["-n", args.namespace])
return command
def run_command(args: argparse.Namespace, command: list[str], *, input_text: str | None = None, check: bool = True) -> subprocess.CompletedProcess[str]:
result = subprocess.run(
command,
input=input_text,
text=True,
capture_output=True,
check=False,
)
if check and result.returncode != 0:
sys.stderr.write(result.stderr)
sys.stderr.write(result.stdout)
raise SystemExit(result.returncode)
return result
def delete_pod(args: argparse.Namespace) -> None:
command = kubectl_base(args) + ["delete", "pod", args.pod_name, "--ignore-not-found=true"]
run_command(args, command, check=False)
def verify_import(args: argparse.Namespace, run_id: str) -> int:
manifest = build_pod_manifest(args, run_id)
delete_pod(args)
create = run_command(args, kubectl_base(args) + ["create", "-f", "-"], input_text=json.dumps(manifest))
if create.stdout.strip():
print(create.stdout.strip(), file=sys.stderr)
try:
wait_command = kubectl_base(args) + [
"wait",
f"--for=jsonpath={{.status.phase}}=Succeeded",
f"pod/{args.pod_name}",
f"--timeout={args.timeout}",
]
wait = run_command(args, wait_command, check=False)
if wait.returncode != 0:
logs = run_command(args, kubectl_base(args) + ["logs", f"pod/{args.pod_name}"], check=False)
if logs.stdout:
sys.stderr.write(logs.stdout)
if logs.stderr:
sys.stderr.write(logs.stderr)
describe = run_command(args, kubectl_base(args) + ["describe", "pod", args.pod_name], check=False)
if describe.stdout:
sys.stderr.write(describe.stdout)
if describe.stderr:
sys.stderr.write(describe.stderr)
return wait.returncode or 1
logs = run_command(args, kubectl_base(args) + ["logs", f"pod/{args.pod_name}"], check=True)
print(logs.stdout.strip())
return 0
finally:
if args.cleanup:
delete_pod(args)
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description="Verify an imported POC operator acceptance run in the platform Mongo store")
parser.add_argument("--run-id", default="")
parser.add_argument("--result-file", default="")
parser.add_argument("--namespace", default=os.getenv("K8S_NAMESPACE") or "droboticflow")
parser.add_argument("--kubeconfig", default=os.getenv("KUBECONFIG") or "")
parser.add_argument("--kubectl", default=os.getenv("KUBECTL") or "kubectl")
parser.add_argument("--mongo-secret", default=os.getenv("POC_ACCEPTANCE_MONGO_SECRET") or "droboticflow-mongo-secret")
parser.add_argument("--mongo-uri-key", default=os.getenv("POC_ACCEPTANCE_MONGO_URI_KEY") or "MONGO_URI")
parser.add_argument("--configmap", default=os.getenv("POC_ACCEPTANCE_CONFIGMAP") or "droboticflow-config")
parser.add_argument("--mongo-db-key", default=os.getenv("POC_ACCEPTANCE_MONGO_DB_KEY") or "MONGO_DB")
parser.add_argument("--mongo-image", default=os.getenv("POC_ACCEPTANCE_MONGO_VERIFY_IMAGE") or DEFAULT_MONGO_IMAGE)
parser.add_argument("--pod-name", default=os.getenv("POC_ACCEPTANCE_VERIFY_POD_NAME") or f"droboticflow-poc-import-verify-{int(time.time())}")
parser.add_argument("--timeout", default=os.getenv("POC_ACCEPTANCE_IMPORT_VERIFY_TIMEOUT") or "180s")
parser.add_argument("--expected-runner-image", default=os.getenv("OPERATOR_RUNNER_IMAGE") or "")
parser.add_argument("--expected-scope", default=os.getenv("POC_ACCEPTANCE_EXPECT_SCOPE") or "")
parser.add_argument("--expected-external", action="append", default=[])
parser.add_argument("--expected-preflight-stage", action="append", default=[])
parser.add_argument("--expected-acceptance-status", default=os.getenv("POC_ACCEPTANCE_EXPECT_STATUS") or "passed")
parser.add_argument("--expected-run-status", default=os.getenv("POC_ACCEPTANCE_EXPECT_RUN_STATUS") or "completed")
parser.add_argument(
"--expected-human-review-task-count",
type=int,
default=int(os.getenv("POC_ACCEPTANCE_EXPECT_HUMAN_REVIEW_TASK_COUNT") or "0"),
)
parser.add_argument("--skip-image-digest", dest="require_image_digest", action="store_false")
parser.add_argument("--skip-real-fixtures", dest="require_real_fixtures", action="store_false")
parser.add_argument("--skip-kubernetes-source", dest="require_kubernetes_source", action="store_false")
parser.add_argument("--skip-repository-policy-clean", dest="require_repository_policy_clean", action="store_false")
parser.add_argument("--no-cleanup", dest="cleanup", action="store_false")
parser.set_defaults(
require_image_digest=True,
require_real_fixtures=True,
require_kubernetes_source=True,
require_repository_policy_clean=True,
cleanup=True,
)
args = parser.parse_args()
args.expected_externals = parse_expected_externals(
[os.getenv("POC_ACCEPTANCE_EXPECT_EXTERNALS") or "", *args.expected_external]
)
args.expected_preflight_stages = parse_expected_preflight_stages(
[os.getenv("POC_ACCEPTANCE_EXPECT_PREFLIGHT_STAGES") or "", *args.expected_preflight_stage]
)
if args.expected_preflight_stages:
if args.expected_acceptance_status == "passed":
args.expected_acceptance_status = "failed"
if args.expected_run_status == "completed":
args.expected_run_status = "failed"
args.require_image_digest = False
args.require_real_fixtures = False
args.require_kubernetes_source = False
validate_k8s_name(args.namespace, "--namespace")
validate_k8s_name(args.mongo_secret, "--mongo-secret")
validate_k8s_name(args.configmap, "--configmap")
validate_k8s_name(args.pod_name, "--pod-name")
validate_image_reference(args.mongo_image, "--mongo-image")
if args.expected_runner_image:
validate_image_reference(args.expected_runner_image, "--expected-runner-image")
if args.expected_human_review_task_count < 0:
raise SystemExit("--expected-human-review-task-count must be zero or positive")
return args
def main() -> int:
args = parse_args()
run_id = resolve_run_id(args)
return verify_import(args, run_id)
if __name__ == "__main__":
sys.exit(main())