#!/usr/bin/env python3 from __future__ import annotations import argparse import ast import datetime as dt import hashlib import json import os import pathlib import re import subprocess from typing import Any ROOT = pathlib.Path(__file__).resolve().parents[1] UNMIGRATED_HISTORICAL_OPERATORS = {"temporal_causal_eval"} RESULT_PREFIX = "DROBOTICFLOW_POC_OPERATOR_ACCEPTANCE=" DEFAULT_REQUIRED_EXTERNALS = "bos,mongo" STAGE_HANDLERS = { "fixture-readiness": "fixture_readiness", "deployment-readiness": "deployment_readiness", "runner-image-readiness": "runner_image_readiness", "strict-preflight-readiness": "strict_preflight_readiness", } REPOSITORY_POLICY_VIOLATION_KEYS = ( "repositoryManifestErrors", "operatorImageContractErrors", "duplicateOperatorIds", "duplicateOperatorNames", "legacyOperators", "builtinHelperCallOperators", "hardcodedExternalApiSources", "helperBuiltinKeys", ) def load_json(path: pathlib.Path) -> dict[str, Any]: text = path.read_text(encoding="utf-8").strip() if not text: return {} payload = json.loads(text) if not isinstance(payload, dict): raise SystemExit(f"{path} must contain a JSON object") return payload def now_id() -> str: return dt.datetime.now(dt.timezone.utc).strftime("%Y%m%d%H%M%S") def now_iso() -> str: return dt.datetime.now(dt.timezone.utc).isoformat().replace("+00:00", "Z") def source_commit() -> str: for env_name in ("POC_ACCEPTANCE_SOURCE_COMMIT", "DROBOTICFLOW_BUILD_COMMIT"): value = os.getenv(env_name, "").strip() if value: return value try: return subprocess.run( ["git", "-C", str(ROOT), "rev-parse", "HEAD"], check=True, capture_output=True, text=True, ).stdout.strip() or "unknown" except (OSError, subprocess.CalledProcessError): return "unknown" def builtin_operator_catalog_metadata() -> tuple[str, int]: catalog_path = ROOT / "internal" / "api" / "builtin_operators_catalog.json" try: catalog = json.loads(catalog_path.read_text(encoding="utf-8")) except (OSError, json.JSONDecodeError) as error: raise SystemExit(f"builtin operator catalog is not readable: {catalog_path}") from error if not isinstance(catalog, list) or not catalog: raise SystemExit("builtin operator catalog must be a non-empty JSON list") canonical_text = json.dumps(catalog, ensure_ascii=False, sort_keys=True, separators=(",", ":")) # Go's encoding/json keeps UTF-8 but escapes HTML-sensitive and line-separator runes. for raw, escaped in ( ("&", "\\u0026"), ("<", "\\u003c"), (">", "\\u003e"), ("\u2028", "\\u2028"), ("\u2029", "\\u2029"), ): canonical_text = canonical_text.replace(raw, escaped) canonical = canonical_text.encode("utf-8") return hashlib.sha256(canonical).hexdigest(), len(catalog) def acceptance_provenance(completed_at: str | None = None) -> dict[str, Any]: catalog_hash, catalog_count = builtin_operator_catalog_metadata() return { "sourceCommit": source_commit(), "catalogHash": catalog_hash, "catalogCount": catalog_count, "completedAt": completed_at or now_iso(), } def required_externals(raw: str) -> list[str]: return sorted({item.strip().lower() for item in raw.split(",") if item.strip()}) def operator_inventory() -> list[str]: operators: list[str] = [] builtin_root = ROOT / "backend" / "operators" / "builtin" for metadata_path in sorted(builtin_root.glob("*/operator.json")): slug = metadata_path.parent.name if slug in UNMIGRATED_HISTORICAL_OPERATORS: continue metadata: dict[str, Any] = {} try: loaded_metadata = json.loads(metadata_path.read_text(encoding="utf-8")) except Exception: loaded_metadata = {} if isinstance(loaded_metadata, dict): metadata = loaded_metadata if metadata.get("type") not in ("builtin", None): continue if not slug: continue operators.append(slug) return sorted(set(operators)) def project_relative(path: pathlib.Path) -> str: try: return str(path.relative_to(ROOT)) except ValueError: return str(path) def operator_helper_builtin_keys() -> list[str]: runner_path = ROOT / "backend" / "operator-runner" / "main.py" if not runner_path.exists(): return [] try: tree = ast.parse(runner_path.read_text(encoding="utf-8")) except SyntaxError as error: return [f"parse-error:{error.msg}"] keys: list[str] = [] for node in tree.body: if not isinstance(node, ast.Assign): continue if not any(isinstance(target, ast.Name) and target.id == "OPERATOR_HELPERS" for target in node.targets): continue if not isinstance(node.value, ast.Dict): continue for key in node.value.keys: if isinstance(key, ast.Constant) and isinstance(key.value, str) and key.value.startswith("builtin_"): keys.append(key.value) return sorted(set(keys)) def static_operator_layout() -> dict[str, Any]: operator_root = ROOT / "backend" / "operators" / "builtin" catalog_path = ROOT / "internal" / "api" / "builtin_operators_catalog.json" # Historical source retained outside the active builtin catalog must not # inflate strict current-operator coverage requirements. metadata_paths = sorted(path for path in operator_root.glob("*/operator.json") if path.parent.name not in UNMIGRATED_HISTORICAL_OPERATORS) sources: list[str] = [] repository_manifests: list[str] = [] repository_manifest_errors: list[str] = [] operator_image_contract_errors: list[str] = [] legacy: list[str] = [] builtin_helper_calls: list[str] = [] hardcoded_external_api_sources: list[str] = [] seen_operator_ids: dict[str, str] = {} duplicate_operator_ids: list[str] = [] seen_operator_names: dict[str, str] = {} duplicate_operator_names: list[str] = [] catalog_items: dict[str, dict[str, Any]] = {} try: loaded_catalog = json.loads(catalog_path.read_text(encoding="utf-8")) if isinstance(loaded_catalog, list): catalog_items = { str(item.get("id", "")).strip(): item for item in loaded_catalog if isinstance(item, dict) and item.get("type") == "builtin" } else: operator_image_contract_errors.append("builtin catalog must be a JSON list") except Exception as error: operator_image_contract_errors.append(f"builtin catalog is not readable ({type(error).__name__}: {error})") for metadata_path in metadata_paths: slug = metadata_path.parent.name operator_path = metadata_path.with_name("operator.py") repository_path = metadata_path.with_name("repository.json") if operator_path.exists(): sources.append(project_relative(operator_path)) source = operator_path.read_text(encoding="utf-8") else: source = "" sources.append(project_relative(operator_path)) repository_manifest_errors.append(f"{slug}: missing operator.py") try: metadata = json.loads(metadata_path.read_text(encoding="utf-8")) except Exception as error: repository_manifest_errors.append(f"{slug}: invalid operator.json ({type(error).__name__}: {error})") metadata = {} if not isinstance(metadata, dict): repository_manifest_errors.append(f"{slug}: operator.json must be an object") metadata = {} operator_id = str(metadata.get("id", "")).strip() operator_name = str(metadata.get("name", "")).strip() if not operator_id: repository_manifest_errors.append(f"{slug}: operator.json id is required") elif operator_id in seen_operator_ids: duplicate_operator_ids.append(operator_id) else: seen_operator_ids[operator_id] = slug if not operator_name: repository_manifest_errors.append(f"{slug}: operator.json name is required") elif operator_name in seen_operator_names: duplicate_operator_names.append(operator_name) else: seen_operator_names[operator_name] = slug if slug == "human_review" and metadata.get("runtimeHandler") != "builtin_human_review": repository_manifest_errors.append("human_review: runtimeHandler must be builtin_human_review") if "LEGACY_HANDLER" in source: legacy.append(slug) if 'helpers["builtin_' in source or "helpers['builtin_" in source: builtin_helper_calls.append(slug) if not repository_path.exists(): repository_manifest_errors.append(f"{slug}: missing repository.json") continue try: repository_manifest = json.loads(repository_path.read_text(encoding="utf-8")) except Exception as error: repository_manifest_errors.append(f"{slug}: invalid repository.json ({type(error).__name__}: {error})") continue if not isinstance(repository_manifest, dict): repository_manifest_errors.append(f"{slug}: repository.json must be an object") continue expected_fields = { "schemaVersion": "droboticflow.operator.repository/v1", "repositoryId": f"builtin/{slug}", "operatorId": metadata.get("id"), "operatorSlug": slug, "sourceRoot": f"backend/operators/builtin/{slug}", "entrypoint": "operator.py", "metadata": "operator.json", "runtimeSource": metadata.get("runtimeSource"), "runtimeHandler": metadata.get("runtimeHandler"), "containerEntrypoint": "backend/operator-runner/main.py", } for key, expected in expected_fields.items(): if repository_manifest.get(key) != expected: repository_manifest_errors.append( f"{slug}: repository.json {key}={repository_manifest.get(key)!r}, expected {expected!r}" ) if "externalApi" in metadata and repository_manifest.get("externalApi") != metadata.get("externalApi"): repository_manifest_errors.append(f"{slug}: repository.json externalApi must match operator.json") catalog_item = catalog_items.get(operator_id) if not catalog_item: operator_image_contract_errors.append(f"{slug}: missing builtin catalog item") catalog_item = {} image = str(catalog_item.get("image") or repository_manifest.get("image") or "").strip() image_id = str(catalog_item.get("imageId") or repository_manifest.get("imageId") or "").strip() runtime_contract = catalog_item.get("runtimeContract") or repository_manifest.get("runtimeContract") if not image: operator_image_contract_errors.append(f"{slug}: builtin image is required") if "droboticflow-operator-runner" in image: operator_image_contract_errors.append(f"{slug}: builtin image must not use shared operator-runner final image") if "droboticflow-operator-" not in image: operator_image_contract_errors.append(f"{slug}: builtin image must use an independent operator image tag") if not re.search(r"@sha256:[a-fA-F0-9]{64}$", image_id): operator_image_contract_errors.append(f"{slug}: imageId must be digest-pinned") if catalog_item.get("containerCommand") != ["python", "/app/backend/operator-runner/main.py"]: operator_image_contract_errors.append(f"{slug}: containerCommand must point at the operator runner entrypoint") if catalog_item.get("containerArgs", []) != []: operator_image_contract_errors.append(f"{slug}: containerArgs must default to []") if not isinstance(runtime_contract, dict): operator_image_contract_errors.append(f"{slug}: runtimeContract is required") elif not runtime_contract.get("imagePolicy", {}).get("independentFinalImage"): operator_image_contract_errors.append(f"{slug}: runtimeContract must require independent final image") command_keys = { str(param.get("key", "")).strip().upper() for param in catalog_item.get("hyperparams", []) if isinstance(param, dict) } if operator_id != "operator-builtin-shell-command" and command_keys.intersection({"COMMAND", "SCRIPT"}): operator_image_contract_errors.append(f"{slug}: non-shell builtin must not expose COMMAND/SCRIPT") acceptance = repository_manifest.get("acceptance") if isinstance(repository_manifest.get("acceptance"), dict) else {} if not acceptance.get("requiresRealInputs"): repository_manifest_errors.append(f"{slug}: repository.json must require real inputs") for key in ("mockOnlyAccepted", "simulationOnlyAccepted", "hardcodedOutputAccepted", "shallowOnlyAccepted"): if acceptance.get(key) is not False: repository_manifest_errors.append(f"{slug}: repository.json acceptance.{key} must be false") if acceptance.get("requiresPersistentWorkspaceEvidence") is not True: repository_manifest_errors.append(f"{slug}: repository.json must require persistent workspace evidence") dependencies = repository_manifest.get("runtimeDependencies") if not isinstance(dependencies, list) or not dependencies: repository_manifest_errors.append(f"{slug}: repository.json must list runtime dependencies") repository_manifests.append(project_relative(repository_path)) return { "operatorFileCount": len(metadata_paths), "repositoryManifestCount": len(repository_manifests), "repositoryManifestErrors": repository_manifest_errors, "operatorImageContractErrors": operator_image_contract_errors, "duplicateOperatorIds": sorted(duplicate_operator_ids), "duplicateOperatorNames": sorted(duplicate_operator_names), "legacyOperators": legacy, "builtinHelperCallOperators": builtin_helper_calls, "hardcodedExternalApiSources": hardcoded_external_api_sources, "helperBuiltinKeys": operator_helper_builtin_keys(), "runtimeSources": sources, "repositoryManifests": repository_manifests, } def operator_repository_summary(static_payload: dict[str, Any]) -> dict[str, Any]: violations: list[str] = [] for key in REPOSITORY_POLICY_VIOLATION_KEYS: for item in static_payload.get(key, []) or []: text = str(item).strip() if text: violations.append(f"{key}:{text}") required_count = int(static_payload.get("operatorFileCount") or 0) repository_count = int(static_payload.get("repositoryManifestCount") or 0) return { "operatorRepositoryRequiredCount": required_count, "operatorRepositoryCount": repository_count, "operatorRepositoryCoverageComplete": required_count > 0 and repository_count == required_count, "operatorRepositoryPolicyClean": len(violations) == 0, "operatorRepositoryPolicyViolations": violations, } def readiness_stage(check_name: str) -> str: if check_name == "fixture-manifest": return "fixture-readiness" if check_name.startswith("deployment:"): return "deployment-readiness" if check_name == "operator-runner-image": return "runner-image-readiness" return "strict-preflight-readiness" def readiness_issues(payload: dict[str, Any]) -> list[tuple[str, str, dict[str, Any]]]: issues: list[tuple[str, str, dict[str, Any]]] = [] for check in payload.get("checks") or []: if not isinstance(check, dict): continue if check.get("status") == "passed" or check.get("required") is False: continue name = str(check.get("name") or "unknown") stage = readiness_stage(name) message = str(check.get("message") or f"{name} did not pass") evidence = check.get("evidence") if isinstance(check.get("evidence"), dict) else {} issues.append((stage, message, {"checkName": name, **evidence})) if issues: return issues status = str(payload.get("status") or "").strip() if status and status != "ready": message = "POC strict readiness did not pass" missing = payload.get("missing") if isinstance(missing, list) and missing: message += ": " + ", ".join(str(item.get("message") if isinstance(item, dict) else item) for item in missing) issues.append(("strict-preflight-readiness", message, {})) return issues def readiness_fixture_sources(payload: dict[str, Any]) -> dict[str, Any]: for check in payload.get("checks") or []: if not isinstance(check, dict): continue if check.get("name") != "fixture-manifest" or check.get("status") != "passed": continue evidence = check.get("evidence") if isinstance(check.get("evidence"), dict) else {} raw_fixtures = evidence.get("fixtures") if isinstance(evidence.get("fixtures"), dict) else {} items: dict[str, dict[str, Any]] = {} for name, raw_path in sorted(raw_fixtures.items()): path = str(raw_path or "").strip() if not path: continue items[str(name)] = {"path": path, "generated": False} if not items or evidence.get("shellCommandConfigured") is not True: return {} return { "allowGeneratedFixtures": False, "fixtureManifestPath": str(evidence.get("path") or "").strip(), "items": items, } return {} def fixture_source_counts(fixture_sources: dict[str, Any]) -> dict[str, Any]: items = fixture_sources.get("items") if isinstance(fixture_sources.get("items"), dict) else {} generated_count = 0 real_count = 0 for item in items.values(): if not isinstance(item, dict): continue if item.get("generated") is True: generated_count += 1 else: real_count += 1 return { "generatedFixtureCount": generated_count, "realFixtureCount": real_count, "allFixturesReal": bool(items) and generated_count == 0 and fixture_sources.get("allowGeneratedFixtures") is not True, } def compact_probe_evidence(payload: dict[str, Any]) -> dict[str, Any]: evidence: dict[str, Any] = { "source": payload.get("source"), "namespace": payload.get("namespace"), "secretName": payload.get("secretName"), "status": payload.get("status"), } context = payload.get("context") if isinstance(context, dict): evidence["context"] = context probes: list[dict[str, Any]] = [] for item in payload.get("probes") or []: if not isinstance(item, dict): continue response = item.get("response") if isinstance(item.get("response"), dict) else {} probes.append( { "slug": item.get("slug"), "status": item.get("status"), "httpStatus": response.get("status"), "bodyKind": response.get("bodyKind"), "accepted": response.get("accepted"), } ) if probes: evidence["probes"] = probes return {key: value for key, value in evidence.items() if value not in (None, "", [], {})} def issue_missing_coverage(stage: str, all_operators: list[str]) -> list[str]: return all_operators def safe_handler(stage: str) -> str: return STAGE_HANDLERS.get(stage, re.sub(r"[^a-z0-9]+", "_", stage.lower()).strip("_") or "strict_preflight") def build_result(args: argparse.Namespace) -> dict[str, Any]: all_operators = operator_inventory() static_payload = static_operator_layout() repository_summary = operator_repository_summary(static_payload) required = required_externals(args.required_externals) source = args.source if source == "readiness": payload = load_json(pathlib.Path(args.input_json)) issues = readiness_issues(payload) namespace = str(payload.get("namespace") or args.namespace) workspace_mount = str(payload.get("workspaceMountPath") or args.workspace_mount_path).rstrip("/") or "/workspace" required = required_externals(",".join(payload.get("requiredExternals") or required)) fixture_sources = readiness_fixture_sources(payload) else: raise SystemExit(f"unsupported source: {source}") if not issues: raise SystemExit("preflight result requires at least one blocking issue") started_at = now_iso() run_id = args.run_id.strip() or f"poc-strict-preflight-{now_id()}" records: list[dict[str, Any]] = [] missing_coverage_set: set[str] = set() failures: list[str] = [] stages: list[str] = [] for stage, reason, evidence in issues: stages.append(stage) failures.append(reason) missing_coverage = issue_missing_coverage(stage, all_operators) missing_coverage_set.update(missing_coverage) record_evidence = dict(evidence) record_evidence["missingOperatorCoverage"] = missing_coverage records.append( { "operator": "poc_acceptance_preflight", "handler": safe_handler(stage), "status": "failed", "error": reason, "evidence": record_evidence, "preflightStage": stage, } ) unique_stages = list(dict.fromkeys(stages)) stage = unique_stages[0] if len(unique_stages) == 1 else "strict-preflight-readiness" missing_coverage = sorted(missing_coverage_set) required_external_systems = sorted({"bos", "mongo"}.intersection(set(required))) fixture_counts = fixture_source_counts(fixture_sources) result = { "runId": run_id, "workspace": f"{workspace_mount}/acceptance/{run_id}", "preflightFailure": True, "preflightStage": stage, "preflightStages": unique_stages, "acceptanceScope": "full" if {"bos", "mongo"}.issubset(set(required)) else "dependency-isolation", "fullExternalAcceptance": False, "operatorCount": len(all_operators), "operatorPassedCount": 0, "coveredOperatorCount": 0, "allOperatorCoverageComplete": False, "blockingOperatorCoverageComplete": False, "missingCoverage": missing_coverage, "blockingMissingCoverage": missing_coverage, "requiredExternals": required, "statusCounts": {"failed": len(records)}, "passedCount": 0, "externalVerificationPassed": [], "externalVerificationMissing": required_external_systems, "externalVerificationPassedCount": 0, "externalVerificationRequiredCount": len(required_external_systems), **fixture_counts, "failedCount": len(records), "skippedCount": 0, "requiredSkippedCount": 0, "optionalSkippedCount": 0, "static": static_payload, **repository_summary, "records": records, "failures": failures, "status": "failed", "namespace": namespace, "startedAt": started_at, "endedAt": started_at, **acceptance_provenance(started_at), } if fixture_sources: result["fixtureSources"] = fixture_sources return result def main() -> int: parser = argparse.ArgumentParser(description="Build a standard POC acceptance result for strict preflight blockers.") parser.add_argument("--source", choices=("readiness",), required=True) parser.add_argument("--input-json", required=True) parser.add_argument("--run-id", default=os.getenv("POC_OPERATOR_ACCEPTANCE_JOB_NAME") or "") parser.add_argument("--namespace", default=os.getenv("K8S_NAMESPACE") or "droboticflow") parser.add_argument("--workspace-mount-path", default=os.getenv("WORKSPACE_MOUNT_PATH") or "/workspace") parser.add_argument("--required-externals", default=os.getenv("POC_ACCEPTANCE_REQUIRED_EXTERNALS") or DEFAULT_REQUIRED_EXTERNALS) args = parser.parse_args() print(RESULT_PREFIX + json.dumps(build_result(args), ensure_ascii=False, sort_keys=True)) return 0 if __name__ == "__main__": raise SystemExit(main())