from __future__ import annotations import argparse import base64 import json import os import pathlib import re import sys import urllib.error import urllib.parse import urllib.request from typing import Any, Callable ROOT = pathlib.Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT / "scripts")) from lib.droboticflow_acceptance import DroboticflowClient from lib.droboticflow_env import api_base_url_from_env from operator_release_manifest import release_image_reference JSON = dict[str, Any] ImageDigestResolver = Callable[[str], str] class ReconcileError(RuntimeError): pass MANIFEST_ACCEPT = ", ".join( [ "application/vnd.oci.image.index.v1+json", "application/vnd.oci.image.manifest.v1+json", "application/vnd.docker.distribution.manifest.list.v2+json", "application/vnd.docker.distribution.manifest.v2+json", ] ) BEARER_CHALLENGE_RE = re.compile(r'(\w+)="([^"]*)"') def item_id(item: JSON) -> str: return str(item.get("id") or "") def quote_id(value: str) -> str: return urllib.parse.quote(value, safe="") def request_json( client: DroboticflowClient, method: str, path: str, payload: Any | None = None, *, expected: tuple[int, ...] = (200, 201, 202, 204), ) -> Any: return client.request_json(method, path, payload, expected=expected, timeout=20, retries=6) def login_if_configured(client: DroboticflowClient, email: str, password: str) -> None: if bool(email) != bool(password): raise ReconcileError("DROBOTICFLOW_ACCEPTANCE_EMAIL and DROBOTICFLOW_ACCEPTANCE_PASSWORD must be set together") if email: client.login_if_configured(email, password, label="reconcile user") def build_settings_payload(settings: JSON, k8s_endpoint: str, k8s_namespace: str) -> JSON: worker = settings.get("worker") defaults = settings.get("defaults") if not isinstance(worker, dict): raise ReconcileError("/api/settings response is missing worker settings") if not isinstance(defaults, dict): raise ReconcileError("/api/settings response is missing default settings") return { "k8s": { "enabled": True, "endpoint": k8s_endpoint, "namespace": k8s_namespace, }, "storage": { "endpoint": "", "buckets": [], }, "worker": { "pollIntervalMs": worker["pollIntervalMs"], "maxConcurrentRuns": worker["maxConcurrentRuns"], }, "defaults": defaults, } def operator_image_context(operator_runner_image: str) -> tuple[str, str]: image = operator_runner_image.strip() if "/" not in image: return "", "" if ":" not in image.rsplit("/", 1)[-1]: raise ReconcileError(f"OPERATOR_RUNNER_IMAGE must include a tag: {operator_runner_image}") registry_prefix, image_name = image.rsplit("/", 1) _, tag = image_name.rsplit(":", 1) return registry_prefix, tag def operator_slug(operator_id: str) -> str: return operator_id.removeprefix("operator-builtin-").replace("-", "_") def poc_operator_image(operator_id: str, registry_prefix: str, tag: str) -> str: slug = operator_slug(operator_id).replace("_", "-") return f"{registry_prefix}/droboticflow-operator-{slug}:{tag}" def image_repository(ref: str) -> str: trimmed = ref.strip() if "@sha256:" in trimmed: trimmed = trimmed.split("@sha256:", 1)[0] if "/" in trimmed and ":" in trimmed.rsplit("/", 1)[-1]: return trimmed.rsplit(":", 1)[0] return trimmed def digest_ref_for_same_repository(image: str, image_id: str) -> str: if "@sha256:" not in image_id: return "" if image_repository(image) != image_repository(image_id): return "" return image_id.strip() def parse_container_image(image: str) -> tuple[str, str, str]: trimmed = image.strip() if "@sha256:" in trimmed: raise ReconcileError(f"image is already digest-pinned: {image}") if "/" not in trimmed: raise ReconcileError(f"image must include a registry host: {image}") registry, remainder = trimmed.split("/", 1) if ":" not in remainder.rsplit("/", 1)[-1]: raise ReconcileError(f"image must include a tag: {image}") repository, tag = remainder.rsplit(":", 1) return registry, repository, tag def image_ref_with_digest(image: str, digest: str) -> str: registry, repository, _ = parse_container_image(image) normalized_digest = digest.strip() if not normalized_digest.startswith("sha256:"): raise ReconcileError(f"registry did not return a sha256 digest for {image}: {digest}") return f"{registry}/{repository}@{normalized_digest}" def docker_auth_header_for_registry(registry: str, docker_config: JSON) -> str: auths = docker_config.get("auths") if not isinstance(auths, dict): return "" candidates = [ registry, f"https://{registry}", f"http://{registry}", ] for candidate in candidates: entry = auths.get(candidate) if not isinstance(entry, dict): continue auth = str(entry.get("auth") or "").strip() if auth: return f"Basic {auth}" username = str(entry.get("username") or "") password = str(entry.get("password") or "") if username or password: encoded = base64.b64encode(f"{username}:{password}".encode("utf-8")).decode("ascii") return f"Basic {encoded}" return "" def docker_config_from_env() -> JSON: raw = os.getenv("REGISTRY_AUTH_CONFIG_JSON", "").strip() if raw: return json.loads(raw) path = os.getenv("REGISTRY_AUTH_FILE", "").strip() if path: return json.loads(pathlib.Path(path).read_text(encoding="utf-8")) return {} def parse_bearer_challenge(header: str) -> dict[str, str]: if not header.strip().lower().startswith("bearer "): return {} return {key: value for key, value in BEARER_CHALLENGE_RE.findall(header)} def registry_bearer_token(challenge: dict[str, str], auth_header: str) -> str: realm = challenge.get("realm", "").strip() if not realm: return "" query = {} for key in ("service", "scope"): value = challenge.get(key, "").strip() if value: query[key] = value url = realm if query: url = f"{realm}?{urllib.parse.urlencode(query)}" request = urllib.request.Request(url, method="GET", headers={"Authorization": auth_header}) with urllib.request.urlopen(request, timeout=30) as response: payload = json.loads(response.read().decode("utf-8")) return str(payload.get("token") or payload.get("access_token") or "").strip() def registry_manifest_digest(url: str, auth_header: str) -> str: headers = {"Accept": MANIFEST_ACCEPT, "Authorization": auth_header} request = urllib.request.Request(url, method="GET", headers=headers) try: with urllib.request.urlopen(request, timeout=30) as response: return response.headers.get("Docker-Content-Digest", "").strip() except urllib.error.HTTPError as error: challenge = parse_bearer_challenge(error.headers.get("WWW-Authenticate", "")) if error.code != 401 or not challenge: raise token = registry_bearer_token(challenge, auth_header) if not token: raise ReconcileError("registry bearer challenge did not return a token") from error bearer_request = urllib.request.Request( url, method="GET", headers={"Accept": MANIFEST_ACCEPT, "Authorization": f"Bearer {token}"}, ) with urllib.request.urlopen(bearer_request, timeout=30) as response: return response.headers.get("Docker-Content-Digest", "").strip() def registry_digest_resolver_from_env() -> ImageDigestResolver | None: raw_enabled = os.getenv("RECONCILE_OPERATOR_IMAGE_DIGESTS", "").strip().lower() has_registry_auth = bool(os.getenv("REGISTRY_AUTH_CONFIG_JSON", "").strip() or os.getenv("REGISTRY_AUTH_FILE", "").strip()) enabled = raw_enabled in {"1", "true", "yes"} or (not raw_enabled and has_registry_auth) if not enabled: return None docker_config = docker_config_from_env() if not docker_config: raise ReconcileError("RECONCILE_OPERATOR_IMAGE_DIGESTS requires REGISTRY_AUTH_CONFIG_JSON or REGISTRY_AUTH_FILE") def resolve(image: str) -> str: registry, repository, tag = parse_container_image(image) auth_header = docker_auth_header_for_registry(registry, docker_config) if not auth_header: raise ReconcileError(f"registry auth config does not contain credentials for {registry}") url = f"https://{registry}/v2/{repository}/manifests/{urllib.parse.quote(tag, safe='')}" digest = registry_manifest_digest(url, auth_header) if not digest: raise ReconcileError(f"registry manifest response did not include Docker-Content-Digest for {image}") return image_ref_with_digest(image, digest) return resolve def builtin_operator_patch( operator: JSON, operator_runner_image: str, *, image_digest_resolver: ImageDigestResolver | None = None, ) -> JSON: identifier = item_id(operator) catalog_image = str(operator.get("image") or "").strip() has_independent_operator_image = ( catalog_image and "droboticflow-operator-" in catalog_image and "droboticflow-operator-runner" not in catalog_image ) if has_independent_operator_image: image, image_id = release_image_reference(operator_slug(identifier)) if image_digest_resolver: resolved_image_id = image_digest_resolver(image) if resolved_image_id != image_id: raise ReconcileError( f"release digest drift for {identifier}: release={image_id} registry={resolved_image_id}" ) else: image = operator_runner_image image_id = image payload = { "name": operator["name"], "type": operator["type"], "category": operator["category"], "description": operator["description"], "inputs": operator.get("inputs") or [], "outputs": operator.get("outputs") or [], "image": image, "imageId": image_id, "containerCommand": operator.get("containerCommand") or [], "containerArgs": operator.get("containerArgs") or [], "runtimeContract": operator.get("runtimeContract") or {}, "skillDefinition": operator.get("skillDefinition") or "", "hyperparams": operator.get("hyperparams") or [], } for optional_key in ("configSchema", "resources"): if optional_key in operator and operator[optional_key] is not None: payload[optional_key] = operator[optional_key] return payload def sync_builtin_operators( client: DroboticflowClient, operator_runner_image: str, *, image_digest_resolver: ImageDigestResolver | None = None, operator_ids: set[str] | None = None, ) -> int: operators = request_json(client, "GET", "/api/operators")["items"] builtin_operators = [item for item in operators if str(item.get("type") or "").strip().lower() == "builtin"] if not builtin_operators: raise ReconcileError("/api/operators did not return any builtin operators") updated = 0 for operator in builtin_operators: identifier = item_id(operator) if not identifier: raise ReconcileError(f"builtin operator {operator.get('name')} is missing id") if operator_ids and identifier not in operator_ids: continue payload = builtin_operator_patch(operator, operator_runner_image, image_digest_resolver=image_digest_resolver) payload_image = str(payload.get("image") or "") payload_image_id = str(payload.get("imageId") or "") if ( str(operator.get("image") or "") == payload_image and str(operator.get("imageId") or "") == payload_image_id and (operator.get("containerCommand") or []) == (payload.get("containerCommand") or []) ): continue print(f"update operator {operator['name']}") request_json(client, "PUT", f"/api/operators/{quote_id(identifier)}", payload) updated += 1 return updated def first_item_by_name(items: list[JSON], name: str) -> JSON | None: for item in items: if str(item.get("name") or "").strip() == name: return item return None def reconcile_bos_connection(client: DroboticflowClient) -> str: endpoint = os.getenv("RECONCILE_BOS_ENDPOINT", "").strip() bucket = os.getenv("RECONCILE_BOS_BUCKET", "").strip().strip("/") access_key_id = os.getenv("RECONCILE_BOS_ACCESS_KEY_ID", "").strip() secret_access_key = os.getenv("RECONCILE_BOS_SECRET_ACCESS_KEY", "").strip() if not endpoint or not access_key_id or not secret_access_key: return "skipped: missing RECONCILE_BOS_ENDPOINT/access key/secret key" name = os.getenv("RECONCILE_BOS_CONNECTION_NAME", "POC BOS Workspace").strip() or "POC BOS Workspace" payload = { "name": name, "description": "POC runtime BOS connection reconciled from Kubernetes secrets.", "provider": "BOS", "endpoint": endpoint, "bucket": bucket, "accessKeyId": access_key_id, "secretAccessKey": secret_access_key, } items = request_json(client, "GET", "/api/storage/connections").get("items", []) existing = first_item_by_name(items if isinstance(items, list) else [], name) if existing: request_json(client, "PUT", f"/api/storage/connections/{quote_id(item_id(existing))}", payload) return f"updated: {item_id(existing)}" created = request_json(client, "POST", "/api/storage/connections", payload, expected=(201,)) return f"created: {item_id(created)}" def reconcile_mongo_connection(client: DroboticflowClient) -> str: uri = os.getenv("RECONCILE_MONGO_URI", "").strip() database = os.getenv("RECONCILE_MONGO_DATABASE", "").strip() if not uri or not database: return "skipped: missing RECONCILE_MONGO_URI/database" name = os.getenv("RECONCILE_MONGO_CONNECTION_NAME", "POC Mongo Runtime").strip() or "POC Mongo Runtime" payload = { "name": name, "description": "POC runtime MongoDB connection reconciled from Kubernetes secrets.", "uri": uri, "database": database, } items = request_json(client, "GET", "/api/mongo/connections").get("items", []) existing = first_item_by_name(items if isinstance(items, list) else [], name) if existing: request_json(client, "PUT", f"/api/mongo/connections/{quote_id(item_id(existing))}", payload) return f"updated: {item_id(existing)}" created = request_json(client, "POST", "/api/mongo/connections", payload, expected=(201,)) return f"created: {item_id(created)}" def reconcile_external_connections(client: DroboticflowClient) -> None: print(f"reconciled BOS storage connection: {reconcile_bos_connection(client)}") print(f"reconciled Mongo connection: {reconcile_mongo_connection(client)}") def reconcile( client: DroboticflowClient, *, k8s_endpoint: str, k8s_namespace: str, operator_runner_image: str, image_digest_resolver: ImageDigestResolver | None = None, ) -> None: settings = request_json(client, "GET", "/api/settings") request_json(client, "PUT", "/api/settings", build_settings_payload(settings, k8s_endpoint, k8s_namespace)) print("updated system settings") updated_count = sync_builtin_operators(client, operator_runner_image, image_digest_resolver=image_digest_resolver) print(f"reconciled builtin operators from API catalog: {updated_count} updated") reconcile_external_connections(client) print("reconciled system data") def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser(description="Reconcile droboticflow system data through the protected API") parser.add_argument("--api-base-url", default=api_base_url_from_env(include_base_url=False)) parser.add_argument("--k8s-endpoint", default=os.getenv("K8S_ENDPOINT", "")) parser.add_argument("--k8s-namespace", default=os.getenv("K8S_NAMESPACE", "droboticflow")) parser.add_argument("--operator-runner-image", default=os.getenv("OPERATOR_RUNNER_IMAGE", "")) parser.add_argument( "--operator-id", action="append", default=[], help="Only validate and reconcile the selected builtin operator; may be repeated.", ) return parser.parse_args() def main() -> None: args = parse_args() if not args.operator_runner_image.strip(): raise SystemExit("OPERATOR_RUNNER_IMAGE is required") if not args.k8s_endpoint.strip(): raise SystemExit("K8S_ENDPOINT is required, for example https://:6443") api_token = os.getenv("DROBOTICFLOW_API_TOKEN", "").strip() client = DroboticflowClient(args.api_base_url, api_token=api_token, error_factory=ReconcileError) if api_token: print("using API token auth for reconcile requests") try: login_if_configured( client, os.getenv("DROBOTICFLOW_ACCEPTANCE_EMAIL", "").strip(), os.getenv("DROBOTICFLOW_ACCEPTANCE_PASSWORD", ""), ) selected_operator_ids = {value.strip() for value in args.operator_id if value.strip()} if selected_operator_ids: updated = sync_builtin_operators( client, args.operator_runner_image, image_digest_resolver=registry_digest_resolver_from_env(), operator_ids=selected_operator_ids, ) print(f"reconciled selected builtin operators: {updated} updated") return reconcile( client, k8s_endpoint=args.k8s_endpoint, k8s_namespace=args.k8s_namespace, operator_runner_image=args.operator_runner_image, image_digest_resolver=registry_digest_resolver_from_env(), ) except ReconcileError as error: raise SystemExit(str(error)) from error if __name__ == "__main__": main()