droboticflow/scripts/reconcile_system_data.py

469 lines
18 KiB
Python

from __future__ import annotations
import argparse
import base64
import json
import os
import pathlib
import re
import sys
import urllib.error
import urllib.parse
import urllib.request
from typing import Any, Callable
ROOT = pathlib.Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "scripts"))
from lib.droboticflow_acceptance import DroboticflowClient
from lib.droboticflow_env import api_base_url_from_env
from operator_release_manifest import release_image_reference
JSON = dict[str, Any]
ImageDigestResolver = Callable[[str], str]
class ReconcileError(RuntimeError):
pass
MANIFEST_ACCEPT = ", ".join(
[
"application/vnd.oci.image.index.v1+json",
"application/vnd.oci.image.manifest.v1+json",
"application/vnd.docker.distribution.manifest.list.v2+json",
"application/vnd.docker.distribution.manifest.v2+json",
]
)
BEARER_CHALLENGE_RE = re.compile(r'(\w+)="([^"]*)"')
def item_id(item: JSON) -> str:
return str(item.get("id") or "")
def quote_id(value: str) -> str:
return urllib.parse.quote(value, safe="")
def request_json(
client: DroboticflowClient,
method: str,
path: str,
payload: Any | None = None,
*,
expected: tuple[int, ...] = (200, 201, 202, 204),
) -> Any:
return client.request_json(method, path, payload, expected=expected, timeout=20, retries=6)
def login_if_configured(client: DroboticflowClient, email: str, password: str) -> None:
if bool(email) != bool(password):
raise ReconcileError("DROBOTICFLOW_ACCEPTANCE_EMAIL and DROBOTICFLOW_ACCEPTANCE_PASSWORD must be set together")
if email:
client.login_if_configured(email, password, label="reconcile user")
def build_settings_payload(settings: JSON, k8s_endpoint: str, k8s_namespace: str) -> JSON:
worker = settings.get("worker")
defaults = settings.get("defaults")
if not isinstance(worker, dict):
raise ReconcileError("/api/settings response is missing worker settings")
if not isinstance(defaults, dict):
raise ReconcileError("/api/settings response is missing default settings")
return {
"k8s": {
"enabled": True,
"endpoint": k8s_endpoint,
"namespace": k8s_namespace,
},
"storage": {
"endpoint": "",
"buckets": [],
},
"worker": {
"pollIntervalMs": worker["pollIntervalMs"],
"maxConcurrentRuns": worker["maxConcurrentRuns"],
},
"defaults": defaults,
}
def operator_image_context(operator_runner_image: str) -> tuple[str, str]:
image = operator_runner_image.strip()
if "/" not in image:
return "", ""
if ":" not in image.rsplit("/", 1)[-1]:
raise ReconcileError(f"OPERATOR_RUNNER_IMAGE must include a tag: {operator_runner_image}")
registry_prefix, image_name = image.rsplit("/", 1)
_, tag = image_name.rsplit(":", 1)
return registry_prefix, tag
def operator_slug(operator_id: str) -> str:
return operator_id.removeprefix("operator-builtin-").replace("-", "_")
def poc_operator_image(operator_id: str, registry_prefix: str, tag: str) -> str:
slug = operator_slug(operator_id).replace("_", "-")
return f"{registry_prefix}/droboticflow-operator-{slug}:{tag}"
def image_repository(ref: str) -> str:
trimmed = ref.strip()
if "@sha256:" in trimmed:
trimmed = trimmed.split("@sha256:", 1)[0]
if "/" in trimmed and ":" in trimmed.rsplit("/", 1)[-1]:
return trimmed.rsplit(":", 1)[0]
return trimmed
def digest_ref_for_same_repository(image: str, image_id: str) -> str:
if "@sha256:" not in image_id:
return ""
if image_repository(image) != image_repository(image_id):
return ""
return image_id.strip()
def parse_container_image(image: str) -> tuple[str, str, str]:
trimmed = image.strip()
if "@sha256:" in trimmed:
raise ReconcileError(f"image is already digest-pinned: {image}")
if "/" not in trimmed:
raise ReconcileError(f"image must include a registry host: {image}")
registry, remainder = trimmed.split("/", 1)
if ":" not in remainder.rsplit("/", 1)[-1]:
raise ReconcileError(f"image must include a tag: {image}")
repository, tag = remainder.rsplit(":", 1)
return registry, repository, tag
def image_ref_with_digest(image: str, digest: str) -> str:
registry, repository, _ = parse_container_image(image)
normalized_digest = digest.strip()
if not normalized_digest.startswith("sha256:"):
raise ReconcileError(f"registry did not return a sha256 digest for {image}: {digest}")
return f"{registry}/{repository}@{normalized_digest}"
def docker_auth_header_for_registry(registry: str, docker_config: JSON) -> str:
auths = docker_config.get("auths")
if not isinstance(auths, dict):
return ""
candidates = [
registry,
f"https://{registry}",
f"http://{registry}",
]
for candidate in candidates:
entry = auths.get(candidate)
if not isinstance(entry, dict):
continue
auth = str(entry.get("auth") or "").strip()
if auth:
return f"Basic {auth}"
username = str(entry.get("username") or "")
password = str(entry.get("password") or "")
if username or password:
encoded = base64.b64encode(f"{username}:{password}".encode("utf-8")).decode("ascii")
return f"Basic {encoded}"
return ""
def docker_config_from_env() -> JSON:
raw = os.getenv("REGISTRY_AUTH_CONFIG_JSON", "").strip()
if raw:
return json.loads(raw)
path = os.getenv("REGISTRY_AUTH_FILE", "").strip()
if path:
return json.loads(pathlib.Path(path).read_text(encoding="utf-8"))
return {}
def parse_bearer_challenge(header: str) -> dict[str, str]:
if not header.strip().lower().startswith("bearer "):
return {}
return {key: value for key, value in BEARER_CHALLENGE_RE.findall(header)}
def registry_bearer_token(challenge: dict[str, str], auth_header: str) -> str:
realm = challenge.get("realm", "").strip()
if not realm:
return ""
query = {}
for key in ("service", "scope"):
value = challenge.get(key, "").strip()
if value:
query[key] = value
url = realm
if query:
url = f"{realm}?{urllib.parse.urlencode(query)}"
request = urllib.request.Request(url, method="GET", headers={"Authorization": auth_header})
with urllib.request.urlopen(request, timeout=30) as response:
payload = json.loads(response.read().decode("utf-8"))
return str(payload.get("token") or payload.get("access_token") or "").strip()
def registry_manifest_digest(url: str, auth_header: str) -> str:
headers = {"Accept": MANIFEST_ACCEPT, "Authorization": auth_header}
request = urllib.request.Request(url, method="GET", headers=headers)
try:
with urllib.request.urlopen(request, timeout=30) as response:
return response.headers.get("Docker-Content-Digest", "").strip()
except urllib.error.HTTPError as error:
challenge = parse_bearer_challenge(error.headers.get("WWW-Authenticate", ""))
if error.code != 401 or not challenge:
raise
token = registry_bearer_token(challenge, auth_header)
if not token:
raise ReconcileError("registry bearer challenge did not return a token") from error
bearer_request = urllib.request.Request(
url,
method="GET",
headers={"Accept": MANIFEST_ACCEPT, "Authorization": f"Bearer {token}"},
)
with urllib.request.urlopen(bearer_request, timeout=30) as response:
return response.headers.get("Docker-Content-Digest", "").strip()
def registry_digest_resolver_from_env() -> ImageDigestResolver | None:
raw_enabled = os.getenv("RECONCILE_OPERATOR_IMAGE_DIGESTS", "").strip().lower()
has_registry_auth = bool(os.getenv("REGISTRY_AUTH_CONFIG_JSON", "").strip() or os.getenv("REGISTRY_AUTH_FILE", "").strip())
enabled = raw_enabled in {"1", "true", "yes"} or (not raw_enabled and has_registry_auth)
if not enabled:
return None
docker_config = docker_config_from_env()
if not docker_config:
raise ReconcileError("RECONCILE_OPERATOR_IMAGE_DIGESTS requires REGISTRY_AUTH_CONFIG_JSON or REGISTRY_AUTH_FILE")
def resolve(image: str) -> str:
registry, repository, tag = parse_container_image(image)
auth_header = docker_auth_header_for_registry(registry, docker_config)
if not auth_header:
raise ReconcileError(f"registry auth config does not contain credentials for {registry}")
url = f"https://{registry}/v2/{repository}/manifests/{urllib.parse.quote(tag, safe='')}"
digest = registry_manifest_digest(url, auth_header)
if not digest:
raise ReconcileError(f"registry manifest response did not include Docker-Content-Digest for {image}")
return image_ref_with_digest(image, digest)
return resolve
def builtin_operator_patch(
operator: JSON,
operator_runner_image: str,
*,
image_digest_resolver: ImageDigestResolver | None = None,
) -> JSON:
identifier = item_id(operator)
catalog_image = str(operator.get("image") or "").strip()
has_independent_operator_image = (
catalog_image and "droboticflow-operator-" in catalog_image and "droboticflow-operator-runner" not in catalog_image
)
if has_independent_operator_image:
image, image_id = release_image_reference(operator_slug(identifier))
if image_digest_resolver:
resolved_image_id = image_digest_resolver(image)
if resolved_image_id != image_id:
raise ReconcileError(
f"release digest drift for {identifier}: release={image_id} registry={resolved_image_id}"
)
else:
image = operator_runner_image
image_id = image
payload = {
"name": operator["name"],
"type": operator["type"],
"category": operator["category"],
"description": operator["description"],
"inputs": operator.get("inputs") or [],
"outputs": operator.get("outputs") or [],
"image": image,
"imageId": image_id,
"containerCommand": operator.get("containerCommand") or [],
"containerArgs": operator.get("containerArgs") or [],
"runtimeContract": operator.get("runtimeContract") or {},
"skillDefinition": operator.get("skillDefinition") or "",
"hyperparams": operator.get("hyperparams") or [],
}
for optional_key in ("configSchema", "resources"):
if optional_key in operator and operator[optional_key] is not None:
payload[optional_key] = operator[optional_key]
return payload
def sync_builtin_operators(
client: DroboticflowClient,
operator_runner_image: str,
*,
image_digest_resolver: ImageDigestResolver | None = None,
operator_ids: set[str] | None = None,
) -> int:
operators = request_json(client, "GET", "/api/operators")["items"]
builtin_operators = [item for item in operators if str(item.get("type") or "").strip().lower() == "builtin"]
if not builtin_operators:
raise ReconcileError("/api/operators did not return any builtin operators")
updated = 0
for operator in builtin_operators:
identifier = item_id(operator)
if not identifier:
raise ReconcileError(f"builtin operator {operator.get('name')} is missing id")
if operator_ids and identifier not in operator_ids:
continue
payload = builtin_operator_patch(operator, operator_runner_image, image_digest_resolver=image_digest_resolver)
payload_image = str(payload.get("image") or "")
payload_image_id = str(payload.get("imageId") or "")
if (
str(operator.get("image") or "") == payload_image
and str(operator.get("imageId") or "") == payload_image_id
and (operator.get("containerCommand") or []) == (payload.get("containerCommand") or [])
):
continue
print(f"update operator {operator['name']}")
request_json(client, "PUT", f"/api/operators/{quote_id(identifier)}", payload)
updated += 1
return updated
def first_item_by_name(items: list[JSON], name: str) -> JSON | None:
for item in items:
if str(item.get("name") or "").strip() == name:
return item
return None
def reconcile_bos_connection(client: DroboticflowClient) -> str:
endpoint = os.getenv("RECONCILE_BOS_ENDPOINT", "").strip()
bucket = os.getenv("RECONCILE_BOS_BUCKET", "").strip().strip("/")
access_key_id = os.getenv("RECONCILE_BOS_ACCESS_KEY_ID", "").strip()
secret_access_key = os.getenv("RECONCILE_BOS_SECRET_ACCESS_KEY", "").strip()
if not endpoint or not access_key_id or not secret_access_key:
return "skipped: missing RECONCILE_BOS_ENDPOINT/access key/secret key"
name = os.getenv("RECONCILE_BOS_CONNECTION_NAME", "POC BOS Workspace").strip() or "POC BOS Workspace"
payload = {
"name": name,
"description": "POC runtime BOS connection reconciled from Kubernetes secrets.",
"provider": "BOS",
"endpoint": endpoint,
"bucket": bucket,
"accessKeyId": access_key_id,
"secretAccessKey": secret_access_key,
}
items = request_json(client, "GET", "/api/storage/connections").get("items", [])
existing = first_item_by_name(items if isinstance(items, list) else [], name)
if existing:
request_json(client, "PUT", f"/api/storage/connections/{quote_id(item_id(existing))}", payload)
return f"updated: {item_id(existing)}"
created = request_json(client, "POST", "/api/storage/connections", payload, expected=(201,))
return f"created: {item_id(created)}"
def reconcile_mongo_connection(client: DroboticflowClient) -> str:
uri = os.getenv("RECONCILE_MONGO_URI", "").strip()
database = os.getenv("RECONCILE_MONGO_DATABASE", "").strip()
if not uri or not database:
return "skipped: missing RECONCILE_MONGO_URI/database"
name = os.getenv("RECONCILE_MONGO_CONNECTION_NAME", "POC Mongo Runtime").strip() or "POC Mongo Runtime"
payload = {
"name": name,
"description": "POC runtime MongoDB connection reconciled from Kubernetes secrets.",
"uri": uri,
"database": database,
}
items = request_json(client, "GET", "/api/mongo/connections").get("items", [])
existing = first_item_by_name(items if isinstance(items, list) else [], name)
if existing:
request_json(client, "PUT", f"/api/mongo/connections/{quote_id(item_id(existing))}", payload)
return f"updated: {item_id(existing)}"
created = request_json(client, "POST", "/api/mongo/connections", payload, expected=(201,))
return f"created: {item_id(created)}"
def reconcile_external_connections(client: DroboticflowClient) -> None:
print(f"reconciled BOS storage connection: {reconcile_bos_connection(client)}")
print(f"reconciled Mongo connection: {reconcile_mongo_connection(client)}")
def reconcile(
client: DroboticflowClient,
*,
k8s_endpoint: str,
k8s_namespace: str,
operator_runner_image: str,
image_digest_resolver: ImageDigestResolver | None = None,
) -> None:
settings = request_json(client, "GET", "/api/settings")
request_json(client, "PUT", "/api/settings", build_settings_payload(settings, k8s_endpoint, k8s_namespace))
print("updated system settings")
updated_count = sync_builtin_operators(client, operator_runner_image, image_digest_resolver=image_digest_resolver)
print(f"reconciled builtin operators from API catalog: {updated_count} updated")
reconcile_external_connections(client)
print("reconciled system data")
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description="Reconcile droboticflow system data through the protected API")
parser.add_argument("--api-base-url", default=api_base_url_from_env(include_base_url=False))
parser.add_argument("--k8s-endpoint", default=os.getenv("K8S_ENDPOINT", ""))
parser.add_argument("--k8s-namespace", default=os.getenv("K8S_NAMESPACE", "droboticflow"))
parser.add_argument("--operator-runner-image", default=os.getenv("OPERATOR_RUNNER_IMAGE", ""))
parser.add_argument(
"--operator-id",
action="append",
default=[],
help="Only validate and reconcile the selected builtin operator; may be repeated.",
)
return parser.parse_args()
def main() -> None:
args = parse_args()
if not args.operator_runner_image.strip():
raise SystemExit("OPERATOR_RUNNER_IMAGE is required")
if not args.k8s_endpoint.strip():
raise SystemExit("K8S_ENDPOINT is required, for example https://<api-server>:6443")
api_token = os.getenv("DROBOTICFLOW_API_TOKEN", "").strip()
client = DroboticflowClient(args.api_base_url, api_token=api_token, error_factory=ReconcileError)
if api_token:
print("using API token auth for reconcile requests")
try:
login_if_configured(
client,
os.getenv("DROBOTICFLOW_ACCEPTANCE_EMAIL", "").strip(),
os.getenv("DROBOTICFLOW_ACCEPTANCE_PASSWORD", ""),
)
selected_operator_ids = {value.strip() for value in args.operator_id if value.strip()}
if selected_operator_ids:
updated = sync_builtin_operators(
client,
args.operator_runner_image,
image_digest_resolver=registry_digest_resolver_from_env(),
operator_ids=selected_operator_ids,
)
print(f"reconciled selected builtin operators: {updated} updated")
return
reconcile(
client,
k8s_endpoint=args.k8s_endpoint,
k8s_namespace=args.k8s_namespace,
operator_runner_image=args.operator_runner_image,
image_digest_resolver=registry_digest_resolver_from_env(),
)
except ReconcileError as error:
raise SystemExit(str(error)) from error
if __name__ == "__main__":
main()