forked from longtao.wu/droboticflow
513 lines
20 KiB
Python
513 lines
20 KiB
Python
from __future__ import annotations
|
|
|
|
import argparse
|
|
import base64
|
|
import json
|
|
import os
|
|
import pathlib
|
|
import re
|
|
import sys
|
|
import urllib.error
|
|
import urllib.parse
|
|
import urllib.request
|
|
from typing import Any, Callable
|
|
|
|
|
|
ROOT = pathlib.Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / "scripts"))
|
|
|
|
from lib.droboticflow_acceptance import DroboticflowClient
|
|
from lib.droboticflow_env import api_base_url_from_env
|
|
from operator_release_manifest import release_image_reference
|
|
|
|
|
|
JSON = dict[str, Any]
|
|
Predicate = Callable[[JSON], bool]
|
|
ImageDigestResolver = Callable[[str], str]
|
|
|
|
|
|
class ReconcileError(RuntimeError):
|
|
pass
|
|
|
|
|
|
MANIFEST_ACCEPT = ", ".join(
|
|
[
|
|
"application/vnd.oci.image.index.v1+json",
|
|
"application/vnd.oci.image.manifest.v1+json",
|
|
"application/vnd.docker.distribution.manifest.list.v2+json",
|
|
"application/vnd.docker.distribution.manifest.v2+json",
|
|
]
|
|
)
|
|
BEARER_CHALLENGE_RE = re.compile(r'(\w+)="([^"]*)"')
|
|
|
|
|
|
def item_id(item: JSON) -> str:
|
|
return str(item.get("id") or "")
|
|
|
|
|
|
def quote_id(value: str) -> str:
|
|
return urllib.parse.quote(value, safe="")
|
|
|
|
|
|
def is_legacy_user(item: JSON) -> bool:
|
|
email = str(item.get("email") or "")
|
|
return item_id(item) in {"1", "2", "3"} or email.endswith("@emboflow.ai")
|
|
|
|
|
|
def is_legacy_project(item: JSON) -> bool:
|
|
return item_id(item) in {"1", "2", "3"} or item.get("description") in {
|
|
"Created by scripts/live-workflow-acceptance.sh",
|
|
}
|
|
|
|
|
|
def is_legacy_template(item: JSON) -> bool:
|
|
return item_id(item).startswith("tpl-")
|
|
|
|
|
|
def is_legacy_operator(item: JSON) -> bool:
|
|
operator_type = str(item.get("type") or "").strip().lower()
|
|
if operator_type == "builtin":
|
|
return False
|
|
image = str(item.get("image") or "")
|
|
return item_id(item).startswith("op-") or image.endswith(":latest")
|
|
|
|
|
|
def request_json(
|
|
client: DroboticflowClient,
|
|
method: str,
|
|
path: str,
|
|
payload: Any | None = None,
|
|
*,
|
|
expected: tuple[int, ...] = (200, 201, 202, 204),
|
|
) -> Any:
|
|
return client.request_json(method, path, payload, expected=expected, timeout=20, retries=6)
|
|
|
|
|
|
def login_if_configured(client: DroboticflowClient, email: str, password: str) -> None:
|
|
if bool(email) != bool(password):
|
|
raise ReconcileError("DROBOTICFLOW_ACCEPTANCE_EMAIL and DROBOTICFLOW_ACCEPTANCE_PASSWORD must be set together")
|
|
if email:
|
|
client.login_if_configured(email, password, label="reconcile user")
|
|
|
|
|
|
def delete_each(client: DroboticflowClient, path_prefix: str, items: list[JSON], predicate: Predicate) -> int:
|
|
deleted = 0
|
|
for item in items:
|
|
if not predicate(item):
|
|
continue
|
|
identifier = item_id(item)
|
|
print(f"delete {path_prefix}/{identifier}")
|
|
request_json(client, "DELETE", f"{path_prefix}/{quote_id(identifier)}", expected=(204, 404))
|
|
deleted += 1
|
|
return deleted
|
|
|
|
|
|
def build_settings_payload(settings: JSON, k8s_endpoint: str, k8s_namespace: str) -> JSON:
|
|
worker = settings.get("worker")
|
|
defaults = settings.get("defaults")
|
|
if not isinstance(worker, dict):
|
|
raise ReconcileError("/api/settings response is missing worker settings")
|
|
if not isinstance(defaults, dict):
|
|
raise ReconcileError("/api/settings response is missing default settings")
|
|
return {
|
|
"k8s": {
|
|
"enabled": True,
|
|
"endpoint": k8s_endpoint,
|
|
"namespace": k8s_namespace,
|
|
},
|
|
"storage": {
|
|
"endpoint": "",
|
|
"buckets": [],
|
|
},
|
|
"worker": {
|
|
"pollIntervalMs": worker["pollIntervalMs"],
|
|
"maxConcurrentRuns": worker["maxConcurrentRuns"],
|
|
},
|
|
"defaults": defaults,
|
|
}
|
|
|
|
|
|
def operator_image_context(operator_runner_image: str) -> tuple[str, str]:
|
|
image = operator_runner_image.strip()
|
|
if "/" not in image:
|
|
return "", ""
|
|
if ":" not in image.rsplit("/", 1)[-1]:
|
|
raise ReconcileError(f"OPERATOR_RUNNER_IMAGE must include a tag: {operator_runner_image}")
|
|
registry_prefix, image_name = image.rsplit("/", 1)
|
|
_, tag = image_name.rsplit(":", 1)
|
|
return registry_prefix, tag
|
|
|
|
|
|
def operator_slug(operator_id: str) -> str:
|
|
return operator_id.removeprefix("operator-builtin-").replace("-", "_")
|
|
|
|
|
|
def poc_operator_image(operator_id: str, registry_prefix: str, tag: str) -> str:
|
|
slug = operator_slug(operator_id).replace("_", "-")
|
|
return f"{registry_prefix}/droboticflow-operator-{slug}:{tag}"
|
|
|
|
|
|
def image_repository(ref: str) -> str:
|
|
trimmed = ref.strip()
|
|
if "@sha256:" in trimmed:
|
|
trimmed = trimmed.split("@sha256:", 1)[0]
|
|
if "/" in trimmed and ":" in trimmed.rsplit("/", 1)[-1]:
|
|
return trimmed.rsplit(":", 1)[0]
|
|
return trimmed
|
|
|
|
|
|
def digest_ref_for_same_repository(image: str, image_id: str) -> str:
|
|
if "@sha256:" not in image_id:
|
|
return ""
|
|
if image_repository(image) != image_repository(image_id):
|
|
return ""
|
|
return image_id.strip()
|
|
|
|
|
|
def parse_container_image(image: str) -> tuple[str, str, str]:
|
|
trimmed = image.strip()
|
|
if "@sha256:" in trimmed:
|
|
raise ReconcileError(f"image is already digest-pinned: {image}")
|
|
if "/" not in trimmed:
|
|
raise ReconcileError(f"image must include a registry host: {image}")
|
|
registry, remainder = trimmed.split("/", 1)
|
|
if ":" not in remainder.rsplit("/", 1)[-1]:
|
|
raise ReconcileError(f"image must include a tag: {image}")
|
|
repository, tag = remainder.rsplit(":", 1)
|
|
return registry, repository, tag
|
|
|
|
|
|
def image_ref_with_digest(image: str, digest: str) -> str:
|
|
registry, repository, _ = parse_container_image(image)
|
|
normalized_digest = digest.strip()
|
|
if not normalized_digest.startswith("sha256:"):
|
|
raise ReconcileError(f"registry did not return a sha256 digest for {image}: {digest}")
|
|
return f"{registry}/{repository}@{normalized_digest}"
|
|
|
|
|
|
def docker_auth_header_for_registry(registry: str, docker_config: JSON) -> str:
|
|
auths = docker_config.get("auths")
|
|
if not isinstance(auths, dict):
|
|
return ""
|
|
candidates = [
|
|
registry,
|
|
f"https://{registry}",
|
|
f"http://{registry}",
|
|
]
|
|
for candidate in candidates:
|
|
entry = auths.get(candidate)
|
|
if not isinstance(entry, dict):
|
|
continue
|
|
auth = str(entry.get("auth") or "").strip()
|
|
if auth:
|
|
return f"Basic {auth}"
|
|
username = str(entry.get("username") or "")
|
|
password = str(entry.get("password") or "")
|
|
if username or password:
|
|
encoded = base64.b64encode(f"{username}:{password}".encode("utf-8")).decode("ascii")
|
|
return f"Basic {encoded}"
|
|
return ""
|
|
|
|
|
|
def docker_config_from_env() -> JSON:
|
|
raw = os.getenv("REGISTRY_AUTH_CONFIG_JSON", "").strip()
|
|
if raw:
|
|
return json.loads(raw)
|
|
path = os.getenv("REGISTRY_AUTH_FILE", "").strip()
|
|
if path:
|
|
return json.loads(pathlib.Path(path).read_text(encoding="utf-8"))
|
|
return {}
|
|
|
|
|
|
def parse_bearer_challenge(header: str) -> dict[str, str]:
|
|
if not header.strip().lower().startswith("bearer "):
|
|
return {}
|
|
return {key: value for key, value in BEARER_CHALLENGE_RE.findall(header)}
|
|
|
|
|
|
def registry_bearer_token(challenge: dict[str, str], auth_header: str) -> str:
|
|
realm = challenge.get("realm", "").strip()
|
|
if not realm:
|
|
return ""
|
|
query = {}
|
|
for key in ("service", "scope"):
|
|
value = challenge.get(key, "").strip()
|
|
if value:
|
|
query[key] = value
|
|
url = realm
|
|
if query:
|
|
url = f"{realm}?{urllib.parse.urlencode(query)}"
|
|
request = urllib.request.Request(url, method="GET", headers={"Authorization": auth_header})
|
|
with urllib.request.urlopen(request, timeout=30) as response:
|
|
payload = json.loads(response.read().decode("utf-8"))
|
|
return str(payload.get("token") or payload.get("access_token") or "").strip()
|
|
|
|
|
|
def registry_manifest_digest(url: str, auth_header: str) -> str:
|
|
headers = {"Accept": MANIFEST_ACCEPT, "Authorization": auth_header}
|
|
request = urllib.request.Request(url, method="GET", headers=headers)
|
|
try:
|
|
with urllib.request.urlopen(request, timeout=30) as response:
|
|
return response.headers.get("Docker-Content-Digest", "").strip()
|
|
except urllib.error.HTTPError as error:
|
|
challenge = parse_bearer_challenge(error.headers.get("WWW-Authenticate", ""))
|
|
if error.code != 401 or not challenge:
|
|
raise
|
|
token = registry_bearer_token(challenge, auth_header)
|
|
if not token:
|
|
raise ReconcileError("registry bearer challenge did not return a token") from error
|
|
bearer_request = urllib.request.Request(
|
|
url,
|
|
method="GET",
|
|
headers={"Accept": MANIFEST_ACCEPT, "Authorization": f"Bearer {token}"},
|
|
)
|
|
with urllib.request.urlopen(bearer_request, timeout=30) as response:
|
|
return response.headers.get("Docker-Content-Digest", "").strip()
|
|
|
|
|
|
def registry_digest_resolver_from_env() -> ImageDigestResolver | None:
|
|
raw_enabled = os.getenv("RECONCILE_OPERATOR_IMAGE_DIGESTS", "").strip().lower()
|
|
has_registry_auth = bool(os.getenv("REGISTRY_AUTH_CONFIG_JSON", "").strip() or os.getenv("REGISTRY_AUTH_FILE", "").strip())
|
|
enabled = raw_enabled in {"1", "true", "yes"} or (not raw_enabled and has_registry_auth)
|
|
if not enabled:
|
|
return None
|
|
docker_config = docker_config_from_env()
|
|
if not docker_config:
|
|
raise ReconcileError("RECONCILE_OPERATOR_IMAGE_DIGESTS requires REGISTRY_AUTH_CONFIG_JSON or REGISTRY_AUTH_FILE")
|
|
|
|
def resolve(image: str) -> str:
|
|
registry, repository, tag = parse_container_image(image)
|
|
auth_header = docker_auth_header_for_registry(registry, docker_config)
|
|
if not auth_header:
|
|
raise ReconcileError(f"registry auth config does not contain credentials for {registry}")
|
|
url = f"https://{registry}/v2/{repository}/manifests/{urllib.parse.quote(tag, safe='')}"
|
|
digest = registry_manifest_digest(url, auth_header)
|
|
if not digest:
|
|
raise ReconcileError(f"registry manifest response did not include Docker-Content-Digest for {image}")
|
|
return image_ref_with_digest(image, digest)
|
|
|
|
return resolve
|
|
|
|
|
|
def builtin_operator_patch(
|
|
operator: JSON,
|
|
operator_runner_image: str,
|
|
*,
|
|
image_digest_resolver: ImageDigestResolver | None = None,
|
|
) -> JSON:
|
|
identifier = item_id(operator)
|
|
catalog_image = str(operator.get("image") or "").strip()
|
|
has_independent_operator_image = (
|
|
catalog_image and "droboticflow-operator-" in catalog_image and "droboticflow-operator-runner" not in catalog_image
|
|
)
|
|
if has_independent_operator_image:
|
|
image, image_id = release_image_reference(operator_slug(identifier))
|
|
if image_digest_resolver:
|
|
resolved_image_id = image_digest_resolver(image)
|
|
if resolved_image_id != image_id:
|
|
raise ReconcileError(
|
|
f"release digest drift for {identifier}: release={image_id} registry={resolved_image_id}"
|
|
)
|
|
else:
|
|
image = operator_runner_image
|
|
image_id = image
|
|
payload = {
|
|
"name": operator["name"],
|
|
"type": operator["type"],
|
|
"category": operator["category"],
|
|
"description": operator["description"],
|
|
"inputs": operator.get("inputs") or [],
|
|
"outputs": operator.get("outputs") or [],
|
|
"image": image,
|
|
"imageId": image_id,
|
|
"containerCommand": operator.get("containerCommand") or [],
|
|
"containerArgs": operator.get("containerArgs") or [],
|
|
"runtimeContract": operator.get("runtimeContract") or {},
|
|
"skillDefinition": operator.get("skillDefinition") or "",
|
|
"hyperparams": operator.get("hyperparams") or [],
|
|
}
|
|
for optional_key in ("configSchema", "resources"):
|
|
if optional_key in operator and operator[optional_key] is not None:
|
|
payload[optional_key] = operator[optional_key]
|
|
return payload
|
|
|
|
|
|
def sync_builtin_operators(
|
|
client: DroboticflowClient,
|
|
operator_runner_image: str,
|
|
*,
|
|
image_digest_resolver: ImageDigestResolver | None = None,
|
|
operator_ids: set[str] | None = None,
|
|
) -> int:
|
|
operators = request_json(client, "GET", "/api/operators")["items"]
|
|
builtin_operators = [item for item in operators if str(item.get("type") or "").strip().lower() == "builtin"]
|
|
if not builtin_operators:
|
|
raise ReconcileError("/api/operators did not return any builtin operators")
|
|
|
|
updated = 0
|
|
for operator in builtin_operators:
|
|
identifier = item_id(operator)
|
|
if not identifier:
|
|
raise ReconcileError(f"builtin operator {operator.get('name')} is missing id")
|
|
if operator_ids and identifier not in operator_ids:
|
|
continue
|
|
payload = builtin_operator_patch(operator, operator_runner_image, image_digest_resolver=image_digest_resolver)
|
|
payload_image = str(payload.get("image") or "")
|
|
payload_image_id = str(payload.get("imageId") or "")
|
|
if (
|
|
str(operator.get("image") or "") == payload_image
|
|
and str(operator.get("imageId") or "") == payload_image_id
|
|
and (operator.get("containerCommand") or []) == (payload.get("containerCommand") or [])
|
|
):
|
|
continue
|
|
print(f"update operator {operator['name']}")
|
|
request_json(client, "PUT", f"/api/operators/{quote_id(identifier)}", payload)
|
|
updated += 1
|
|
return updated
|
|
|
|
|
|
def first_item_by_name(items: list[JSON], name: str) -> JSON | None:
|
|
for item in items:
|
|
if str(item.get("name") or "").strip() == name:
|
|
return item
|
|
return None
|
|
|
|
|
|
def reconcile_bos_connection(client: DroboticflowClient) -> str:
|
|
endpoint = os.getenv("RECONCILE_BOS_ENDPOINT", "").strip()
|
|
bucket = os.getenv("RECONCILE_BOS_BUCKET", "").strip().strip("/")
|
|
access_key_id = os.getenv("RECONCILE_BOS_ACCESS_KEY_ID", "").strip()
|
|
secret_access_key = os.getenv("RECONCILE_BOS_SECRET_ACCESS_KEY", "").strip()
|
|
if not endpoint or not access_key_id or not secret_access_key:
|
|
return "skipped: missing RECONCILE_BOS_ENDPOINT/access key/secret key"
|
|
|
|
name = os.getenv("RECONCILE_BOS_CONNECTION_NAME", "POC BOS Workspace").strip() or "POC BOS Workspace"
|
|
payload = {
|
|
"name": name,
|
|
"description": "POC runtime BOS connection reconciled from Kubernetes secrets.",
|
|
"provider": "BOS",
|
|
"endpoint": endpoint,
|
|
"bucket": bucket,
|
|
"accessKeyId": access_key_id,
|
|
"secretAccessKey": secret_access_key,
|
|
}
|
|
items = request_json(client, "GET", "/api/storage/connections").get("items", [])
|
|
existing = first_item_by_name(items if isinstance(items, list) else [], name)
|
|
if existing:
|
|
request_json(client, "PUT", f"/api/storage/connections/{quote_id(item_id(existing))}", payload)
|
|
return f"updated: {item_id(existing)}"
|
|
created = request_json(client, "POST", "/api/storage/connections", payload, expected=(201,))
|
|
return f"created: {item_id(created)}"
|
|
|
|
|
|
def reconcile_mongo_connection(client: DroboticflowClient) -> str:
|
|
uri = os.getenv("RECONCILE_MONGO_URI", "").strip()
|
|
database = os.getenv("RECONCILE_MONGO_DATABASE", "").strip()
|
|
if not uri or not database:
|
|
return "skipped: missing RECONCILE_MONGO_URI/database"
|
|
|
|
name = os.getenv("RECONCILE_MONGO_CONNECTION_NAME", "POC Mongo Runtime").strip() or "POC Mongo Runtime"
|
|
payload = {
|
|
"name": name,
|
|
"description": "POC runtime MongoDB connection reconciled from Kubernetes secrets.",
|
|
"uri": uri,
|
|
"database": database,
|
|
}
|
|
items = request_json(client, "GET", "/api/mongo/connections").get("items", [])
|
|
existing = first_item_by_name(items if isinstance(items, list) else [], name)
|
|
if existing:
|
|
request_json(client, "PUT", f"/api/mongo/connections/{quote_id(item_id(existing))}", payload)
|
|
return f"updated: {item_id(existing)}"
|
|
created = request_json(client, "POST", "/api/mongo/connections", payload, expected=(201,))
|
|
return f"created: {item_id(created)}"
|
|
|
|
|
|
def reconcile_external_connections(client: DroboticflowClient) -> None:
|
|
print(f"reconciled BOS storage connection: {reconcile_bos_connection(client)}")
|
|
print(f"reconciled Mongo connection: {reconcile_mongo_connection(client)}")
|
|
|
|
|
|
def reconcile(
|
|
client: DroboticflowClient,
|
|
*,
|
|
k8s_endpoint: str,
|
|
k8s_namespace: str,
|
|
operator_runner_image: str,
|
|
image_digest_resolver: ImageDigestResolver | None = None,
|
|
) -> None:
|
|
users = request_json(client, "GET", "/api/users")["items"]
|
|
projects = request_json(client, "GET", "/api/projects")["items"]
|
|
templates = request_json(client, "GET", "/api/templates")["items"]
|
|
operators = request_json(client, "GET", "/api/operators")["items"]
|
|
settings = request_json(client, "GET", "/api/settings")
|
|
|
|
delete_each(client, "/api/users", users, is_legacy_user)
|
|
delete_each(client, "/api/projects", projects, is_legacy_project)
|
|
delete_each(client, "/api/templates", templates, is_legacy_template)
|
|
delete_each(client, "/api/operators", operators, is_legacy_operator)
|
|
|
|
request_json(client, "PUT", "/api/settings", build_settings_payload(settings, k8s_endpoint, k8s_namespace))
|
|
print("updated system settings")
|
|
|
|
updated_count = sync_builtin_operators(client, operator_runner_image, image_digest_resolver=image_digest_resolver)
|
|
print(f"reconciled builtin operators from API catalog: {updated_count} updated")
|
|
reconcile_external_connections(client)
|
|
print("reconciled system data")
|
|
|
|
|
|
def parse_args() -> argparse.Namespace:
|
|
parser = argparse.ArgumentParser(description="Reconcile droboticflow system data through the protected API")
|
|
parser.add_argument("--api-base-url", default=api_base_url_from_env(include_base_url=False))
|
|
parser.add_argument("--k8s-endpoint", default=os.getenv("K8S_ENDPOINT", ""))
|
|
parser.add_argument("--k8s-namespace", default=os.getenv("K8S_NAMESPACE", "droboticflow"))
|
|
parser.add_argument("--operator-runner-image", default=os.getenv("OPERATOR_RUNNER_IMAGE", ""))
|
|
parser.add_argument(
|
|
"--operator-id",
|
|
action="append",
|
|
default=[],
|
|
help="Only validate and reconcile the selected builtin operator; may be repeated.",
|
|
)
|
|
return parser.parse_args()
|
|
|
|
|
|
def main() -> None:
|
|
args = parse_args()
|
|
if not args.operator_runner_image.strip():
|
|
raise SystemExit("OPERATOR_RUNNER_IMAGE is required")
|
|
if not args.k8s_endpoint.strip():
|
|
raise SystemExit("K8S_ENDPOINT is required, for example https://<api-server>:6443")
|
|
|
|
api_token = os.getenv("DROBOTICFLOW_API_TOKEN", "").strip()
|
|
client = DroboticflowClient(args.api_base_url, api_token=api_token, error_factory=ReconcileError)
|
|
if api_token:
|
|
print("using API token auth for reconcile requests")
|
|
try:
|
|
login_if_configured(
|
|
client,
|
|
os.getenv("DROBOTICFLOW_ACCEPTANCE_EMAIL", "").strip(),
|
|
os.getenv("DROBOTICFLOW_ACCEPTANCE_PASSWORD", ""),
|
|
)
|
|
selected_operator_ids = {value.strip() for value in args.operator_id if value.strip()}
|
|
if selected_operator_ids:
|
|
updated = sync_builtin_operators(
|
|
client,
|
|
args.operator_runner_image,
|
|
image_digest_resolver=registry_digest_resolver_from_env(),
|
|
operator_ids=selected_operator_ids,
|
|
)
|
|
print(f"reconciled selected builtin operators: {updated} updated")
|
|
return
|
|
reconcile(
|
|
client,
|
|
k8s_endpoint=args.k8s_endpoint,
|
|
k8s_namespace=args.k8s_namespace,
|
|
operator_runner_image=args.operator_runner_image,
|
|
image_digest_resolver=registry_digest_resolver_from_env(),
|
|
)
|
|
except ReconcileError as error:
|
|
raise SystemExit(str(error)) from error
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|