225 lines
8.2 KiB
Python
225 lines
8.2 KiB
Python
from __future__ import annotations
|
|
|
|
import argparse
|
|
import os
|
|
import pathlib
|
|
import sys
|
|
import urllib.parse
|
|
from typing import Any, Callable
|
|
|
|
|
|
ROOT = pathlib.Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / "scripts"))
|
|
|
|
from lib.droboticflow_acceptance import DroboticflowClient
|
|
from lib.droboticflow_env import api_base_url_from_env
|
|
|
|
|
|
JSON = dict[str, Any]
|
|
Predicate = Callable[[JSON], bool]
|
|
|
|
|
|
class ReconcileError(RuntimeError):
|
|
pass
|
|
|
|
|
|
def item_id(item: JSON) -> str:
|
|
return str(item.get("id") or "")
|
|
|
|
|
|
def quote_id(value: str) -> str:
|
|
return urllib.parse.quote(value, safe="")
|
|
|
|
|
|
def is_legacy_user(item: JSON) -> bool:
|
|
email = str(item.get("email") or "")
|
|
return item_id(item) in {"1", "2", "3"} or email.endswith("@emboflow.ai")
|
|
|
|
|
|
def is_legacy_project(item: JSON) -> bool:
|
|
return item_id(item) in {"1", "2", "3"} or item.get("description") in {
|
|
"Created by scripts/live-workflow-acceptance.sh",
|
|
}
|
|
|
|
|
|
def is_legacy_template(item: JSON) -> bool:
|
|
return item_id(item).startswith("tpl-")
|
|
|
|
|
|
def is_legacy_operator(item: JSON) -> bool:
|
|
operator_type = str(item.get("type") or "").strip().lower()
|
|
if operator_type == "builtin":
|
|
return False
|
|
image = str(item.get("image") or "")
|
|
return item_id(item).startswith("op-") or image.endswith(":latest")
|
|
|
|
|
|
def request_json(
|
|
client: DroboticflowClient,
|
|
method: str,
|
|
path: str,
|
|
payload: Any | None = None,
|
|
*,
|
|
expected: tuple[int, ...] = (200, 201, 202, 204),
|
|
) -> Any:
|
|
return client.request_json(method, path, payload, expected=expected, timeout=20, retries=6)
|
|
|
|
|
|
def login_if_configured(client: DroboticflowClient, email: str, password: str) -> None:
|
|
if bool(email) != bool(password):
|
|
raise ReconcileError("DROBOTICFLOW_ACCEPTANCE_EMAIL and DROBOTICFLOW_ACCEPTANCE_PASSWORD must be set together")
|
|
if email:
|
|
client.login_if_configured(email, password, label="reconcile user")
|
|
|
|
|
|
def delete_each(client: DroboticflowClient, path_prefix: str, items: list[JSON], predicate: Predicate) -> int:
|
|
deleted = 0
|
|
for item in items:
|
|
if not predicate(item):
|
|
continue
|
|
identifier = item_id(item)
|
|
print(f"delete {path_prefix}/{identifier}")
|
|
request_json(client, "DELETE", f"{path_prefix}/{quote_id(identifier)}", expected=(204, 404))
|
|
deleted += 1
|
|
return deleted
|
|
|
|
|
|
def build_settings_payload(settings: JSON, k8s_endpoint: str, k8s_namespace: str) -> JSON:
|
|
worker = settings.get("worker")
|
|
defaults = settings.get("defaults")
|
|
if not isinstance(worker, dict):
|
|
raise ReconcileError("/api/settings response is missing worker settings")
|
|
if not isinstance(defaults, dict):
|
|
raise ReconcileError("/api/settings response is missing default settings")
|
|
return {
|
|
"k8s": {
|
|
"enabled": True,
|
|
"endpoint": k8s_endpoint,
|
|
"namespace": k8s_namespace,
|
|
},
|
|
"storage": {
|
|
"endpoint": "",
|
|
"buckets": [],
|
|
},
|
|
"worker": {
|
|
"pollIntervalMs": worker["pollIntervalMs"],
|
|
"maxConcurrentRuns": worker["maxConcurrentRuns"],
|
|
},
|
|
"defaults": defaults,
|
|
}
|
|
|
|
|
|
def builtin_operator_patch(operator: JSON, operator_runner_image: str) -> JSON:
|
|
catalog_image = str(operator.get("image") or "").strip()
|
|
catalog_image_id = str(operator.get("imageId") or "").strip()
|
|
image = (
|
|
catalog_image
|
|
if catalog_image and "droboticflow-operator-" in catalog_image and "droboticflow-operator-runner" not in catalog_image
|
|
else operator_runner_image
|
|
)
|
|
payload = {
|
|
"name": operator["name"],
|
|
"type": operator["type"],
|
|
"category": operator["category"],
|
|
"description": operator["description"],
|
|
"inputs": operator.get("inputs") or [],
|
|
"outputs": operator.get("outputs") or [],
|
|
"image": image,
|
|
"imageId": catalog_image_id or image,
|
|
"containerCommand": operator.get("containerCommand") or [],
|
|
"containerArgs": operator.get("containerArgs") or [],
|
|
"runtimeContract": operator.get("runtimeContract") or {},
|
|
"skillDefinition": operator.get("skillDefinition") or "",
|
|
"hyperparams": operator.get("hyperparams") or [],
|
|
}
|
|
for optional_key in ("configSchema", "resources"):
|
|
if optional_key in operator and operator[optional_key] is not None:
|
|
payload[optional_key] = operator[optional_key]
|
|
return payload
|
|
|
|
|
|
def sync_builtin_operators(client: DroboticflowClient, operator_runner_image: str) -> int:
|
|
operators = request_json(client, "GET", "/api/operators")["items"]
|
|
builtin_operators = [item for item in operators if str(item.get("type") or "").strip().lower() == "builtin"]
|
|
if not builtin_operators:
|
|
raise ReconcileError("/api/operators did not return any builtin operators")
|
|
|
|
updated = 0
|
|
for operator in builtin_operators:
|
|
identifier = item_id(operator)
|
|
if not identifier:
|
|
raise ReconcileError(f"builtin operator {operator.get('name')} is missing id")
|
|
payload = builtin_operator_patch(operator, operator_runner_image)
|
|
payload_image = str(payload.get("image") or "")
|
|
payload_image_id = str(payload.get("imageId") or "")
|
|
if (
|
|
str(operator.get("image") or "") == payload_image
|
|
and str(operator.get("imageId") or "") == payload_image_id
|
|
and (operator.get("containerCommand") or []) == (payload.get("containerCommand") or [])
|
|
):
|
|
continue
|
|
print(f"update operator {operator['name']}")
|
|
request_json(client, "PUT", f"/api/operators/{quote_id(identifier)}", payload)
|
|
updated += 1
|
|
return updated
|
|
|
|
|
|
def reconcile(client: DroboticflowClient, *, k8s_endpoint: str, k8s_namespace: str, operator_runner_image: str) -> None:
|
|
users = request_json(client, "GET", "/api/users")["items"]
|
|
projects = request_json(client, "GET", "/api/projects")["items"]
|
|
templates = request_json(client, "GET", "/api/templates")["items"]
|
|
operators = request_json(client, "GET", "/api/operators")["items"]
|
|
settings = request_json(client, "GET", "/api/settings")
|
|
|
|
delete_each(client, "/api/users", users, is_legacy_user)
|
|
delete_each(client, "/api/projects", projects, is_legacy_project)
|
|
delete_each(client, "/api/templates", templates, is_legacy_template)
|
|
delete_each(client, "/api/operators", operators, is_legacy_operator)
|
|
|
|
request_json(client, "PUT", "/api/settings", build_settings_payload(settings, k8s_endpoint, k8s_namespace))
|
|
print("updated system settings")
|
|
|
|
updated_count = sync_builtin_operators(client, operator_runner_image)
|
|
print(f"reconciled builtin operators from API catalog: {updated_count} updated")
|
|
print("reconciled system data")
|
|
|
|
|
|
def parse_args() -> argparse.Namespace:
|
|
parser = argparse.ArgumentParser(description="Reconcile droboticflow system data through the protected API")
|
|
parser.add_argument("--api-base-url", default=api_base_url_from_env(include_base_url=False))
|
|
parser.add_argument("--k8s-endpoint", default=os.getenv("K8S_ENDPOINT", ""))
|
|
parser.add_argument("--k8s-namespace", default=os.getenv("K8S_NAMESPACE", "droboticflow"))
|
|
parser.add_argument("--operator-runner-image", default=os.getenv("OPERATOR_RUNNER_IMAGE", ""))
|
|
return parser.parse_args()
|
|
|
|
|
|
def main() -> None:
|
|
args = parse_args()
|
|
if not args.operator_runner_image.strip():
|
|
raise SystemExit("OPERATOR_RUNNER_IMAGE is required")
|
|
if not args.k8s_endpoint.strip():
|
|
raise SystemExit("K8S_ENDPOINT is required, for example https://<api-server>:6443")
|
|
|
|
api_token = os.getenv("DROBOTICFLOW_API_TOKEN", "").strip()
|
|
client = DroboticflowClient(args.api_base_url, api_token=api_token, error_factory=ReconcileError)
|
|
if api_token:
|
|
print("using API token auth for reconcile requests")
|
|
try:
|
|
login_if_configured(
|
|
client,
|
|
os.getenv("DROBOTICFLOW_ACCEPTANCE_EMAIL", "").strip(),
|
|
os.getenv("DROBOTICFLOW_ACCEPTANCE_PASSWORD", ""),
|
|
)
|
|
reconcile(
|
|
client,
|
|
k8s_endpoint=args.k8s_endpoint,
|
|
k8s_namespace=args.k8s_namespace,
|
|
operator_runner_image=args.operator_runner_image,
|
|
)
|
|
except ReconcileError as error:
|
|
raise SystemExit(str(error)) from error
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|