droboticflow/scripts/reconcile_system_data.py
2026-06-16 20:44:44 +08:00

205 lines
7.3 KiB
Python

from __future__ import annotations
import argparse
import os
import pathlib
import sys
import urllib.parse
from typing import Any, Callable
ROOT = pathlib.Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "scripts"))
from lib.droboticflow_acceptance import DroboticflowClient
JSON = dict[str, Any]
Predicate = Callable[[JSON], bool]
class ReconcileError(RuntimeError):
pass
def item_id(item: JSON) -> str:
return str(item.get("id") or "")
def quote_id(value: str) -> str:
return urllib.parse.quote(value, safe="")
def is_legacy_user(item: JSON) -> bool:
email = str(item.get("email") or "")
return item_id(item) in {"1", "2", "3"} or email.endswith("@emboflow.ai")
def is_legacy_project(item: JSON) -> bool:
return item_id(item) in {"1", "2", "3"} or item.get("description") == "Created by scripts/smoke-test.sh"
def is_legacy_template(item: JSON) -> bool:
return item_id(item).startswith("tpl-")
def is_legacy_operator(item: JSON) -> bool:
operator_type = str(item.get("type") or "").strip().lower()
if operator_type == "builtin":
return False
image = str(item.get("image") or "")
return item_id(item).startswith("op-") or image.endswith(":latest")
def request_json(
client: DroboticflowClient,
method: str,
path: str,
payload: Any | None = None,
*,
expected: tuple[int, ...] = (200, 201, 202, 204),
) -> Any:
return client.request_json(method, path, payload, expected=expected, timeout=20, retries=6)
def login_if_configured(client: DroboticflowClient, email: str, password: str) -> None:
if bool(email) != bool(password):
raise ReconcileError("DROBOTICFLOW_SMOKE_EMAIL and DROBOTICFLOW_SMOKE_PASSWORD must be set together")
if email:
client.login_if_configured(email, password, label="reconcile user")
def delete_each(client: DroboticflowClient, path_prefix: str, items: list[JSON], predicate: Predicate) -> int:
deleted = 0
for item in items:
if not predicate(item):
continue
identifier = item_id(item)
print(f"delete {path_prefix}/{identifier}")
request_json(client, "DELETE", f"{path_prefix}/{quote_id(identifier)}", expected=(204, 404))
deleted += 1
return deleted
def build_settings_payload(settings: JSON, k8s_endpoint: str, k8s_namespace: str) -> JSON:
worker = settings.get("worker")
defaults = settings.get("defaults")
if not isinstance(worker, dict):
raise ReconcileError("/api/settings response is missing worker settings")
if not isinstance(defaults, dict):
raise ReconcileError("/api/settings response is missing default settings")
return {
"k8s": {
"enabled": True,
"endpoint": k8s_endpoint,
"namespace": k8s_namespace,
},
"storage": {
"endpoint": "",
"buckets": [],
},
"worker": {
"pollIntervalMs": worker["pollIntervalMs"],
"maxConcurrentRuns": worker["maxConcurrentRuns"],
},
"defaults": defaults,
}
def builtin_operator_patch(operator: JSON, operator_runner_image: str) -> JSON:
payload = {
"name": operator["name"],
"type": operator["type"],
"category": operator["category"],
"description": operator["description"],
"inputs": operator.get("inputs") or [],
"outputs": operator.get("outputs") or [],
"image": operator_runner_image,
"skillDefinition": operator.get("skillDefinition") or "",
"hyperparams": operator.get("hyperparams") or [],
}
for optional_key in ("configSchema", "resources"):
if optional_key in operator and operator[optional_key] is not None:
payload[optional_key] = operator[optional_key]
return payload
def sync_builtin_operators(client: DroboticflowClient, operator_runner_image: str) -> int:
operators = request_json(client, "GET", "/api/operators")["items"]
builtin_operators = [item for item in operators if str(item.get("type") or "").strip().lower() == "builtin"]
if not builtin_operators:
raise ReconcileError("/api/operators did not return any builtin operators")
updated = 0
for operator in builtin_operators:
identifier = item_id(operator)
if not identifier:
raise ReconcileError(f"builtin operator {operator.get('name')} is missing id")
payload = builtin_operator_patch(operator, operator_runner_image)
if str(operator.get("image") or "") == operator_runner_image:
continue
print(f"update operator {operator['name']}")
request_json(client, "PUT", f"/api/operators/{quote_id(identifier)}", payload)
updated += 1
return updated
def reconcile(client: DroboticflowClient, *, k8s_endpoint: str, k8s_namespace: str, operator_runner_image: str) -> None:
users = request_json(client, "GET", "/api/users")["items"]
projects = request_json(client, "GET", "/api/projects")["items"]
templates = request_json(client, "GET", "/api/templates")["items"]
operators = request_json(client, "GET", "/api/operators")["items"]
settings = request_json(client, "GET", "/api/settings")
delete_each(client, "/api/users", users, is_legacy_user)
delete_each(client, "/api/projects", projects, is_legacy_project)
delete_each(client, "/api/templates", templates, is_legacy_template)
delete_each(client, "/api/operators", operators, is_legacy_operator)
request_json(client, "PUT", "/api/settings", build_settings_payload(settings, k8s_endpoint, k8s_namespace))
print("updated system settings")
updated_count = sync_builtin_operators(client, operator_runner_image)
print(f"reconciled builtin operators from API catalog: {updated_count} updated")
print("reconciled system data")
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description="Reconcile droboticflow system data through the protected API")
parser.add_argument("--api-base-url", default=os.getenv("API_BASE_URL", "http://127.0.0.1:3001"))
parser.add_argument("--k8s-endpoint", default=os.getenv("K8S_ENDPOINT", ""))
parser.add_argument("--k8s-namespace", default=os.getenv("K8S_NAMESPACE", "droboticflow"))
parser.add_argument("--operator-runner-image", default=os.getenv("OPERATOR_RUNNER_IMAGE", ""))
return parser.parse_args()
def main() -> None:
args = parse_args()
if not args.operator_runner_image.strip():
raise SystemExit("OPERATOR_RUNNER_IMAGE is required")
if not args.k8s_endpoint.strip():
raise SystemExit("K8S_ENDPOINT is required, for example https://<api-server>:6443")
api_token = os.getenv("DROBOTICFLOW_API_TOKEN", "").strip()
client = DroboticflowClient(args.api_base_url, api_token=api_token, error_factory=ReconcileError)
if api_token:
print("using API token auth for reconcile requests")
try:
login_if_configured(
client,
os.getenv("DROBOTICFLOW_SMOKE_EMAIL", "").strip(),
os.getenv("DROBOTICFLOW_SMOKE_PASSWORD", ""),
)
reconcile(
client,
k8s_endpoint=args.k8s_endpoint,
k8s_namespace=args.k8s_namespace,
operator_runner_image=args.operator_runner_image,
)
except ReconcileError as error:
raise SystemExit(str(error)) from error
if __name__ == "__main__":
main()