349 lines
15 KiB
Bash
Executable File
349 lines
15 KiB
Bash
Executable File
#!/usr/bin/env bash
|
||
set -euo pipefail
|
||
|
||
API_BASE_URL="${1:-${API_BASE_URL:-http://127.0.0.1:3001}}"
|
||
K8S_ENDPOINT="${K8S_ENDPOINT:-https://120.48.161.230:6443}"
|
||
K8S_NAMESPACE="${K8S_NAMESPACE:-droboticflow}"
|
||
OPERATOR_RUNNER_IMAGE="${OPERATOR_RUNNER_IMAGE:-}"
|
||
|
||
if [[ -z "$OPERATOR_RUNNER_IMAGE" && -f /tmp/droboticflow-images-latest.env ]]; then
|
||
# shellcheck disable=SC1091
|
||
source /tmp/droboticflow-images-latest.env
|
||
fi
|
||
|
||
: "${OPERATOR_RUNNER_IMAGE:?OPERATOR_RUNNER_IMAGE is required}"
|
||
|
||
python3 - "$API_BASE_URL" "$K8S_ENDPOINT" "$K8S_NAMESPACE" "$OPERATOR_RUNNER_IMAGE" <<'PY'
|
||
import http.client
|
||
import json
|
||
import time
|
||
import sys
|
||
import urllib.error
|
||
import urllib.parse
|
||
import urllib.request
|
||
|
||
api_base_url = sys.argv[1].rstrip("/")
|
||
k8s_endpoint = sys.argv[2]
|
||
k8s_namespace = sys.argv[3]
|
||
operator_runner_image = sys.argv[4]
|
||
|
||
|
||
def request(method, path, payload=None, expected=(200, 201, 202, 204)):
|
||
body = None
|
||
headers = {"Content-Type": "application/json"}
|
||
if payload is not None:
|
||
body = json.dumps(payload).encode("utf-8")
|
||
last_error = None
|
||
for attempt in range(1, 7):
|
||
req = urllib.request.Request(f"{api_base_url}{path}", data=body, headers=headers, method=method)
|
||
try:
|
||
with urllib.request.urlopen(req, timeout=20) as response:
|
||
status = response.getcode()
|
||
raw = response.read().decode("utf-8")
|
||
if status not in expected:
|
||
raise SystemExit(f"{method} {path} returned unexpected status {status}: {raw}")
|
||
if not raw:
|
||
return None
|
||
return json.loads(raw)
|
||
except urllib.error.HTTPError as error:
|
||
raw = error.read().decode("utf-8", errors="replace")
|
||
raise SystemExit(f"{method} {path} failed with {error.code}: {raw}") from error
|
||
except (urllib.error.URLError, http.client.RemoteDisconnected, TimeoutError) as error:
|
||
last_error = error
|
||
if attempt == 6:
|
||
break
|
||
time.sleep(min(attempt, 3))
|
||
raise SystemExit(f"{method} {path} failed after retries: {last_error}")
|
||
|
||
|
||
def delete_each(path_prefix, items, predicate):
|
||
for item in items:
|
||
if not predicate(item):
|
||
continue
|
||
print(f"delete {path_prefix}/{item['id']}")
|
||
request("DELETE", f"{path_prefix}/{urllib.parse.quote(item['id'])}", expected=(204, 404))
|
||
|
||
|
||
users = request("GET", "/api/users")["items"]
|
||
projects = request("GET", "/api/projects")["items"]
|
||
templates = request("GET", "/api/templates")["items"]
|
||
operators = request("GET", "/api/operators")["items"]
|
||
settings = request("GET", "/api/settings")
|
||
|
||
delete_each(
|
||
"/api/users",
|
||
users,
|
||
lambda item: item.get("id") in {"1", "2", "3"} or item.get("email", "").endswith("@emboflow.ai"),
|
||
)
|
||
delete_each(
|
||
"/api/projects",
|
||
projects,
|
||
lambda item: item.get("id") in {"1", "2", "3"} or item.get("description") == "Created by scripts/smoke-test.sh",
|
||
)
|
||
delete_each(
|
||
"/api/templates",
|
||
templates,
|
||
lambda item: item.get("id", "").startswith("tpl-"),
|
||
)
|
||
delete_each(
|
||
"/api/operators",
|
||
operators,
|
||
lambda item: item.get("id", "").startswith("op-") or str(item.get("image", "")).endswith(":latest"),
|
||
)
|
||
|
||
request(
|
||
"PUT",
|
||
"/api/settings",
|
||
{
|
||
"k8s": {
|
||
"enabled": True,
|
||
"endpoint": k8s_endpoint,
|
||
"namespace": k8s_namespace,
|
||
},
|
||
"storage": {
|
||
"endpoint": "",
|
||
"buckets": [],
|
||
},
|
||
"worker": {
|
||
"pollIntervalMs": settings["worker"]["pollIntervalMs"],
|
||
"maxConcurrentRuns": settings["worker"]["maxConcurrentRuns"],
|
||
},
|
||
"defaults": settings["defaults"],
|
||
},
|
||
)
|
||
print("updated system settings")
|
||
|
||
builtin_operators = [
|
||
{
|
||
"name": "真实数据源",
|
||
"type": "builtin",
|
||
"category": "IO",
|
||
"description": "从 SOURCE_TEXT、SOURCE_JSON、SOURCE_PATH 或上游输入生成真实源数据清单。",
|
||
"inputs": [],
|
||
"outputs": [{"id": "out-1", "name": "manifest", "type": "json", "transmission": "bus"}],
|
||
"image": operator_runner_image,
|
||
"skillDefinition": "工作流起点。读取真实输入源并生成结构化清单,供下游节点消费。",
|
||
"hyperparams": [
|
||
{"key": "SOURCE_TEXT", "value": "", "description": "直接注入文本内容"},
|
||
{"key": "SOURCE_JSON", "value": "", "description": "直接注入 JSON 内容"},
|
||
{"key": "SOURCE_PATH", "value": "", "description": "从本地路径读取真实数据"},
|
||
],
|
||
},
|
||
{
|
||
"name": "模型推理",
|
||
"type": "builtin",
|
||
"category": "AI",
|
||
"description": "执行真实节点输入汇总、命令调用或模型镜像推理。",
|
||
"inputs": [{"id": "in-1", "name": "input", "type": "json", "transmission": "bus"}],
|
||
"outputs": [{"id": "out-1", "name": "result", "type": "json", "transmission": "bus"}],
|
||
"image": operator_runner_image,
|
||
"skillDefinition": "执行通用推理节点;若提供 COMMAND 或 SCRIPT,会真实执行命令并收集输出。",
|
||
"hyperparams": [
|
||
{"key": "COMMAND", "value": "", "description": "容器内执行的命令"},
|
||
{"key": "SCRIPT", "value": "", "description": "可执行脚本内容"},
|
||
],
|
||
},
|
||
{
|
||
"name": "数据合并",
|
||
"type": "builtin",
|
||
"category": "MATH",
|
||
"description": "将多个 JSON 输入合并成统一结果。",
|
||
"inputs": [
|
||
{"id": "in-1", "name": "left", "type": "json", "transmission": "bus"},
|
||
{"id": "in-2", "name": "right", "type": "json", "transmission": "bus"},
|
||
],
|
||
"outputs": [{"id": "out-1", "name": "merged", "type": "json", "transmission": "bus"}],
|
||
"image": operator_runner_image,
|
||
"skillDefinition": "对多个 JSON 输入执行真实合并。",
|
||
},
|
||
{
|
||
"name": "结果上传",
|
||
"type": "builtin",
|
||
"category": "IO",
|
||
"description": "收集上游真实输出并生成上传结果清单。",
|
||
"inputs": [{"id": "in-1", "name": "result", "type": "json", "transmission": "bus"}],
|
||
"outputs": [],
|
||
"image": operator_runner_image,
|
||
"skillDefinition": "复制或汇总上游产物,生成上传结果清单。",
|
||
},
|
||
{
|
||
"name": "人工审核",
|
||
"type": "builtin",
|
||
"category": "HUMAN",
|
||
"description": "将真实输入送入人工审核并等待审核动作恢复流程。",
|
||
"inputs": [{"id": "in-1", "name": "candidate", "type": "json", "transmission": "bus"}],
|
||
"outputs": [
|
||
{"id": "approved", "name": "approved", "type": "json", "transmission": "bus"},
|
||
{"id": "rejected", "name": "rejected", "type": "json", "transmission": "bus"},
|
||
],
|
||
"image": operator_runner_image,
|
||
"skillDefinition": "基于真实输入生成审核任务并等待人工处理。",
|
||
},
|
||
{
|
||
"name": "目录扫描",
|
||
"type": "builtin",
|
||
"category": "IO",
|
||
"description": "对真实目录做文件统计、扩展名分布和样例文件扫描。",
|
||
"inputs": [{"id": "in-1", "name": "input", "type": "json", "transmission": "bus"}],
|
||
"outputs": [{"id": "out-1", "name": "summary", "type": "json", "transmission": "bus"}],
|
||
"image": operator_runner_image,
|
||
"skillDefinition": "读取 SOURCE_PATH 或上游 source manifest,输出真实目录概览。",
|
||
"hyperparams": [
|
||
{"key": "SOURCE_PATH", "value": "", "description": "待扫描目录"},
|
||
],
|
||
},
|
||
{
|
||
"name": "CSV 转 JSON",
|
||
"type": "builtin",
|
||
"category": "DATA",
|
||
"description": "将真实 CSV/TSV 文件转换成结构化 JSON 行列表。",
|
||
"inputs": [{"id": "in-1", "name": "input", "type": "json", "transmission": "bus"}],
|
||
"outputs": [{"id": "out-1", "name": "table", "type": "json", "transmission": "bus"}],
|
||
"image": operator_runner_image,
|
||
"skillDefinition": "读取 SOURCE_PATH 或上游 source manifest 指向的 csv/tsv 文件,输出完整 JSON 行数据。",
|
||
"hyperparams": [
|
||
{"key": "SOURCE_PATH", "value": "", "description": "csv/tsv 文件路径"},
|
||
],
|
||
},
|
||
{
|
||
"name": "JSON 转 CSV",
|
||
"type": "builtin",
|
||
"category": "DATA",
|
||
"description": "将 JSON 行数据导出为真实 CSV 文件。",
|
||
"inputs": [{"id": "in-1", "name": "input", "type": "json", "transmission": "bus"}],
|
||
"outputs": [{"id": "out-1", "name": "csv", "type": "file", "transmission": "bus"}],
|
||
"image": operator_runner_image,
|
||
"skillDefinition": "把上游 JSON items/entries 或单条对象导出为真实 CSV 文件,并附带摘要。",
|
||
},
|
||
{
|
||
"name": "JSON 字段提取",
|
||
"type": "builtin",
|
||
"category": "DATA",
|
||
"description": "按字段路径从 JSON 输入中提取指定值。",
|
||
"inputs": [{"id": "in-1", "name": "input", "type": "json", "transmission": "bus"}],
|
||
"outputs": [{"id": "out-1", "name": "extracted", "type": "json", "transmission": "bus"}],
|
||
"image": operator_runner_image,
|
||
"skillDefinition": "根据 FIELD_PATH 从 JSON 输入中提取字段。",
|
||
"hyperparams": [
|
||
{"key": "FIELD_PATH", "value": "", "description": "点分字段路径,例如 metrics.score"},
|
||
],
|
||
},
|
||
{
|
||
"name": "JSON 过滤",
|
||
"type": "builtin",
|
||
"category": "DATA",
|
||
"description": "按字段条件过滤 JSON 列表。",
|
||
"inputs": [{"id": "in-1", "name": "input", "type": "json", "transmission": "bus"}],
|
||
"outputs": [{"id": "out-1", "name": "filtered", "type": "json", "transmission": "bus"}],
|
||
"image": operator_runner_image,
|
||
"skillDefinition": "按 FIELD_PATH + OPERATOR + EXPECTED 条件真实过滤 JSON items。",
|
||
"hyperparams": [
|
||
{"key": "FIELD_PATH", "value": "", "description": "待判断字段路径"},
|
||
{"key": "OPERATOR", "value": "eq", "description": "eq/ne/contains/gt/gte/lt/lte/exists/missing"},
|
||
{"key": "EXPECTED", "value": "", "description": "字符串比较值"},
|
||
{"key": "EXPECTED_JSON", "value": "", "description": "JSON 格式比较值,优先于 EXPECTED"},
|
||
],
|
||
},
|
||
{
|
||
"name": "JSON 映射重命名",
|
||
"type": "builtin",
|
||
"category": "DATA",
|
||
"description": "按映射表批量重命名 JSON 字段。",
|
||
"inputs": [{"id": "in-1", "name": "input", "type": "json", "transmission": "bus"}],
|
||
"outputs": [{"id": "out-1", "name": "mapped", "type": "json", "transmission": "bus"}],
|
||
"image": operator_runner_image,
|
||
"skillDefinition": "根据 FIELD_MAP_JSON 真实执行字段名重映射。",
|
||
"hyperparams": [
|
||
{"key": "FIELD_MAP_JSON", "value": "{}", "description": "JSON 字段映射,例如 {\"old\":\"new\"}"},
|
||
],
|
||
},
|
||
{
|
||
"name": "文本模板渲染",
|
||
"type": "builtin",
|
||
"category": "UTIL",
|
||
"description": "使用上游 JSON 上下文渲染真实文本模板文件。",
|
||
"inputs": [{"id": "in-1", "name": "input", "type": "json", "transmission": "bus"}],
|
||
"outputs": [{"id": "out-1", "name": "rendered", "type": "file", "transmission": "bus"}],
|
||
"image": operator_runner_image,
|
||
"skillDefinition": "支持 {{field}} 和 Python format 风格占位符,输出真实文本文件。",
|
||
"hyperparams": [
|
||
{"key": "TEMPLATE_TEXT", "value": "", "description": "模板文本"},
|
||
{"key": "OUTPUT_NAME", "value": "rendered.txt", "description": "输出文件名"},
|
||
],
|
||
},
|
||
{
|
||
"name": "条件分流",
|
||
"type": "builtin",
|
||
"category": "LOGIC",
|
||
"description": "按字段条件将输入拆分到 matched / unmatched 两个真实输出端口。",
|
||
"inputs": [{"id": "in-1", "name": "input", "type": "json", "transmission": "bus"}],
|
||
"outputs": [
|
||
{"id": "matched", "name": "matched", "type": "json", "transmission": "bus"},
|
||
{"id": "unmatched", "name": "unmatched", "type": "json", "transmission": "bus"},
|
||
],
|
||
"image": operator_runner_image,
|
||
"skillDefinition": "按 FIELD_PATH + OPERATOR + EXPECTED 条件把真实数据分流到两个输出端口。",
|
||
"hyperparams": [
|
||
{"key": "FIELD_PATH", "value": "", "description": "待判断字段路径"},
|
||
{"key": "OPERATOR", "value": "eq", "description": "eq/ne/contains/gt/gte/lt/lte/exists/missing"},
|
||
{"key": "EXPECTED", "value": "", "description": "字符串比较值"},
|
||
{"key": "EXPECTED_JSON", "value": "", "description": "JSON 格式比较值,优先于 EXPECTED"},
|
||
],
|
||
},
|
||
{
|
||
"name": "LeRobot 数据集概览",
|
||
"type": "builtin",
|
||
"category": "ROBOTICS",
|
||
"description": "读取真实 LeRobot 数据集目录,输出 meta/tasks/parquet/videos 概览。",
|
||
"inputs": [{"id": "in-1", "name": "input", "type": "json", "transmission": "bus"}],
|
||
"outputs": [{"id": "out-1", "name": "summary", "type": "json", "transmission": "bus"}],
|
||
"image": operator_runner_image,
|
||
"skillDefinition": "面向 LeRobot 数据集目录的真实概览算子,适合机器人示教数据检查。",
|
||
"hyperparams": [
|
||
{"key": "SOURCE_PATH", "value": "", "description": "LeRobot 数据集根目录"},
|
||
],
|
||
},
|
||
{
|
||
"name": "ROSBag 数据概览",
|
||
"type": "builtin",
|
||
"category": "ROBOTICS",
|
||
"description": "读取真实 rosbag1 / rosbag2 / mcap / db3 数据并输出概览。",
|
||
"inputs": [{"id": "in-1", "name": "input", "type": "json", "transmission": "bus"}],
|
||
"outputs": [{"id": "out-1", "name": "summary", "type": "json", "transmission": "bus"}],
|
||
"image": operator_runner_image,
|
||
"skillDefinition": "支持 rosbag db3 topic/message 概览,以及 bag/mcap 文件摘要。",
|
||
"hyperparams": [
|
||
{"key": "SOURCE_PATH", "value": "", "description": "bag/db3/mcap 文件或 rosbag2 目录"},
|
||
],
|
||
},
|
||
{
|
||
"name": "Shell 命令执行",
|
||
"type": "builtin",
|
||
"category": "UTIL",
|
||
"description": "在 operator-runner 容器中真实执行 shell 命令并收集输出文件。",
|
||
"inputs": [{"id": "in-1", "name": "input", "type": "json", "transmission": "bus"}],
|
||
"outputs": [{"id": "out-1", "name": "result", "type": "file", "transmission": "bus"}],
|
||
"image": operator_runner_image,
|
||
"skillDefinition": "执行 COMMAND 或 SCRIPT,自动收集输出目录产物并生成预览元数据。",
|
||
"hyperparams": [
|
||
{"key": "COMMAND", "value": "", "description": "容器内执行的命令"},
|
||
{"key": "SCRIPT", "value": "", "description": "可执行脚本内容"},
|
||
],
|
||
},
|
||
]
|
||
|
||
operators = request("GET", "/api/operators")["items"]
|
||
operators_by_name = {item["name"]: item for item in operators}
|
||
|
||
for operator in builtin_operators:
|
||
existing = operators_by_name.get(operator["name"])
|
||
if existing:
|
||
print(f"update operator {operator['name']}")
|
||
request("PUT", f"/api/operators/{urllib.parse.quote(existing['id'])}", operator)
|
||
else:
|
||
print(f"create operator {operator['name']}")
|
||
request("POST", "/api/operators", operator, expected=(201,))
|
||
|
||
print("reconciled system data")
|
||
PY
|