droboticflow/scripts/reconcile-system-data.sh

349 lines
15 KiB
Bash
Executable File
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

#!/usr/bin/env bash
set -euo pipefail
API_BASE_URL="${1:-${API_BASE_URL:-http://127.0.0.1:3001}}"
K8S_ENDPOINT="${K8S_ENDPOINT:-https://120.48.161.230:6443}"
K8S_NAMESPACE="${K8S_NAMESPACE:-droboticflow}"
OPERATOR_RUNNER_IMAGE="${OPERATOR_RUNNER_IMAGE:-}"
if [[ -z "$OPERATOR_RUNNER_IMAGE" && -f /tmp/droboticflow-images-latest.env ]]; then
# shellcheck disable=SC1091
source /tmp/droboticflow-images-latest.env
fi
: "${OPERATOR_RUNNER_IMAGE:?OPERATOR_RUNNER_IMAGE is required}"
python3 - "$API_BASE_URL" "$K8S_ENDPOINT" "$K8S_NAMESPACE" "$OPERATOR_RUNNER_IMAGE" <<'PY'
import http.client
import json
import time
import sys
import urllib.error
import urllib.parse
import urllib.request
api_base_url = sys.argv[1].rstrip("/")
k8s_endpoint = sys.argv[2]
k8s_namespace = sys.argv[3]
operator_runner_image = sys.argv[4]
def request(method, path, payload=None, expected=(200, 201, 202, 204)):
body = None
headers = {"Content-Type": "application/json"}
if payload is not None:
body = json.dumps(payload).encode("utf-8")
last_error = None
for attempt in range(1, 7):
req = urllib.request.Request(f"{api_base_url}{path}", data=body, headers=headers, method=method)
try:
with urllib.request.urlopen(req, timeout=20) as response:
status = response.getcode()
raw = response.read().decode("utf-8")
if status not in expected:
raise SystemExit(f"{method} {path} returned unexpected status {status}: {raw}")
if not raw:
return None
return json.loads(raw)
except urllib.error.HTTPError as error:
raw = error.read().decode("utf-8", errors="replace")
raise SystemExit(f"{method} {path} failed with {error.code}: {raw}") from error
except (urllib.error.URLError, http.client.RemoteDisconnected, TimeoutError) as error:
last_error = error
if attempt == 6:
break
time.sleep(min(attempt, 3))
raise SystemExit(f"{method} {path} failed after retries: {last_error}")
def delete_each(path_prefix, items, predicate):
for item in items:
if not predicate(item):
continue
print(f"delete {path_prefix}/{item['id']}")
request("DELETE", f"{path_prefix}/{urllib.parse.quote(item['id'])}", expected=(204, 404))
users = request("GET", "/api/users")["items"]
projects = request("GET", "/api/projects")["items"]
templates = request("GET", "/api/templates")["items"]
operators = request("GET", "/api/operators")["items"]
settings = request("GET", "/api/settings")
delete_each(
"/api/users",
users,
lambda item: item.get("id") in {"1", "2", "3"} or item.get("email", "").endswith("@emboflow.ai"),
)
delete_each(
"/api/projects",
projects,
lambda item: item.get("id") in {"1", "2", "3"} or item.get("description") == "Created by scripts/smoke-test.sh",
)
delete_each(
"/api/templates",
templates,
lambda item: item.get("id", "").startswith("tpl-"),
)
delete_each(
"/api/operators",
operators,
lambda item: item.get("id", "").startswith("op-") or str(item.get("image", "")).endswith(":latest"),
)
request(
"PUT",
"/api/settings",
{
"k8s": {
"enabled": True,
"endpoint": k8s_endpoint,
"namespace": k8s_namespace,
},
"storage": {
"endpoint": "",
"buckets": [],
},
"worker": {
"pollIntervalMs": settings["worker"]["pollIntervalMs"],
"maxConcurrentRuns": settings["worker"]["maxConcurrentRuns"],
},
"defaults": settings["defaults"],
},
)
print("updated system settings")
builtin_operators = [
{
"name": "真实数据源",
"type": "builtin",
"category": "IO",
"description": "从 SOURCE_TEXT、SOURCE_JSON、SOURCE_PATH 或上游输入生成真实源数据清单。",
"inputs": [],
"outputs": [{"id": "out-1", "name": "manifest", "type": "json", "transmission": "bus"}],
"image": operator_runner_image,
"skillDefinition": "工作流起点。读取真实输入源并生成结构化清单,供下游节点消费。",
"hyperparams": [
{"key": "SOURCE_TEXT", "value": "", "description": "直接注入文本内容"},
{"key": "SOURCE_JSON", "value": "", "description": "直接注入 JSON 内容"},
{"key": "SOURCE_PATH", "value": "", "description": "从本地路径读取真实数据"},
],
},
{
"name": "模型推理",
"type": "builtin",
"category": "AI",
"description": "执行真实节点输入汇总、命令调用或模型镜像推理。",
"inputs": [{"id": "in-1", "name": "input", "type": "json", "transmission": "bus"}],
"outputs": [{"id": "out-1", "name": "result", "type": "json", "transmission": "bus"}],
"image": operator_runner_image,
"skillDefinition": "执行通用推理节点;若提供 COMMAND 或 SCRIPT,会真实执行命令并收集输出。",
"hyperparams": [
{"key": "COMMAND", "value": "", "description": "容器内执行的命令"},
{"key": "SCRIPT", "value": "", "description": "可执行脚本内容"},
],
},
{
"name": "数据合并",
"type": "builtin",
"category": "MATH",
"description": "将多个 JSON 输入合并成统一结果。",
"inputs": [
{"id": "in-1", "name": "left", "type": "json", "transmission": "bus"},
{"id": "in-2", "name": "right", "type": "json", "transmission": "bus"},
],
"outputs": [{"id": "out-1", "name": "merged", "type": "json", "transmission": "bus"}],
"image": operator_runner_image,
"skillDefinition": "对多个 JSON 输入执行真实合并。",
},
{
"name": "结果上传",
"type": "builtin",
"category": "IO",
"description": "收集上游真实输出并生成上传结果清单。",
"inputs": [{"id": "in-1", "name": "result", "type": "json", "transmission": "bus"}],
"outputs": [],
"image": operator_runner_image,
"skillDefinition": "复制或汇总上游产物,生成上传结果清单。",
},
{
"name": "人工审核",
"type": "builtin",
"category": "HUMAN",
"description": "将真实输入送入人工审核并等待审核动作恢复流程。",
"inputs": [{"id": "in-1", "name": "candidate", "type": "json", "transmission": "bus"}],
"outputs": [
{"id": "approved", "name": "approved", "type": "json", "transmission": "bus"},
{"id": "rejected", "name": "rejected", "type": "json", "transmission": "bus"},
],
"image": operator_runner_image,
"skillDefinition": "基于真实输入生成审核任务并等待人工处理。",
},
{
"name": "目录扫描",
"type": "builtin",
"category": "IO",
"description": "对真实目录做文件统计、扩展名分布和样例文件扫描。",
"inputs": [{"id": "in-1", "name": "input", "type": "json", "transmission": "bus"}],
"outputs": [{"id": "out-1", "name": "summary", "type": "json", "transmission": "bus"}],
"image": operator_runner_image,
"skillDefinition": "读取 SOURCE_PATH 或上游 source manifest,输出真实目录概览。",
"hyperparams": [
{"key": "SOURCE_PATH", "value": "", "description": "待扫描目录"},
],
},
{
"name": "CSV 转 JSON",
"type": "builtin",
"category": "DATA",
"description": "将真实 CSV/TSV 文件转换成结构化 JSON 行列表。",
"inputs": [{"id": "in-1", "name": "input", "type": "json", "transmission": "bus"}],
"outputs": [{"id": "out-1", "name": "table", "type": "json", "transmission": "bus"}],
"image": operator_runner_image,
"skillDefinition": "读取 SOURCE_PATH 或上游 source manifest 指向的 csv/tsv 文件,输出完整 JSON 行数据。",
"hyperparams": [
{"key": "SOURCE_PATH", "value": "", "description": "csv/tsv 文件路径"},
],
},
{
"name": "JSON 转 CSV",
"type": "builtin",
"category": "DATA",
"description": "将 JSON 行数据导出为真实 CSV 文件。",
"inputs": [{"id": "in-1", "name": "input", "type": "json", "transmission": "bus"}],
"outputs": [{"id": "out-1", "name": "csv", "type": "file", "transmission": "bus"}],
"image": operator_runner_image,
"skillDefinition": "把上游 JSON items/entries 或单条对象导出为真实 CSV 文件,并附带摘要。",
},
{
"name": "JSON 字段提取",
"type": "builtin",
"category": "DATA",
"description": "按字段路径从 JSON 输入中提取指定值。",
"inputs": [{"id": "in-1", "name": "input", "type": "json", "transmission": "bus"}],
"outputs": [{"id": "out-1", "name": "extracted", "type": "json", "transmission": "bus"}],
"image": operator_runner_image,
"skillDefinition": "根据 FIELD_PATH 从 JSON 输入中提取字段。",
"hyperparams": [
{"key": "FIELD_PATH", "value": "", "description": "点分字段路径,例如 metrics.score"},
],
},
{
"name": "JSON 过滤",
"type": "builtin",
"category": "DATA",
"description": "按字段条件过滤 JSON 列表。",
"inputs": [{"id": "in-1", "name": "input", "type": "json", "transmission": "bus"}],
"outputs": [{"id": "out-1", "name": "filtered", "type": "json", "transmission": "bus"}],
"image": operator_runner_image,
"skillDefinition": "按 FIELD_PATH + OPERATOR + EXPECTED 条件真实过滤 JSON items。",
"hyperparams": [
{"key": "FIELD_PATH", "value": "", "description": "待判断字段路径"},
{"key": "OPERATOR", "value": "eq", "description": "eq/ne/contains/gt/gte/lt/lte/exists/missing"},
{"key": "EXPECTED", "value": "", "description": "字符串比较值"},
{"key": "EXPECTED_JSON", "value": "", "description": "JSON 格式比较值,优先于 EXPECTED"},
],
},
{
"name": "JSON 映射重命名",
"type": "builtin",
"category": "DATA",
"description": "按映射表批量重命名 JSON 字段。",
"inputs": [{"id": "in-1", "name": "input", "type": "json", "transmission": "bus"}],
"outputs": [{"id": "out-1", "name": "mapped", "type": "json", "transmission": "bus"}],
"image": operator_runner_image,
"skillDefinition": "根据 FIELD_MAP_JSON 真实执行字段名重映射。",
"hyperparams": [
{"key": "FIELD_MAP_JSON", "value": "{}", "description": "JSON 字段映射,例如 {\"old\":\"new\"}"},
],
},
{
"name": "文本模板渲染",
"type": "builtin",
"category": "UTIL",
"description": "使用上游 JSON 上下文渲染真实文本模板文件。",
"inputs": [{"id": "in-1", "name": "input", "type": "json", "transmission": "bus"}],
"outputs": [{"id": "out-1", "name": "rendered", "type": "file", "transmission": "bus"}],
"image": operator_runner_image,
"skillDefinition": "支持 {{field}} 和 Python format 风格占位符,输出真实文本文件。",
"hyperparams": [
{"key": "TEMPLATE_TEXT", "value": "", "description": "模板文本"},
{"key": "OUTPUT_NAME", "value": "rendered.txt", "description": "输出文件名"},
],
},
{
"name": "条件分流",
"type": "builtin",
"category": "LOGIC",
"description": "按字段条件将输入拆分到 matched / unmatched 两个真实输出端口。",
"inputs": [{"id": "in-1", "name": "input", "type": "json", "transmission": "bus"}],
"outputs": [
{"id": "matched", "name": "matched", "type": "json", "transmission": "bus"},
{"id": "unmatched", "name": "unmatched", "type": "json", "transmission": "bus"},
],
"image": operator_runner_image,
"skillDefinition": "按 FIELD_PATH + OPERATOR + EXPECTED 条件把真实数据分流到两个输出端口。",
"hyperparams": [
{"key": "FIELD_PATH", "value": "", "description": "待判断字段路径"},
{"key": "OPERATOR", "value": "eq", "description": "eq/ne/contains/gt/gte/lt/lte/exists/missing"},
{"key": "EXPECTED", "value": "", "description": "字符串比较值"},
{"key": "EXPECTED_JSON", "value": "", "description": "JSON 格式比较值,优先于 EXPECTED"},
],
},
{
"name": "LeRobot 数据集概览",
"type": "builtin",
"category": "ROBOTICS",
"description": "读取真实 LeRobot 数据集目录,输出 meta/tasks/parquet/videos 概览。",
"inputs": [{"id": "in-1", "name": "input", "type": "json", "transmission": "bus"}],
"outputs": [{"id": "out-1", "name": "summary", "type": "json", "transmission": "bus"}],
"image": operator_runner_image,
"skillDefinition": "面向 LeRobot 数据集目录的真实概览算子,适合机器人示教数据检查。",
"hyperparams": [
{"key": "SOURCE_PATH", "value": "", "description": "LeRobot 数据集根目录"},
],
},
{
"name": "ROSBag 数据概览",
"type": "builtin",
"category": "ROBOTICS",
"description": "读取真实 rosbag1 / rosbag2 / mcap / db3 数据并输出概览。",
"inputs": [{"id": "in-1", "name": "input", "type": "json", "transmission": "bus"}],
"outputs": [{"id": "out-1", "name": "summary", "type": "json", "transmission": "bus"}],
"image": operator_runner_image,
"skillDefinition": "支持 rosbag db3 topic/message 概览,以及 bag/mcap 文件摘要。",
"hyperparams": [
{"key": "SOURCE_PATH", "value": "", "description": "bag/db3/mcap 文件或 rosbag2 目录"},
],
},
{
"name": "Shell 命令执行",
"type": "builtin",
"category": "UTIL",
"description": "在 operator-runner 容器中真实执行 shell 命令并收集输出文件。",
"inputs": [{"id": "in-1", "name": "input", "type": "json", "transmission": "bus"}],
"outputs": [{"id": "out-1", "name": "result", "type": "file", "transmission": "bus"}],
"image": operator_runner_image,
"skillDefinition": "执行 COMMAND 或 SCRIPT,自动收集输出目录产物并生成预览元数据。",
"hyperparams": [
{"key": "COMMAND", "value": "", "description": "容器内执行的命令"},
{"key": "SCRIPT", "value": "", "description": "可执行脚本内容"},
],
},
]
operators = request("GET", "/api/operators")["items"]
operators_by_name = {item["name"]: item for item in operators}
for operator in builtin_operators:
existing = operators_by_name.get(operator["name"])
if existing:
print(f"update operator {operator['name']}")
request("PUT", f"/api/operators/{urllib.parse.quote(existing['id'])}", operator)
else:
print(f"create operator {operator['name']}")
request("POST", "/api/operators", operator, expected=(201,))
print("reconciled system data")
PY