Include the accumulated workflow scheduling, run history, subtree rerun, operator UI, deployment, data asset, object storage, database browser, and documentation changes currently in the worktree.
691 lines
30 KiB
Go
691 lines
30 KiB
Go
package api
|
||
|
||
import (
|
||
"context"
|
||
"strings"
|
||
|
||
"droboticflow/internal/domain"
|
||
)
|
||
|
||
const (
|
||
builtinRealSourceOperatorID = "operator-builtin-real-source"
|
||
builtinBOSScanOperatorID = "operator-builtin-bos-incremental-scan"
|
||
builtinModelOperatorID = "operator-builtin-model-inference"
|
||
builtinMergeOperatorID = "operator-builtin-merge"
|
||
builtinUploadOperatorID = "operator-builtin-upload"
|
||
builtinHumanReviewOperatorID = "operator-builtin-human-review"
|
||
builtinDirectoryScanOperatorID = "operator-builtin-directory-scan"
|
||
builtinCSVToJSONOperatorID = "operator-builtin-csv-to-json"
|
||
builtinJSONToCSVOperatorID = "operator-builtin-json-to-csv"
|
||
builtinJSONExtractOperatorID = "operator-builtin-json-field-extract"
|
||
builtinJSONFilterOperatorID = "operator-builtin-json-filter"
|
||
builtinJSONMapOperatorID = "operator-builtin-json-map-rename"
|
||
builtinTemplateRenderOperatorID = "operator-builtin-template-render"
|
||
builtinRouteOperatorID = "operator-builtin-conditional-route"
|
||
builtinDatasetIndexOperatorID = "operator-builtin-xplatform-dataset-index"
|
||
builtinDatasetValidateOperatorID = "operator-builtin-xplatform-dataset-validate"
|
||
builtinVideoQualityOperatorID = "operator-builtin-video-quality"
|
||
builtinVideoFilterOperatorID = "operator-builtin-video-filter"
|
||
builtinVideoSliceOperatorID = "operator-builtin-video-slice"
|
||
builtinVideoAnnotateOperatorID = "operator-builtin-video-annotate"
|
||
builtinPackageOperatorID = "operator-builtin-package"
|
||
builtinSamplingOperatorID = "operator-builtin-audit-sample"
|
||
builtinMongoWriteOperatorID = "operator-builtin-mongodb-write"
|
||
builtinLeRobotSummaryOperatorID = "operator-builtin-lerobot-summary"
|
||
builtinROSBagSummaryOperatorID = "operator-builtin-rosbag-summary"
|
||
builtinShellCommandOperatorID = "operator-builtin-shell-command"
|
||
)
|
||
|
||
var reservedMongoCollections = map[string]struct{}{
|
||
"projects": {},
|
||
"operators": {},
|
||
"operatorTombstones": {},
|
||
"workflows": {},
|
||
"templates": {},
|
||
"runs": {},
|
||
"sourceStates": {},
|
||
"auditTasks": {},
|
||
"samplingTasks": {},
|
||
"users": {},
|
||
"storageConnections": {},
|
||
"mongoConnections": {},
|
||
"settings": {},
|
||
}
|
||
|
||
func jsonPort(id string, name string, description string) domain.PortDefinition {
|
||
return domain.PortDefinition{ID: id, Name: name, Type: domain.PortTypeJSON, Transmission: "bus", Description: description}
|
||
}
|
||
|
||
func filePort(id string, name string, description string) domain.PortDefinition {
|
||
return domain.PortDefinition{ID: id, Name: name, Type: domain.PortTypeFile, Transmission: "bus", Description: description}
|
||
}
|
||
|
||
func builtinOperatorRecords(now string, runnerImage string) []domain.OperatorRecord {
|
||
return []domain.OperatorRecord{
|
||
{
|
||
ID: builtinRealSourceOperatorID,
|
||
Name: "真实数据源",
|
||
Type: "builtin",
|
||
Category: "IO",
|
||
Description: "从 SOURCE_TEXT、SOURCE_JSON、SOURCE_PATH 或上游输入生成真实源数据清单。",
|
||
Inputs: nil,
|
||
Outputs: []domain.PortDefinition{jsonPort("out-1", "manifest", "源数据清单")},
|
||
Image: runnerImage,
|
||
SkillDefinition: "工作流起点。读取真实输入源并生成结构化清单,供下游节点消费。",
|
||
Hyperparams: []domain.HyperParam{
|
||
{Key: "SOURCE_TEXT", Value: "", Description: "直接注入文本内容", Type: "string"},
|
||
{Key: "SOURCE_JSON", Value: "", Description: "直接注入 JSON 内容", Type: "string"},
|
||
{Key: "SOURCE_PATH", Value: "", Description: "从路径读取真实数据", Type: "string"},
|
||
},
|
||
CreatedAt: now,
|
||
UpdatedAt: now,
|
||
},
|
||
{
|
||
ID: builtinBOSScanOperatorID,
|
||
Name: "BOS 增量扫描",
|
||
Type: "builtin",
|
||
Category: "IO",
|
||
Description: "按 bucket/prefix 增量扫描 BOS,仅输出本次新增对象并提交 checkpoint 给 runtime。",
|
||
Inputs: nil,
|
||
Outputs: []domain.PortDefinition{jsonPort("out-1", "manifest", "增量对象清单")},
|
||
Image: runnerImage,
|
||
SkillDefinition: "统一工作流里的有状态源算子。每次 run 只产出 BOS 中新增的对象,状态由 runtime 在 run 成功后提交。",
|
||
Hyperparams: []domain.HyperParam{
|
||
{Key: "DATA_SOURCE", Value: "BOS", Description: "固定为 BOS", Type: "string"},
|
||
{Key: "SOURCE_PATH", Value: "", Description: "完整 bos://bucket/prefix 路径", Type: "string"},
|
||
{Key: "BOS_ENDPOINT", Value: "https://bj.bcebos.com", Description: "BOS Endpoint", Type: "string"},
|
||
{Key: "BOS_ACCESS_KEY_ID", Value: "", Description: "BOS Access Key ID", Type: "string"},
|
||
{Key: "BOS_SECRET_ACCESS_KEY", Value: "", Description: "BOS Secret Access Key", Type: "string"},
|
||
{Key: "SCAN_MODE", Value: "INCREMENTAL", Description: "固定为 INCREMENTAL", Type: "string"},
|
||
},
|
||
CreatedAt: now,
|
||
UpdatedAt: now,
|
||
},
|
||
{
|
||
ID: builtinModelOperatorID,
|
||
Name: "模型推理",
|
||
Type: "builtin",
|
||
Category: "AI",
|
||
Description: "执行真实节点输入汇总、命令调用或模型镜像推理。",
|
||
Inputs: []domain.PortDefinition{jsonPort("in-1", "input", "输入数据")},
|
||
Outputs: []domain.PortDefinition{jsonPort("out-1", "result", "推理结果")},
|
||
Image: runnerImage,
|
||
SkillDefinition: "执行通用推理节点;若提供 COMMAND 或 SCRIPT,会真实执行命令并收集输出。",
|
||
Hyperparams: []domain.HyperParam{
|
||
{Key: "COMMAND", Value: "", Description: "容器内执行的命令", Type: "string"},
|
||
{Key: "SCRIPT", Value: "", Description: "可执行脚本内容", Type: "string"},
|
||
},
|
||
CreatedAt: now,
|
||
UpdatedAt: now,
|
||
},
|
||
{
|
||
ID: builtinMergeOperatorID,
|
||
Name: "数据合并",
|
||
Type: "builtin",
|
||
Category: "MATH",
|
||
Description: "将多个 JSON 输入合并成统一结果。",
|
||
Inputs: []domain.PortDefinition{jsonPort("in-1", "left", "左输入"), jsonPort("in-2", "right", "右输入")},
|
||
Outputs: []domain.PortDefinition{jsonPort("out-1", "merged", "合并结果")},
|
||
Image: runnerImage,
|
||
SkillDefinition: "对多个 JSON 输入执行真实合并。",
|
||
CreatedAt: now,
|
||
UpdatedAt: now,
|
||
},
|
||
{
|
||
ID: builtinUploadOperatorID,
|
||
Name: "结果上传",
|
||
Type: "builtin",
|
||
Category: "IO",
|
||
Description: "收集上游真实输出并生成上传结果清单。",
|
||
Inputs: []domain.PortDefinition{jsonPort("in-1", "result", "待上传结果")},
|
||
Outputs: nil,
|
||
Image: runnerImage,
|
||
SkillDefinition: "复制或汇总上游产物,生成上传结果清单。",
|
||
CreatedAt: now,
|
||
UpdatedAt: now,
|
||
},
|
||
{
|
||
ID: builtinHumanReviewOperatorID,
|
||
Name: "人工审核",
|
||
Type: "builtin",
|
||
Category: "HUMAN",
|
||
Description: "将真实输入送入人工审核并等待审核动作恢复流程。",
|
||
Inputs: []domain.PortDefinition{jsonPort("in-1", "candidate", "候选数据")},
|
||
Outputs: []domain.PortDefinition{
|
||
jsonPort("approved", "approved", "审核通过"),
|
||
jsonPort("rejected", "rejected", "审核拒绝"),
|
||
},
|
||
Image: runnerImage,
|
||
SkillDefinition: "基于真实输入生成审核任务并等待人工处理。",
|
||
CreatedAt: now,
|
||
UpdatedAt: now,
|
||
},
|
||
{
|
||
ID: builtinDirectoryScanOperatorID,
|
||
Name: "目录扫描",
|
||
Type: "builtin",
|
||
Category: "IO",
|
||
Description: "对真实目录做文件统计、扩展名分布和样例文件扫描。",
|
||
Inputs: []domain.PortDefinition{jsonPort("in-1", "input", "目录输入")},
|
||
Outputs: []domain.PortDefinition{jsonPort("out-1", "summary", "目录概览")},
|
||
Image: runnerImage,
|
||
SkillDefinition: "读取 SOURCE_PATH 或上游 source manifest,输出真实目录概览。",
|
||
Hyperparams: []domain.HyperParam{
|
||
{Key: "SOURCE_PATH", Value: "", Description: "待扫描目录", Type: "string"},
|
||
},
|
||
CreatedAt: now,
|
||
UpdatedAt: now,
|
||
},
|
||
{
|
||
ID: builtinCSVToJSONOperatorID,
|
||
Name: "CSV 转 JSON",
|
||
Type: "builtin",
|
||
Category: "DATA",
|
||
Description: "将真实 CSV/TSV 文件转换成结构化 JSON 行列表。",
|
||
Inputs: []domain.PortDefinition{jsonPort("in-1", "input", "CSV/TSV 输入")},
|
||
Outputs: []domain.PortDefinition{jsonPort("out-1", "table", "JSON 行列表")},
|
||
Image: runnerImage,
|
||
SkillDefinition: "读取 SOURCE_PATH 或上游 source manifest 指向的 csv/tsv 文件,输出完整 JSON 行数据。",
|
||
Hyperparams: []domain.HyperParam{
|
||
{Key: "SOURCE_PATH", Value: "", Description: "csv/tsv 文件路径", Type: "string"},
|
||
},
|
||
CreatedAt: now,
|
||
UpdatedAt: now,
|
||
},
|
||
{
|
||
ID: builtinJSONToCSVOperatorID,
|
||
Name: "JSON 转 CSV",
|
||
Type: "builtin",
|
||
Category: "DATA",
|
||
Description: "将 JSON 行数据导出为真实 CSV 文件。",
|
||
Inputs: []domain.PortDefinition{jsonPort("in-1", "input", "JSON 行列表")},
|
||
Outputs: []domain.PortDefinition{filePort("out-1", "csv", "CSV 文件")},
|
||
Image: runnerImage,
|
||
SkillDefinition: "把上游 JSON items/entries 或单条对象导出为真实 CSV 文件,并附带摘要。",
|
||
CreatedAt: now,
|
||
UpdatedAt: now,
|
||
},
|
||
{
|
||
ID: builtinJSONExtractOperatorID,
|
||
Name: "JSON 字段提取",
|
||
Type: "builtin",
|
||
Category: "DATA",
|
||
Description: "按字段路径从 JSON 输入中提取指定值。",
|
||
Inputs: []domain.PortDefinition{jsonPort("in-1", "input", "JSON 输入")},
|
||
Outputs: []domain.PortDefinition{jsonPort("out-1", "extracted", "提取结果")},
|
||
Image: runnerImage,
|
||
SkillDefinition: "根据 FIELD_PATH 从 JSON 输入中提取字段。",
|
||
Hyperparams: []domain.HyperParam{
|
||
{Key: "FIELD_PATH", Value: "", Description: "点分字段路径,例如 metrics.score", Type: "string"},
|
||
},
|
||
CreatedAt: now,
|
||
UpdatedAt: now,
|
||
},
|
||
{
|
||
ID: builtinJSONFilterOperatorID,
|
||
Name: "JSON 过滤",
|
||
Type: "builtin",
|
||
Category: "DATA",
|
||
Description: "按字段条件过滤 JSON 列表。",
|
||
Inputs: []domain.PortDefinition{jsonPort("in-1", "input", "JSON 输入")},
|
||
Outputs: []domain.PortDefinition{jsonPort("out-1", "filtered", "过滤结果")},
|
||
Image: runnerImage,
|
||
SkillDefinition: "按 FIELD_PATH + OPERATOR + EXPECTED 条件真实过滤 JSON items。",
|
||
Hyperparams: []domain.HyperParam{
|
||
{Key: "FIELD_PATH", Value: "", Description: "待判断字段路径", Type: "string"},
|
||
{Key: "OPERATOR", Value: "eq", Description: "eq/ne/contains/gt/gte/lt/lte/exists/missing", Type: "string"},
|
||
{Key: "EXPECTED", Value: "", Description: "字符串比较值", Type: "string"},
|
||
{Key: "EXPECTED_JSON", Value: "", Description: "JSON 格式比较值,优先于 EXPECTED", Type: "string"},
|
||
},
|
||
CreatedAt: now,
|
||
UpdatedAt: now,
|
||
},
|
||
{
|
||
ID: builtinJSONMapOperatorID,
|
||
Name: "JSON 映射重命名",
|
||
Type: "builtin",
|
||
Category: "DATA",
|
||
Description: "按映射表批量重命名 JSON 字段。",
|
||
Inputs: []domain.PortDefinition{jsonPort("in-1", "input", "JSON 输入")},
|
||
Outputs: []domain.PortDefinition{jsonPort("out-1", "mapped", "映射结果")},
|
||
Image: runnerImage,
|
||
SkillDefinition: "根据 FIELD_MAP_JSON 真实执行字段名重映射。",
|
||
Hyperparams: []domain.HyperParam{
|
||
{Key: "FIELD_MAP_JSON", Value: "{}", Description: "JSON 字段映射,例如 {\"old\":\"new\"}", Type: "string"},
|
||
},
|
||
CreatedAt: now,
|
||
UpdatedAt: now,
|
||
},
|
||
{
|
||
ID: builtinTemplateRenderOperatorID,
|
||
Name: "文本模板渲染",
|
||
Type: "builtin",
|
||
Category: "UTIL",
|
||
Description: "使用上游 JSON 上下文渲染真实文本模板文件。",
|
||
Inputs: []domain.PortDefinition{jsonPort("in-1", "input", "JSON 输入")},
|
||
Outputs: []domain.PortDefinition{filePort("out-1", "rendered", "渲染后的文本文件")},
|
||
Image: runnerImage,
|
||
SkillDefinition: "支持 {{field}} 和 Python format 风格占位符,输出真实文本文件。",
|
||
Hyperparams: []domain.HyperParam{
|
||
{Key: "TEMPLATE_TEXT", Value: "", Description: "模板文本", Type: "string"},
|
||
{Key: "OUTPUT_NAME", Value: "rendered.txt", Description: "输出文件名", Type: "string"},
|
||
},
|
||
CreatedAt: now,
|
||
UpdatedAt: now,
|
||
},
|
||
{
|
||
ID: builtinRouteOperatorID,
|
||
Name: "条件分流",
|
||
Type: "builtin",
|
||
Category: "LOGIC",
|
||
Description: "按字段条件将输入拆分到 matched / unmatched 两个真实输出端口。",
|
||
Inputs: []domain.PortDefinition{jsonPort("in-1", "input", "JSON 输入")},
|
||
Outputs: []domain.PortDefinition{
|
||
jsonPort("matched", "matched", "命中规则的数据"),
|
||
jsonPort("unmatched", "unmatched", "未命中规则的数据"),
|
||
},
|
||
Image: runnerImage,
|
||
SkillDefinition: "按 FIELD_PATH + OPERATOR + EXPECTED 条件把真实数据分流到两个输出端口。",
|
||
Hyperparams: []domain.HyperParam{
|
||
{Key: "FIELD_PATH", Value: "", Description: "待判断字段路径", Type: "string"},
|
||
{Key: "OPERATOR", Value: "eq", Description: "eq/ne/contains/gt/gte/lt/lte/exists/missing", Type: "string"},
|
||
{Key: "EXPECTED", Value: "", Description: "字符串比较值", Type: "string"},
|
||
{Key: "EXPECTED_JSON", Value: "", Description: "JSON 格式比较值,优先于 EXPECTED", Type: "string"},
|
||
},
|
||
CreatedAt: now,
|
||
UpdatedAt: now,
|
||
},
|
||
{
|
||
ID: builtinDatasetIndexOperatorID,
|
||
Name: "X 平台数据集索引",
|
||
Type: "builtin",
|
||
Category: "DATASET",
|
||
Description: "扫描 X-platform 采集目录,解析 meta.json、video_meta.json 和视频文件索引。",
|
||
Inputs: []domain.PortDefinition{jsonPort("in-1", "input", "目录输入")},
|
||
Outputs: []domain.PortDefinition{jsonPort("out-1", "dataset", "数据集索引")},
|
||
Image: runnerImage,
|
||
SkillDefinition: "面向 BJ_***/DJI_***/meta.json/video_meta.json 结构的真实数据集索引算子,输出采集目录和视频清单。",
|
||
Hyperparams: []domain.HyperParam{
|
||
{Key: "SOURCE_PATH", Value: "", Description: "X-platform 根目录或单个采集目录", Type: "string"},
|
||
},
|
||
CreatedAt: now,
|
||
UpdatedAt: now,
|
||
},
|
||
{
|
||
ID: builtinDatasetValidateOperatorID,
|
||
Name: "采集目录校验",
|
||
Type: "builtin",
|
||
Category: "DATASET",
|
||
Description: "校验采集目录是否包含 meta.json、video_meta.json,以及可选 intrinsics.json。",
|
||
Inputs: []domain.PortDefinition{jsonPort("in-1", "input", "数据集索引")},
|
||
Outputs: []domain.PortDefinition{jsonPort("out-1", "validated", "校验结果")},
|
||
Image: runnerImage,
|
||
SkillDefinition: "对 X-platform 数据集索引结果执行真实结构校验,输出每个采集目录的有效性报告。",
|
||
Hyperparams: []domain.HyperParam{
|
||
{Key: "REQUIRE_INTRINSICS", Value: "false", Description: "是否要求 intrinsics.json 必须存在", Type: "string"},
|
||
},
|
||
CreatedAt: now,
|
||
UpdatedAt: now,
|
||
},
|
||
{
|
||
ID: builtinVideoQualityOperatorID,
|
||
Name: "视频质检",
|
||
Type: "builtin",
|
||
Category: "VIDEO",
|
||
Description: "基于真实视频帧计算清晰度、抖动度和无手率估计。",
|
||
Inputs: []domain.PortDefinition{jsonPort("in-1", "input", "视频清单")},
|
||
Outputs: []domain.PortDefinition{jsonPort("out-1", "quality", "质检结果")},
|
||
Image: runnerImage,
|
||
SkillDefinition: "使用 OpenCV 对真实 mp4 做质检分析,输出 clarity / shake / handOcclusion / passed。",
|
||
Hyperparams: []domain.HyperParam{
|
||
{Key: "MIN_CLARITY", Value: "0.8", Description: "清晰度最低阈值", Type: "string"},
|
||
{Key: "MAX_SHAKE", Value: "0.1", Description: "抖动度最高阈值", Type: "string"},
|
||
{Key: "MAX_HAND_OCCLUSION", Value: "0.1", Description: "无手率最高阈值", Type: "string"},
|
||
{Key: "CLARITY_SCALE", Value: "35", Description: "清晰度归一化尺度", Type: "string"},
|
||
},
|
||
CreatedAt: now,
|
||
UpdatedAt: now,
|
||
},
|
||
{
|
||
ID: builtinVideoFilterOperatorID,
|
||
Name: "数据过滤",
|
||
Type: "builtin",
|
||
Category: "VIDEO",
|
||
Description: "按日期、地点、场景和质检阈值过滤有效视频。",
|
||
Inputs: []domain.PortDefinition{jsonPort("in-1", "input", "视频与元数据输入")},
|
||
Outputs: []domain.PortDefinition{jsonPort("out-1", "filtered", "过滤结果")},
|
||
Image: runnerImage,
|
||
SkillDefinition: "针对 X-platform 质检结果执行真实筛选,输出符合条件的视频列表。",
|
||
Hyperparams: []domain.HyperParam{
|
||
{Key: "DATE_START", Value: "", Description: "开始日期,支持 YYYY-MM-DD / YYYYMMDD", Type: "string"},
|
||
{Key: "DATE_END", Value: "", Description: "结束日期,支持 YYYY-MM-DD / YYYYMMDD", Type: "string"},
|
||
{Key: "PLACE", Value: "", Description: "采集地点", Type: "string"},
|
||
{Key: "TYPE", Value: "", Description: "采集场景类型", Type: "string"},
|
||
{Key: "MIN_CLARITY", Value: "0.8", Description: "清晰度最低阈值", Type: "string"},
|
||
{Key: "MAX_SHAKE", Value: "0.1", Description: "抖动度最高阈值", Type: "string"},
|
||
{Key: "MAX_HAND_OCCLUSION", Value: "0.1", Description: "无手率最高阈值", Type: "string"},
|
||
},
|
||
CreatedAt: now,
|
||
UpdatedAt: now,
|
||
},
|
||
{
|
||
ID: builtinVideoSliceOperatorID,
|
||
Name: "视频切片",
|
||
Type: "builtin",
|
||
Category: "VIDEO",
|
||
Description: "将真实视频按固定时长切分为多个 mp4 片段。",
|
||
Inputs: []domain.PortDefinition{jsonPort("in-1", "input", "视频输入")},
|
||
Outputs: []domain.PortDefinition{jsonPort("out-1", "clips", "切片结果")},
|
||
Image: runnerImage,
|
||
SkillDefinition: "使用 ffmpeg/OpenCV 将真实视频切成固定长度片段,并输出 clip manifest。",
|
||
Hyperparams: []domain.HyperParam{
|
||
{Key: "SEGMENT_DURATION_SECONDS", Value: "300", Description: "每个切片的秒数", Type: "string"},
|
||
},
|
||
CreatedAt: now,
|
||
UpdatedAt: now,
|
||
},
|
||
{
|
||
ID: builtinVideoAnnotateOperatorID,
|
||
Name: "视频标注",
|
||
Type: "builtin",
|
||
Category: "VIDEO",
|
||
Description: "基于真实视频片段生成阶段划分、动作标签和对象提示。",
|
||
Inputs: []domain.PortDefinition{jsonPort("in-1", "input", "切片输入")},
|
||
Outputs: []domain.PortDefinition{jsonPort("out-1", "annotations", "标注结果")},
|
||
Image: runnerImage,
|
||
SkillDefinition: "读取切片视频并做真实帧间运动分析,输出 annotation JSON 和动作分段。",
|
||
Hyperparams: []domain.HyperParam{
|
||
{Key: "FRAME_INTERVAL_SECONDS", Value: "1.0", Description: "采样帧间隔秒数", Type: "string"},
|
||
},
|
||
CreatedAt: now,
|
||
UpdatedAt: now,
|
||
},
|
||
{
|
||
ID: builtinPackageOperatorID,
|
||
Name: "成品打包",
|
||
Type: "builtin",
|
||
Category: "VIDEO",
|
||
Description: "将切片视频、质检结果、标注结果和配置文件组装成成品数据包。",
|
||
Inputs: []domain.PortDefinition{jsonPort("in-1", "input", "切片/标注/质检结果")},
|
||
Outputs: []domain.PortDefinition{jsonPort("out-1", "packages", "成品包清单")},
|
||
Image: runnerImage,
|
||
SkillDefinition: "按 clip 组织真实 package 目录,输出 packages manifest 和所有成品文件。",
|
||
CreatedAt: now,
|
||
UpdatedAt: now,
|
||
},
|
||
{
|
||
ID: builtinSamplingOperatorID,
|
||
Name: "抽检采样",
|
||
Type: "builtin",
|
||
Category: "QA",
|
||
Description: "按质检和标注规则对成品数据执行真实抽检采样。",
|
||
Inputs: []domain.PortDefinition{jsonPort("in-1", "input", "成品包输入")},
|
||
Outputs: []domain.PortDefinition{jsonPort("out-1", "samples", "抽检样本清单")},
|
||
Image: runnerImage,
|
||
SkillDefinition: "根据 RULES_JSON 对成品数据包做规则筛选和稳定抽样,输出抽检任务清单。",
|
||
Hyperparams: []domain.HyperParam{
|
||
{Key: "RULES_JSON", Value: "[]", Description: "抽检规则数组,包含 field/operator/value/samplePercent", Type: "string"},
|
||
},
|
||
CreatedAt: now,
|
||
UpdatedAt: now,
|
||
},
|
||
{
|
||
ID: builtinMongoWriteOperatorID,
|
||
Name: "MongoDB 写入",
|
||
Type: "builtin",
|
||
Category: "DATA",
|
||
Description: "将上游 JSON 数据写入用户配置的业务 MongoDB,支持批量插入和按键 upsert,并允许显式覆盖连接。",
|
||
Inputs: []domain.PortDefinition{
|
||
jsonPort("input", "input", "待写入的 JSON 文档或 items 列表"),
|
||
},
|
||
Outputs: []domain.PortDefinition{
|
||
jsonPort("result", "result", "写入结果摘要"),
|
||
},
|
||
Image: runnerImage,
|
||
SkillDefinition: "将 JSON 文档写入业务 MongoDB 集合。",
|
||
Hyperparams: []domain.HyperParam{
|
||
{Key: "MONGO_COLLECTION", Value: "", Description: "目标集合名称(禁止写系统保留集合)", Type: "string"},
|
||
{Key: "MONGO_MODE", Value: "insert_many", Description: "写入模式:insert_many 或 upsert", Type: "string"},
|
||
{Key: "MONGO_CONNECTION_ID", Value: "", Description: "业务 MongoDB 连接 ID;推荐优先配置", Type: "string"},
|
||
{Key: "MONGO_TARGET", Value: "business", Description: "兼容字段;默认 business。若显式传入 URI/DB,会优先使用显式连接", Type: "string"},
|
||
{Key: "UPSERT_KEY", Value: "_id", Description: "upsert 模式使用的唯一键", Type: "string"},
|
||
{Key: "DOCUMENT_PATH", Value: "", Description: "从输入 payload 中提取待写入文档的字段路径", Type: "string"},
|
||
{Key: "MONGO_DB", Value: "", Description: "可选数据库名;显式填写后会覆盖 business/platform 默认连接", Type: "string"},
|
||
{Key: "MONGO_URI", Value: "", Description: "可选 Mongo URI;显式填写后会覆盖 business/platform 默认连接", Type: "string"},
|
||
},
|
||
CreatedAt: now,
|
||
UpdatedAt: now,
|
||
},
|
||
{
|
||
ID: builtinLeRobotSummaryOperatorID,
|
||
Name: "LeRobot 数据集概览",
|
||
Type: "builtin",
|
||
Category: "ROBOTICS",
|
||
Description: "读取真实 LeRobot 数据集目录,输出 meta/tasks/parquet/videos 概览。",
|
||
Inputs: []domain.PortDefinition{jsonPort("in-1", "input", "目录输入")},
|
||
Outputs: []domain.PortDefinition{jsonPort("out-1", "summary", "概览结果")},
|
||
Image: runnerImage,
|
||
SkillDefinition: "面向 LeRobot 数据集目录的真实概览算子,适合机器人示教数据检查。",
|
||
Hyperparams: []domain.HyperParam{
|
||
{Key: "SOURCE_PATH", Value: "", Description: "LeRobot 数据集根目录", Type: "string"},
|
||
},
|
||
CreatedAt: now,
|
||
UpdatedAt: now,
|
||
},
|
||
{
|
||
ID: builtinROSBagSummaryOperatorID,
|
||
Name: "ROSBag 数据概览",
|
||
Type: "builtin",
|
||
Category: "ROBOTICS",
|
||
Description: "读取真实 rosbag1 / rosbag2 / mcap / db3 数据并输出概览。",
|
||
Inputs: []domain.PortDefinition{jsonPort("in-1", "input", "bag 输入")},
|
||
Outputs: []domain.PortDefinition{jsonPort("out-1", "summary", "概览结果")},
|
||
Image: runnerImage,
|
||
SkillDefinition: "支持 rosbag db3 topic/message 概览,以及 bag/mcap 文件摘要。",
|
||
Hyperparams: []domain.HyperParam{
|
||
{Key: "SOURCE_PATH", Value: "", Description: "bag/db3/mcap 文件或 rosbag2 目录", Type: "string"},
|
||
},
|
||
CreatedAt: now,
|
||
UpdatedAt: now,
|
||
},
|
||
{
|
||
ID: builtinShellCommandOperatorID,
|
||
Name: "Shell 命令执行",
|
||
Type: "builtin",
|
||
Category: "UTIL",
|
||
Description: "在 operator-runner 容器中真实执行 shell 命令并收集输出文件。",
|
||
Inputs: []domain.PortDefinition{jsonPort("in-1", "input", "输入数据")},
|
||
Outputs: []domain.PortDefinition{filePort("out-1", "result", "执行结果文件")},
|
||
Image: runnerImage,
|
||
SkillDefinition: "执行 COMMAND 或 SCRIPT,自动收集输出目录产物并生成预览元数据。",
|
||
Hyperparams: []domain.HyperParam{
|
||
{Key: "COMMAND", Value: "", Description: "容器内执行的命令", Type: "string"},
|
||
{Key: "SCRIPT", Value: "", Description: "可执行脚本内容", Type: "string"},
|
||
},
|
||
CreatedAt: now,
|
||
UpdatedAt: now,
|
||
},
|
||
}
|
||
}
|
||
|
||
func findBuiltinOperator(items []domain.OperatorRecord, id string, name string) *domain.OperatorRecord {
|
||
for index := range items {
|
||
item := items[index]
|
||
if item.ID == id || strings.TrimSpace(item.Name) == strings.TrimSpace(name) {
|
||
copy := item
|
||
return ©
|
||
}
|
||
}
|
||
return nil
|
||
}
|
||
|
||
type builtinOperatorDeletionSet struct {
|
||
ids map[string]struct{}
|
||
names map[string]struct{}
|
||
}
|
||
|
||
func normalizeBuiltinOperatorDeletionValue(value string) string {
|
||
return strings.ToLower(strings.TrimSpace(value))
|
||
}
|
||
|
||
func builtinOperatorDeletionKey(id string, name string) string {
|
||
if normalizedName := normalizeBuiltinOperatorDeletionValue(name); normalizedName != "" {
|
||
return "name:" + normalizedName
|
||
}
|
||
if normalizedID := normalizeBuiltinOperatorDeletionValue(id); normalizedID != "" {
|
||
return "id:" + normalizedID
|
||
}
|
||
return ""
|
||
}
|
||
|
||
func newBuiltinOperatorDeletionSet(items []domain.BuiltinOperatorDeletionRecord) builtinOperatorDeletionSet {
|
||
set := builtinOperatorDeletionSet{
|
||
ids: make(map[string]struct{}),
|
||
names: make(map[string]struct{}),
|
||
}
|
||
for _, item := range items {
|
||
if normalizedID := normalizeBuiltinOperatorDeletionValue(item.OperatorID); normalizedID != "" {
|
||
set.ids[normalizedID] = struct{}{}
|
||
}
|
||
if normalizedName := normalizeBuiltinOperatorDeletionValue(item.OperatorName); normalizedName != "" {
|
||
set.names[normalizedName] = struct{}{}
|
||
}
|
||
}
|
||
return set
|
||
}
|
||
|
||
func (set builtinOperatorDeletionSet) Has(item domain.OperatorRecord) bool {
|
||
if normalizedID := normalizeBuiltinOperatorDeletionValue(item.ID); normalizedID != "" {
|
||
if _, ok := set.ids[normalizedID]; ok {
|
||
return true
|
||
}
|
||
}
|
||
if normalizedName := normalizeBuiltinOperatorDeletionValue(item.Name); normalizedName != "" {
|
||
if _, ok := set.names[normalizedName]; ok {
|
||
return true
|
||
}
|
||
}
|
||
return false
|
||
}
|
||
|
||
func filterDeletedBuiltinOperators(items []domain.OperatorRecord, deletions builtinOperatorDeletionSet) []domain.OperatorRecord {
|
||
filtered := make([]domain.OperatorRecord, 0, len(items))
|
||
for _, item := range items {
|
||
if strings.EqualFold(strings.TrimSpace(item.Type), "builtin") && deletions.Has(item) {
|
||
continue
|
||
}
|
||
filtered = append(filtered, item)
|
||
}
|
||
return filtered
|
||
}
|
||
|
||
func upsertBuiltinOperator(ctx context.Context, server *server, items []domain.OperatorRecord, builtin domain.OperatorRecord) ([]domain.OperatorRecord, error) {
|
||
existing := findBuiltinOperator(items, builtin.ID, builtin.Name)
|
||
if existing == nil {
|
||
if err := server.store.UpsertOperator(ctx, builtin); err != nil {
|
||
return items, err
|
||
}
|
||
return append(items, builtin), nil
|
||
}
|
||
|
||
updated := *existing
|
||
updated.ID = builtin.ID
|
||
updated.Name = builtin.Name
|
||
updated.Type = builtin.Type
|
||
updated.Category = builtin.Category
|
||
updated.Description = builtin.Description
|
||
updated.Inputs = builtin.Inputs
|
||
updated.Outputs = builtin.Outputs
|
||
updated.ConfigSchema = builtin.ConfigSchema
|
||
updated.Image = builtin.Image
|
||
updated.SkillDefinition = builtin.SkillDefinition
|
||
updated.Hyperparams = builtin.Hyperparams
|
||
updated.Resources = builtin.Resources
|
||
updated.UpdatedAt = builtin.UpdatedAt
|
||
if strings.TrimSpace(updated.CreatedAt) == "" {
|
||
updated.CreatedAt = builtin.CreatedAt
|
||
}
|
||
if err := server.store.UpsertOperator(ctx, updated); err != nil {
|
||
return items, err
|
||
}
|
||
|
||
for index := range items {
|
||
if items[index].ID == existing.ID || strings.TrimSpace(items[index].Name) == strings.TrimSpace(existing.Name) {
|
||
items[index] = updated
|
||
return items, nil
|
||
}
|
||
}
|
||
return append(items, updated), nil
|
||
}
|
||
|
||
func (server *server) ensureBuiltinOperators(ctx context.Context) error {
|
||
items, err := server.store.ListOperators(ctx)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
deletions, err := server.store.ListBuiltinOperatorDeletions(ctx)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
deletionSet := newBuiltinOperatorDeletionSet(deletions)
|
||
|
||
now := nowISO()
|
||
for _, builtin := range builtinOperatorRecords(now, server.operatorRunnerImage) {
|
||
if deletionSet.Has(builtin) {
|
||
continue
|
||
}
|
||
items, err = upsertBuiltinOperator(ctx, server, items, builtin)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
}
|
||
return server.refreshBuiltinOperatorImages(ctx, deletionSet)
|
||
}
|
||
|
||
func (server *server) refreshBuiltinOperatorImages(ctx context.Context, deletions builtinOperatorDeletionSet) error {
|
||
runnerImage := strings.TrimSpace(server.operatorRunnerImage)
|
||
if runnerImage == "" {
|
||
return nil
|
||
}
|
||
items, err := server.store.ListOperators(ctx)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
now := nowISO()
|
||
for _, item := range items {
|
||
if strings.TrimSpace(strings.ToLower(item.Type)) != "builtin" {
|
||
continue
|
||
}
|
||
if deletions.Has(item) {
|
||
continue
|
||
}
|
||
if strings.TrimSpace(item.Image) == runnerImage {
|
||
continue
|
||
}
|
||
item.Image = runnerImage
|
||
item.UpdatedAt = now
|
||
if strings.TrimSpace(item.CreatedAt) == "" {
|
||
item.CreatedAt = now
|
||
}
|
||
if err := server.store.UpsertOperator(ctx, item); err != nil {
|
||
return err
|
||
}
|
||
}
|
||
return nil
|
||
}
|
||
|
||
func (server *server) listAvailableOperators(ctx context.Context) ([]domain.OperatorRecord, error) {
|
||
if err := server.ensureBuiltinOperators(ctx); err != nil {
|
||
return nil, err
|
||
}
|
||
items, err := server.store.ListOperators(ctx)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
deletions, err := server.store.ListBuiltinOperatorDeletions(ctx)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
return filterDeletedBuiltinOperators(items, newBuiltinOperatorDeletionSet(deletions)), nil
|
||
}
|