diff --git a/backend/operator-runner/main.py b/backend/operator-runner/main.py index 50db25d1..8e7b672d 100644 --- a/backend/operator-runner/main.py +++ b/backend/operator-runner/main.py @@ -195,6 +195,18 @@ def load_input_artifacts() -> list[dict[str, Any]]: return [] +def load_input_bindings() -> list[dict[str, Any]]: + input_bindings_path = os.getenv("INPUT_BINDINGS_PATH", "").strip() + if input_bindings_path: + loaded = load_json_file(input_bindings_path, []) + if isinstance(loaded, list): + return json_compatible(loaded) + loaded = load_json_env("INPUT_BINDINGS_JSON", []) + if isinstance(loaded, list): + return json_compatible(loaded) + return [] + + def _first_numeric(payload: dict[str, Any], keys: tuple[str, ...]) -> int | float | None: for key in keys: value = payload.get(key) @@ -5410,6 +5422,7 @@ def main() -> int: node_data = load_json_env("NODE_DATA_JSON", {}) inputs = load_input_artifacts() + input_bindings = load_input_bindings() started_at = now_iso() start_cpu, start_memory = resource_snapshot() sampler = ResourceSampler(start_cpu, start_memory, started_at) @@ -5421,7 +5434,10 @@ def main() -> int: logs.append({"timestamp": now_iso(), "level": level, "message": message, "operator": node_label}) log(f"[operator-runner][{level.lower()}] {message}") - append_log("INFO", f"run={run_id} workflow={workflow_id} node={node_id} output_dir={output_dir}") + append_log( + "INFO", + f"run={run_id} workflow={workflow_id} node={node_id} output_dir={output_dir} input_bindings={len(input_bindings)}", + ) append_log("INFO", "Preparing operator execution context") try: diff --git a/backend/operator-runner/test_main.py b/backend/operator-runner/test_main.py index 4912b6ce..1e3cc08c 100644 --- a/backend/operator-runner/test_main.py +++ b/backend/operator-runner/test_main.py @@ -482,6 +482,19 @@ class RuntimeFileContractTests(unittest.TestCase): self.assertEqual(len(loaded), 1) self.assertEqual(loaded[0]["id"], "artifact-from-file") + def test_load_input_bindings_preserves_target_port_and_payload(self): + with tempfile.TemporaryDirectory() as temp_dir: + manifest_path = pathlib.Path(temp_dir) / "input-bindings.json" + manifest_path.write_text( + '[{"targetPortId":"config","workflowInputId":"config","payload":{"mode":"inline","type":"json","inline":{"ok":true}}}]', + encoding="utf-8", + ) + with mock.patch.dict(os.environ, {"INPUT_BINDINGS_PATH": str(manifest_path)}, clear=False): + loaded = MODULE.load_input_bindings() + + self.assertEqual(loaded[0]["targetPortId"], "config") + self.assertTrue(loaded[0]["payload"]["inline"]["ok"]) + def test_main_writes_result_and_stage_manifest_files(self): with tempfile.TemporaryDirectory() as temp_dir: node_root = pathlib.Path(temp_dir) diff --git a/docs/api.md b/docs/api.md index c9cf8086..e1adf428 100644 --- a/docs/api.md +++ b/docs/api.md @@ -105,6 +105,8 @@ Project names are trimmed and required on create and update. Project status defa - `PUT /api/workflows/:id` - `DELETE /api/workflows/:id` - `POST /api/workflows/:id/run` +- `POST /api/workflows/:id/release` +- `POST /api/workflows/:id/versions` `PUT /api/workflows/:id` @@ -127,6 +129,33 @@ Workflow names are trimmed and required on create and update. Workflow `type` ac `POST /api/workflows/:id/run` +工作流可通过 `workflowInputs` 把没有前置边节点的输入端口暴露为运行接口。输入端口支持 `required`;已发布定义要求根节点必选端口绑定到工作流输入,内部必选端口绑定到上游边。运行请求携带本次具体值: + +```json +{ + "trigger": "manual", + "inputs": { + "config": { "mode": "inline", "type": "json", "inline": { "threshold": 0.5 } }, + "dataset": { + "mode": "ref", + "type": "file", + "ref": { + "provider": "BOS", + "connectionId": "storage-1", + "bucket": "robot-data", + "kind": "directory", + "prefix": "incoming/session-1/", + "manifestKey": "incoming/session-1/manifest.json" + } + } + } +} +``` + +`inline` 仅允许 JSON,序列化后最多 64 KiB;`ref` 只保存无凭证的对象存储定位和完整性元数据。目录 ref 可以暂不带 `manifestKey`,Worker 会在根节点启动前列举 prefix 并写入本次 run 的快照 manifest。 + +工作流定义另有 `definitionState: draft | released`,不复用会随运行变化的 `status`。`release` 完成必选输入和 artifact 存储校验后冻结定义;released 工作流继续接受不同的根输入运行,但 `PUT` 返回 `workflow_definition_locked`。`versions` 从 released 定义创建同族的新草稿版本。 + ```json { "trigger": "manual" diff --git a/docs/operator-authoring.md b/docs/operator-authoring.md index 9e856826..69515564 100644 --- a/docs/operator-authoring.md +++ b/docs/operator-authoring.md @@ -18,7 +18,7 @@ Drobodata 的算子执行分为三层: 2. Go worker 负责调度 DAG、收集上游 artifact、启动节点执行 3. Python operator-runner 真正执行算子逻辑,并把结果回传给 worker -注意:算子之间传输的不是共享内存,也不是直接传文件句柄,而是 `artifact`。 +注意:算子之间传输的不是跨 Job 共享内存或直接文件句柄,而是带端口语义的 artifact。新协议支持小 JSON 的值传递(`payload.mode=inline`)以及对象、目录、大 JSON 和二进制的引用传递(`payload.mode=ref`)。 执行链路如下: @@ -26,10 +26,10 @@ Drobodata 的算子执行分为三层: 2. 上游节点产出 `ExecutionArtifact[]` 3. worker 把这些 artifact 持久化到当前 run 的执行记录中 4. 下游节点启动前,worker 按 DAG 边关系筛选对应的 artifact -5. worker 把这些 artifact 写入共享工作区的 `input-artifacts.json` +5. worker 把这些 artifact 写入共享工作区的 `input-artifacts.json`,并把按 `targetPortId` 分组后的绑定写入 `input-bindings.json` 6. worker 通过 `INPUT_ARTIFACTS_PATH` 把输入文件路径传给下游算子 -也就是说,跨算子的数据传输核心仍然是 artifact 契约,但主链路已经改成“共享 PVC 上的文件化 manifest + 轻量环境变量指针”,不再依赖大体积 env 直传。 +算子可通过 `INPUT_BINDINGS_PATH` 获得端口到 payload/artifact 的准确映射;为兼容历史算子,`INPUT_ARTIFACTS_PATH` 仍保留扁平列表。小清单会另外提供 `INPUT_BINDINGS_JSON` / `INPUT_ARTIFACTS_JSON`,超过 64 KiB 时只传文件路径。 ## 2. 算子运行时会收到什么 @@ -109,6 +109,12 @@ worker 再通过结果文件读取日志、artifact、资源采样,并回写 artifact 是算子之间的统一数据交换格式。一个 artifact 可能包含以下字段: +- `payload.mode=inline`:仅用于不超过 64 KiB 的 JSON 值,并随 run 记录持久化 +- `payload.mode=ref`:保存 `connectionId/bucket/key|prefix/manifestKey/versionId/etag/sha256` +- `ref.kind=object|directory`:区分单对象和整个目录;目录 manifest 最后写入,作为完整提交标记 + +Worker 在节点完成后统一规范化输出:有 artifact 存储策略时,大 JSON 和二进制上传 BOS;目录先上传文件再写 manifest;已返回合法 ref 的算子不会被重复上传。发布后的工作流必须能解析平台默认或工作流覆盖的 artifact 存储。 + - `id` - `runId` - `nodeId` diff --git a/internal/api/server.go b/internal/api/server.go index d66825da..63d22c02 100644 --- a/internal/api/server.go +++ b/internal/api/server.go @@ -353,6 +353,7 @@ func sanitizeWorkflow(item *domain.WorkflowRecord) *domain.WorkflowRecord { return nil } copy := *item + domain.NormalizeWorkflowDefinitionMetadata(©) copy.ExecutionScope = normalizeWorkflowExecutionScope(copy.ExecutionScope) copy.Nodes = cloneWorkflowNodes(item.Nodes) copy.ScanConfig = sanitizeScanConfig(copy.ScanConfig) @@ -366,6 +367,7 @@ func sanitizeWorkflows(items []domain.WorkflowRecord) []domain.WorkflowRecord { sanitized := make([]domain.WorkflowRecord, 0, len(items)) for _, item := range items { copy := item + domain.NormalizeWorkflowDefinitionMetadata(©) copy.ExecutionScope = normalizeWorkflowExecutionScope(copy.ExecutionScope) copy.Nodes = cloneWorkflowNodes(item.Nodes) copy.ScanConfig = sanitizeScanConfig(copy.ScanConfig) @@ -1124,6 +1126,15 @@ func sanitizeTemplate(item *domain.TemplateRecord) *domain.TemplateRecord { return nil } copy := *item + if copy.DefinitionState == "" { + copy.DefinitionState = domain.WorkflowDefinitionStateDraft + } + if copy.DefinitionVersion <= 0 { + copy.DefinitionVersion = 1 + } + if strings.TrimSpace(copy.DefinitionFamilyID) == "" { + copy.DefinitionFamilyID = copy.ID + } copy.ExecutionScope = normalizeWorkflowExecutionScope(copy.ExecutionScope) copy.Nodes = cloneWorkflowNodes(item.Nodes) copy.ScanConfig = sanitizeScanConfig(copy.ScanConfig) @@ -1137,6 +1148,15 @@ func sanitizeTemplates(items []domain.TemplateRecord) []domain.TemplateRecord { sanitized := make([]domain.TemplateRecord, 0, len(items)) for _, item := range items { copy := item + if copy.DefinitionState == "" { + copy.DefinitionState = domain.WorkflowDefinitionStateDraft + } + if copy.DefinitionVersion <= 0 { + copy.DefinitionVersion = 1 + } + if strings.TrimSpace(copy.DefinitionFamilyID) == "" { + copy.DefinitionFamilyID = copy.ID + } copy.ExecutionScope = normalizeWorkflowExecutionScope(copy.ExecutionScope) copy.Nodes = cloneWorkflowNodes(item.Nodes) copy.ScanConfig = sanitizeScanConfig(copy.ScanConfig) @@ -1339,6 +1359,8 @@ func NewHandlerWithConfig(dataStore store.Store, aiService ai.Service, appConfig router.Put("/api/projects/{id}/storage-path-favorites", instance.handleProjectStoragePathFavoritesUpdate) router.Post("/api/projects/{id}/workflows", instance.handleProjectWorkflowsCreate) router.Put("/api/workflows/{id}", instance.handleWorkflowsUpdate) + router.Post("/api/workflows/{id}/release", instance.handleWorkflowRelease) + router.Post("/api/workflows/{id}/versions", instance.handleWorkflowCreateVersion) router.Delete("/api/workflows/{id}", instance.handleWorkflowsDelete) router.Delete("/api/workflows/{id}/nodes/{nodeId}/source-state", instance.handleWorkflowNodeSourceStateDelete) router.Post("/api/workflows/{id}/run", instance.handleWorkflowRun) @@ -3260,6 +3282,14 @@ func (server *server) handleProjectWorkflowsCreate(writer http.ResponseWriter, r renderValidationError(writer, issues...) return } + if issues := validateWorkflowInputContract(payload.Nodes, payload.Edges, payload.WorkflowInputs, false); len(issues) > 0 { + renderValidationError(writer, issues...) + return + } + if issues := server.validateArtifactStoragePolicy(request.Context(), payload.OrganizationID, payload.ArtifactStorage, "artifactStorage"); len(issues) > 0 { + renderValidationError(writer, issues...) + return + } if issues := server.validateWorkflowOperatorPublication(request.Context(), payload.Nodes, payload.ExecutionScope); len(issues) > 0 { renderValidationError(writer, issues...) return @@ -3343,22 +3373,24 @@ func (server *server) handleWorkflowNodeSourceStateDelete(writer http.ResponseWr } type workflowPatch struct { - Name *string `json:"name"` - Description *string `json:"description"` - Type *domain.WorkflowType `json:"type"` - Kind *domain.WorkflowKind `json:"kind"` - ExecutionScope *domain.WorkflowExecutionScope `json:"executionScope"` - ScanConfig *domain.ScanConfig `json:"scanConfig"` - Hyperparams *[]domain.HyperParam `json:"hyperparams"` - MaxConcurrentRuns *int `json:"maxConcurrentRuns"` - NodeTimeoutSeconds *int `json:"nodeTimeoutSeconds"` - RunTimeoutSeconds *int `json:"runTimeoutSeconds"` - MaxRetryAttempts *int `json:"maxRetryAttempts"` - RetryBackoffSeconds *int `json:"retryBackoffSeconds"` - Schedule *domain.WorkflowSchedule `json:"schedule"` - Nodes *[]domain.WorkflowNode `json:"nodes"` - Edges *[]domain.WorkflowEdge `json:"edges"` - Status *domain.WorkflowStatus `json:"status"` + Name *string `json:"name"` + Description *string `json:"description"` + Type *domain.WorkflowType `json:"type"` + Kind *domain.WorkflowKind `json:"kind"` + ExecutionScope *domain.WorkflowExecutionScope `json:"executionScope"` + ScanConfig *domain.ScanConfig `json:"scanConfig"` + Hyperparams *[]domain.HyperParam `json:"hyperparams"` + MaxConcurrentRuns *int `json:"maxConcurrentRuns"` + NodeTimeoutSeconds *int `json:"nodeTimeoutSeconds"` + RunTimeoutSeconds *int `json:"runTimeoutSeconds"` + MaxRetryAttempts *int `json:"maxRetryAttempts"` + RetryBackoffSeconds *int `json:"retryBackoffSeconds"` + Schedule *domain.WorkflowSchedule `json:"schedule"` + Nodes *[]domain.WorkflowNode `json:"nodes"` + Edges *[]domain.WorkflowEdge `json:"edges"` + Status *domain.WorkflowStatus `json:"status"` + WorkflowInputs *[]domain.WorkflowInputDefinition `json:"workflowInputs"` + ArtifactStorage *domain.ArtifactStoragePolicy `json:"artifactStorage"` } func scanConfigRequestFromWorkflow(name string, description string, schedule *domain.WorkflowSchedule, scanConfig *domain.ScanConfig) scanConfigRequest { @@ -3664,6 +3696,56 @@ func validateWorkflowTopology(nodes []domain.WorkflowNode, edges []domain.Workfl return issues } +func validateWorkflowInputContract(nodes []domain.WorkflowNode, edges []domain.WorkflowEdge, inputs []domain.WorkflowInputDefinition, release bool) []validationIssue { + contractIssues := domain.ValidateWorkflowInputContract(nodes, edges, inputs, release) + issues := make([]validationIssue, 0, len(contractIssues)) + for _, item := range contractIssues { + issues = append(issues, validationIssue{Field: item.Field, Code: item.Code, Message: item.Message}) + } + return issues +} + +func (server *server) validateArtifactStoragePolicy(ctx context.Context, organizationID string, policy *domain.ArtifactStoragePolicy, field string) []validationIssue { + if policy == nil { + return nil + } + issues := make([]validationIssue, 0, 2) + policy.ConnectionID = strings.TrimSpace(policy.ConnectionID) + policy.Bucket = strings.TrimSpace(policy.Bucket) + policy.PrefixTemplate = strings.Trim(strings.TrimSpace(policy.PrefixTemplate), "/") + if policy.ConnectionID == "" && policy.Bucket == "" && policy.PrefixTemplate == "" { + return nil + } + if policy.ConnectionID == "" { + issues = append(issues, validationIssue{Field: field + ".connectionId", Code: "artifact_storage_connection_required", Message: "artifact storage connectionId is required"}) + } + if policy.Bucket == "" { + issues = append(issues, validationIssue{Field: field + ".bucket", Code: "artifact_storage_bucket_required", Message: "artifact storage bucket is required"}) + } + if len(issues) > 0 { + return issues + } + connection, err := server.store.GetStorageConnection(ctx, policy.ConnectionID) + if err != nil || connection == nil || normalizedTenantID(connection.OrganizationID) != normalizedTenantID(organizationID) { + return []validationIssue{{Field: field + ".connectionId", Code: "artifact_storage_connection_invalid", Message: "artifact storage connectionId must reference an accessible storage connection"}} + } + return nil +} + +func (server *server) validateRunInputConnections(ctx context.Context, organizationID string, inputs map[string]domain.ArtifactPayload) []validationIssue { + issues := make([]validationIssue, 0) + for inputID, payload := range inputs { + if payload.Mode != domain.ArtifactTransferModeRef || payload.Ref == nil { + continue + } + connection, err := server.store.GetStorageConnection(ctx, strings.TrimSpace(payload.Ref.ConnectionID)) + if err != nil || connection == nil || normalizedTenantID(connection.OrganizationID) != normalizedTenantID(organizationID) { + issues = append(issues, validationIssue{Field: "inputs." + inputID + ".ref.connectionId", Code: "workflow_input_connection_invalid", Message: "input ref connectionId must reference an accessible storage connection"}) + } + } + return issues +} + func validateWorkflowNodeResources(nodes []domain.WorkflowNode) []validationIssue { issues := make([]validationIssue, 0) for index, node := range nodes { @@ -3746,6 +3828,14 @@ func (server *server) handleWorkflowsUpdate(writer http.ResponseWriter, request if !server.ensureTenantAccess(writer, request, item.OrganizationID, "Workflow") { return } + domain.NormalizeWorkflowDefinitionMetadata(item) + if item.DefinitionState == domain.WorkflowDefinitionStateReleased { + renderJSON(writer, http.StatusConflict, map[string]string{ + "error": "工作流定义已发布并锁定,请创建新版本后修改", + "code": "workflow_definition_locked", + }) + return + } active, err := server.store.HasActiveRun(request.Context(), item.ID) if err != nil { renderError(writer, http.StatusInternalServerError, err) @@ -3857,6 +3947,16 @@ func (server *server) handleWorkflowsUpdate(writer http.ResponseWriter, request if payload.Edges != nil { item.Edges = *payload.Edges } + if payload.WorkflowInputs != nil { + item.WorkflowInputs = *payload.WorkflowInputs + } + if payload.ArtifactStorage != nil { + if strings.TrimSpace(payload.ArtifactStorage.ConnectionID) == "" && strings.TrimSpace(payload.ArtifactStorage.Bucket) == "" { + item.ArtifactStorage = nil + } else { + item.ArtifactStorage = payload.ArtifactStorage + } + } if item.Kind == domain.WorkflowKindScan || item.ScanConfig != nil || payload.ScanConfig != nil || domain.FindScanSourceNode(item.Nodes) != nil { existingScanConfig := item.ScanConfig effectiveScanConfig := item.ScanConfig @@ -3890,6 +3990,14 @@ func (server *server) handleWorkflowsUpdate(writer http.ResponseWriter, request renderValidationError(writer, issues...) return } + if issues := validateWorkflowInputContract(item.Nodes, item.Edges, item.WorkflowInputs, false); len(issues) > 0 { + renderValidationError(writer, issues...) + return + } + if issues := server.validateArtifactStoragePolicy(request.Context(), item.OrganizationID, item.ArtifactStorage, "artifactStorage"); len(issues) > 0 { + renderValidationError(writer, issues...) + return + } if issues := server.validateWorkflowOperatorPublication(request.Context(), item.Nodes, item.ExecutionScope); len(issues) > 0 { renderValidationError(writer, issues...) return @@ -3902,6 +4010,137 @@ func (server *server) handleWorkflowsUpdate(writer http.ResponseWriter, request renderJSON(writer, http.StatusOK, sanitizeWorkflow(item)) } +func (server *server) effectiveArtifactStorage(ctx context.Context, workflow *domain.WorkflowRecord) (*domain.ArtifactStoragePolicy, error) { + if workflow != nil && workflow.ArtifactStorage != nil && strings.TrimSpace(workflow.ArtifactStorage.ConnectionID) != "" { + copy := *workflow.ArtifactStorage + return ©, nil + } + settings, err := server.store.GetSettings(ctx) + if err != nil { + return nil, err + } + if settings == nil || strings.TrimSpace(settings.ArtifactStorage.ConnectionID) == "" || strings.TrimSpace(settings.ArtifactStorage.Bucket) == "" { + return nil, nil + } + return &domain.ArtifactStoragePolicy{ + ConnectionID: settings.ArtifactStorage.ConnectionID, + Bucket: settings.ArtifactStorage.Bucket, + PrefixTemplate: settings.ArtifactStorage.PrefixTemplate, + }, nil +} + +func (server *server) handleWorkflowRelease(writer http.ResponseWriter, request *http.Request) { + workflow, err := server.store.GetWorkflow(request.Context(), chi.URLParam(request, "id")) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + if workflow == nil { + renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Workflow not found"}) + return + } + if !server.ensureTenantAccess(writer, request, workflow.OrganizationID, "Workflow") { + return + } + domain.NormalizeWorkflowDefinitionMetadata(workflow) + if workflow.DefinitionState == domain.WorkflowDefinitionStateReleased { + renderJSON(writer, http.StatusOK, sanitizeWorkflow(workflow)) + return + } + if active, err := server.store.HasActiveRun(request.Context(), workflow.ID); err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } else if active { + renderJSON(writer, http.StatusConflict, map[string]string{"error": "工作流运行中,不能发布定义", "code": "workflow_run_active"}) + return + } + issues := validateWorkflowTopology(workflow.Nodes, workflow.Edges) + issues = append(issues, validateWorkflowInputContract(workflow.Nodes, workflow.Edges, workflow.WorkflowInputs, true)...) + if domain.WorkflowScheduleEnabled(workflow.Schedule) { + for index, input := range workflow.WorkflowInputs { + if input.Required && input.DefaultPayload == nil { + issues = append(issues, validationIssue{Field: fmt.Sprintf("workflowInputs[%d].defaultPayload", index), Code: "scheduled_input_default_required", Message: "scheduled workflows require defaults for every required workflow input"}) + } + } + } + storage, storageErr := server.effectiveArtifactStorage(request.Context(), workflow) + if storageErr != nil { + renderError(writer, http.StatusInternalServerError, storageErr) + return + } + if storage == nil { + issues = append(issues, validationIssue{Field: "artifactStorage", Code: "artifact_storage_required", Message: "released workflows require platform or workflow artifact storage"}) + } else { + issues = append(issues, server.validateArtifactStoragePolicy(request.Context(), workflow.OrganizationID, storage, "artifactStorage")...) + } + if len(issues) > 0 { + renderValidationError(writer, issues...) + return + } + workflow.DefinitionState = domain.WorkflowDefinitionStateReleased + workflow.UpdatedAt = nowISO() + if err := server.store.UpsertWorkflow(request.Context(), *workflow); err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + renderJSON(writer, http.StatusOK, sanitizeWorkflow(workflow)) +} + +func (server *server) handleWorkflowCreateVersion(writer http.ResponseWriter, request *http.Request) { + source, err := server.store.GetWorkflow(request.Context(), chi.URLParam(request, "id")) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + if source == nil { + renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Workflow not found"}) + return + } + if !server.ensureTenantAccess(writer, request, source.OrganizationID, "Workflow") { + return + } + domain.NormalizeWorkflowDefinitionMetadata(source) + if source.DefinitionState != domain.WorkflowDefinitionStateReleased { + renderJSON(writer, http.StatusConflict, map[string]string{"error": "only released workflows can create a new version", "code": "workflow_not_released"}) + return + } + created, err := server.store.CreateWorkflow(request.Context(), domain.CreateWorkflowInput{ + OrganizationID: source.OrganizationID, + ProjectID: source.ProjectID, + Name: source.Name, + Description: source.Description, + Type: source.Type, + Kind: source.Kind, + Hidden: source.Hidden, + ExecutionScope: source.ExecutionScope, + ScanConfig: source.ScanConfig, + Hyperparams: source.Hyperparams, + MaxConcurrentRuns: source.MaxConcurrentRuns, + NodeTimeoutSeconds: source.NodeTimeoutSeconds, + RunTimeoutSeconds: source.RunTimeoutSeconds, + MaxRetryAttempts: source.MaxRetryAttempts, + RetryBackoffSeconds: source.RetryBackoffSeconds, + Nodes: source.Nodes, + Edges: source.Edges, + Schedule: source.Schedule, + WorkflowInputs: source.WorkflowInputs, + ArtifactStorage: source.ArtifactStorage, + }) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + created.DefinitionVersion = source.DefinitionVersion + 1 + created.DefinitionFamilyID = source.DefinitionFamilyID + created.SupersedesWorkflowID = source.ID + created.DefinitionState = domain.WorkflowDefinitionStateDraft + if err := server.store.UpsertWorkflow(request.Context(), *created); err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + renderJSON(writer, http.StatusCreated, sanitizeWorkflow(created)) +} + func (server *server) handleWorkflowsDelete(writer http.ResponseWriter, request *http.Request) { item, err := server.store.GetWorkflow(request.Context(), chi.URLParam(request, "id")) if err != nil { @@ -3928,7 +4167,8 @@ func (server *server) handleWorkflowsDelete(writer http.ResponseWriter, request } type runRequest struct { - Trigger domain.RunTrigger `json:"trigger"` + Trigger domain.RunTrigger `json:"trigger"` + Inputs map[string]domain.ArtifactPayload `json:"inputs,omitempty"` } func validRunTrigger(value domain.RunTrigger) bool { @@ -3936,6 +4176,10 @@ func validRunTrigger(value domain.RunTrigger) bool { } func (server *server) createRunWithLightweightManifests(ctx context.Context, workflow *domain.WorkflowRecord, trigger domain.RunTrigger, taskIDs ...string) (*domain.RunRecord, error) { + return server.createRunWithInputsAndLightweightManifests(ctx, workflow, trigger, nil, taskIDs...) +} + +func (server *server) createRunWithInputsAndLightweightManifests(ctx context.Context, workflow *domain.WorkflowRecord, trigger domain.RunTrigger, inputs map[string]domain.ArtifactPayload, taskIDs ...string) (*domain.RunRecord, error) { if workflow == nil { return nil, fmt.Errorf("workflow not found") } @@ -3943,7 +4187,7 @@ func (server *server) createRunWithLightweightManifests(ctx context.Context, wor if err != nil { return nil, err } - run, err := server.store.CreateRun(ctx, workflow.ID, trigger, taskIDs...) + run, err := server.store.CreateRunWithInputs(ctx, workflow.ID, trigger, inputs, taskIDs...) if err != nil { return nil, err } @@ -4034,11 +4278,24 @@ func (server *server) handleWorkflowRun(writer http.ResponseWriter, request *htt renderValidationError(writer, issues...) return } + if issues := validateWorkflowInputContract(workflow.Nodes, workflow.Edges, workflow.WorkflowInputs, true); len(issues) > 0 { + renderValidationError(writer, issues...) + return + } + resolvedInputs, _, resolveErr := domain.ResolveWorkflowRunInputs(*workflow, payload.Inputs) + if resolveErr != nil { + renderValidationError(writer, validationIssue{Field: "inputs", Code: "workflow_inputs_invalid", Message: resolveErr.Error()}) + return + } + if issues := server.validateRunInputConnections(request.Context(), workflow.OrganizationID, resolvedInputs); len(issues) > 0 { + renderValidationError(writer, issues...) + return + } if issues := server.validateWorkflowOperatorPublication(request.Context(), workflow.Nodes, normalizeWorkflowExecutionScope(workflow.ExecutionScope)); len(issues) > 0 { renderValidationError(writer, issues...) return } - run, err := server.createRunWithLightweightManifests(request.Context(), workflow, payload.Trigger) + run, err := server.createRunWithInputsAndLightweightManifests(request.Context(), workflow, payload.Trigger, payload.Inputs) if err != nil { if renderRunCreationError(writer, err) { return @@ -4176,7 +4433,7 @@ func (server *server) handleRunsRerunSubtree(writer http.ResponseWriter, request } } - childRun, err := server.createRunWithLightweightManifests(request.Context(), workflow, domain.RunTriggerRerunSubtree, sourceRun.TaskID) + childRun, err := server.createRunWithInputsAndLightweightManifests(request.Context(), workflow, domain.RunTriggerRerunSubtree, sourceRun.Inputs, sourceRun.TaskID) if err != nil { if renderRunCreationError(writer, err) { return @@ -4284,6 +4541,14 @@ func (server *server) handleTemplatesCreate(writer http.ResponseWriter, request renderValidationError(writer, issues...) return } + if issues := validateWorkflowInputContract(payload.Nodes, payload.Edges, payload.WorkflowInputs, false); len(issues) > 0 { + renderValidationError(writer, issues...) + return + } + if issues := server.validateArtifactStoragePolicy(request.Context(), payload.OrganizationID, payload.ArtifactStorage, "artifactStorage"); len(issues) > 0 { + renderValidationError(writer, issues...) + return + } if issues := server.validateWorkflowOperatorPublication(request.Context(), payload.Nodes, payload.ExecutionScope); len(issues) > 0 { renderValidationError(writer, issues...) return @@ -4313,21 +4578,23 @@ func (server *server) handleTemplatesGet(writer http.ResponseWriter, request *ht } type templatePatch struct { - Name *string `json:"name"` - Description *string `json:"description"` - Type *domain.WorkflowType `json:"type"` - Kind *domain.WorkflowKind `json:"kind"` - ExecutionScope *domain.WorkflowExecutionScope `json:"executionScope"` - ScanConfig *domain.ScanConfig `json:"scanConfig"` - Hyperparams *[]domain.HyperParam `json:"hyperparams"` - MaxConcurrentRuns *int `json:"maxConcurrentRuns"` - NodeTimeoutSeconds *int `json:"nodeTimeoutSeconds"` - RunTimeoutSeconds *int `json:"runTimeoutSeconds"` - MaxRetryAttempts *int `json:"maxRetryAttempts"` - RetryBackoffSeconds *int `json:"retryBackoffSeconds"` - Schedule *domain.WorkflowSchedule `json:"schedule"` - Nodes *[]domain.WorkflowNode `json:"nodes"` - Edges *[]domain.WorkflowEdge `json:"edges"` + Name *string `json:"name"` + Description *string `json:"description"` + Type *domain.WorkflowType `json:"type"` + Kind *domain.WorkflowKind `json:"kind"` + ExecutionScope *domain.WorkflowExecutionScope `json:"executionScope"` + ScanConfig *domain.ScanConfig `json:"scanConfig"` + Hyperparams *[]domain.HyperParam `json:"hyperparams"` + MaxConcurrentRuns *int `json:"maxConcurrentRuns"` + NodeTimeoutSeconds *int `json:"nodeTimeoutSeconds"` + RunTimeoutSeconds *int `json:"runTimeoutSeconds"` + MaxRetryAttempts *int `json:"maxRetryAttempts"` + RetryBackoffSeconds *int `json:"retryBackoffSeconds"` + Schedule *domain.WorkflowSchedule `json:"schedule"` + Nodes *[]domain.WorkflowNode `json:"nodes"` + Edges *[]domain.WorkflowEdge `json:"edges"` + WorkflowInputs *[]domain.WorkflowInputDefinition `json:"workflowInputs"` + ArtifactStorage *domain.ArtifactStoragePolicy `json:"artifactStorage"` } func (server *server) handleTemplatesUpdate(writer http.ResponseWriter, request *http.Request) { @@ -4433,6 +4700,16 @@ func (server *server) handleTemplatesUpdate(writer http.ResponseWriter, request if payload.Edges != nil { item.Edges = *payload.Edges } + if payload.WorkflowInputs != nil { + item.WorkflowInputs = *payload.WorkflowInputs + } + if payload.ArtifactStorage != nil { + if strings.TrimSpace(payload.ArtifactStorage.ConnectionID) == "" && strings.TrimSpace(payload.ArtifactStorage.Bucket) == "" { + item.ArtifactStorage = nil + } else { + item.ArtifactStorage = payload.ArtifactStorage + } + } if item.Kind == domain.WorkflowKindScan || item.ScanConfig != nil || payload.ScanConfig != nil || domain.FindScanSourceNode(item.Nodes) != nil { existingScanConfig := item.ScanConfig effectiveScanConfig := item.ScanConfig @@ -4462,6 +4739,14 @@ func (server *server) handleTemplatesUpdate(writer http.ResponseWriter, request renderValidationError(writer, issues...) return } + if issues := validateWorkflowInputContract(item.Nodes, item.Edges, item.WorkflowInputs, false); len(issues) > 0 { + renderValidationError(writer, issues...) + return + } + if issues := server.validateArtifactStoragePolicy(request.Context(), item.OrganizationID, item.ArtifactStorage, "artifactStorage"); len(issues) > 0 { + renderValidationError(writer, issues...) + return + } if issues := server.validateWorkflowOperatorPublication(request.Context(), item.Nodes, item.ExecutionScope); len(issues) > 0 { renderValidationError(writer, issues...) return @@ -4698,6 +4983,7 @@ func normalizePortDefinitions(items []domain.PortDefinition) []domain.PortDefini ID: strings.TrimSpace(item.ID), Name: strings.TrimSpace(item.Name), Type: domain.PortType(strings.ToLower(strings.TrimSpace(string(item.Type)))), + Required: item.Required, Description: strings.TrimSpace(item.Description), Transmission: strings.ToLower(strings.TrimSpace(item.Transmission)), }) @@ -7621,6 +7907,11 @@ type settingsPatch struct { PollIntervalMS *int `json:"pollIntervalMs"` MaxConcurrentRuns *int `json:"maxConcurrentRuns"` } `json:"worker"` + ArtifactStorage *struct { + ConnectionID *string `json:"connectionId"` + Bucket *string `json:"bucket"` + PrefixTemplate *string `json:"prefixTemplate"` + } `json:"artifactStorage"` AI *struct { Provider *string `json:"provider"` BaseURL *string `json:"baseURL"` @@ -7849,6 +8140,28 @@ func (server *server) handleSettingsUpdate(writer http.ResponseWriter, request * item.K8s.Kubeconfig = strings.TrimSpace(*payload.K8s.Kubeconfig) } } + if payload.ArtifactStorage != nil { + if payload.ArtifactStorage.ConnectionID != nil { + item.ArtifactStorage.ConnectionID = strings.TrimSpace(*payload.ArtifactStorage.ConnectionID) + } + if payload.ArtifactStorage.Bucket != nil { + item.ArtifactStorage.Bucket = strings.TrimSpace(*payload.ArtifactStorage.Bucket) + } + if payload.ArtifactStorage.PrefixTemplate != nil { + item.ArtifactStorage.PrefixTemplate = strings.Trim(strings.TrimSpace(*payload.ArtifactStorage.PrefixTemplate), "/") + } + if item.ArtifactStorage.ConnectionID != "" || item.ArtifactStorage.Bucket != "" { + policy := &domain.ArtifactStoragePolicy{ + ConnectionID: item.ArtifactStorage.ConnectionID, + Bucket: item.ArtifactStorage.Bucket, + PrefixTemplate: item.ArtifactStorage.PrefixTemplate, + } + if issues := server.validateArtifactStoragePolicy(request.Context(), item.OrganizationID, policy, "artifactStorage"); len(issues) > 0 { + renderValidationError(writer, issues...) + return + } + } + } if strings.TrimSpace(item.K8s.WorkspacePVCName) == "" { renderValidationError(writer, validationIssue{Field: "k8s.workspacePVCName", Message: "workspacePVCName is required"}) return @@ -8018,6 +8331,7 @@ func sanitizeSettings(item *domain.SettingsRecord) *domain.SettingsView { ID: item.ID, K8s: item.K8s, Worker: item.Worker, + ArtifactStorage: item.ArtifactStorage, AI: item.AI, OperatorGeneration: item.OperatorGeneration, BusinessMongo: item.BusinessMongo, diff --git a/internal/api/server_test.go b/internal/api/server_test.go index 98692c6a..0b41760f 100644 --- a/internal/api/server_test.go +++ b/internal/api/server_test.go @@ -11063,3 +11063,83 @@ func TestFreshAcceptanceSplitsBusinessAndOperatorCoverageGates(t *testing.T) { t.Fatalf("stale artifact must not remain passed or blocked: strict=%#v business=%#v coverage=%#v", strict, business, coverage) } } + +func TestWorkflowDefinitionReleaseLocksUpdatesAndCreatesNewVersion(t *testing.T) { + ctx := context.Background() + dataStore := store.NewMemoryStore() + project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "输入契约项目", Description: "workflow inputs"}) + if err != nil { + t.Fatalf("create project: %v", err) + } + connection, err := dataStore.CreateStorageConnection(ctx, domain.CreateStorageConnectionInput{ + Name: "Artifact BOS", Provider: domain.StorageProviderBOS, Endpoint: "https://bos.example.test", AccessKeyID: "ak", SecretAccessKey: "sk", + }) + if err != nil { + t.Fatalf("create storage connection: %v", err) + } + workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{ + ProjectID: project.ID, Name: "输入契约", Description: "release", Type: domain.WorkflowTypeBatch, + Nodes: []domain.WorkflowNode{{ID: "source", Type: "operator", Data: domain.WorkflowNodeData{ + Label: "Source", Inputs: []domain.PortDefinition{{ID: "config", Name: "Config", Type: domain.PortTypeJSON, Required: true}}, + }}}, + WorkflowInputs: []domain.WorkflowInputDefinition{{ + ID: "config", Name: "Config", Type: domain.PortTypeJSON, Required: true, + Bindings: []domain.WorkflowInputTarget{{NodeID: "source", TargetPortID: "config"}}, + }}, + ArtifactStorage: &domain.ArtifactStoragePolicy{ConnectionID: connection.ID, Bucket: "robot-data"}, + }) + if err != nil { + t.Fatalf("create workflow: %v", err) + } + handler := NewHandler(dataStore, nil) + release := httptest.NewRecorder() + handler.ServeHTTP(release, httptest.NewRequest(http.MethodPost, "/api/workflows/"+workflow.ID+"/release", nil)) + if release.Code != http.StatusOK || !strings.Contains(release.Body.String(), `"definitionState":"released"`) { + t.Fatalf("release workflow: status=%d body=%s", release.Code, release.Body.String()) + } + update := httptest.NewRecorder() + handler.ServeHTTP(update, httptest.NewRequest(http.MethodPut, "/api/workflows/"+workflow.ID, strings.NewReader(`{"name":"mutated"}`))) + if update.Code != http.StatusConflict || !strings.Contains(update.Body.String(), "workflow_definition_locked") { + t.Fatalf("released update should be locked: status=%d body=%s", update.Code, update.Body.String()) + } + version := httptest.NewRecorder() + handler.ServeHTTP(version, httptest.NewRequest(http.MethodPost, "/api/workflows/"+workflow.ID+"/versions", nil)) + if version.Code != http.StatusCreated || !strings.Contains(version.Body.String(), `"definitionVersion":2`) || !strings.Contains(version.Body.String(), `"definitionState":"draft"`) { + t.Fatalf("create workflow version: status=%d body=%s", version.Code, version.Body.String()) + } +} + +func TestWorkflowRunAcceptsAndPersistsInlineRootInput(t *testing.T) { + ctx := context.Background() + dataStore := store.NewMemoryStore() + project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "Run Input", Description: "inline"}) + if err != nil { + t.Fatalf("create project: %v", err) + } + workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{ + ProjectID: project.ID, Name: "Inline Run", Description: "inline", Type: domain.WorkflowTypeBatch, + Nodes: []domain.WorkflowNode{{ID: "source", Type: "operator", Data: domain.WorkflowNodeData{ + Label: "Source", Inputs: []domain.PortDefinition{{ID: "config", Name: "Config", Type: domain.PortTypeJSON, Required: true}}, + }}}, + WorkflowInputs: []domain.WorkflowInputDefinition{{ + ID: "config", Name: "Config", Type: domain.PortTypeJSON, Required: true, + Bindings: []domain.WorkflowInputTarget{{NodeID: "source", TargetPortID: "config"}}, + }}, + }) + if err != nil { + t.Fatalf("create workflow: %v", err) + } + payload := `{"trigger":"manual","inputs":{"config":{"mode":"inline","type":"json","inline":{"threshold":0.5}}}}` + recorder := httptest.NewRecorder() + NewHandler(dataStore, nil).ServeHTTP(recorder, httptest.NewRequest(http.MethodPost, "/api/workflows/"+workflow.ID+"/run", strings.NewReader(payload))) + if recorder.Code != http.StatusAccepted { + t.Fatalf("create run: status=%d body=%s", recorder.Code, recorder.Body.String()) + } + var run domain.RunRecord + if err := json.Unmarshal(recorder.Body.Bytes(), &run); err != nil { + t.Fatalf("decode run: %v", err) + } + if run.Inputs["config"].Mode != domain.ArtifactTransferModeInline || len(run.ResolvedInputBindings) != 1 || run.ResolvedInputBindings[0].TargetPortID != "config" { + t.Fatalf("run input snapshot missing: %#v", run) + } +} diff --git a/internal/domain/types.go b/internal/domain/types.go index 10b8cae0..47fc378a 100644 --- a/internal/domain/types.go +++ b/internal/domain/types.go @@ -146,10 +146,94 @@ type PortDefinition struct { ID string `json:"id" bson:"id"` Name string `json:"name" bson:"name"` Type PortType `json:"type" bson:"type"` + Required bool `json:"required" bson:"required"` Description string `json:"description,omitempty" bson:"description,omitempty"` Transmission string `json:"transmission,omitempty" bson:"transmission,omitempty"` } +type WorkflowDefinitionState string + +const ( + WorkflowDefinitionStateDraft WorkflowDefinitionState = "draft" + WorkflowDefinitionStateReleased WorkflowDefinitionState = "released" +) + +type ArtifactTransferMode string + +const ( + ArtifactTransferModeInline ArtifactTransferMode = "inline" + ArtifactTransferModeRef ArtifactTransferMode = "ref" +) + +type ArtifactRefKind string + +const ( + ArtifactRefKindObject ArtifactRefKind = "object" + ArtifactRefKindDirectory ArtifactRefKind = "directory" +) + +// ArtifactReference is a credential-free pointer to durable object storage. +type ArtifactReference struct { + Provider StorageProvider `json:"provider" bson:"provider"` + ConnectionID string `json:"connectionId" bson:"connectionId"` + Bucket string `json:"bucket" bson:"bucket"` + Key string `json:"key,omitempty" bson:"key,omitempty"` + Prefix string `json:"prefix,omitempty" bson:"prefix,omitempty"` + ManifestKey string `json:"manifestKey,omitempty" bson:"manifestKey,omitempty"` + VersionID string `json:"versionId,omitempty" bson:"versionId,omitempty"` + ETag string `json:"etag,omitempty" bson:"etag,omitempty"` + SizeBytes int64 `json:"sizeBytes,omitempty" bson:"sizeBytes,omitempty"` + SHA256 string `json:"sha256,omitempty" bson:"sha256,omitempty"` + MimeType string `json:"mimeType,omitempty" bson:"mimeType,omitempty"` + Kind ArtifactRefKind `json:"kind" bson:"kind"` +} + +// ArtifactPayload is the common value/reference carrier used by workflow +// inputs and execution artifacts. Mode selects exactly one of Inline and Ref. +type ArtifactPayload struct { + Mode ArtifactTransferMode `json:"mode" bson:"mode"` + Type PortType `json:"type" bson:"type"` + Inline any `json:"inline,omitempty" bson:"inline,omitempty"` + Ref *ArtifactReference `json:"ref,omitempty" bson:"ref,omitempty"` +} + +type WorkflowInputTarget struct { + NodeID string `json:"nodeId" bson:"nodeId"` + TargetPortID string `json:"targetPortId" bson:"targetPortId"` +} + +type WorkflowInputDefinition struct { + ID string `json:"id" bson:"id"` + Name string `json:"name" bson:"name"` + Description string `json:"description,omitempty" bson:"description,omitempty"` + Type PortType `json:"type" bson:"type"` + Required bool `json:"required" bson:"required"` + DefaultPayload *ArtifactPayload `json:"defaultPayload,omitempty" bson:"defaultPayload,omitempty"` + Bindings []WorkflowInputTarget `json:"bindings" bson:"bindings"` +} + +type ArtifactStoragePolicy struct { + ConnectionID string `json:"connectionId" bson:"connectionId"` + Bucket string `json:"bucket" bson:"bucket"` + PrefixTemplate string `json:"prefixTemplate,omitempty" bson:"prefixTemplate,omitempty"` +} + +type ResolvedInputBinding struct { + WorkflowInputID string `json:"workflowInputId" bson:"workflowInputId"` + NodeID string `json:"nodeId" bson:"nodeId"` + TargetPortID string `json:"targetPortId" bson:"targetPortId"` + Payload ArtifactPayload `json:"payload" bson:"payload"` +} + +type NodeInputBinding struct { + TargetPortID string `json:"targetPortId" bson:"targetPortId"` + WorkflowInputID string `json:"workflowInputId,omitempty" bson:"workflowInputId,omitempty"` + SourceNodeID string `json:"sourceNodeId,omitempty" bson:"sourceNodeId,omitempty"` + SourcePortID string `json:"sourcePortId,omitempty" bson:"sourcePortId,omitempty"` + Payload *ArtifactPayload `json:"payload,omitempty" bson:"payload,omitempty"` + Artifacts []ExecutionArtifact `json:"artifacts,omitempty" bson:"artifacts,omitempty"` +} + type HyperParam struct { Key string `json:"key" bson:"key"` Value string `json:"value" bson:"value"` @@ -312,52 +396,64 @@ type WorkflowSchedule struct { } type WorkflowRecord struct { - ID string `json:"id" bson:"id"` - OrganizationID string `json:"organizationId,omitempty" bson:"organizationId,omitempty"` - ProjectID string `json:"projectId" bson:"projectId"` - Name string `json:"name" bson:"name"` - Description string `json:"description" bson:"description"` - Type WorkflowType `json:"type" bson:"type"` - Kind WorkflowKind `json:"kind,omitempty" bson:"kind,omitempty"` - Hidden bool `json:"hidden,omitempty" bson:"hidden,omitempty"` - ExecutionScope WorkflowExecutionScope `json:"executionScope,omitempty" bson:"executionScope,omitempty"` - ScanConfig *ScanConfig `json:"scanConfig,omitempty" bson:"scanConfig,omitempty"` - Hyperparams []HyperParam `json:"hyperparams,omitempty" bson:"hyperparams,omitempty"` - MaxConcurrentRuns int `json:"maxConcurrentRuns,omitempty" bson:"maxConcurrentRuns,omitempty"` - NodeTimeoutSeconds int `json:"nodeTimeoutSeconds,omitempty" bson:"nodeTimeoutSeconds,omitempty"` - RunTimeoutSeconds int `json:"runTimeoutSeconds,omitempty" bson:"runTimeoutSeconds,omitempty"` - MaxRetryAttempts int `json:"maxRetryAttempts,omitempty" bson:"maxRetryAttempts,omitempty"` - RetryBackoffSeconds int `json:"retryBackoffSeconds,omitempty" bson:"retryBackoffSeconds,omitempty"` - Schedule *WorkflowSchedule `json:"schedule,omitempty" bson:"schedule,omitempty"` - Nodes []WorkflowNode `json:"nodes" bson:"nodes"` - Edges []WorkflowEdge `json:"edges" bson:"edges"` - Status WorkflowStatus `json:"status" bson:"status"` - LastRunID string `json:"lastRunId,omitempty" bson:"lastRunId,omitempty"` - LastRunAt string `json:"lastRunAt,omitempty" bson:"lastRunAt,omitempty"` - CreatedAt string `json:"createdAt" bson:"createdAt"` - UpdatedAt string `json:"updatedAt" bson:"updatedAt"` + ID string `json:"id" bson:"id"` + OrganizationID string `json:"organizationId,omitempty" bson:"organizationId,omitempty"` + ProjectID string `json:"projectId" bson:"projectId"` + Name string `json:"name" bson:"name"` + Description string `json:"description" bson:"description"` + Type WorkflowType `json:"type" bson:"type"` + Kind WorkflowKind `json:"kind,omitempty" bson:"kind,omitempty"` + Hidden bool `json:"hidden,omitempty" bson:"hidden,omitempty"` + ExecutionScope WorkflowExecutionScope `json:"executionScope,omitempty" bson:"executionScope,omitempty"` + ScanConfig *ScanConfig `json:"scanConfig,omitempty" bson:"scanConfig,omitempty"` + Hyperparams []HyperParam `json:"hyperparams,omitempty" bson:"hyperparams,omitempty"` + MaxConcurrentRuns int `json:"maxConcurrentRuns,omitempty" bson:"maxConcurrentRuns,omitempty"` + NodeTimeoutSeconds int `json:"nodeTimeoutSeconds,omitempty" bson:"nodeTimeoutSeconds,omitempty"` + RunTimeoutSeconds int `json:"runTimeoutSeconds,omitempty" bson:"runTimeoutSeconds,omitempty"` + MaxRetryAttempts int `json:"maxRetryAttempts,omitempty" bson:"maxRetryAttempts,omitempty"` + RetryBackoffSeconds int `json:"retryBackoffSeconds,omitempty" bson:"retryBackoffSeconds,omitempty"` + Schedule *WorkflowSchedule `json:"schedule,omitempty" bson:"schedule,omitempty"` + DefinitionState WorkflowDefinitionState `json:"definitionState" bson:"definitionState"` + DefinitionVersion int `json:"definitionVersion" bson:"definitionVersion"` + DefinitionFamilyID string `json:"definitionFamilyId" bson:"definitionFamilyId"` + SupersedesWorkflowID string `json:"supersedesWorkflowId,omitempty" bson:"supersedesWorkflowId,omitempty"` + WorkflowInputs []WorkflowInputDefinition `json:"workflowInputs,omitempty" bson:"workflowInputs,omitempty"` + ArtifactStorage *ArtifactStoragePolicy `json:"artifactStorage,omitempty" bson:"artifactStorage,omitempty"` + Nodes []WorkflowNode `json:"nodes" bson:"nodes"` + Edges []WorkflowEdge `json:"edges" bson:"edges"` + Status WorkflowStatus `json:"status" bson:"status"` + LastRunID string `json:"lastRunId,omitempty" bson:"lastRunId,omitempty"` + LastRunAt string `json:"lastRunAt,omitempty" bson:"lastRunAt,omitempty"` + CreatedAt string `json:"createdAt" bson:"createdAt"` + UpdatedAt string `json:"updatedAt" bson:"updatedAt"` } type TemplateRecord struct { - ID string `json:"id" bson:"id"` - OrganizationID string `json:"organizationId,omitempty" bson:"organizationId,omitempty"` - Name string `json:"name" bson:"name"` - Description string `json:"description" bson:"description"` - Type WorkflowType `json:"type" bson:"type"` - Kind WorkflowKind `json:"kind,omitempty" bson:"kind,omitempty"` - ExecutionScope WorkflowExecutionScope `json:"executionScope,omitempty" bson:"executionScope,omitempty"` - ScanConfig *ScanConfig `json:"scanConfig,omitempty" bson:"scanConfig,omitempty"` - Hyperparams []HyperParam `json:"hyperparams,omitempty" bson:"hyperparams,omitempty"` - MaxConcurrentRuns int `json:"maxConcurrentRuns,omitempty" bson:"maxConcurrentRuns,omitempty"` - NodeTimeoutSeconds int `json:"nodeTimeoutSeconds,omitempty" bson:"nodeTimeoutSeconds,omitempty"` - RunTimeoutSeconds int `json:"runTimeoutSeconds,omitempty" bson:"runTimeoutSeconds,omitempty"` - MaxRetryAttempts int `json:"maxRetryAttempts,omitempty" bson:"maxRetryAttempts,omitempty"` - RetryBackoffSeconds int `json:"retryBackoffSeconds,omitempty" bson:"retryBackoffSeconds,omitempty"` - Schedule *WorkflowSchedule `json:"schedule,omitempty" bson:"schedule,omitempty"` - Nodes []WorkflowNode `json:"nodes" bson:"nodes"` - Edges []WorkflowEdge `json:"edges" bson:"edges"` - CreatedAt string `json:"createdAt" bson:"createdAt"` - UpdatedAt string `json:"updatedAt" bson:"updatedAt"` + ID string `json:"id" bson:"id"` + OrganizationID string `json:"organizationId,omitempty" bson:"organizationId,omitempty"` + Name string `json:"name" bson:"name"` + Description string `json:"description" bson:"description"` + Type WorkflowType `json:"type" bson:"type"` + Kind WorkflowKind `json:"kind,omitempty" bson:"kind,omitempty"` + ExecutionScope WorkflowExecutionScope `json:"executionScope,omitempty" bson:"executionScope,omitempty"` + ScanConfig *ScanConfig `json:"scanConfig,omitempty" bson:"scanConfig,omitempty"` + Hyperparams []HyperParam `json:"hyperparams,omitempty" bson:"hyperparams,omitempty"` + MaxConcurrentRuns int `json:"maxConcurrentRuns,omitempty" bson:"maxConcurrentRuns,omitempty"` + NodeTimeoutSeconds int `json:"nodeTimeoutSeconds,omitempty" bson:"nodeTimeoutSeconds,omitempty"` + RunTimeoutSeconds int `json:"runTimeoutSeconds,omitempty" bson:"runTimeoutSeconds,omitempty"` + MaxRetryAttempts int `json:"maxRetryAttempts,omitempty" bson:"maxRetryAttempts,omitempty"` + RetryBackoffSeconds int `json:"retryBackoffSeconds,omitempty" bson:"retryBackoffSeconds,omitempty"` + Schedule *WorkflowSchedule `json:"schedule,omitempty" bson:"schedule,omitempty"` + DefinitionState WorkflowDefinitionState `json:"definitionState" bson:"definitionState"` + DefinitionVersion int `json:"definitionVersion" bson:"definitionVersion"` + DefinitionFamilyID string `json:"definitionFamilyId" bson:"definitionFamilyId"` + SupersedesTemplateID string `json:"supersedesTemplateId,omitempty" bson:"supersedesTemplateId,omitempty"` + WorkflowInputs []WorkflowInputDefinition `json:"workflowInputs,omitempty" bson:"workflowInputs,omitempty"` + ArtifactStorage *ArtifactStoragePolicy `json:"artifactStorage,omitempty" bson:"artifactStorage,omitempty"` + Nodes []WorkflowNode `json:"nodes" bson:"nodes"` + Edges []WorkflowEdge `json:"edges" bson:"edges"` + CreatedAt string `json:"createdAt" bson:"createdAt"` + UpdatedAt string `json:"updatedAt" bson:"updatedAt"` } type LogEntryRecord struct { @@ -400,6 +496,7 @@ type ExecutionArtifact struct { Path string `json:"path" bson:"path"` Metadata *ArtifactMetadata `json:"metadata,omitempty" bson:"metadata,omitempty"` Data map[string]any `json:"data,omitempty" bson:"data,omitempty"` + Payload *ArtifactPayload `json:"payload,omitempty" bson:"payload,omitempty"` } // UnmarshalJSON keeps the public artifact contract compatible with older @@ -497,6 +594,7 @@ type NodeExecutionRecord struct { StageManifestPath string `json:"stageManifestPath,omitempty" bson:"stageManifestPath,omitempty"` InputArtifacts []ExecutionArtifact `json:"inputArtifacts,omitempty" bson:"inputArtifacts,omitempty"` InputLineage []InputLineageRecord `json:"inputLineage,omitempty" bson:"inputLineage,omitempty"` + InputBindings []NodeInputBinding `json:"inputBindings,omitempty" bson:"inputBindings,omitempty"` Logs []LogEntryRecord `json:"logs" bson:"logs"` Artifacts []ExecutionArtifact `json:"artifacts" bson:"artifacts"` DataRefs []DataRefRecord `json:"dataRefs,omitempty" bson:"dataRefs,omitempty"` @@ -635,40 +733,43 @@ type RunProgress struct { } type RunRecord struct { - ID string `json:"id" bson:"id"` - OrganizationID string `json:"organizationId,omitempty" bson:"organizationId,omitempty"` - WorkflowID string `json:"workflowId" bson:"workflowId"` - WorkflowName string `json:"workflowName" bson:"workflowName"` - ProjectID string `json:"projectId" bson:"projectId"` - ProjectName string `json:"projectName" bson:"projectName"` - TaskID string `json:"taskId,omitempty" bson:"taskId,omitempty"` - Trigger RunTrigger `json:"trigger" bson:"trigger"` - Attempt int `json:"attempt,omitempty" bson:"attempt,omitempty"` - ParentRunID string `json:"parentRunId,omitempty" bson:"parentRunId,omitempty"` - RestartFromNodeID string `json:"restartFromNodeId,omitempty" bson:"restartFromNodeId,omitempty"` - AvailableAt string `json:"availableAt,omitempty" bson:"availableAt,omitempty"` - ExecutionLeaseOwner string `json:"executionLeaseOwner,omitempty" bson:"executionLeaseOwner,omitempty"` - ExecutionLeaseExpiresAt string `json:"executionLeaseExpiresAt,omitempty" bson:"executionLeaseExpiresAt,omitempty"` - Status RunStatus `json:"status" bson:"status"` - StopMode RunStopMode `json:"stopMode,omitempty" bson:"stopMode,omitempty"` - StopRequestedAt string `json:"stopRequestedAt,omitempty" bson:"stopRequestedAt,omitempty"` - PreviousRunAt string `json:"previousRunAt,omitempty" bson:"previousRunAt,omitempty"` - CurrentRunAt string `json:"currentRunAt,omitempty" bson:"currentRunAt,omitempty"` - StartedAt string `json:"startedAt,omitempty" bson:"startedAt,omitempty"` - EndedAt string `json:"endedAt,omitempty" bson:"endedAt,omitempty"` - DurationMS int64 `json:"durationMs,omitempty" bson:"durationMs,omitempty"` - CurrentNodeID string `json:"currentNodeId,omitempty" bson:"currentNodeId,omitempty"` - NodeExecutions []NodeExecutionRecord `json:"nodeExecutions" bson:"nodeExecutions"` - Logs []LogEntryRecord `json:"logs" bson:"logs"` - ResourceSeries []ResourceSample `json:"resourceSeries" bson:"resourceSeries"` - Artifacts []ExecutionArtifact `json:"artifacts" bson:"artifacts"` - K8sRefs []K8sRef `json:"k8sRefs" bson:"k8sRefs"` - Metadata *RunMetadata `json:"metadata,omitempty" bson:"metadata,omitempty"` - Failure *RunFailure `json:"failure,omitempty" bson:"failure,omitempty"` - Progress *RunProgress `json:"progress,omitempty" bson:"progress,omitempty"` - Error string `json:"error,omitempty" bson:"error,omitempty"` - CreatedAt string `json:"createdAt" bson:"createdAt"` - UpdatedAt string `json:"updatedAt" bson:"updatedAt"` + ID string `json:"id" bson:"id"` + OrganizationID string `json:"organizationId,omitempty" bson:"organizationId,omitempty"` + WorkflowID string `json:"workflowId" bson:"workflowId"` + WorkflowName string `json:"workflowName" bson:"workflowName"` + ProjectID string `json:"projectId" bson:"projectId"` + ProjectName string `json:"projectName" bson:"projectName"` + TaskID string `json:"taskId,omitempty" bson:"taskId,omitempty"` + Trigger RunTrigger `json:"trigger" bson:"trigger"` + Attempt int `json:"attempt,omitempty" bson:"attempt,omitempty"` + DefinitionVersion int `json:"definitionVersion,omitempty" bson:"definitionVersion,omitempty"` + Inputs map[string]ArtifactPayload `json:"inputs,omitempty" bson:"inputs,omitempty"` + ResolvedInputBindings []ResolvedInputBinding `json:"resolvedInputBindings,omitempty" bson:"resolvedInputBindings,omitempty"` + ParentRunID string `json:"parentRunId,omitempty" bson:"parentRunId,omitempty"` + RestartFromNodeID string `json:"restartFromNodeId,omitempty" bson:"restartFromNodeId,omitempty"` + AvailableAt string `json:"availableAt,omitempty" bson:"availableAt,omitempty"` + ExecutionLeaseOwner string `json:"executionLeaseOwner,omitempty" bson:"executionLeaseOwner,omitempty"` + ExecutionLeaseExpiresAt string `json:"executionLeaseExpiresAt,omitempty" bson:"executionLeaseExpiresAt,omitempty"` + Status RunStatus `json:"status" bson:"status"` + StopMode RunStopMode `json:"stopMode,omitempty" bson:"stopMode,omitempty"` + StopRequestedAt string `json:"stopRequestedAt,omitempty" bson:"stopRequestedAt,omitempty"` + PreviousRunAt string `json:"previousRunAt,omitempty" bson:"previousRunAt,omitempty"` + CurrentRunAt string `json:"currentRunAt,omitempty" bson:"currentRunAt,omitempty"` + StartedAt string `json:"startedAt,omitempty" bson:"startedAt,omitempty"` + EndedAt string `json:"endedAt,omitempty" bson:"endedAt,omitempty"` + DurationMS int64 `json:"durationMs,omitempty" bson:"durationMs,omitempty"` + CurrentNodeID string `json:"currentNodeId,omitempty" bson:"currentNodeId,omitempty"` + NodeExecutions []NodeExecutionRecord `json:"nodeExecutions" bson:"nodeExecutions"` + Logs []LogEntryRecord `json:"logs" bson:"logs"` + ResourceSeries []ResourceSample `json:"resourceSeries" bson:"resourceSeries"` + Artifacts []ExecutionArtifact `json:"artifacts" bson:"artifacts"` + K8sRefs []K8sRef `json:"k8sRefs" bson:"k8sRefs"` + Metadata *RunMetadata `json:"metadata,omitempty" bson:"metadata,omitempty"` + Failure *RunFailure `json:"failure,omitempty" bson:"failure,omitempty"` + Progress *RunProgress `json:"progress,omitempty" bson:"progress,omitempty"` + Error string `json:"error,omitempty" bson:"error,omitempty"` + CreatedAt string `json:"createdAt" bson:"createdAt"` + UpdatedAt string `json:"updatedAt" bson:"updatedAt"` } type RunSummary struct { @@ -808,6 +909,12 @@ type StorageSettings struct { Buckets []string `json:"buckets" bson:"buckets"` } +type ArtifactStorageSettings struct { + ConnectionID string `json:"connectionId,omitempty" bson:"connectionId,omitempty"` + Bucket string `json:"bucket,omitempty" bson:"bucket,omitempty"` + PrefixTemplate string `json:"prefixTemplate,omitempty" bson:"prefixTemplate,omitempty"` +} + type AISettings struct { Provider string `json:"provider" bson:"provider"` BaseURL string `json:"baseURL" bson:"baseURL"` @@ -881,6 +988,7 @@ type SettingsRecord struct { K8s K8sSettings `json:"k8s" bson:"k8s"` Worker WorkerSettings `json:"worker" bson:"worker"` Storage StorageSettings `json:"storage" bson:"storage"` + ArtifactStorage ArtifactStorageSettings `json:"artifactStorage,omitempty" bson:"artifactStorage,omitempty"` AI AISettings `json:"ai" bson:"ai"` OperatorGeneration OperatorGenerationSettings `json:"operatorGeneration" bson:"operatorGeneration"` BusinessMongo BusinessMongoSettings `json:"businessMongo" bson:"businessMongo"` @@ -894,6 +1002,7 @@ type SettingsView struct { ID string `json:"id"` K8s K8sSettings `json:"k8s"` Worker WorkerSettings `json:"worker"` + ArtifactStorage ArtifactStorageSettings `json:"artifactStorage,omitempty"` AI AISettings `json:"ai"` OperatorGeneration OperatorGenerationSettings `json:"operatorGeneration"` BusinessMongo BusinessMongoSettings `json:"businessMongo"` @@ -1033,43 +1142,47 @@ type CreateMongoConnectionInput struct { } type CreateWorkflowInput struct { - OrganizationID string `json:"organizationId,omitempty"` - ProjectID string `json:"projectId"` - Name string `json:"name"` - Description string `json:"description"` - Type WorkflowType `json:"type"` - Kind WorkflowKind `json:"kind,omitempty"` - Hidden bool `json:"hidden,omitempty"` - ExecutionScope WorkflowExecutionScope `json:"executionScope,omitempty"` - ScanConfig *ScanConfig `json:"scanConfig,omitempty"` - Hyperparams []HyperParam `json:"hyperparams,omitempty"` - MaxConcurrentRuns int `json:"maxConcurrentRuns,omitempty"` - NodeTimeoutSeconds int `json:"nodeTimeoutSeconds,omitempty"` - RunTimeoutSeconds int `json:"runTimeoutSeconds,omitempty"` - MaxRetryAttempts int `json:"maxRetryAttempts,omitempty"` - RetryBackoffSeconds int `json:"retryBackoffSeconds,omitempty"` - Nodes []WorkflowNode `json:"nodes"` - Edges []WorkflowEdge `json:"edges"` - Schedule *WorkflowSchedule `json:"schedule,omitempty"` + OrganizationID string `json:"organizationId,omitempty"` + ProjectID string `json:"projectId"` + Name string `json:"name"` + Description string `json:"description"` + Type WorkflowType `json:"type"` + Kind WorkflowKind `json:"kind,omitempty"` + Hidden bool `json:"hidden,omitempty"` + ExecutionScope WorkflowExecutionScope `json:"executionScope,omitempty"` + ScanConfig *ScanConfig `json:"scanConfig,omitempty"` + Hyperparams []HyperParam `json:"hyperparams,omitempty"` + MaxConcurrentRuns int `json:"maxConcurrentRuns,omitempty"` + NodeTimeoutSeconds int `json:"nodeTimeoutSeconds,omitempty"` + RunTimeoutSeconds int `json:"runTimeoutSeconds,omitempty"` + MaxRetryAttempts int `json:"maxRetryAttempts,omitempty"` + RetryBackoffSeconds int `json:"retryBackoffSeconds,omitempty"` + Nodes []WorkflowNode `json:"nodes"` + Edges []WorkflowEdge `json:"edges"` + Schedule *WorkflowSchedule `json:"schedule,omitempty"` + WorkflowInputs []WorkflowInputDefinition `json:"workflowInputs,omitempty"` + ArtifactStorage *ArtifactStoragePolicy `json:"artifactStorage,omitempty"` } type CreateTemplateInput struct { - OrganizationID string `json:"organizationId,omitempty"` - Name string `json:"name"` - Description string `json:"description"` - Type WorkflowType `json:"type"` - Kind WorkflowKind `json:"kind,omitempty"` - ExecutionScope WorkflowExecutionScope `json:"executionScope,omitempty"` - ScanConfig *ScanConfig `json:"scanConfig,omitempty"` - Hyperparams []HyperParam `json:"hyperparams,omitempty"` - MaxConcurrentRuns int `json:"maxConcurrentRuns,omitempty"` - NodeTimeoutSeconds int `json:"nodeTimeoutSeconds,omitempty"` - RunTimeoutSeconds int `json:"runTimeoutSeconds,omitempty"` - MaxRetryAttempts int `json:"maxRetryAttempts,omitempty"` - RetryBackoffSeconds int `json:"retryBackoffSeconds,omitempty"` - Schedule *WorkflowSchedule `json:"schedule,omitempty"` - Nodes []WorkflowNode `json:"nodes"` - Edges []WorkflowEdge `json:"edges"` + OrganizationID string `json:"organizationId,omitempty"` + Name string `json:"name"` + Description string `json:"description"` + Type WorkflowType `json:"type"` + Kind WorkflowKind `json:"kind,omitempty"` + ExecutionScope WorkflowExecutionScope `json:"executionScope,omitempty"` + ScanConfig *ScanConfig `json:"scanConfig,omitempty"` + Hyperparams []HyperParam `json:"hyperparams,omitempty"` + MaxConcurrentRuns int `json:"maxConcurrentRuns,omitempty"` + NodeTimeoutSeconds int `json:"nodeTimeoutSeconds,omitempty"` + RunTimeoutSeconds int `json:"runTimeoutSeconds,omitempty"` + MaxRetryAttempts int `json:"maxRetryAttempts,omitempty"` + RetryBackoffSeconds int `json:"retryBackoffSeconds,omitempty"` + Schedule *WorkflowSchedule `json:"schedule,omitempty"` + Nodes []WorkflowNode `json:"nodes"` + Edges []WorkflowEdge `json:"edges"` + WorkflowInputs []WorkflowInputDefinition `json:"workflowInputs,omitempty"` + ArtifactStorage *ArtifactStoragePolicy `json:"artifactStorage,omitempty"` } type CreateOperatorInput struct { diff --git a/internal/domain/workflow_inputs.go b/internal/domain/workflow_inputs.go new file mode 100644 index 00000000..60170766 --- /dev/null +++ b/internal/domain/workflow_inputs.go @@ -0,0 +1,259 @@ +package domain + +import ( + "encoding/json" + "fmt" + "strings" +) + +const MaxInlinePayloadBytes = 64 * 1024 + +func NormalizeWorkflowDefinitionMetadata(workflow *WorkflowRecord) { + if workflow == nil { + return + } + if workflow.DefinitionState == "" { + workflow.DefinitionState = WorkflowDefinitionStateDraft + } + if workflow.DefinitionVersion <= 0 { + workflow.DefinitionVersion = 1 + } + if strings.TrimSpace(workflow.DefinitionFamilyID) == "" { + workflow.DefinitionFamilyID = workflow.ID + } +} + +type WorkflowInputContractIssue struct { + Field string + Code string + Message string +} + +func ValidateArtifactPayload(payload ArtifactPayload, expectedType PortType) error { + if payload.Type == "" { + payload.Type = expectedType + } + if expectedType != "" && payload.Type != expectedType { + return fmt.Errorf("payload type %q does not match input type %q", payload.Type, expectedType) + } + switch payload.Mode { + case ArtifactTransferModeInline: + if payload.Type != PortTypeJSON { + return fmt.Errorf("inline payloads are only supported for json inputs") + } + if payload.Inline == nil { + return fmt.Errorf("inline payload value is required") + } + if payload.Ref != nil { + return fmt.Errorf("inline payload must not include ref") + } + encoded, err := json.Marshal(payload.Inline) + if err != nil { + return fmt.Errorf("inline payload must be valid json: %w", err) + } + if len(encoded) > MaxInlinePayloadBytes { + return fmt.Errorf("inline payload exceeds %d bytes; use ref mode", MaxInlinePayloadBytes) + } + case ArtifactTransferModeRef: + if payload.Ref == nil { + return fmt.Errorf("ref payload requires ref") + } + if payload.Inline != nil { + return fmt.Errorf("ref payload must not include inline") + } + ref := payload.Ref + if ref.Provider != StorageProviderBOS { + return fmt.Errorf("ref.provider must be BOS") + } + if strings.TrimSpace(ref.ConnectionID) == "" { + return fmt.Errorf("ref.connectionId is required") + } + if strings.TrimSpace(ref.Bucket) == "" { + return fmt.Errorf("ref.bucket is required") + } + switch ref.Kind { + case ArtifactRefKindObject: + if strings.TrimSpace(ref.Key) == "" { + return fmt.Errorf("object ref.key is required") + } + case ArtifactRefKindDirectory: + if payload.Type != PortTypeFile { + return fmt.Errorf("directory refs require a file input") + } + if strings.TrimSpace(ref.Prefix) == "" { + return fmt.Errorf("directory ref.prefix is required") + } + default: + return fmt.Errorf("ref.kind must be object or directory") + } + default: + return fmt.Errorf("payload mode must be inline or ref") + } + return nil +} + +func WorkflowRootNodeIDs(nodes []WorkflowNode, edges []WorkflowEdge) map[string]struct{} { + rootIDs := make(map[string]struct{}, len(nodes)) + for _, node := range nodes { + rootIDs[node.ID] = struct{}{} + } + for _, edge := range edges { + delete(rootIDs, edge.Target) + } + return rootIDs +} + +func workflowInputPort(node *WorkflowNode, portID string) *PortDefinition { + if node == nil { + return nil + } + for index := range node.Data.Inputs { + if strings.TrimSpace(node.Data.Inputs[index].ID) == strings.TrimSpace(portID) { + return &node.Data.Inputs[index] + } + } + return nil +} + +func edgeTargetsPort(edge WorkflowEdge, node WorkflowNode, port PortDefinition) bool { + if edge.Target != node.ID { + return false + } + targetHandle := strings.TrimSpace(edge.TargetHandle) + if targetHandle != "" { + return targetHandle == strings.TrimSpace(port.ID) + } + return len(node.Data.Inputs) == 1 +} + +func ValidateWorkflowInputContract(nodes []WorkflowNode, edges []WorkflowEdge, definitions []WorkflowInputDefinition, release bool) []WorkflowInputContractIssue { + issues := make([]WorkflowInputContractIssue, 0) + rootIDs := WorkflowRootNodeIDs(nodes, edges) + nodeByID := make(map[string]*WorkflowNode, len(nodes)) + for index := range nodes { + nodeByID[nodes[index].ID] = &nodes[index] + } + definitionIDs := make(map[string]struct{}, len(definitions)) + boundTargets := map[string]string{} + for index, definition := range definitions { + field := fmt.Sprintf("workflowInputs[%d]", index) + id := strings.TrimSpace(definition.ID) + if id == "" { + issues = append(issues, WorkflowInputContractIssue{Field: field + ".id", Code: "workflow_input_id_required", Message: "workflow input id is required"}) + } else if _, exists := definitionIDs[id]; exists { + issues = append(issues, WorkflowInputContractIssue{Field: field + ".id", Code: "workflow_input_id_duplicate", Message: "workflow input id must be unique"}) + } else { + definitionIDs[id] = struct{}{} + } + if strings.TrimSpace(definition.Name) == "" { + issues = append(issues, WorkflowInputContractIssue{Field: field + ".name", Code: "workflow_input_name_required", Message: "workflow input name is required"}) + } + if definition.DefaultPayload != nil { + if err := ValidateArtifactPayload(*definition.DefaultPayload, definition.Type); err != nil { + issues = append(issues, WorkflowInputContractIssue{Field: field + ".defaultPayload", Code: "workflow_input_payload_invalid", Message: err.Error()}) + } + } + if len(definition.Bindings) == 0 { + issues = append(issues, WorkflowInputContractIssue{Field: field + ".bindings", Code: "workflow_input_binding_required", Message: "workflow input requires at least one root-node binding"}) + } + for bindingIndex, binding := range definition.Bindings { + bindingField := fmt.Sprintf("%s.bindings[%d]", field, bindingIndex) + node := nodeByID[strings.TrimSpace(binding.NodeID)] + if node == nil { + issues = append(issues, WorkflowInputContractIssue{Field: bindingField + ".nodeId", Code: "workflow_input_node_missing", Message: "workflow input binding node does not exist"}) + continue + } + if _, root := rootIDs[node.ID]; !root { + issues = append(issues, WorkflowInputContractIssue{Field: bindingField + ".nodeId", Code: "workflow_input_node_not_root", Message: "workflow inputs may only bind nodes without incoming edges"}) + } + port := workflowInputPort(node, binding.TargetPortID) + if port == nil { + issues = append(issues, WorkflowInputContractIssue{Field: bindingField + ".targetPortId", Code: "workflow_input_port_missing", Message: "workflow input binding port does not exist"}) + continue + } + if port.Type != definition.Type { + issues = append(issues, WorkflowInputContractIssue{Field: bindingField + ".targetPortId", Code: "workflow_input_type_mismatch", Message: "workflow input type must match target port type"}) + } + if port.Required && !definition.Required { + issues = append(issues, WorkflowInputContractIssue{Field: field + ".required", Code: "workflow_input_required_mismatch", Message: "a required target port cannot be exposed as optional"}) + } + targetKey := node.ID + "\x00" + port.ID + if previous, exists := boundTargets[targetKey]; exists && previous != id { + issues = append(issues, WorkflowInputContractIssue{Field: bindingField, Code: "workflow_input_binding_duplicate", Message: "a root input port may only be bound by one workflow input"}) + } else { + boundTargets[targetKey] = id + } + } + } + if !release { + return issues + } + for _, node := range nodes { + _, root := rootIDs[node.ID] + for _, port := range node.Data.Inputs { + if !port.Required { + continue + } + if root { + if _, bound := boundTargets[node.ID+"\x00"+port.ID]; !bound { + issues = append(issues, WorkflowInputContractIssue{Field: "workflowInputs", Code: "required_root_input_unbound", Message: fmt.Sprintf("required root input %s.%s must be exposed as a workflow input", node.ID, port.ID)}) + } + continue + } + connected := false + for _, edge := range edges { + if edgeTargetsPort(edge, node, port) { + connected = true + break + } + } + if !connected { + issues = append(issues, WorkflowInputContractIssue{Field: "edges", Code: "required_input_unconnected", Message: fmt.Sprintf("required input %s.%s must have an upstream edge", node.ID, port.ID)}) + } + } + } + return issues +} + +func ResolveWorkflowRunInputs(workflow WorkflowRecord, provided map[string]ArtifactPayload) (map[string]ArtifactPayload, []ResolvedInputBinding, error) { + definitions := make(map[string]WorkflowInputDefinition, len(workflow.WorkflowInputs)) + for _, definition := range workflow.WorkflowInputs { + definitions[definition.ID] = definition + } + for id := range provided { + if _, ok := definitions[id]; !ok { + return nil, nil, fmt.Errorf("unknown workflow input %q", id) + } + } + resolvedInputs := make(map[string]ArtifactPayload) + bindings := make([]ResolvedInputBinding, 0) + for _, definition := range workflow.WorkflowInputs { + payload, ok := provided[definition.ID] + if !ok && definition.DefaultPayload != nil { + payload = *definition.DefaultPayload + ok = true + } + if !ok { + if definition.Required { + return nil, nil, fmt.Errorf("required workflow input %q is missing", definition.ID) + } + continue + } + if payload.Type == "" { + payload.Type = definition.Type + } + if err := ValidateArtifactPayload(payload, definition.Type); err != nil { + return nil, nil, fmt.Errorf("workflow input %q: %w", definition.ID, err) + } + resolvedInputs[definition.ID] = payload + for _, target := range definition.Bindings { + bindings = append(bindings, ResolvedInputBinding{ + WorkflowInputID: definition.ID, + NodeID: target.NodeID, + TargetPortID: target.TargetPortID, + Payload: payload, + }) + } + } + return resolvedInputs, bindings, nil +} diff --git a/internal/domain/workflow_inputs_test.go b/internal/domain/workflow_inputs_test.go new file mode 100644 index 00000000..83ee6133 --- /dev/null +++ b/internal/domain/workflow_inputs_test.go @@ -0,0 +1,60 @@ +package domain + +import ( + "strings" + "testing" +) + +func TestValidateArtifactPayloadSupportsInlineAndDirectoryRefs(t *testing.T) { + inline := ArtifactPayload{Mode: ArtifactTransferModeInline, Type: PortTypeJSON, Inline: map[string]any{"ok": true}} + if err := ValidateArtifactPayload(inline, PortTypeJSON); err != nil { + t.Fatalf("validate inline payload: %v", err) + } + ref := ArtifactPayload{Mode: ArtifactTransferModeRef, Type: PortTypeFile, Ref: &ArtifactReference{ + Provider: StorageProviderBOS, ConnectionID: "storage-1", Bucket: "robot-data", Prefix: "incoming/run", Kind: ArtifactRefKindDirectory, + }} + if err := ValidateArtifactPayload(ref, PortTypeFile); err != nil { + t.Fatalf("validate directory ref: %v", err) + } + tooLarge := ArtifactPayload{Mode: ArtifactTransferModeInline, Type: PortTypeJSON, Inline: strings.Repeat("x", MaxInlinePayloadBytes)} + if err := ValidateArtifactPayload(tooLarge, PortTypeJSON); err == nil || !strings.Contains(err.Error(), "exceeds") { + t.Fatalf("expected inline size failure, got %v", err) + } +} + +func TestValidateWorkflowInputContractRequiresReleasedRootAndInternalInputs(t *testing.T) { + nodes := []WorkflowNode{ + {ID: "source", Data: WorkflowNodeData{Inputs: []PortDefinition{{ID: "config", Name: "config", Type: PortTypeJSON, Required: true}}, Outputs: []PortDefinition{{ID: "out", Name: "out", Type: PortTypeJSON}}}}, + {ID: "sink", Data: WorkflowNodeData{Inputs: []PortDefinition{{ID: "in", Name: "in", Type: PortTypeJSON, Required: true}}}}, + } + edges := []WorkflowEdge{{ID: "edge", Source: "source", SourceHandle: "out", Target: "sink", TargetHandle: "in"}} + issues := ValidateWorkflowInputContract(nodes, edges, nil, true) + if len(issues) != 1 || issues[0].Code != "required_root_input_unbound" { + t.Fatalf("unexpected release issues: %#v", issues) + } + definitions := []WorkflowInputDefinition{{ + ID: "config", Name: "Config", Type: PortTypeJSON, Required: true, + Bindings: []WorkflowInputTarget{{NodeID: "source", TargetPortID: "config"}}, + }} + if issues := ValidateWorkflowInputContract(nodes, edges, definitions, true); len(issues) != 0 { + t.Fatalf("expected valid contract, got %#v", issues) + } +} + +func TestResolveWorkflowRunInputsUsesDefaultsAndPreservesTargetBindings(t *testing.T) { + workflow := WorkflowRecord{WorkflowInputs: []WorkflowInputDefinition{{ + ID: "config", Name: "Config", Type: PortTypeJSON, Required: true, + DefaultPayload: &ArtifactPayload{Mode: ArtifactTransferModeInline, Type: PortTypeJSON, Inline: map[string]any{"mode": "default"}}, + Bindings: []WorkflowInputTarget{{NodeID: "a", TargetPortID: "config"}, {NodeID: "b", TargetPortID: "config"}}, + }}} + resolved, bindings, err := ResolveWorkflowRunInputs(workflow, nil) + if err != nil { + t.Fatalf("resolve defaults: %v", err) + } + if len(resolved) != 1 || len(bindings) != 2 { + t.Fatalf("unexpected resolved inputs: %#v %#v", resolved, bindings) + } + if bindings[0].TargetPortID != "config" || bindings[1].NodeID != "b" { + t.Fatalf("target bindings were not preserved: %#v", bindings) + } +} diff --git a/internal/store/memory.go b/internal/store/memory.go index 3c499283..025c138b 100644 --- a/internal/store/memory.go +++ b/internal/store/memory.go @@ -656,6 +656,7 @@ func (store *MemoryStore) GetWorkflow(_ context.Context, id string) (*domain.Wor return nil, nil } cloned := clone(workflow) + domain.NormalizeWorkflowDefinitionMetadata(&cloned) return &cloned, nil } @@ -681,12 +682,17 @@ func (store *MemoryStore) CreateWorkflow(_ context.Context, input domain.CreateW MaxRetryAttempts: input.MaxRetryAttempts, RetryBackoffSeconds: input.RetryBackoffSeconds, Schedule: clone(input.Schedule), + DefinitionState: domain.WorkflowDefinitionStateDraft, + DefinitionVersion: 1, + WorkflowInputs: clone(input.WorkflowInputs), + ArtifactStorage: clone(input.ArtifactStorage), Nodes: clone(input.Nodes), Edges: clone(input.Edges), Status: domain.WorkflowStatusDraft, CreatedAt: now, UpdatedAt: now, } + item.DefinitionFamilyID = item.ID store.workflows[item.ID] = clone(item) cloned := clone(item) return &cloned, nil @@ -805,11 +811,16 @@ func (store *MemoryStore) CreateTemplate(_ context.Context, input domain.CreateT MaxRetryAttempts: input.MaxRetryAttempts, RetryBackoffSeconds: input.RetryBackoffSeconds, Schedule: clone(input.Schedule), + DefinitionState: domain.WorkflowDefinitionStateDraft, + DefinitionVersion: 1, + WorkflowInputs: clone(input.WorkflowInputs), + ArtifactStorage: clone(input.ArtifactStorage), Nodes: clone(input.Nodes), Edges: clone(input.Edges), CreatedAt: now, UpdatedAt: now, } + item.DefinitionFamilyID = item.ID store.templates[item.ID] = clone(item) cloned := clone(item) return &cloned, nil @@ -1196,7 +1207,11 @@ func (store *MemoryStore) GetRun(_ context.Context, id string) (*domain.RunRecor return &cloned, nil } -func (store *MemoryStore) CreateRun(_ context.Context, workflowID string, trigger domain.RunTrigger, taskIDs ...string) (*domain.RunRecord, error) { +func (store *MemoryStore) CreateRun(ctx context.Context, workflowID string, trigger domain.RunTrigger, taskIDs ...string) (*domain.RunRecord, error) { + return store.CreateRunWithInputs(ctx, workflowID, trigger, nil, taskIDs...) +} + +func (store *MemoryStore) CreateRunWithInputs(_ context.Context, workflowID string, trigger domain.RunTrigger, inputs map[string]domain.ArtifactPayload, taskIDs ...string) (*domain.RunRecord, error) { store.mu.Lock() defer store.mu.Unlock() workflow, ok := store.workflows[workflowID] @@ -1222,26 +1237,33 @@ func (store *MemoryStore) CreateRun(_ context.Context, workflowID string, trigge availableAt = time.Now().UTC().Add(24 * time.Hour).Format(time.RFC3339Nano) } previousRunAt := workflow.LastRunAt + resolvedInputs, resolvedBindings, err := domain.ResolveWorkflowRunInputs(workflow, inputs) + if err != nil { + return nil, err + } run := domain.RunRecord{ - ID: domain.NewID("run"), - OrganizationID: workflow.OrganizationID, - WorkflowID: workflow.ID, - WorkflowName: workflow.Name, - ProjectID: workflow.ProjectID, - ProjectName: projectName, - Trigger: trigger, - Attempt: 1, - AvailableAt: availableAt, - Status: domain.RunStatusQueued, - PreviousRunAt: previousRunAt, - CurrentRunAt: now, - NodeExecutions: []domain.NodeExecutionRecord{}, - Logs: []domain.LogEntryRecord{}, - ResourceSeries: []domain.ResourceSample{}, - Artifacts: []domain.ExecutionArtifact{}, - K8sRefs: []domain.K8sRef{}, - CreatedAt: now, - UpdatedAt: now, + ID: domain.NewID("run"), + OrganizationID: workflow.OrganizationID, + WorkflowID: workflow.ID, + WorkflowName: workflow.Name, + ProjectID: workflow.ProjectID, + ProjectName: projectName, + Trigger: trigger, + Attempt: 1, + DefinitionVersion: workflow.DefinitionVersion, + Inputs: clone(resolvedInputs), + ResolvedInputBindings: clone(resolvedBindings), + AvailableAt: availableAt, + Status: domain.RunStatusQueued, + PreviousRunAt: previousRunAt, + CurrentRunAt: now, + NodeExecutions: []domain.NodeExecutionRecord{}, + Logs: []domain.LogEntryRecord{}, + ResourceSeries: []domain.ResourceSample{}, + Artifacts: []domain.ExecutionArtifact{}, + K8sRefs: []domain.K8sRef{}, + CreatedAt: now, + UpdatedAt: now, } if len(taskIDs) > 0 { run.TaskID = strings.TrimSpace(taskIDs[0]) diff --git a/internal/store/mongo.go b/internal/store/mongo.go index d64f9cc4..dc3b7e95 100644 --- a/internal/store/mongo.go +++ b/internal/store/mongo.go @@ -492,6 +492,7 @@ func (store *MongoStore) GetWorkflow(ctx context.Context, id string) (*domain.Wo if err != nil { return nil, err } + domain.NormalizeWorkflowDefinitionMetadata(&item) return &item, nil } @@ -515,12 +516,17 @@ func (store *MongoStore) CreateWorkflow(ctx context.Context, input domain.Create MaxRetryAttempts: input.MaxRetryAttempts, RetryBackoffSeconds: input.RetryBackoffSeconds, Schedule: input.Schedule, + DefinitionState: domain.WorkflowDefinitionStateDraft, + DefinitionVersion: 1, + WorkflowInputs: input.WorkflowInputs, + ArtifactStorage: input.ArtifactStorage, Nodes: input.Nodes, Edges: input.Edges, Status: domain.WorkflowStatusDraft, CreatedAt: now, UpdatedAt: now, } + item.DefinitionFamilyID = item.ID if err := store.UpsertWorkflow(ctx, item); err != nil { return nil, err } @@ -614,11 +620,16 @@ func (store *MongoStore) CreateTemplate(ctx context.Context, input domain.Create MaxRetryAttempts: input.MaxRetryAttempts, RetryBackoffSeconds: input.RetryBackoffSeconds, Schedule: input.Schedule, + DefinitionState: domain.WorkflowDefinitionStateDraft, + DefinitionVersion: 1, + WorkflowInputs: input.WorkflowInputs, + ArtifactStorage: input.ArtifactStorage, Nodes: input.Nodes, Edges: input.Edges, CreatedAt: now, UpdatedAt: now, } + item.DefinitionFamilyID = item.ID if err := store.UpsertTemplate(ctx, item); err != nil { return nil, err } @@ -1158,6 +1169,10 @@ func (store *MongoStore) GetRun(ctx context.Context, id string) (*domain.RunReco } func (store *MongoStore) CreateRun(ctx context.Context, workflowID string, trigger domain.RunTrigger, taskIDs ...string) (*domain.RunRecord, error) { + return store.CreateRunWithInputs(ctx, workflowID, trigger, nil, taskIDs...) +} + +func (store *MongoStore) CreateRunWithInputs(ctx context.Context, workflowID string, trigger domain.RunTrigger, inputs map[string]domain.ArtifactPayload, taskIDs ...string) (*domain.RunRecord, error) { workflow, err := store.GetWorkflow(ctx, workflowID) if err != nil { return nil, err @@ -1184,26 +1199,33 @@ func (store *MongoStore) CreateRun(ctx context.Context, workflowID string, trigg availableAt = time.Now().UTC().Add(24 * time.Hour).Format(time.RFC3339Nano) } previousRunAt := workflow.LastRunAt + resolvedInputs, resolvedBindings, err := domain.ResolveWorkflowRunInputs(*workflow, inputs) + if err != nil { + return nil, err + } run := domain.RunRecord{ - ID: domain.NewID("run"), - OrganizationID: workflow.OrganizationID, - WorkflowID: workflow.ID, - WorkflowName: workflow.Name, - ProjectID: workflow.ProjectID, - ProjectName: projectName, - Trigger: trigger, - Attempt: 1, - AvailableAt: availableAt, - Status: domain.RunStatusQueued, - PreviousRunAt: previousRunAt, - CurrentRunAt: now, - NodeExecutions: []domain.NodeExecutionRecord{}, - Logs: []domain.LogEntryRecord{}, - ResourceSeries: []domain.ResourceSample{}, - Artifacts: []domain.ExecutionArtifact{}, - K8sRefs: []domain.K8sRef{}, - CreatedAt: now, - UpdatedAt: now, + ID: domain.NewID("run"), + OrganizationID: workflow.OrganizationID, + WorkflowID: workflow.ID, + WorkflowName: workflow.Name, + ProjectID: workflow.ProjectID, + ProjectName: projectName, + Trigger: trigger, + Attempt: 1, + DefinitionVersion: workflow.DefinitionVersion, + Inputs: resolvedInputs, + ResolvedInputBindings: resolvedBindings, + AvailableAt: availableAt, + Status: domain.RunStatusQueued, + PreviousRunAt: previousRunAt, + CurrentRunAt: now, + NodeExecutions: []domain.NodeExecutionRecord{}, + Logs: []domain.LogEntryRecord{}, + ResourceSeries: []domain.ResourceSample{}, + Artifacts: []domain.ExecutionArtifact{}, + K8sRefs: []domain.K8sRef{}, + CreatedAt: now, + UpdatedAt: now, } if len(taskIDs) > 0 { run.TaskID = strings.TrimSpace(taskIDs[0]) diff --git a/internal/store/mongo_migrations.go b/internal/store/mongo_migrations.go index de637f5b..7b22515b 100644 --- a/internal/store/mongo_migrations.go +++ b/internal/store/mongo_migrations.go @@ -25,6 +25,9 @@ func (store *MongoStore) RunMigrations(ctx context.Context) error { if err := store.normalizeLegacyDefaultOrganizationReferences(ctx); err != nil { return err } + if err := store.normalizeWorkflowDefinitionMetadata(ctx); err != nil { + return err + } if err := store.deduplicateOperatorRecordsByID(ctx); err != nil { return err } @@ -34,6 +37,23 @@ func (store *MongoStore) RunMigrations(ctx context.Context) error { return nil } +func (store *MongoStore) normalizeWorkflowDefinitionMetadata(ctx context.Context) error { + update := mongo.Pipeline{bson.D{{Key: "$set", Value: bson.D{ + {Key: "definitionState", Value: bson.D{{Key: "$ifNull", Value: bson.A{"$definitionState", string(domain.WorkflowDefinitionStateDraft)}}}}, + {Key: "definitionVersion", Value: bson.D{{Key: "$ifNull", Value: bson.A{"$definitionVersion", 1}}}}, + {Key: "definitionFamilyId", Value: bson.D{{Key: "$ifNull", Value: bson.A{"$definitionFamilyId", "$id"}}}}, + }}}} + if _, err := store.workflows.UpdateMany( + ctx, + bson.M{}, + update, + ); err != nil { + return err + } + _, err := store.templates.UpdateMany(ctx, bson.M{}, update) + return err +} + func (store *MongoStore) clearExternalBusinessWriteTokenValues(ctx context.Context) error { _, err := store.settings.UpdateMany( ctx, diff --git a/internal/store/store.go b/internal/store/store.go index cfc0e08d..e083c30b 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -81,6 +81,7 @@ type Store interface { HasActiveRun(context.Context, string) (bool, error) GetRun(context.Context, string) (*domain.RunRecord, error) CreateRun(context.Context, string, domain.RunTrigger, ...string) (*domain.RunRecord, error) + CreateRunWithInputs(context.Context, string, domain.RunTrigger, map[string]domain.ArtifactPayload, ...string) (*domain.RunRecord, error) ClaimNextQueuedRun(context.Context, string, time.Duration) (*domain.RunRecord, error) ClaimQueuedRun(context.Context, string, string, time.Duration) (*domain.RunRecord, error) AcquireRunLease(context.Context, string, string, time.Duration) (*domain.RunRecord, bool, error) diff --git a/internal/worker/artifact_storage.go b/internal/worker/artifact_storage.go new file mode 100644 index 00000000..ea048a57 --- /dev/null +++ b/internal/worker/artifact_storage.go @@ -0,0 +1,427 @@ +package worker + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "mime" + "os" + "path/filepath" + "sort" + "strings" + + "droboticflow/internal/domain" + "droboticflow/internal/objectsync" + + "github.com/baidubce/bce-sdk-go/services/bos" + bosapi "github.com/baidubce/bce-sdk-go/services/bos/api" +) + +const defaultArtifactPrefixTemplate = "artifacts/{organizationId}/{projectId}/{workflowId}/{runId}/{nodeId}" + +type directoryManifestEntry struct { + Path string `json:"path"` + Key string `json:"key"` + SizeBytes int64 `json:"sizeBytes"` + SHA256 string `json:"sha256,omitempty"` +} + +type directoryManifest struct { + SchemaVersion int `json:"schemaVersion"` + Kind string `json:"kind"` + Source map[string]string `json:"source,omitempty"` + FileCount int `json:"fileCount"` + TotalBytes int64 `json:"totalBytes"` + Entries []directoryManifestEntry `json:"entries"` +} + +func compactArtifactData(data map[string]any) map[string]any { + if len(data) == 0 { + return nil + } + compact := map[string]any{} + for key, value := range data { + if key == "preview" || key == "reportSummary" || key == "dataRef" || key == "count" || strings.HasSuffix(key, "Count") { + compact[key] = value + } + } + if len(compact) == 0 { + return nil + } + return compact +} + +func normalizeArtifactKey(value string) string { + parts := strings.FieldsFunc(strings.TrimSpace(value), func(r rune) bool { + return r == '\\' || r == '/' + }) + cleaned := make([]string, 0, len(parts)) + for _, part := range parts { + part = artifactSlug(part) + if part != "" && part != "." && part != ".." { + cleaned = append(cleaned, part) + } + } + return strings.Join(cleaned, "/") +} + +func artifactSlug(value string) string { + value = strings.TrimSpace(value) + if value == "" { + return "artifact" + } + var builder strings.Builder + for _, r := range value { + switch { + case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z', r >= '0' && r <= '9', r == '.', r == '_', r == '-': + builder.WriteRune(r) + default: + builder.WriteRune('-') + } + } + return strings.Trim(builder.String(), "-") +} + +func artifactPrefix(policy domain.ArtifactStoragePolicy, run *domain.RunRecord, nodeID string) string { + template := strings.Trim(strings.TrimSpace(policy.PrefixTemplate), "/") + if template == "" { + template = defaultArtifactPrefixTemplate + } + replacements := map[string]string{ + "{organizationId}": run.OrganizationID, + "{projectId}": run.ProjectID, + "{workflowId}": run.WorkflowID, + "{runId}": run.ID, + "{nodeId}": nodeID, + } + for token, value := range replacements { + template = strings.ReplaceAll(template, token, artifactSlug(value)) + } + return normalizeArtifactKey(template) +} + +func (runtime *Runtime) artifactStoragePolicy(ctx context.Context, workflow *domain.WorkflowRecord) (*domain.ArtifactStoragePolicy, error) { + if workflow != nil && workflow.ArtifactStorage != nil && strings.TrimSpace(workflow.ArtifactStorage.ConnectionID) != "" { + copy := *workflow.ArtifactStorage + return ©, nil + } + settings, err := runtime.store.GetSettings(ctx) + if err != nil { + return nil, err + } + if settings == nil || strings.TrimSpace(settings.ArtifactStorage.ConnectionID) == "" || strings.TrimSpace(settings.ArtifactStorage.Bucket) == "" { + return nil, nil + } + return &domain.ArtifactStoragePolicy{ + ConnectionID: settings.ArtifactStorage.ConnectionID, + Bucket: settings.ArtifactStorage.Bucket, + PrefixTemplate: settings.ArtifactStorage.PrefixTemplate, + }, nil +} + +func (runtime *Runtime) artifactStorageConnection(ctx context.Context, connectionID string, organizationID string) (*domain.StorageConnectionRecord, error) { + connection, err := runtime.store.GetStorageConnection(ctx, strings.TrimSpace(connectionID)) + if err != nil { + return nil, err + } + if connection == nil { + return nil, fmt.Errorf("artifact storage connection %s not found", connectionID) + } + if strings.TrimSpace(connection.OrganizationID) != "" && strings.TrimSpace(organizationID) != "" && connection.OrganizationID != organizationID { + return nil, fmt.Errorf("artifact storage connection belongs to another organization") + } + copy := *connection + if strings.TrimSpace(copy.Endpoint) == "" { + copy.Endpoint = runtime.globalBOSEndpoint + } + return ©, nil +} + +func fileSHA256(path string) (string, error) { + file, err := os.Open(path) + if err != nil { + return "", err + } + defer file.Close() + hash := sha256.New() + if _, err := io.Copy(hash, file); err != nil { + return "", err + } + return hex.EncodeToString(hash.Sum(nil)), nil +} + +func putArtifactFile(ctx context.Context, publisher objectsync.Publisher, connection domain.StorageConnectionRecord, bucket, key, path string) (objectsync.ObjectMetadata, string, error) { + info, err := os.Stat(path) + if err != nil { + return objectsync.ObjectMetadata{}, "", err + } + sha, err := fileSHA256(path) + if err != nil { + return objectsync.ObjectMetadata{}, "", err + } + file, err := os.Open(path) + if err != nil { + return objectsync.ObjectMetadata{}, "", err + } + defer file.Close() + contentType := mime.TypeByExtension(strings.ToLower(filepath.Ext(path))) + if contentType == "" { + contentType = "application/octet-stream" + } + meta, err := publisher.PutInternal(ctx, connection, bucket, key, file, info.Size(), contentType, map[string]string{"sha256": sha}) + return meta, sha, err +} + +func (runtime *Runtime) externalizeDirectory(ctx context.Context, publisher objectsync.Publisher, connection domain.StorageConnectionRecord, policy domain.ArtifactStoragePolicy, run *domain.RunRecord, nodeID string, artifact domain.ExecutionArtifact) (domain.ExecutionArtifact, error) { + basePrefix := artifactPrefix(policy, run, nodeID) + "/" + artifactSlug(artifact.ID) + entries := make([]directoryManifestEntry, 0) + var totalBytes int64 + err := filepath.WalkDir(artifact.Path, func(path string, entry os.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + if entry.IsDir() { + return nil + } + relative, err := filepath.Rel(artifact.Path, path) + if err != nil { + return err + } + key := basePrefix + "/files/" + normalizeArtifactKey(filepath.ToSlash(relative)) + meta, sha, err := putArtifactFile(ctx, publisher, connection, policy.Bucket, key, path) + if err != nil { + return err + } + entries = append(entries, directoryManifestEntry{Path: filepath.ToSlash(relative), Key: key, SizeBytes: meta.SizeBytes, SHA256: sha}) + totalBytes += meta.SizeBytes + return nil + }) + if err != nil { + return artifact, err + } + sort.Slice(entries, func(i, j int) bool { return entries[i].Path < entries[j].Path }) + manifest := directoryManifest{SchemaVersion: 1, Kind: "directory", FileCount: len(entries), TotalBytes: totalBytes, Entries: entries} + payload, err := json.Marshal(manifest) + if err != nil { + return artifact, err + } + manifestKey := basePrefix + "/manifest.json" + meta, err := publisher.PutInternal(ctx, connection, policy.Bucket, manifestKey, bytes.NewReader(payload), int64(len(payload)), "application/json", nil) + if err != nil { + return artifact, err + } + ref := &domain.ArtifactReference{Provider: domain.StorageProviderBOS, ConnectionID: policy.ConnectionID, Bucket: policy.Bucket, Prefix: basePrefix + "/files", ManifestKey: manifestKey, VersionID: meta.VersionID, ETag: meta.ETag, SizeBytes: totalBytes, Kind: domain.ArtifactRefKindDirectory} + artifact.Path = "bos://" + policy.Bucket + "/" + ref.Prefix + artifact.SizeBytes = totalBytes + artifact.Payload = &domain.ArtifactPayload{Mode: domain.ArtifactTransferModeRef, Type: artifact.Type, Ref: ref} + artifact.Data = compactArtifactData(artifact.Data) + return artifact, nil +} + +func (runtime *Runtime) normalizeOutputArtifacts(ctx context.Context, workflow *domain.WorkflowRecord, run *domain.RunRecord, nodeID string, artifacts []domain.ExecutionArtifact) ([]domain.ExecutionArtifact, error) { + policy, err := runtime.artifactStoragePolicy(ctx, workflow) + if err != nil { + return nil, err + } + var connection *domain.StorageConnectionRecord + var publisher objectsync.Publisher + if policy != nil { + connection, err = runtime.artifactStorageConnection(ctx, policy.ConnectionID, run.OrganizationID) + if err != nil { + return nil, err + } + publisher = objectsync.BOSPublisher{GlobalEndpoint: runtime.globalBOSEndpoint} + } + result := make([]domain.ExecutionArtifact, len(artifacts)) + for index, artifact := range artifacts { + result[index] = artifact + if artifact.Payload != nil && artifact.Payload.Mode == domain.ArtifactTransferModeRef { + if err := domain.ValidateArtifactPayload(*artifact.Payload, artifact.Type); err != nil { + return nil, fmt.Errorf("artifact_ref_invalid: %w", err) + } + ref := artifact.Payload.Ref + if ref.Kind == domain.ArtifactRefKindDirectory && strings.TrimSpace(ref.ManifestKey) == "" { + return nil, fmt.Errorf("manifest_incomplete: directory artifact ref requires manifestKey") + } + refConnection, err := runtime.artifactStorageConnection(ctx, ref.ConnectionID, run.OrganizationID) + if err != nil { + return nil, fmt.Errorf("artifact_ref_invalid: %w", err) + } + checkKey := ref.Key + if ref.Kind == domain.ArtifactRefKindDirectory { + checkKey = ref.ManifestKey + } + meta, err := (objectsync.BOSPublisher{GlobalEndpoint: runtime.globalBOSEndpoint}).Head(ctx, *refConnection, ref.Bucket, checkKey) + if err != nil || !meta.Exists { + if err == nil { + err = fmt.Errorf("object does not exist") + } + return nil, fmt.Errorf("artifact_ref_invalid: %w", err) + } + continue + } + info, statErr := os.Stat(artifact.Path) + if statErr == nil && info.Mode().IsRegular() && artifact.Type == domain.PortTypeJSON && info.Size() <= domain.MaxInlinePayloadBytes { + payloadBytes, readErr := os.ReadFile(artifact.Path) + if readErr != nil { + return nil, readErr + } + var inline any + if json.Unmarshal(payloadBytes, &inline) == nil { + result[index].Payload = &domain.ArtifactPayload{Mode: domain.ArtifactTransferModeInline, Type: domain.PortTypeJSON, Inline: inline} + continue + } + } + if statErr != nil && strings.HasPrefix(strings.ToLower(strings.TrimSpace(artifact.Path)), "bos://") { + location := strings.TrimPrefix(strings.TrimSpace(artifact.Path), "bos://") + parts := strings.SplitN(location, "/", 2) + if len(parts) == 2 && strings.TrimSpace(parts[0]) != "" && strings.TrimSpace(parts[1]) != "" { + connectionID := "" + if workflow != nil { + if node := domain.GetNodeByID(workflow.Nodes, nodeID); node != nil { + connectionID = strings.TrimSpace(node.Data.ConnectionID) + } + } + if connectionID == "" && policy != nil { + connectionID = policy.ConnectionID + } + if connectionID != "" { + ref := &domain.ArtifactReference{Provider: domain.StorageProviderBOS, ConnectionID: connectionID, Bucket: parts[0], Key: parts[1], SizeBytes: artifact.SizeBytes, Kind: domain.ArtifactRefKindObject} + result[index].Payload = &domain.ArtifactPayload{Mode: domain.ArtifactTransferModeRef, Type: artifact.Type, Ref: ref} + result[index].Data = compactArtifactData(artifact.Data) + continue + } + } + } + if policy == nil { + continue // Draft/legacy workflows retain workspace-path compatibility. + } + if statErr != nil { + if workflow != nil && workflow.DefinitionState == domain.WorkflowDefinitionStateReleased { + return nil, fmt.Errorf("artifact_externalization_failed: artifact path %s is not accessible to the worker and no durable ref was provided", artifact.Path) + } + continue + } + if info.IsDir() { + externalized, externalizeErr := runtime.externalizeDirectory(ctx, publisher, *connection, *policy, run, nodeID, artifact) + if externalizeErr != nil { + return nil, fmt.Errorf("artifact_externalization_failed: %w", externalizeErr) + } + result[index] = externalized + continue + } + key := artifactPrefix(*policy, run, nodeID) + "/" + artifactSlug(artifact.ID) + "/" + artifactSlug(filepath.Base(artifact.Path)) + meta, sha, uploadErr := putArtifactFile(ctx, publisher, *connection, policy.Bucket, key, artifact.Path) + if uploadErr != nil { + return nil, fmt.Errorf("artifact_externalization_failed: %w", uploadErr) + } + ref := &domain.ArtifactReference{Provider: domain.StorageProviderBOS, ConnectionID: policy.ConnectionID, Bucket: policy.Bucket, Key: key, VersionID: meta.VersionID, ETag: meta.ETag, SizeBytes: meta.SizeBytes, SHA256: sha, Kind: domain.ArtifactRefKindObject} + result[index].Path = "bos://" + policy.Bucket + "/" + key + result[index].SizeBytes = meta.SizeBytes + result[index].Payload = &domain.ArtifactPayload{Mode: domain.ArtifactTransferModeRef, Type: artifact.Type, Ref: ref} + result[index].Data = compactArtifactData(artifact.Data) + } + return result, nil +} + +func (runtime *Runtime) resolveDirectoryRunInputs(ctx context.Context, run *domain.RunRecord) error { + if run == nil { + return nil + } + changed := false + resolvedByInputID := map[string]*domain.ArtifactReference{} + for index := range run.ResolvedInputBindings { + binding := &run.ResolvedInputBindings[index] + if binding.Payload.Mode != domain.ArtifactTransferModeRef || binding.Payload.Ref == nil { + continue + } + if resolved, ok := resolvedByInputID[binding.WorkflowInputID]; ok { + refCopy := *resolved + binding.Payload.Ref = &refCopy + changed = true + continue + } + ref := binding.Payload.Ref + connection, err := runtime.artifactStorageConnection(ctx, ref.ConnectionID, run.OrganizationID) + if err != nil { + return fmt.Errorf("input_resolution_failed: %w", err) + } + publisher := objectsync.BOSPublisher{GlobalEndpoint: runtime.globalBOSEndpoint} + if ref.Kind == domain.ArtifactRefKindObject || strings.TrimSpace(ref.ManifestKey) != "" { + checkKey := ref.Key + if ref.Kind == domain.ArtifactRefKindDirectory { + checkKey = ref.ManifestKey + } + meta, headErr := publisher.Head(ctx, *connection, ref.Bucket, checkKey) + if headErr != nil || !meta.Exists { + if headErr == nil { + headErr = fmt.Errorf("object does not exist") + } + return fmt.Errorf("input_resolution_failed: %w", headErr) + } + refCopy := *ref + resolvedByInputID[binding.WorkflowInputID] = &refCopy + continue + } + client, err := bos.NewClient(connection.AccessKeyID, connection.SecretAccessKey, connection.Endpoint) + if err != nil { + return fmt.Errorf("input_resolution_failed: %w", err) + } + requestPrefix := strings.Trim(strings.TrimSpace(ref.Prefix), "/") + "/" + entries := make([]directoryManifestEntry, 0) + marker := "" + for { + page, listErr := client.ListObjectsWithContext(ctx, ref.Bucket, &bosapi.ListObjectsArgs{Prefix: requestPrefix, Marker: marker, MaxKeys: 1000}) + if listErr != nil { + return fmt.Errorf("input_resolution_failed: %w", listErr) + } + if page == nil { + break + } + for _, object := range page.Contents { + key := strings.TrimSpace(object.Key) + if key == "" || strings.Contains(key, "/_droboticflow/manifests/") { + continue + } + entries = append(entries, directoryManifestEntry{Path: strings.TrimPrefix(key, requestPrefix), Key: key, SizeBytes: int64(object.Size)}) + } + if !page.IsTruncated || strings.TrimSpace(page.NextMarker) == "" || page.NextMarker == marker { + break + } + marker = page.NextMarker + } + sort.Slice(entries, func(i, j int) bool { return entries[i].Path < entries[j].Path }) + var totalBytes int64 + for _, entry := range entries { + totalBytes += entry.SizeBytes + } + manifest := directoryManifest{SchemaVersion: 1, Kind: "directory", Source: map[string]string{"bucket": ref.Bucket, "prefix": ref.Prefix}, FileCount: len(entries), TotalBytes: totalBytes, Entries: entries} + payload, _ := json.Marshal(manifest) + manifestKey := strings.Trim(requestPrefix, "/") + "/_droboticflow/manifests/" + artifactSlug(run.ID) + ".json" + meta, putErr := publisher.PutInternal(ctx, *connection, ref.Bucket, manifestKey, bytes.NewReader(payload), int64(len(payload)), "application/json", nil) + if putErr != nil { + return fmt.Errorf("input_resolution_failed: %w", putErr) + } + ref.ManifestKey = manifestKey + ref.SizeBytes = totalBytes + ref.ETag = meta.ETag + ref.VersionID = meta.VersionID + binding.Payload.Ref = ref + refCopy := *ref + resolvedByInputID[binding.WorkflowInputID] = &refCopy + if input, ok := run.Inputs[binding.WorkflowInputID]; ok && input.Ref != nil { + input.Ref = ref + run.Inputs[binding.WorkflowInputID] = input + } + changed = true + } + if changed { + run.UpdatedAt = nowISO() + return runtime.store.UpsertRun(ctx, *run) + } + return nil +} diff --git a/internal/worker/artifact_storage_test.go b/internal/worker/artifact_storage_test.go new file mode 100644 index 00000000..8e9015cd --- /dev/null +++ b/internal/worker/artifact_storage_test.go @@ -0,0 +1,58 @@ +package worker + +import ( + "context" + "os" + "path/filepath" + "testing" + + "droboticflow/internal/domain" + "droboticflow/internal/store" +) + +func TestGatherNodeInputsIncludesWorkflowPayloadByTargetPort(t *testing.T) { + run := &domain.RunRecord{ID: "run-1", ResolvedInputBindings: []domain.ResolvedInputBinding{{ + WorkflowInputID: "config", NodeID: "source", TargetPortID: "in-config", + Payload: domain.ArtifactPayload{Mode: domain.ArtifactTransferModeInline, Type: domain.PortTypeJSON, Inline: map[string]any{"ok": true}}, + }}} + workflow := domain.WorkflowRecord{Nodes: []domain.WorkflowNode{{ID: "source"}}} + artifacts, lineage, bindings := gatherNodeInputs(workflow, run, nil, "source") + if len(lineage) != 0 || len(artifacts) != 1 || len(bindings) != 1 { + t.Fatalf("unexpected root inputs: artifacts=%#v lineage=%#v bindings=%#v", artifacts, lineage, bindings) + } + if bindings[0].TargetPortID != "in-config" || artifacts[0].Payload == nil || artifacts[0].Payload.Mode != domain.ArtifactTransferModeInline { + t.Fatalf("workflow payload was not bound to target port: %#v %#v", bindings[0], artifacts[0]) + } +} + +func TestNormalizeOutputArtifactsInlinesSmallJSONWithoutArtifactStorage(t *testing.T) { + dataStore := store.NewMemoryStore() + runtime, err := newTestRuntime(t, dataStore) + if err != nil { + t.Fatalf("new runtime: %v", err) + } + path := filepath.Join(t.TempDir(), "result.json") + if err := os.WriteFile(path, []byte(`{"ok":true}`), 0o644); err != nil { + t.Fatalf("write json: %v", err) + } + run := &domain.RunRecord{ID: "run-1", WorkflowID: "workflow-1"} + artifacts, err := runtime.normalizeOutputArtifacts(context.Background(), &domain.WorkflowRecord{}, run, "node-1", []domain.ExecutionArtifact{{ + ID: "artifact-1", RunID: run.ID, NodeID: "node-1", Name: "result.json", Type: domain.PortTypeJSON, Path: path, + }}) + if err != nil { + t.Fatalf("normalize artifacts: %v", err) + } + if len(artifacts) != 1 || artifacts[0].Payload == nil || artifacts[0].Payload.Mode != domain.ArtifactTransferModeInline { + t.Fatalf("expected inline json payload, got %#v", artifacts) + } +} + +func TestValidateRequiredNodeInputsRejectsMissingArtifact(t *testing.T) { + node := domain.WorkflowNode{ID: "sink", Data: domain.WorkflowNodeData{Inputs: []domain.PortDefinition{{ID: "in", Type: domain.PortTypeJSON, Required: true}}}} + if err := validateRequiredNodeInputs(node, nil); err == nil { + t.Fatal("expected required input failure") + } + if err := validateRequiredNodeInputs(node, []domain.NodeInputBinding{{TargetPortID: "in", Payload: &domain.ArtifactPayload{Mode: domain.ArtifactTransferModeInline, Type: domain.PortTypeJSON, Inline: true}}}); err != nil { + t.Fatalf("expected payload binding to satisfy required input: %v", err) + } +} diff --git a/internal/worker/executor.go b/internal/worker/executor.go index 3f0d031e..5ddb0a5a 100644 --- a/internal/worker/executor.go +++ b/internal/worker/executor.go @@ -25,6 +25,7 @@ type nodeWorkspace struct { Root string OutputDir string InputArtifactsPath string + InputBindingsPath string ResultPath string StageManifestPath string } @@ -127,6 +128,26 @@ func cloneInputLineage(items []domain.InputLineageRecord) []domain.InputLineageR return cloned } +func cloneInputBindings(items []domain.NodeInputBinding) []domain.NodeInputBinding { + if items == nil { + return nil + } + cloned := make([]domain.NodeInputBinding, len(items)) + for index, item := range items { + cloned[index] = item + cloned[index].Artifacts = cloneArtifacts(item.Artifacts) + if item.Payload != nil { + payload := *item.Payload + if item.Payload.Ref != nil { + ref := *item.Payload.Ref + payload.Ref = &ref + } + cloned[index].Payload = &payload + } + } + return cloned +} + func runTimestampEnvValue(value string) string { trimmed := strings.TrimSpace(value) if trimmed == "" { @@ -214,6 +235,120 @@ func inputArtifactsFromLineage(lineage []domain.InputLineageRecord) []domain.Exe return items } +func artifactFromWorkflowInput(run *domain.RunRecord, binding domain.ResolvedInputBinding) domain.ExecutionArtifact { + payload := binding.Payload + artifact := domain.ExecutionArtifact{ + ID: run.ID + "-input-" + sanitizeEnvKey(binding.WorkflowInputID) + "-" + sanitizeEnvKey(binding.TargetPortID), + RunID: run.ID, + NodeID: binding.NodeID, + PortID: binding.TargetPortID, + PortName: binding.TargetPortID, + Name: binding.WorkflowInputID, + Type: payload.Type, + Payload: &payload, + } + if payload.Mode == domain.ArtifactTransferModeInline { + encoded, _ := json.Marshal(payload.Inline) + artifact.SizeBytes = int64(len(encoded)) + artifact.Size = fmt.Sprintf("%d B", len(encoded)) + if data, ok := payload.Inline.(map[string]any); ok { + artifact.Data = data + } else { + artifact.Data = map[string]any{"value": payload.Inline} + } + return artifact + } + if payload.Ref != nil { + ref := payload.Ref + artifact.SizeBytes = ref.SizeBytes + artifact.Size = fmt.Sprintf("%d B", ref.SizeBytes) + if ref.Kind == domain.ArtifactRefKindDirectory { + artifact.Path = "bos://" + ref.Bucket + "/" + strings.TrimPrefix(ref.Prefix, "/") + } else { + artifact.Path = "bos://" + ref.Bucket + "/" + strings.TrimPrefix(ref.Key, "/") + } + artifact.Metadata = &domain.ArtifactMetadata{SHA256: ref.SHA256, MimeType: ref.MimeType} + artifact.Data = map[string]any{"source": map[string]any{"path": artifact.Path}} + } + return artifact +} + +func workflowInputBindingsForNode(run *domain.RunRecord, nodeID string) ([]domain.NodeInputBinding, []domain.ExecutionArtifact) { + if run == nil { + return nil, nil + } + bindings := make([]domain.NodeInputBinding, 0) + artifacts := make([]domain.ExecutionArtifact, 0) + for _, item := range run.ResolvedInputBindings { + if item.NodeID != nodeID { + continue + } + artifact := artifactFromWorkflowInput(run, item) + payload := item.Payload + bindings = append(bindings, domain.NodeInputBinding{ + TargetPortID: item.TargetPortID, + WorkflowInputID: item.WorkflowInputID, + Payload: &payload, + Artifacts: []domain.ExecutionArtifact{artifact}, + }) + artifacts = append(artifacts, artifact) + } + return bindings, artifacts +} + +func lineageInputBindings(lineage []domain.InputLineageRecord) []domain.NodeInputBinding { + items := make([]domain.NodeInputBinding, 0) + indexByKey := map[string]int{} + for _, item := range lineage { + key := item.SourceNodeID + "\x00" + item.SourceHandle + "\x00" + item.TargetHandle + if index, ok := indexByKey[key]; ok { + items[index].Artifacts = append(items[index].Artifacts, item.Artifact) + continue + } + indexByKey[key] = len(items) + items = append(items, domain.NodeInputBinding{ + TargetPortID: item.TargetHandle, + SourceNodeID: item.SourceNodeID, + SourcePortID: item.SourceHandle, + Artifacts: []domain.ExecutionArtifact{item.Artifact}, + }) + } + return items +} + +func validateRequiredNodeInputs(node domain.WorkflowNode, bindings []domain.NodeInputBinding) error { + for _, port := range node.Data.Inputs { + if !port.Required { + continue + } + present := false + for _, binding := range bindings { + targetPortID := strings.TrimSpace(binding.TargetPortID) + if targetPortID != strings.TrimSpace(port.ID) && !(targetPortID == "" && len(node.Data.Inputs) == 1) { + continue + } + if binding.Payload != nil || len(binding.Artifacts) > 0 { + present = true + break + } + } + if !present { + return fmt.Errorf("required_input_missing: node %s input %s has no value or upstream artifact", node.ID, port.ID) + } + } + return nil +} + +func gatherNodeInputs(workflow domain.WorkflowRecord, run *domain.RunRecord, executionMap map[string]domain.NodeExecutionRecord, nodeID string) ([]domain.ExecutionArtifact, []domain.InputLineageRecord, []domain.NodeInputBinding) { + lineage := gatherInputLineage(workflow, executionMap, nodeID) + artifacts := inputArtifactsFromLineage(lineage) + bindings := lineageInputBindings(lineage) + rootBindings, rootArtifacts := workflowInputBindingsForNode(run, nodeID) + bindings = append(bindings, rootBindings...) + artifacts = append(artifacts, rootArtifacts...) + return artifacts, lineage, bindings +} + func gatherInputArtifacts(workflow domain.WorkflowRecord, executionMap map[string]domain.NodeExecutionRecord, nodeID string) []domain.ExecutionArtifact { return inputArtifactsFromLineage(gatherInputLineage(workflow, executionMap, nodeID)) } @@ -309,6 +444,7 @@ func prepareNodeWorkspace(workspaceRoot string, runID string, nodeID string) (no Root: nodeRoot, OutputDir: outputDir, InputArtifactsPath: filepath.Join(nodeRoot, "input-artifacts.json"), + InputBindingsPath: filepath.Join(nodeRoot, "input-bindings.json"), ResultPath: filepath.Join(nodeRoot, "result.json"), StageManifestPath: filepath.Join(nodeRoot, "stage-manifest.json"), }, nil @@ -352,7 +488,7 @@ func applyOperatorCommandPolicyEnv(env map[string]string) { } func buildNodeEnv(run *domain.RunRecord, workflow *domain.WorkflowRecord, node domain.WorkflowNode, inputArtifacts []domain.ExecutionArtifact, workspace nodeWorkspace, settings *domain.SettingsRecord) (map[string]string, error) { - return buildNodeEnvWithResources(run, workflow, node, inputArtifacts, workspace, effectiveNodeResources{ + return buildNodeEnvWithResources(run, workflow, node, inputArtifacts, nil, workspace, effectiveNodeResources{ CPULimit: node.Data.CPU, MemoryLimit: node.Data.Memory, GPULimit: node.Data.GPU, @@ -368,7 +504,7 @@ type effectiveNodeResources struct { SharedMemorySize string } -func buildNodeEnvWithResources(run *domain.RunRecord, workflow *domain.WorkflowRecord, node domain.WorkflowNode, inputArtifacts []domain.ExecutionArtifact, workspace nodeWorkspace, resources effectiveNodeResources) (map[string]string, error) { +func buildNodeEnvWithResources(run *domain.RunRecord, workflow *domain.WorkflowRecord, node domain.WorkflowNode, inputArtifacts []domain.ExecutionArtifact, inputBindings []domain.NodeInputBinding, workspace nodeWorkspace, resources effectiveNodeResources) (map[string]string, error) { if inputArtifacts == nil { inputArtifacts = []domain.ExecutionArtifact{} } @@ -383,6 +519,19 @@ func buildNodeEnvWithResources(run *domain.RunRecord, workflow *domain.WorkflowR if err := writeJSONFile(workspace.InputArtifactsPath, inputArtifacts); err != nil { return nil, err } + if inputBindings == nil { + inputBindings = []domain.NodeInputBinding{} + } + if strings.TrimSpace(workspace.InputBindingsPath) == "" { + workspace.InputBindingsPath = filepath.Join(workspace.Root, "input-bindings.json") + } + inputBindingsJSON, err := json.Marshal(inputBindings) + if err != nil { + return nil, err + } + if err := writeJSONFile(workspace.InputBindingsPath, inputBindings); err != nil { + return nil, err + } env := map[string]string{ "RUN_ID": run.ID, "WORKFLOW_ID": run.WorkflowID, @@ -393,6 +542,7 @@ func buildNodeEnvWithResources(run *domain.RunRecord, workflow *domain.WorkflowR "NODE_DESCRIPTION": node.Data.Description, "NODE_DATA_JSON": string(nodeDataJSON), "INPUT_ARTIFACTS_PATH": workspace.InputArtifactsPath, + "INPUT_BINDINGS_PATH": workspace.InputBindingsPath, "RESULT_OUTPUT_PATH": workspace.ResultPath, "STAGE_MANIFEST_PATH": workspace.StageManifestPath, "OUTPUT_DIR": workspace.OutputDir, @@ -411,6 +561,9 @@ func buildNodeEnvWithResources(run *domain.RunRecord, workflow *domain.WorkflowR if len(inputArtifactsJSON) <= inputArtifactsInlineLimit { env["INPUT_ARTIFACTS_JSON"] = string(inputArtifactsJSON) } + if len(inputBindingsJSON) <= inputArtifactsInlineLimit { + env["INPUT_BINDINGS_JSON"] = string(inputBindingsJSON) + } if node.Data.Bucket != "" { env["STORAGE_BUCKET"] = node.Data.Bucket } @@ -636,7 +789,10 @@ func (runtime *Runtime) resolveNodeBusinessStage(ctx context.Context, run *domai return node } -func (runtime *Runtime) executeNode(ctx context.Context, run *domain.RunRecord, node domain.WorkflowNode, inputArtifacts []domain.ExecutionArtifact) (executionResult, *k8s.JobReference, error) { +func (runtime *Runtime) executeNode(ctx context.Context, run *domain.RunRecord, node domain.WorkflowNode, inputArtifacts []domain.ExecutionArtifact, inputBindings []domain.NodeInputBinding) (executionResult, *k8s.JobReference, error) { + if err := validateRequiredNodeInputs(node, inputBindings); err != nil { + return executionResult{}, nil, err + } node = runtime.resolveNodeBusinessStage(ctx, run, node) workspace, err := prepareNodeWorkspace(runtime.workspaceRoot, run.ID, node.ID) if err != nil { @@ -668,7 +824,7 @@ func (runtime *Runtime) executeNode(ctx context.Context, run *domain.RunRecord, return executionResult{}, nil, err } } - env, err := buildNodeEnvWithResources(run, workflow, node, inputArtifacts, workspace, resources) + env, err := buildNodeEnvWithResources(run, workflow, node, inputArtifacts, inputBindings, workspace, resources) if err != nil { return executionResult{}, nil, err } @@ -716,6 +872,9 @@ func (runtime *Runtime) executeNode(ctx context.Context, run *domain.RunRecord, _ = runtime.store.UpdateRunProgress(ctx, run.ID, progress) } parsed, err := parseExecutionOutput(launched.Output, node.Data.Label) + if err == nil { + parsed.Artifacts, err = runtime.normalizeOutputArtifacts(ctx, workflow, run, node.ID, parsed.Artifacts) + } if err == nil { parsed.StageManifestPath = workspace.StageManifestPath parsed.DataRefs, err = loadStageManifestDataRefs(workspace.StageManifestPath) diff --git a/internal/worker/runtime.go b/internal/worker/runtime.go index 3ae6d2e5..d7c938ae 100644 --- a/internal/worker/runtime.go +++ b/internal/worker/runtime.go @@ -587,7 +587,7 @@ func applyExecution(run *domain.RunRecord, execution domain.NodeExecutionRecord) } } -func applyNodeExecutionSnapshot(execution *domain.NodeExecutionRecord, node domain.WorkflowNode, inputArtifacts []domain.ExecutionArtifact, inputLineage []domain.InputLineageRecord) { +func applyNodeExecutionSnapshot(execution *domain.NodeExecutionRecord, node domain.WorkflowNode, inputArtifacts []domain.ExecutionArtifact, inputLineage []domain.InputLineageRecord, inputBindings []domain.NodeInputBinding) { if execution == nil { return } @@ -607,6 +607,9 @@ func applyNodeExecutionSnapshot(execution *domain.NodeExecutionRecord, node doma if inputLineage != nil { execution.InputLineage = cloneInputLineage(inputLineage) } + if inputBindings != nil { + execution.InputBindings = cloneInputBindings(inputBindings) + } } func globalConcurrencyLimit(settings *domain.SettingsRecord) int { @@ -1387,8 +1390,7 @@ func (runtime *Runtime) recoverSucceededJob(ctx context.Context, run *domain.Run for _, item := range run.NodeExecutions { executionMap[item.NodeID] = item } - inputLineage := gatherInputLineage(*workflow, executionMap, node.ID) - inputArtifacts := inputArtifactsFromLineage(inputLineage) + inputArtifacts, inputLineage, inputBindings := gatherNodeInputs(*workflow, run, executionMap, node.ID) _, existing := findNodeExecution(run, node.ID) execution := domain.NodeExecutionRecord{ NodeID: node.ID, @@ -1404,7 +1406,7 @@ func (runtime *Runtime) recoverSucceededJob(ctx context.Context, run *domain.Run if existing != nil { execution = *existing } - applyNodeExecutionSnapshot(&execution, *node, inputArtifacts, inputLineage) + applyNodeExecutionSnapshot(&execution, *node, inputArtifacts, inputLineage, inputBindings) parsed, err := parseExecutionOutput(inspection.Output, node.Data.Label) if err != nil { return err @@ -2195,7 +2197,7 @@ func (runtime *Runtime) enqueueRetryRun(ctx context.Context, workflow *domain.Wo if err != nil { return nil, err } - retryRun, err := runtime.store.CreateRun(ctx, workflow.ID, domain.RunTriggerRetry, failedRun.TaskID) + retryRun, err := runtime.store.CreateRunWithInputs(ctx, workflow.ID, domain.RunTriggerRetry, failedRun.Inputs, failedRun.TaskID) if err != nil { return nil, err } @@ -2528,6 +2530,16 @@ func (runtime *Runtime) processRun(ctx context.Context, claimed *domain.RunRecor if isTerminalRunStatus(currentRun.Status) { return currentRun, nil } + if err := runtime.resolveDirectoryRunInputs(ctx, currentRun); err != nil { + message := err.Error() + currentRun.Status = domain.RunStatusFailed + currentRun.Error = message + currentRun.Failure = &domain.RunFailure{Code: "input_resolution_failed", Stage: "input_resolution", Message: message, Retryable: true, Suggestion: "检查对象存储连接、目录前缀和写入 manifest 的权限"} + currentRun.EndedAt = nowISO() + currentRun.UpdatedAt = nowISO() + _ = runtime.store.UpsertRun(ctx, *currentRun) + return currentRun, nil + } settings, err := runtime.store.GetSettings(ctx) if err != nil { return nil, err @@ -2717,8 +2729,7 @@ func (runtime *Runtime) processRun(ctx context.Context, claimed *domain.RunRecor continue } - inputLineage := gatherInputLineage(*workflow, executionMap, node.ID) - inputArtifacts := inputArtifactsFromLineage(inputLineage) + inputArtifacts, inputLineage, inputBindings := gatherNodeInputs(*workflow, currentRun, executionMap, node.ID) executionAttempt := runExecutionAttempt(currentRun) executionLogs := []domain.LogEntryRecord{logEntry("INFO", "Starting node execution", node.Data.Label)} if previous, ok := executionMap[node.ID]; ok && missingJobRecoveryCount(&previous) > 0 { @@ -2739,7 +2750,7 @@ func (runtime *Runtime) processRun(ctx context.Context, claimed *domain.RunRecor ResourceSeries: []domain.ResourceSample{}, StageManifestPath: "", } - applyNodeExecutionSnapshot(&execution, *node, inputArtifacts, inputLineage) + applyNodeExecutionSnapshot(&execution, *node, inputArtifacts, inputLineage, inputBindings) executionMap[node.ID] = execution applyExecution(currentRun, execution) runningNodes[node.ID] = struct{}{} @@ -2761,6 +2772,7 @@ func (runtime *Runtime) processRun(ctx context.Context, claimed *domain.RunRecor continue } inputSnapshot := cloneArtifacts(inputArtifacts) + inputBindingsSnapshot := cloneInputBindings(inputBindings) nodeExecutionCtx := executionCtx cancelNodeExecution := func() {} if timeout := workflowNodeTimeout(workflow); timeout > 0 { @@ -2778,7 +2790,7 @@ func (runtime *Runtime) processRun(ctx context.Context, claimed *domain.RunRecor progressWG.Add(1) go runtime.runNodeProgressReporter(progressCtx, runSnapshot.ID, nodeSnapshot.ID, sampleGPU, func() { progressWG.Done() }) } - result, reference, err := runtime.executeNode(nodeExecutionCtx, &runSnapshot, nodeSnapshot, inputSnapshot) + result, reference, err := runtime.executeNode(nodeExecutionCtx, &runSnapshot, nodeSnapshot, inputSnapshot, inputBindingsSnapshot) progressCancel() progressWG.Wait() results <- nodeExecutionOutcome{ @@ -2820,8 +2832,7 @@ func (runtime *Runtime) processRun(ctx context.Context, claimed *domain.RunRecor if node == nil { continue } - inputLineage := gatherInputLineage(*workflow, executionMap, node.ID) - inputArtifacts := inputArtifactsFromLineage(inputLineage) + inputArtifacts, inputLineage, inputBindings := gatherNodeInputs(*workflow, currentRun, executionMap, node.ID) execution := executionMap[node.ID] if execution.NodeID == "" { execution = domain.NodeExecutionRecord{ @@ -2836,7 +2847,7 @@ func (runtime *Runtime) processRun(ctx context.Context, claimed *domain.RunRecor ResourceSeries: []domain.ResourceSample{}, } } - applyNodeExecutionSnapshot(&execution, *node, inputArtifacts, inputLineage) + applyNodeExecutionSnapshot(&execution, *node, inputArtifacts, inputLineage, inputBindings) task := domain.AuditTaskRecord{ ID: domain.NewID("audit"), OrganizationID: currentRun.OrganizationID, diff --git a/internal/worker/runtime_test.go b/internal/worker/runtime_test.go index 4babf052..35de0860 100644 --- a/internal/worker/runtime_test.go +++ b/internal/worker/runtime_test.go @@ -449,7 +449,7 @@ func TestExecuteNodeFallsBackToOperatorResourceDefaults(t *testing.T) { }, nil }, }) - if _, _, err := runtime.executeNode(ctx, run, workflow.Nodes[0], nil); err != nil { + if _, _, err := runtime.executeNode(ctx, run, workflow.Nodes[0], nil, nil); err != nil { t.Fatalf("execute node: %v", err) } if launchCount != 1 { diff --git a/src/lib/templates.js b/src/lib/templates.js index ceb372e0..acca241a 100644 --- a/src/lib/templates.js +++ b/src/lib/templates.js @@ -23,6 +23,12 @@ export function buildTemplatePayload(source, overrides = {}) { maxRetryAttempts: source.maxRetryAttempts ?? 0, retryBackoffSeconds: source.retryBackoffSeconds ?? 0, schedule: overrides.schedule ?? deepClone(source.schedule), + ...(overrides.workflowInputs !== undefined || source.workflowInputs !== undefined + ? { workflowInputs: overrides.workflowInputs ?? deepClone(source.workflowInputs || []) } + : {}), + ...(overrides.artifactStorage !== undefined || source.artifactStorage !== undefined + ? { artifactStorage: overrides.artifactStorage ?? deepClone(source.artifactStorage) } + : {}), nodes: deepClone(source.nodes || []), edges: deepClone(source.edges || []), }; @@ -48,6 +54,12 @@ export function buildWorkflowPayload(source, overrides = {}) { maxRetryAttempts: overrides.maxRetryAttempts ?? source.maxRetryAttempts ?? 0, retryBackoffSeconds: overrides.retryBackoffSeconds ?? source.retryBackoffSeconds ?? 0, schedule: overrides.schedule ?? deepClone(source.schedule), + ...(overrides.workflowInputs !== undefined || source.workflowInputs !== undefined + ? { workflowInputs: overrides.workflowInputs ?? deepClone(source.workflowInputs || []) } + : {}), + ...(overrides.artifactStorage !== undefined || source.artifactStorage !== undefined + ? { artifactStorage: overrides.artifactStorage ?? deepClone(source.artifactStorage) } + : {}), nodes: overrides.nodes ?? deepClone(source.nodes ?? []), edges: overrides.edges ?? deepClone(source.edges ?? []), }; @@ -73,6 +85,12 @@ export function buildWorkflowFromTemplate(template, overrides = {}) { maxRetryAttempts: overrides.maxRetryAttempts ?? template.maxRetryAttempts ?? 0, retryBackoffSeconds: overrides.retryBackoffSeconds ?? template.retryBackoffSeconds ?? 0, schedule: overrides.schedule ?? deepClone(template.schedule), + ...(overrides.workflowInputs !== undefined || template.workflowInputs !== undefined + ? { workflowInputs: overrides.workflowInputs ?? deepClone(template.workflowInputs || []) } + : {}), + ...(overrides.artifactStorage !== undefined || template.artifactStorage !== undefined + ? { artifactStorage: overrides.artifactStorage ?? deepClone(template.artifactStorage) } + : {}), nodes: overrides.nodes ?? deepClone(template.nodes ?? []), edges: overrides.edges ?? deepClone(template.edges ?? []), }; @@ -98,6 +116,12 @@ export function buildWorkflowCopyPayload(workflow, overrides = {}) { maxRetryAttempts: overrides.maxRetryAttempts ?? workflow.maxRetryAttempts ?? 0, retryBackoffSeconds: overrides.retryBackoffSeconds ?? workflow.retryBackoffSeconds ?? 0, schedule: overrides.schedule ?? deepClone(workflow.schedule), + ...(overrides.workflowInputs !== undefined || workflow.workflowInputs !== undefined + ? { workflowInputs: overrides.workflowInputs ?? deepClone(workflow.workflowInputs || []) } + : {}), + ...(overrides.artifactStorage !== undefined || workflow.artifactStorage !== undefined + ? { artifactStorage: overrides.artifactStorage ?? deepClone(workflow.artifactStorage) } + : {}), nodes: overrides.nodes ?? deepClone(workflow.nodes ?? []), edges: overrides.edges ?? deepClone(workflow.edges ?? []), }; diff --git a/src/lib/workflow-editor.js b/src/lib/workflow-editor.js index 4059654e..565d4b7e 100644 --- a/src/lib/workflow-editor.js +++ b/src/lib/workflow-editor.js @@ -1,6 +1,12 @@ const ACTIVE_RUN_STATUSES = new Set(["queued", "running", "stopping", "waiting_for_audit"]); -export function getWorkflowEditorLockState({ isReadOnly, activeRunStatus, isRunContextLoading, isExecuting }) { +export function getWorkflowEditorLockState({ + isReadOnly, + activeRunStatus, + isRunContextLoading, + isExecuting, + definitionState, +}) { if (ACTIVE_RUN_STATUSES.has(activeRunStatus)) { return { locked: true, @@ -15,6 +21,13 @@ export function getWorkflowEditorLockState({ isReadOnly, activeRunStatus, isRunC message: "当前页面为只读模式,无法修改或保存工作流。", }; } + if (definitionState === "released") { + return { + locked: true, + reason: "released", + message: "工作流定义已发布并冻结;可以运行或创建新版本,但不能直接修改。", + }; + } if (isExecuting) { return { locked: true, @@ -109,6 +122,18 @@ export function buildSelectedNodeEnvRows({ nodeHyperparams = [], workflowHyperpa return [...customRows, ...inheritedRows, ...(systemRows || [])]; } -export function isWorkflowSettingsLocked({ isReadOnly, activeRunStatus, isRunContextLoading, isExecuting }) { - return getWorkflowEditorLockState({ isReadOnly, activeRunStatus, isRunContextLoading, isExecuting }).locked; +export function isWorkflowSettingsLocked({ + isReadOnly, + activeRunStatus, + isRunContextLoading, + isExecuting, + definitionState, +}) { + return getWorkflowEditorLockState({ + isReadOnly, + activeRunStatus, + isRunContextLoading, + isExecuting, + definitionState, + }).locked; } diff --git a/src/lib/workflow-inputs.js b/src/lib/workflow-inputs.js new file mode 100644 index 00000000..8e2bc6d7 --- /dev/null +++ b/src/lib/workflow-inputs.js @@ -0,0 +1,140 @@ +function normalize(value) { + return String(value || "").trim(); +} + +export function rootNodeIds(nodes, edges) { + const roots = new Set((nodes || []).map((node) => normalize(node?.id)).filter(Boolean)); + for (const edge of edges || []) { + roots.delete(normalize(edge?.target)); + } + return roots; +} + +export function syncRootWorkflowInputs(nodes, edges, existingInputs = []) { + const roots = rootNodeIds(nodes, edges); + const existingByTarget = new Map(); + for (const input of existingInputs || []) { + for (const binding of input?.bindings || []) { + existingByTarget.set(`${normalize(binding.nodeId)}::${normalize(binding.targetPortId)}`, input); + } + } + + const result = []; + const consumed = new Set(); + for (const node of nodes || []) { + if (!roots.has(normalize(node?.id))) { + continue; + } + for (const port of node?.data?.inputs || []) { + const nodeId = normalize(node.id); + const portId = normalize(port?.id); + if (!portId) { + continue; + } + const targetKey = `${nodeId}::${portId}`; + const current = existingByTarget.get(targetKey); + if (current && !consumed.has(current.id)) { + result.push({ + ...current, + type: port.type, + required: Boolean(port.required || current.required), + bindings: (current.bindings || []).filter((binding) => { + const boundNode = (nodes || []).find((candidate) => candidate.id === binding.nodeId); + return roots.has(normalize(binding.nodeId)) && (boundNode?.data?.inputs || []).some((item) => item.id === binding.targetPortId); + }), + }); + consumed.add(current.id); + continue; + } + result.push({ + id: `input-${nodeId}-${portId}`, + name: normalize(port.name) || `${node?.data?.label || nodeId} / ${portId}`, + description: port.description || "", + type: port.type, + required: Boolean(port.required), + bindings: [{ nodeId, targetPortId: portId }], + }); + } + } + return result; +} + +export function workflowInputDraft(payload) { + if (payload?.mode === "inline") { + return { + mode: "inline", + inlineText: JSON.stringify(payload.inline ?? null, null, 2), + connectionId: "", + bucket: "", + objectKind: "object", + key: "", + prefix: "", + manifestKey: "", + }; + } + return { + mode: "ref", + inlineText: "", + connectionId: payload?.ref?.connectionId || "", + bucket: payload?.ref?.bucket || "", + objectKind: payload?.ref?.kind || "object", + key: payload?.ref?.key || "", + prefix: payload?.ref?.prefix || "", + manifestKey: payload?.ref?.manifestKey || "", + }; +} + +export function parseWorkflowRunInputs(definitions, drafts) { + const inputs = {}; + for (const definition of definitions || []) { + const draft = drafts?.[definition.id]; + if (!draft) { + if (definition.required && !definition.defaultPayload) { + throw new Error(`必填输入“${definition.name}”不能为空`); + } + continue; + } + if (draft.mode === "inline") { + if (!String(draft.inlineText || "").trim()) { + if (definition.required && !definition.defaultPayload) { + throw new Error(`必填输入“${definition.name}”不能为空`); + } + continue; + } + let inline; + try { + inline = JSON.parse(draft.inlineText); + } catch { + throw new Error(`输入“${definition.name}”必须是合法 JSON`); + } + inputs[definition.id] = { mode: "inline", type: definition.type, inline }; + continue; + } + const connectionId = normalize(draft.connectionId); + const bucket = normalize(draft.bucket); + const kind = draft.objectKind === "directory" ? "directory" : "object"; + const key = normalize(draft.key); + const prefix = normalize(draft.prefix); + if (!connectionId && !bucket && !key && !prefix) { + if (definition.required && !definition.defaultPayload) { + throw new Error(`必填输入“${definition.name}”不能为空`); + } + continue; + } + if (!connectionId || !bucket || (kind === "object" ? !key : !prefix)) { + throw new Error(`输入“${definition.name}”的对象存储引用不完整`); + } + inputs[definition.id] = { + mode: "ref", + type: definition.type, + ref: { + provider: "BOS", + connectionId, + bucket, + kind, + ...(kind === "object" ? { key } : { prefix, manifestKey: normalize(draft.manifestKey) || undefined }), + }, + }; + } + return inputs; +} diff --git a/src/pages/Operators.tsx b/src/pages/Operators.tsx index 3a5ef1d5..960befea 100644 --- a/src/pages/Operators.tsx +++ b/src/pages/Operators.tsx @@ -364,6 +364,7 @@ export default function Operators() { name: `端口 ${nextIndex}`, type: "json", transmission: "bus", + ...(type === "inputs" ? { required: true } : {}), }; return { ...current, [type]: [...(current[type] || []), port] }; }); @@ -1082,7 +1083,7 @@ export default function Operators() { containerArgs: generated.containerArgs || [], runtimeContract, resources: generated.resources, - inputs: generated.inputs.map((input) => ({ id: createPortID(), ...input })), + inputs: generated.inputs.map((input) => ({ id: createPortID(), required: true, ...input })), outputs: generated.outputs.map((output) => ({ id: createPortID(), ...output })), hyperparams: generated.hyperparams || [], skillDefinition: generated.skillDefinition, @@ -2184,7 +2185,7 @@ export default function Operators() { {newOp.inputs?.map((port, idx) => (
+ 大 JSON、图片、视频、Tensor、文件和目录默认上传到这里;工作流可以单独覆盖。 +
+