package worker import ( "context" "encoding/json" "fmt" "maps" "os" "path/filepath" "regexp" "strings" "time" "droboticflow/internal/domain" "droboticflow/internal/k8s" "droboticflow/internal/lightweight" ) const resultPrefix = "DROBOTICFLOW_RESULT=" const resultPathPrefix = "DROBOTICFLOW_RESULT_PATH=" const inputArtifactsInlineLimit = 64 * 1024 var inferenceServiceImagePattern = regexp.MustCompile(`^\S+@sha256:[a-f0-9]{64}$`) type nodeWorkspace struct { RunRoot string Root string OutputDir string InputArtifactsPath string InputBindingsPath string ResultPath string StageManifestPath string } func setNodeHyperparamValue(node *domain.WorkflowNode, key, value string) { if node == nil { return } for index := range node.Data.Hyperparams { if strings.TrimSpace(node.Data.Hyperparams[index].Key) == key { node.Data.Hyperparams[index].Value = value return } } node.Data.Hyperparams = append(node.Data.Hyperparams, domain.HyperParam{Key: key, Value: value, Type: "string"}) } func inferenceSidecarForNode(run *domain.RunRecord, node *domain.WorkflowNode) (sidecar *k8s.InferenceSidecar, err error) { defer func() { if sidecar != nil && node != nil { limits := applyFanyuResourceProfile(*node, effectiveNodeResources{CPURequest: sidecar.CPURequest, CPULimit: sidecar.CPULimit, MemoryRequest: sidecar.MemoryRequest, MemoryLimit: sidecar.MemoryLimit, GPULimit: sidecar.GPULimit}) sidecar.CPULimit, sidecar.MemoryLimit = limits.CPULimit, limits.MemoryLimit } }() if node == nil { return nil, nil } params := nodeHyperparamsMap(*node) switch strings.TrimSpace(node.Data.OperatorID) { case "operator-builtin-wilor-hand-pose": image := strings.TrimSpace(params["WILOR_SERVICE_IMAGE_ID"]) if !inferenceServiceImagePattern.MatchString(image) { return nil, fmt.Errorf("WiLoR on-demand service requires digest-pinned WILOR_SERVICE_IMAGE_ID") } setNodeHyperparamValue(node, "WILOR_API_BASE_URL", "http://127.0.0.1:8080") return &k8s.InferenceSidecar{Name: "wilor-inference", Image: image, Port: 8080, HealthPath: "/api/health", CPURequest: "2", CPULimit: "4", MemoryRequest: "16Gi", MemoryLimit: "24Gi", GPULimit: "1"}, nil case "operator-builtin-head-pose": image := strings.TrimSpace(params["HEADPOSE_SERVICE_IMAGE_ID"]) if !inferenceServiceImagePattern.MatchString(image) { return nil, fmt.Errorf("HeadPose on-demand service requires digest-pinned HEADPOSE_SERVICE_IMAGE_ID") } setNodeHyperparamValue(node, "HEADPOSE_API_BASE_URL", "http://127.0.0.1:8080") outputRoot := "/workspace/system/headpose-service" if run != nil { outputRoot = filepath.Join("/workspace/system/runs", run.ID, "nodes", node.ID, "headpose-service") } return &k8s.InferenceSidecar{ Name: "headpose-inference", Image: image, Port: 8080, HealthPath: "/health", CPURequest: "1", CPULimit: "2", MemoryRequest: "2Gi", MemoryLimit: "4Gi", Env: map[string]string{"HEADPOSE_MODEL_SHA256": params["HEADPOSE_MODEL_SHA256"], "HEADPOSE_LICENSE_ID": params["HEADPOSE_LICENSE_ID"], "HEADPOSE_OUTPUT_ROOT": outputRoot, "HEADPOSE_GPU_ID": "-1"}, }, nil default: return nil, nil } } type executionResult struct { Logs []domain.LogEntryRecord `json:"logs"` Artifacts []domain.ExecutionArtifact `json:"artifacts"` DataRefs []domain.DataRefRecord `json:"dataRefs,omitempty"` DataMetrics *domain.DataMetricsRecord `json:"dataMetrics,omitempty"` StageManifestPath string `json:"stageManifestPath,omitempty"` ResourceSeries []domain.ResourceSample `json:"resourceSeries"` } type stageManifestRecord struct { DataRefs []domain.DataRefRecord `json:"dataRefs"` DataMetrics *domain.DataMetricsRecord `json:"dataMetrics,omitempty"` } var protectedSystemEnv = map[string]struct{}{ "WORKFLOW_PREVIOUS_RUN_TIMESTAMP": {}, "WORKFLOW_CURRENT_RUN_TIMESTAMP": {}, "PARENT_RUN_ID": {}, "PARENT_RUN_WORKSPACE_ROOT": {}, "PARENT_NODE_OUTPUT_DIR": {}, "OPERATOR_ALLOW_SHELL_COMMANDS": {}, "OPERATOR_COMMAND_ALLOWLIST_REGEX": {}, "OPERATOR_COMMAND_DENYLIST_REGEX": {}, "DROBOTICFLOW_QWEN_NODE_CACHE_ENABLED": {}, "DROBOTICFLOW_QWEN_NODE_CACHE_ROOT": {}, "DROBOTICFLOW_QWEN_NODE_CACHE_STALE_STAGING_SECONDS": {}, } var operatorCommandPolicyEnvKeys = []string{ "OPERATOR_ALLOW_SHELL_COMMANDS", "OPERATOR_COMMAND_ALLOWLIST_REGEX", "OPERATOR_COMMAND_DENYLIST_REGEX", } var workflowEnvLocation = loadWorkflowEnvLocation() var invalidEnvCharacters = regexp.MustCompile(`[^A-Z0-9_]+`) func loadWorkflowEnvLocation() *time.Location { location, err := time.LoadLocation("Asia/Shanghai") if err != nil { return time.FixedZone("Asia/Shanghai", 8*60*60) } return location } func sanitizeEnvKey(value string) string { normalized := strings.ToUpper(strings.TrimSpace(value)) if normalized == "" { return "" } normalized = invalidEnvCharacters.ReplaceAllString(normalized, "_") normalized = strings.Trim(normalized, "_") if normalized == "" { return "" } if normalized[0] >= '0' && normalized[0] <= '9' { normalized = "HP_" + normalized } return normalized } func cloneArtifacts(items []domain.ExecutionArtifact) []domain.ExecutionArtifact { cloned := make([]domain.ExecutionArtifact, 0, len(items)) for _, item := range items { cloned = append(cloned, cloneArtifact(item)) } return cloned } func tagDebugArtifact(run *domain.RunRecord, artifact domain.ExecutionArtifact) domain.ExecutionArtifact { if run == nil || run.Mode != domain.RunModeDebug { return artifact } artifact.Mode = domain.RunModeDebug artifact.Temporary = true if artifact.Metadata == nil { artifact.Metadata = &domain.ArtifactMetadata{} } artifact.Metadata.Temporary = true return artifact } func tagDebugArtifacts(run *domain.RunRecord, artifacts []domain.ExecutionArtifact) []domain.ExecutionArtifact { if run == nil || run.Mode != domain.RunModeDebug { return artifacts } for i := range artifacts { artifacts[i] = tagDebugArtifact(run, artifacts[i]) } return artifacts } func routeAuditArtifacts( node domain.WorkflowNode, inputArtifacts []domain.ExecutionArtifact, edges []domain.WorkflowEdge, ) []domain.ExecutionArtifact { handles := make([]string, 0, len(edges)) seenHandles := map[string]struct{}{} for _, edge := range edges { handle := strings.TrimSpace(edge.SourceHandle) if handle == "" { continue } if _, exists := seenHandles[handle]; exists { continue } seenHandles[handle] = struct{}{} handles = append(handles, handle) } if len(handles) == 0 { return cloneArtifacts(inputArtifacts) } portNames := make(map[string]string, len(node.Data.Outputs)) for _, output := range node.Data.Outputs { portNames[strings.TrimSpace(output.ID)] = strings.TrimSpace(output.Name) } routed := make([]domain.ExecutionArtifact, 0, len(inputArtifacts)*len(handles)) for _, handle := range handles { portName := portNames[handle] if portName == "" { portName = handle } for _, input := range inputArtifacts { artifact := cloneArtifact(input) artifact.PortID = handle artifact.PortName = portName routed = append(routed, artifact) } } return routed } func cloneStringSlice(items []string) []string { if items == nil { return nil } cloned := make([]string, len(items)) copy(cloned, items) return cloned } func cloneArtifact(item domain.ExecutionArtifact) domain.ExecutionArtifact { cloned := item if item.Metadata != nil { metadata := *item.Metadata cloned.Metadata = &metadata } if item.Data != nil { cloned.Data = maps.Clone(item.Data) } return cloned } func cloneInputLineage(items []domain.InputLineageRecord) []domain.InputLineageRecord { cloned := make([]domain.InputLineageRecord, 0, len(items)) for _, item := range items { record := item record.Artifact = cloneArtifact(item.Artifact) cloned = append(cloned, record) } return cloned } func cloneInputBindings(items []domain.NodeInputBinding) []domain.NodeInputBinding { if items == nil { return nil } cloned := make([]domain.NodeInputBinding, len(items)) for index, item := range items { cloned[index] = item cloned[index].Artifacts = cloneArtifacts(item.Artifacts) if item.Payload != nil { payload := *item.Payload if item.Payload.Ref != nil { ref := *item.Payload.Ref payload.Ref = &ref } cloned[index].Payload = &payload } } return cloned } func runTimestampEnvValue(value string) string { trimmed := strings.TrimSpace(value) if trimmed == "" { return "" } parsed, err := time.Parse(time.RFC3339Nano, trimmed) if err != nil { return "" } return parsed.In(workflowEnvLocation).Format(time.RFC3339) } func firstNonEmpty(values ...string) string { for _, value := range values { if strings.TrimSpace(value) != "" { return value } } return "" } func inputArtifactLineageKey(item domain.InputLineageRecord) string { artifact := item.Artifact key := artifact.ID if key == "" { key = artifact.NodeID + ":" + artifact.Name } return fmt.Sprintf("%s:%s:%s:%s:%s", item.WorkflowEdgeID, item.SourceNodeID, item.SourceHandle, item.TargetHandle, key) } func gatherInputLineage(workflow domain.WorkflowRecord, executionMap map[string]domain.NodeExecutionRecord, nodeID string) []domain.InputLineageRecord { items := make([]domain.InputLineageRecord, 0) seen := map[string]struct{}{} for _, edge := range domain.GetIncomingEdges(workflow.Edges, nodeID) { execution, ok := executionMap[edge.Source] if !ok { continue } requiresHandle := strings.TrimSpace(edge.SourceHandle) != "" hasPortAwareArtifacts := false for _, artifact := range execution.Artifacts { if strings.TrimSpace(artifact.PortID) != "" { hasPortAwareArtifacts = true break } } for _, artifact := range execution.Artifacts { if requiresHandle && hasPortAwareArtifacts && artifact.PortID != edge.SourceHandle { continue } item := domain.InputLineageRecord{ WorkflowEdgeID: edge.ID, SourceNodeID: edge.Source, SourceHandle: edge.SourceHandle, TargetNodeID: edge.Target, TargetHandle: edge.TargetHandle, Artifact: artifact, } key := inputArtifactLineageKey(item) if _, exists := seen[key]; exists { continue } seen[key] = struct{}{} items = append(items, item) } } return items } func inputArtifactsFromLineage(lineage []domain.InputLineageRecord) []domain.ExecutionArtifact { items := make([]domain.ExecutionArtifact, 0) seen := map[string]struct{}{} for _, item := range lineage { artifact := item.Artifact key := artifact.ID if key == "" { key = artifact.NodeID + ":" + artifact.Name } if _, exists := seen[key]; exists { continue } seen[key] = struct{}{} items = append(items, artifact) } return items } func artifactFromWorkflowInput(run *domain.RunRecord, binding domain.ResolvedInputBinding) domain.ExecutionArtifact { payload := binding.Payload artifact := domain.ExecutionArtifact{ ID: run.ID + "-input-" + sanitizeEnvKey(binding.WorkflowInputID) + "-" + sanitizeEnvKey(binding.TargetPortID), RunID: run.ID, NodeID: binding.NodeID, PortID: binding.TargetPortID, PortName: binding.TargetPortID, Name: binding.WorkflowInputID, Type: payload.Type, Payload: &payload, } if payload.Mode == domain.ArtifactTransferModeInline { encoded, _ := json.Marshal(payload.Inline) artifact.SizeBytes = int64(len(encoded)) artifact.Size = fmt.Sprintf("%d B", len(encoded)) if data, ok := payload.Inline.(map[string]any); ok { artifact.Data = data } else { artifact.Data = map[string]any{"value": payload.Inline} } return tagDebugArtifact(run, artifact) } if payload.Ref != nil { ref := payload.Ref artifact.SizeBytes = ref.SizeBytes artifact.Size = fmt.Sprintf("%d B", ref.SizeBytes) if ref.Kind == domain.ArtifactRefKindDirectory { artifact.Path = "bos://" + ref.Bucket + "/" + strings.TrimPrefix(ref.Prefix, "/") } else { artifact.Path = "bos://" + ref.Bucket + "/" + strings.TrimPrefix(ref.Key, "/") } artifact.Metadata = &domain.ArtifactMetadata{SHA256: ref.SHA256, MimeType: ref.MimeType} artifact.Data = map[string]any{"source": map[string]any{"path": artifact.Path}} } return tagDebugArtifact(run, artifact) } func workflowInputBindingsForNode(run *domain.RunRecord, nodeID string) ([]domain.NodeInputBinding, []domain.ExecutionArtifact) { if run == nil { return nil, nil } bindings := make([]domain.NodeInputBinding, 0) artifacts := make([]domain.ExecutionArtifact, 0) for _, item := range run.ResolvedInputBindings { if item.NodeID != nodeID { continue } artifact := artifactFromWorkflowInput(run, item) payload := item.Payload bindings = append(bindings, domain.NodeInputBinding{ TargetPortID: item.TargetPortID, WorkflowInputID: item.WorkflowInputID, Payload: &payload, Artifacts: []domain.ExecutionArtifact{artifact}, }) artifacts = append(artifacts, artifact) } return bindings, artifacts } func lineageInputBindings(lineage []domain.InputLineageRecord) []domain.NodeInputBinding { items := make([]domain.NodeInputBinding, 0) indexByKey := map[string]int{} for _, item := range lineage { key := item.SourceNodeID + "\x00" + item.SourceHandle + "\x00" + item.TargetHandle if index, ok := indexByKey[key]; ok { items[index].Artifacts = append(items[index].Artifacts, item.Artifact) continue } indexByKey[key] = len(items) items = append(items, domain.NodeInputBinding{ TargetPortID: item.TargetHandle, SourceNodeID: item.SourceNodeID, SourcePortID: item.SourceHandle, Artifacts: []domain.ExecutionArtifact{item.Artifact}, }) } return items } func validateRequiredNodeInputs(node domain.WorkflowNode, bindings []domain.NodeInputBinding) error { for _, port := range node.Data.Inputs { if !port.Required { continue } present := false for _, binding := range bindings { targetPortID := strings.TrimSpace(binding.TargetPortID) if targetPortID != strings.TrimSpace(port.ID) && !(targetPortID == "" && len(node.Data.Inputs) == 1) { continue } if binding.Payload != nil || len(binding.Artifacts) > 0 { present = true break } } if !present { return fmt.Errorf("required_input_missing: node %s input %s has no value or upstream artifact", node.ID, port.ID) } } return nil } func artifactFromNodeOverrideInput(run *domain.RunRecord, nodeID string, portKey string, payload domain.ArtifactPayload) (domain.NodeInputBinding, domain.ExecutionArtifact) { runID := "" if run != nil { runID = run.ID } artifact := domain.ExecutionArtifact{ ID: runID + "-override-" + sanitizeEnvKey(nodeID) + "-" + sanitizeEnvKey(portKey), RunID: runID, NodeID: nodeID, PortID: portKey, PortName: portKey, Name: portKey, Type: payload.Type, Payload: &payload, } if payload.Mode == domain.ArtifactTransferModeInline { encoded, _ := json.Marshal(payload.Inline) artifact.SizeBytes = int64(len(encoded)) artifact.Size = fmt.Sprintf("%d B", len(encoded)) if data, ok := payload.Inline.(map[string]any); ok { artifact.Data = data } else { artifact.Data = map[string]any{"value": payload.Inline} } } else if payload.Ref != nil { ref := payload.Ref artifact.SizeBytes = ref.SizeBytes artifact.Size = fmt.Sprintf("%d B", ref.SizeBytes) if ref.Kind == domain.ArtifactRefKindDirectory { artifact.Path = "bos://" + ref.Bucket + "/" + strings.TrimPrefix(ref.Prefix, "/") } else { artifact.Path = "bos://" + ref.Bucket + "/" + strings.TrimPrefix(ref.Key, "/") } artifact.Metadata = &domain.ArtifactMetadata{SHA256: ref.SHA256, MimeType: ref.MimeType} artifact.Data = map[string]any{"source": map[string]any{"path": artifact.Path}} } binding := domain.NodeInputBinding{ TargetPortID: portKey, Payload: &payload, Artifacts: []domain.ExecutionArtifact{artifact}, } return binding, tagDebugArtifact(run, artifact) } func applyNodeOverridesToNode(node domain.WorkflowNode, override domain.NodeOverride) domain.WorkflowNode { cloned := node // 1. Hyperparams: merge / override by key if len(override.Hyperparams) > 0 { hpMap := make(map[string]domain.HyperParam, len(node.Data.Hyperparams)+len(override.Hyperparams)) for _, hp := range node.Data.Hyperparams { hpMap[hp.Key] = hp } for _, hp := range override.Hyperparams { hpMap[hp.Key] = hp } newHyperparams := make([]domain.HyperParam, 0, len(hpMap)) seen := make(map[string]bool, len(hpMap)) for _, hp := range node.Data.Hyperparams { if updated, ok := hpMap[hp.Key]; ok { newHyperparams = append(newHyperparams, updated) seen[hp.Key] = true } } for _, hp := range override.Hyperparams { if !seen[hp.Key] { newHyperparams = append(newHyperparams, hp) seen[hp.Key] = true } } cloned.Data.Hyperparams = newHyperparams } // 2. Operator image if strings.TrimSpace(override.Image) != "" { cloned.Data.Image = strings.TrimSpace(override.Image) } // 3. Container command and args if len(override.ContainerCommand) > 0 { cloned.Data.ContainerCommand = cloneStringSlice(override.ContainerCommand) } if len(override.ContainerArgs) > 0 { cloned.Data.ContainerArgs = cloneStringSlice(override.ContainerArgs) } return cloned } func gatherNodeInputs(workflow domain.WorkflowRecord, run *domain.RunRecord, executionMap map[string]domain.NodeExecutionRecord, nodeID string) ([]domain.ExecutionArtifact, []domain.InputLineageRecord, []domain.NodeInputBinding) { lineage := gatherInputLineage(workflow, executionMap, nodeID) artifacts := inputArtifactsFromLineage(lineage) bindings := lineageInputBindings(lineage) rootBindings, rootArtifacts := workflowInputBindingsForNode(run, nodeID) bindings = append(bindings, rootBindings...) artifacts = append(artifacts, rootArtifacts...) if run != nil && len(run.NodeOverrides) > 0 { if override, ok := run.NodeOverrides[nodeID]; ok && len(override.Inputs) > 0 { for portKey, payload := range override.Inputs { overrideBinding, overrideArtifact := artifactFromNodeOverrideInput(run, nodeID, portKey, payload) replaced := false for i, b := range bindings { if strings.TrimSpace(b.TargetPortID) == strings.TrimSpace(portKey) { bindings[i] = overrideBinding replaced = true break } } if !replaced { bindings = append(bindings, overrideBinding) } artifactReplaced := false for i, a := range artifacts { if strings.TrimSpace(a.PortID) == strings.TrimSpace(portKey) { artifacts[i] = overrideArtifact artifactReplaced = true break } } if !artifactReplaced { artifacts = append(artifacts, overrideArtifact) } } } } return artifacts, lineage, bindings } func gatherInputArtifacts(workflow domain.WorkflowRecord, executionMap map[string]domain.NodeExecutionRecord, nodeID string) []domain.ExecutionArtifact { return inputArtifactsFromLineage(gatherInputLineage(workflow, executionMap, nodeID)) } func extractConfidence(artifact domain.ExecutionArtifact) (float64, bool) { if artifact.Data == nil { return 0, false } for _, key := range []string{"confidence", "score"} { value, ok := artifact.Data[key] if !ok { continue } switch typed := value.(type) { case float64: return typed, true case int: return float64(typed), true } } return 0, false } func extractObjectHints(artifact domain.ExecutionArtifact) []string { if artifact.Data == nil { return nil } items := make([]string, 0) for _, key := range []string{"detectedObjects", "labels", "entries"} { value, ok := artifact.Data[key] if !ok { continue } switch typed := value.(type) { case []any: for _, candidate := range typed { text := strings.TrimSpace(fmt.Sprint(candidate)) if text != "" { items = append(items, text) } } case []string: for _, candidate := range typed { if trimmed := strings.TrimSpace(candidate); trimmed != "" { items = append(items, trimmed) } } } } return items } func createAuditPayload(node domain.WorkflowNode, inputArtifacts []domain.ExecutionArtifact) domain.AuditTaskPayload { payload := domain.AuditTaskPayload{ Reason: fmt.Sprintf("%s requires explicit approval before downstream execution", node.Data.Label), DetectedObjects: []string{}, } for _, artifact := range inputArtifacts { if payload.Image == "" { switch { case artifact.Metadata != nil && artifact.Metadata.Thumbnail != "": payload.Image = artifact.Metadata.Thumbnail case artifact.Type == domain.PortTypeImage || artifact.Type == domain.PortTypeVideo: payload.Image = artifact.Path } } if payload.Confidence == 0 { if confidence, ok := extractConfidence(artifact); ok { payload.Confidence = confidence } } payload.DetectedObjects = append(payload.DetectedObjects, extractObjectHints(artifact)...) } if len(payload.DetectedObjects) == 0 { payload.DetectedObjects = []string{fmt.Sprintf("%d upstream artifact(s)", len(inputArtifacts))} } return payload } func prepareNodeWorkspace(workspaceRoot string, runID string, nodeID string) (nodeWorkspace, error) { root := strings.TrimSpace(workspaceRoot) if root == "" { root = filepath.Join(os.TempDir(), "droboticflow") } runRoot := filepath.Join(root, "system", "runs", runID) nodeRoot := filepath.Join(runRoot, "nodes", nodeID) outputDir := filepath.Join(nodeRoot, "outputs") if err := os.MkdirAll(outputDir, 0o755); err != nil { return nodeWorkspace{}, err } return nodeWorkspace{ RunRoot: runRoot, Root: nodeRoot, OutputDir: outputDir, InputArtifactsPath: filepath.Join(nodeRoot, "input-artifacts.json"), InputBindingsPath: filepath.Join(nodeRoot, "input-bindings.json"), ResultPath: filepath.Join(nodeRoot, "result.json"), StageManifestPath: filepath.Join(nodeRoot, "stage-manifest.json"), }, nil } func shouldInjectPlatformMongoEnv(node domain.WorkflowNode) bool { label := strings.ToLower(strings.TrimSpace(node.Data.Label)) return strings.Contains(label, "mongodb") || strings.Contains(label, "mongo") } func writeJSONFile(path string, value any) error { data, err := json.Marshal(value) if err != nil { return err } if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { return err } return os.WriteFile(path, data, 0o644) } func applyHyperparamsToEnv(env map[string]string, items []domain.HyperParam) { for _, item := range items { key := sanitizeEnvKey(item.Key) if key == "" { continue } if _, blocked := protectedSystemEnv[key]; blocked { continue } env[key] = item.Value } } func applyOperatorCommandPolicyEnv(env map[string]string) { for _, key := range operatorCommandPolicyEnvKeys { if value := strings.TrimSpace(os.Getenv(key)); value != "" { env[key] = value } } } func buildNodeEnv(run *domain.RunRecord, workflow *domain.WorkflowRecord, node domain.WorkflowNode, inputArtifacts []domain.ExecutionArtifact, workspace nodeWorkspace, settings *domain.SettingsRecord) (map[string]string, error) { return buildNodeEnvWithResources(run, workflow, node, inputArtifacts, nil, workspace, effectiveNodeResources{ CPULimit: node.Data.CPU, MemoryLimit: node.Data.Memory, GPULimit: node.Data.GPU, }) } type effectiveNodeResources struct { CPURequest string CPULimit string MemoryRequest string MemoryLimit string GPULimit string SharedMemorySize string } func buildNodeEnvWithResources(run *domain.RunRecord, workflow *domain.WorkflowRecord, node domain.WorkflowNode, inputArtifacts []domain.ExecutionArtifact, inputBindings []domain.NodeInputBinding, workspace nodeWorkspace, resources effectiveNodeResources) (map[string]string, error) { if inputArtifacts == nil { inputArtifacts = []domain.ExecutionArtifact{} } safeNodeData := node.Data safeNodeData.Hyperparams = make([]domain.HyperParam, 0, len(node.Data.Hyperparams)) for _, hyperparam := range node.Data.Hyperparams { if isSensitiveNodeEnvKey(hyperparam.Key) { continue } safeNodeData.Hyperparams = append(safeNodeData.Hyperparams, hyperparam) } nodeDataJSON, err := json.Marshal(safeNodeData) if err != nil { return nil, err } inputArtifactsJSON, err := json.Marshal(inputArtifacts) if err != nil { return nil, err } if err := writeJSONFile(workspace.InputArtifactsPath, inputArtifacts); err != nil { return nil, err } if inputBindings == nil { inputBindings = []domain.NodeInputBinding{} } if strings.TrimSpace(workspace.InputBindingsPath) == "" { workspace.InputBindingsPath = filepath.Join(workspace.Root, "input-bindings.json") } inputBindingsJSON, err := json.Marshal(inputBindings) if err != nil { return nil, err } if err := writeJSONFile(workspace.InputBindingsPath, inputBindings); err != nil { return nil, err } env := map[string]string{ "RUN_ID": run.ID, "WORKFLOW_ID": run.WorkflowID, "NODE_ID": node.ID, "NODE_LABEL": node.Data.Label, "NODE_TYPE": node.Type, "NODE_CATEGORY": node.Data.Category, "NODE_DESCRIPTION": node.Data.Description, "NODE_DATA_JSON": string(nodeDataJSON), "INPUT_ARTIFACTS_PATH": workspace.InputArtifactsPath, "INPUT_BINDINGS_PATH": workspace.InputBindingsPath, "RESULT_OUTPUT_PATH": workspace.ResultPath, "STAGE_MANIFEST_PATH": workspace.StageManifestPath, "OUTPUT_DIR": workspace.OutputDir, "WORKSPACE_ROOT": workspace.RunRoot, } if parentRunID := strings.TrimSpace(run.ParentRunID); parentRunID != "" { parentRunRoot := filepath.Join(filepath.Dir(workspace.RunRoot), filepath.Base(parentRunID)) env["PARENT_RUN_ID"] = parentRunID env["PARENT_RUN_WORKSPACE_ROOT"] = parentRunRoot env["PARENT_NODE_OUTPUT_DIR"] = filepath.Join(parentRunRoot, "nodes", node.ID, "outputs") } if run.Trigger == domain.RunTriggerScheduled { env["WORKFLOW_CURRENT_RUN_TIMESTAMP"] = runTimestampEnvValue(firstNonEmpty(run.CurrentRunAt, run.CreatedAt)) env["WORKFLOW_PREVIOUS_RUN_TIMESTAMP"] = runTimestampEnvValue(run.PreviousRunAt) } if len(inputArtifactsJSON) <= inputArtifactsInlineLimit { env["INPUT_ARTIFACTS_JSON"] = string(inputArtifactsJSON) } if len(inputBindingsJSON) <= inputArtifactsInlineLimit { env["INPUT_BINDINGS_JSON"] = string(inputBindingsJSON) } if node.Data.Bucket != "" { env["STORAGE_BUCKET"] = node.Data.Bucket } if node.Data.Prefix != "" { env["STORAGE_PREFIX"] = node.Data.Prefix } if node.Data.Pattern != "" { env["STORAGE_PATTERN"] = node.Data.Pattern } if resources.CPULimit != "" { env["CPU_LIMIT"] = resources.CPULimit } if resources.MemoryLimit != "" { env["MEMORY_LIMIT"] = resources.MemoryLimit } if resources.GPULimit != "" { env["GPU_LIMIT"] = resources.GPULimit } if node.Data.Image != "" { env["NODE_IMAGE"] = node.Data.Image } if handler := runtimeContractHandler(node.Data.RuntimeContract); handler != "" { env["DROBOTICFLOW_OPERATOR_HANDLER"] = handler } if shouldInjectPlatformMongoEnv(node) { if value := strings.TrimSpace(os.Getenv("MONGO_URI")); value != "" { env["PLATFORM_MONGO_URI"] = value } if value := strings.TrimSpace(os.Getenv("MONGO_DB")); value != "" { env["PLATFORM_MONGO_DB"] = value } } if workflow != nil { applyHyperparamsToEnv(env, workflow.Hyperparams) } applyHyperparamsToEnv(env, node.Data.Hyperparams) applyOperatorCommandPolicyEnv(env) return env, nil } func isSensitiveNodeEnvKey(key string) bool { normalized := strings.ToUpper(strings.TrimSpace(key)) if normalized == "" { return false } if normalized == "MONGO_URI" || normalized == "PLATFORM_MONGO_URI" { return true } for _, fragment := range []string{"PASSWORD", "SECRET", "PRIVATE_KEY", "ACCESS_KEY_ID", "API_KEY", "AUTH_TOKEN"} { if strings.Contains(normalized, fragment) { return true } } return strings.HasSuffix(normalized, "_TOKEN") || strings.HasSuffix(normalized, "_CREDENTIALS") } func partitionSensitiveNodeEnv(env map[string]string) (map[string]string, map[string]string) { public := make(map[string]string, len(env)) sensitive := make(map[string]string) for key, value := range env { if isSensitiveNodeEnvKey(key) { sensitive[key] = value continue } public[key] = value } return public, sensitive } func runtimeContractHandler(contract map[string]any) string { value, ok := contract["handler"] if !ok { return "" } handler := strings.TrimSpace(fmt.Sprint(value)) if strings.HasPrefix(handler, "builtin_") { return strings.TrimPrefix(handler, "builtin_") } return handler } func runtimeContractLongRunning(contract map[string]any) bool { value, ok := contract["longRunning"].(bool) return ok && value } func lightweightOperatorMount(contract map[string]any) (string, string) { config := lightweight.ConfigFromContract(contract) if !config.Enabled { return "", "" } return config.CodeSubPath, config.MountPath } func runtimeContractString(items map[string]any, key string) string { value, ok := items[key] if !ok || value == nil { return "" } return fmt.Sprint(value) } func operatorResourceDefaults(ctx context.Context, dataStore interface { GetOperator(context.Context, string) (*domain.OperatorRecord, error) }, node domain.WorkflowNode) *domain.OperatorResources { operatorID := strings.TrimSpace(node.Data.OperatorID) if operatorID == "" || dataStore == nil { return nil } operator, err := dataStore.GetOperator(ctx, operatorID) if err != nil || operator == nil { return nil } return operator.Resources } func resolveEffectiveNodeResources(node domain.WorkflowNode, settings *domain.SettingsRecord, operatorResources *domain.OperatorResources) effectiveNodeResources { defaults := domain.DefaultResources{} if settings != nil { defaults = settings.Defaults } resources := effectiveNodeResources{ CPURequest: defaults.CPURequest, CPULimit: defaults.CPULimit, MemoryRequest: defaults.MemoryRequest, MemoryLimit: defaults.MemoryLimit, GPULimit: defaults.GPULimit, } if operatorResources != nil { resources.CPURequest = firstNonEmpty(operatorResources.CPURequest, resources.CPURequest) resources.CPULimit = firstNonEmpty(operatorResources.CPULimit, resources.CPULimit) resources.MemoryRequest = firstNonEmpty(operatorResources.MemoryRequest, resources.MemoryRequest) resources.MemoryLimit = firstNonEmpty(operatorResources.MemoryLimit, resources.MemoryLimit) resources.GPULimit = firstNonEmpty(operatorResources.GPULimit, resources.GPULimit) resources.SharedMemorySize = firstNonEmpty(operatorResources.SharedMemorySize, resources.SharedMemorySize) } resources.CPULimit = firstNonEmpty(node.Data.CPU, resources.CPULimit) resources.MemoryLimit = firstNonEmpty(node.Data.Memory, resources.MemoryLimit) resources.GPULimit = firstNonEmpty(node.Data.GPU, resources.GPULimit) resources.SharedMemorySize = firstNonEmpty(node.Data.SharedMemorySize, resources.SharedMemorySize) return applyFanyuResourceProfile(node, resources) } func parseExecutionOutput(stdout string, nodeLabel string) (executionResult, error) { result := executionResult{ Logs: []domain.LogEntryRecord{}, Artifacts: []domain.ExecutionArtifact{}, ResourceSeries: []domain.ResourceSample{}, } lines := strings.Split(strings.ReplaceAll(stdout, "\r\n", "\n"), "\n") structuredPayload := "" structuredPayloadPath := "" rawLogs := make([]domain.LogEntryRecord, 0) for _, line := range lines { trimmed := strings.TrimSpace(line) if trimmed == "" { continue } if strings.HasPrefix(trimmed, resultPrefix) { structuredPayload = strings.TrimPrefix(trimmed, resultPrefix) continue } if strings.HasPrefix(trimmed, resultPathPrefix) { structuredPayloadPath = strings.TrimSpace(strings.TrimPrefix(trimmed, resultPathPrefix)) continue } rawLogs = append(rawLogs, logEntry("INFO", trimmed, nodeLabel)) } if structuredPayloadPath != "" { payloadBytes, err := os.ReadFile(structuredPayloadPath) if err != nil { result.Logs = rawLogs return result, fmt.Errorf("read operator result path: %w", err) } structuredPayload = strings.TrimSpace(string(payloadBytes)) } if structuredPayload == "" { result.Logs = rawLogs return result, fmt.Errorf("operator execution did not emit %s or %s payload", resultPrefix, resultPathPrefix) } var parsed executionResult if err := json.Unmarshal([]byte(structuredPayload), &parsed); err != nil { return result, fmt.Errorf("decode operator result: %w", err) } if len(parsed.Logs) > 0 { result.Logs = append(result.Logs, parsed.Logs...) } else { result.Logs = append(result.Logs, rawLogs...) } result.Artifacts = parsed.Artifacts result.ResourceSeries = parsed.ResourceSeries return result, nil } func loadStageManifestDataRefs(path string) ([]domain.DataRefRecord, error) { trimmed := strings.TrimSpace(path) if trimmed == "" { return nil, nil } payload, err := os.ReadFile(trimmed) if err != nil { return nil, err } var manifest stageManifestRecord if err := json.Unmarshal(payload, &manifest); err != nil { return nil, err } if manifest.DataRefs == nil { return []domain.DataRefRecord{}, nil } return manifest.DataRefs, nil } func loadStageManifestDataMetrics(path string) (*domain.DataMetricsRecord, error) { trimmed := strings.TrimSpace(path) if trimmed == "" { return nil, nil } payload, err := os.ReadFile(trimmed) if err != nil { return nil, err } var manifest stageManifestRecord if err := json.Unmarshal(payload, &manifest); err != nil { return nil, err } return manifest.DataMetrics, nil } func (runtime *Runtime) resolveNodeBusinessStage(ctx context.Context, run *domain.RunRecord, node domain.WorkflowNode) domain.WorkflowNode { if run != nil && strings.TrimSpace(run.TaskID) != "" { if task, err := runtime.store.GetCollectionTask(ctx, run.TaskID); err == nil && task != nil { for stage, nodeID := range task.StageBindings { if strings.TrimSpace(nodeID) == node.ID { node.Data.BusinessStage = stage node.Data.MetricBoundary = true return node } } } } if node.Data.BusinessStage != "" { node.Data.MetricBoundary = true return node } if strings.TrimSpace(node.Data.OperatorID) != "" { if operator, err := runtime.store.GetOperator(ctx, node.Data.OperatorID); err == nil && operator != nil && operator.BusinessStage != "" { node.Data.BusinessStage = operator.BusinessStage node.Data.MetricBoundary = true } } return node } func (runtime *Runtime) executeNode(ctx context.Context, run *domain.RunRecord, node domain.WorkflowNode, inputArtifacts []domain.ExecutionArtifact, inputBindings []domain.NodeInputBinding) (executionResult, *k8s.JobReference, error) { if node.Data.OperatorID == "operator-builtin-package" && strings.TrimSpace(os.Getenv("FANYU_SLICE_DISCARD_PACKAGE_IMAGE")) != "" && discardPackageImageOverride(node) == "" { return executionResult{}, nil, fmt.Errorf("FANYU_SLICE_DISCARD_PACKAGE_IMAGE must be a digest-pinned package image") } if run != nil && len(run.NodeOverrides) > 0 { if override, ok := run.NodeOverrides[node.ID]; ok { node = applyNodeOverridesToNode(node, override) } } if err := runtime.preflightV4OriginalSource(ctx, run, node); err != nil { return executionResult{}, nil, err } if err := validateRequiredNodeInputs(node, inputBindings); err != nil { return executionResult{}, nil, err } node = runtime.resolveNodeBusinessStage(ctx, run, node) inferenceSidecar, err := inferenceSidecarForNode(run, &node) if err != nil { return executionResult{}, nil, err } workspace, err := prepareNodeWorkspace(runtime.workspaceRoot, run.ID, node.ID) if err != nil { return executionResult{}, nil, err } workflow, workflowErr := runtime.store.GetWorkflow(ctx, run.WorkflowID) if workflowErr != nil { workflow = nil } if run.Metadata != nil && run.Metadata.CaptureDeliveryPackage != nil { if workflowErr != nil { return executionResult{}, nil, workflowErr } workflow, err = deliveryRuntimeWorkflow(workflow, run) if err != nil || workflow == nil { if err == nil { err = fmt.Errorf("missing delivery workflow") } return executionResult{}, nil, err } } settings, err := runtime.store.GetSettings(ctx) if err != nil { return executionResult{}, nil, err } resources := resolveEffectiveNodeResources(node, settings, operatorResourceDefaults(ctx, runtime.store, node)) lightweightConfig := lightweight.ConfigFromContract(node.Data.RuntimeContract) if lightweightConfig.Enabled { if err := lightweight.ValidateRuntimeConfig(lightweight.RuntimeConfig{ NodeID: node.ID, NodeLabel: node.Data.Label, Image: node.Data.Image, CodeSubPath: lightweightConfig.CodeSubPath, MountPath: lightweightConfig.MountPath, Command: node.Data.ContainerCommand, Args: node.Data.ContainerArgs, ArgsConfigured: lightweightConfig.ContainerArgsConfigured, WorkspaceRoot: runtime.workspaceRoot, Mounts: lightweightConfig.Mounts, }); err != nil { return executionResult{}, nil, err } } env, err := buildNodeEnvWithResources(run, workflow, node, inputArtifacts, inputBindings, workspace, resources) if err != nil { return executionResult{}, nil, err } env, sensitiveEnv := partitionSensitiveNodeEnv(env) codeSubPath, codeMountPath := lightweightConfig.CodeSubPath, lightweightConfig.MountPath if codeSubPath != "" { env["LIGHTWEIGHT_OPERATOR_CODE_SUBPATH"] = codeSubPath env["LIGHTWEIGHT_OPERATOR_CODE_DIR"] = codeMountPath } pvcMounts := make([]k8s.PVCMount, 0, len(lightweightConfig.Mounts)) for _, item := range lightweightConfig.Mounts { pvcMounts = append(pvcMounts, k8s.PVCMount{Name: item.Name, PVCName: item.PVCName, SubPath: item.SubPath, MountPath: item.MountPath, ReadOnly: item.ReadOnly}) } gpuNodeSelector := map[string]string(nil) env, pvcMounts, gpuNodeSelector, err = runtime.qwenNodeCache.apply(node, env, pvcMounts) if err != nil { return executionResult{}, nil, err } nodeTimeoutSeconds := 0 if workflow != nil { nodeTimeoutSeconds = workflow.NodeTimeoutSeconds } launchGPULimit := resources.GPULimit if inferenceSidecar != nil && strings.TrimSpace(inferenceSidecar.GPULimit) != "" { launchGPULimit = inferenceSidecar.GPULimit } if workflow != nil && workflow.ProjectID != "" { p, e := runtime.store.GetProject(ctx, workflow.ProjectID) if e != nil { return executionResult{}, nil, e } if p == nil || p.DeletionTaskID != "" { return executionResult{}, nil, fmt.Errorf("project deletion in progress") } } image, frozen, err := frozenDeliveryPackageImage(run, node) if err == nil && !frozen { image, err = sourceIndexPackageImage(node, inputArtifacts, runtime.operatorRunnerImage) } if err != nil { return executionResult{}, nil, err } launched, err := runtime.launcher.Launch(ctx, k8s.LaunchInput{ Image: image, Command: cloneStringSlice(node.Data.ContainerCommand), Args: cloneStringSlice(node.Data.ContainerArgs), RunID: run.ID, WorkflowID: run.WorkflowID, NodeID: node.ID, NodeLabel: node.Data.Label, Attempt: run.Attempt, TimeoutSeconds: nodeTimeoutSeconds, Env: env, SensitiveEnv: sensitiveEnv, CPURequest: resources.CPURequest, CPULimit: resources.CPULimit, MemoryRequest: resources.MemoryRequest, MemoryLimit: resources.MemoryLimit, GPULimit: launchGPULimit, GPUNodeSelector: gpuNodeSelector, SharedMemorySize: resources.SharedMemorySize, LongRunning: runtimeContractLongRunning(node.Data.RuntimeContract), CodeSubPath: codeSubPath, CodeMountPath: codeMountPath, CodeReadOnly: lightweightConfig.CodeReadOnly, PVCMounts: pvcMounts, OperatorRunnerImage: runtime.operatorRunnerImage, InferenceSidecar: inferenceSidecar, }) if err != nil { return executionResult{}, nil, err } if progress, progressErr := latestStructuredProgress(launched.Output); progressErr == nil && progress != nil { _ = runtime.store.UpdateRunProgress(ctx, run.ID, progress) } parsed, err := parseExecutionOutput(launched.Output, node.Data.Label) if err == nil { parsed.Artifacts, err = runtime.normalizeOutputArtifacts(ctx, workflow, run, node.ID, parsed.Artifacts) } if err == nil { parsed.Artifacts = tagDebugArtifacts(run, parsed.Artifacts) } if err == nil { parsed.StageManifestPath = workspace.StageManifestPath parsed.DataRefs, err = loadStageManifestDataRefs(workspace.StageManifestPath) if err == nil && parsed.DataMetrics == nil { parsed.DataMetrics, err = loadStageManifestDataMetrics(workspace.StageManifestPath) } } // Custom-container operators bypass operator-runner's ResourceSampler, so they // emit no resourceSeries in result.json. The launcher samples their GPU usage // itself (see k8s.Launch); fall back to those platform-collected samples when // the operator did not self-report. Builtin operator-runner pods self-report a // non-empty series, so their own (more complete, CPU+memory included) samples // are preserved. if len(parsed.ResourceSeries) == 0 && len(launched.ResourceSeries) > 0 { parsed.ResourceSeries = make([]domain.ResourceSample, 0, len(launched.ResourceSeries)) for _, sample := range launched.ResourceSeries { parsed.ResourceSeries = append(parsed.ResourceSeries, domain.ResourceSample{ Timestamp: sample.Timestamp, GPU: sample.GPU, GPUMemoryUsedMiB: sample.GPUMemoryUsedMiB, GPUMemoryTotalMiB: sample.GPUMemoryTotalMiB, GPUName: sample.GPUName, GPUCount: sample.GPUCount, }) } } return parsed, launched.Reference, err }