package worker import ( "context" "encoding/json" "fmt" "os" "path/filepath" "regexp" "strings" "time" "droboticflow/internal/domain" "droboticflow/internal/k8s" ) const resultPrefix = "DROBOTICFLOW_RESULT=" const resultPathPrefix = "DROBOTICFLOW_RESULT_PATH=" const inputArtifactsInlineLimit = 64 * 1024 type nodeWorkspace struct { RunRoot string Root string OutputDir string InputArtifactsPath string ResultPath string StageManifestPath string } type executionResult struct { Logs []domain.LogEntryRecord `json:"logs"` Artifacts []domain.ExecutionArtifact `json:"artifacts"` DataRefs []domain.DataRefRecord `json:"dataRefs,omitempty"` StageManifestPath string `json:"stageManifestPath,omitempty"` ResourceSeries []domain.ResourceSample `json:"resourceSeries"` } type stageManifestRecord struct { DataRefs []domain.DataRefRecord `json:"dataRefs"` } var protectedSystemEnv = map[string]struct{}{ "WORKFLOW_PREVIOUS_RUN_TIMESTAMP": {}, "WORKFLOW_CURRENT_RUN_TIMESTAMP": {}, "OPERATOR_ALLOW_SHELL_COMMANDS": {}, "OPERATOR_COMMAND_ALLOWLIST_REGEX": {}, "OPERATOR_COMMAND_DENYLIST_REGEX": {}, } var operatorCommandPolicyEnvKeys = []string{ "OPERATOR_ALLOW_SHELL_COMMANDS", "OPERATOR_COMMAND_ALLOWLIST_REGEX", "OPERATOR_COMMAND_DENYLIST_REGEX", } var workflowEnvLocation = loadWorkflowEnvLocation() var invalidEnvCharacters = regexp.MustCompile(`[^A-Z0-9_]+`) func loadWorkflowEnvLocation() *time.Location { location, err := time.LoadLocation("Asia/Shanghai") if err != nil { return time.FixedZone("Asia/Shanghai", 8*60*60) } return location } func sanitizeEnvKey(value string) string { normalized := strings.ToUpper(strings.TrimSpace(value)) if normalized == "" { return "" } normalized = invalidEnvCharacters.ReplaceAllString(normalized, "_") normalized = strings.Trim(normalized, "_") if normalized == "" { return "" } if normalized[0] >= '0' && normalized[0] <= '9' { normalized = "HP_" + normalized } return normalized } func cloneArtifacts(items []domain.ExecutionArtifact) []domain.ExecutionArtifact { cloned := make([]domain.ExecutionArtifact, 0, len(items)) for _, item := range items { cloned = append(cloned, item) } return cloned } func runTimestampEnvValue(value string) string { trimmed := strings.TrimSpace(value) if trimmed == "" { return "" } parsed, err := time.Parse(time.RFC3339Nano, trimmed) if err != nil { return "" } return parsed.In(workflowEnvLocation).Format(time.RFC3339) } func firstNonEmpty(values ...string) string { for _, value := range values { if strings.TrimSpace(value) != "" { return value } } return "" } func gatherInputArtifacts(workflow domain.WorkflowRecord, executionMap map[string]domain.NodeExecutionRecord, nodeID string) []domain.ExecutionArtifact { items := make([]domain.ExecutionArtifact, 0) seen := map[string]struct{}{} for _, edge := range domain.GetIncomingEdges(workflow.Edges, nodeID) { execution, ok := executionMap[edge.Source] if !ok { continue } requiresHandle := strings.TrimSpace(edge.SourceHandle) != "" hasPortAwareArtifacts := false for _, artifact := range execution.Artifacts { if strings.TrimSpace(artifact.PortID) != "" { hasPortAwareArtifacts = true break } } for _, artifact := range execution.Artifacts { if requiresHandle && hasPortAwareArtifacts && artifact.PortID != edge.SourceHandle { continue } key := artifact.ID if key == "" { key = artifact.NodeID + ":" + artifact.Name } if _, exists := seen[key]; exists { continue } seen[key] = struct{}{} items = append(items, artifact) } } return items } func extractConfidence(artifact domain.ExecutionArtifact) (float64, bool) { if artifact.Data == nil { return 0, false } for _, key := range []string{"confidence", "score"} { value, ok := artifact.Data[key] if !ok { continue } switch typed := value.(type) { case float64: return typed, true case int: return float64(typed), true } } return 0, false } func extractObjectHints(artifact domain.ExecutionArtifact) []string { if artifact.Data == nil { return nil } items := make([]string, 0) for _, key := range []string{"detectedObjects", "labels", "entries"} { value, ok := artifact.Data[key] if !ok { continue } switch typed := value.(type) { case []any: for _, candidate := range typed { text := strings.TrimSpace(fmt.Sprint(candidate)) if text != "" { items = append(items, text) } } case []string: for _, candidate := range typed { if trimmed := strings.TrimSpace(candidate); trimmed != "" { items = append(items, trimmed) } } } } return items } func createAuditPayload(node domain.WorkflowNode, inputArtifacts []domain.ExecutionArtifact) domain.AuditTaskPayload { payload := domain.AuditTaskPayload{ Reason: fmt.Sprintf("%s requires explicit approval before downstream execution", node.Data.Label), DetectedObjects: []string{}, } for _, artifact := range inputArtifacts { if payload.Image == "" { switch { case artifact.Metadata != nil && artifact.Metadata.Thumbnail != "": payload.Image = artifact.Metadata.Thumbnail case artifact.Type == domain.PortTypeImage || artifact.Type == domain.PortTypeVideo: payload.Image = artifact.Path } } if payload.Confidence == 0 { if confidence, ok := extractConfidence(artifact); ok { payload.Confidence = confidence } } payload.DetectedObjects = append(payload.DetectedObjects, extractObjectHints(artifact)...) } if len(payload.DetectedObjects) == 0 { payload.DetectedObjects = []string{fmt.Sprintf("%d upstream artifact(s)", len(inputArtifacts))} } return payload } func prepareNodeWorkspace(workspaceRoot string, runID string, nodeID string) (nodeWorkspace, error) { root := strings.TrimSpace(workspaceRoot) if root == "" { root = filepath.Join(os.TempDir(), "droboticflow") } runRoot := filepath.Join(root, "runs", runID) nodeRoot := filepath.Join(runRoot, "nodes", nodeID) outputDir := filepath.Join(nodeRoot, "outputs") if err := os.MkdirAll(outputDir, 0o755); err != nil { return nodeWorkspace{}, err } return nodeWorkspace{ RunRoot: runRoot, Root: nodeRoot, OutputDir: outputDir, InputArtifactsPath: filepath.Join(nodeRoot, "input-artifacts.json"), ResultPath: filepath.Join(nodeRoot, "result.json"), StageManifestPath: filepath.Join(nodeRoot, "stage-manifest.json"), }, nil } func shouldInjectPlatformMongoEnv(node domain.WorkflowNode) bool { label := strings.ToLower(strings.TrimSpace(node.Data.Label)) return strings.Contains(label, "mongodb") || strings.Contains(label, "mongo") } func writeJSONFile(path string, value any) error { data, err := json.Marshal(value) if err != nil { return err } if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { return err } return os.WriteFile(path, data, 0o644) } func applyHyperparamsToEnv(env map[string]string, items []domain.HyperParam) { for _, item := range items { key := sanitizeEnvKey(item.Key) if key == "" { continue } if _, blocked := protectedSystemEnv[key]; blocked { continue } env[key] = item.Value } } func applyOperatorCommandPolicyEnv(env map[string]string) { for _, key := range operatorCommandPolicyEnvKeys { if value := strings.TrimSpace(os.Getenv(key)); value != "" { env[key] = value } } } func buildNodeEnv(run *domain.RunRecord, workflow *domain.WorkflowRecord, node domain.WorkflowNode, inputArtifacts []domain.ExecutionArtifact, workspace nodeWorkspace, settings *domain.SettingsRecord) (map[string]string, error) { if inputArtifacts == nil { inputArtifacts = []domain.ExecutionArtifact{} } nodeDataJSON, err := json.Marshal(node.Data) if err != nil { return nil, err } inputArtifactsJSON, err := json.Marshal(inputArtifacts) if err != nil { return nil, err } if err := writeJSONFile(workspace.InputArtifactsPath, inputArtifacts); err != nil { return nil, err } env := map[string]string{ "RUN_ID": run.ID, "WORKFLOW_ID": run.WorkflowID, "NODE_ID": node.ID, "NODE_LABEL": node.Data.Label, "NODE_TYPE": node.Type, "NODE_CATEGORY": node.Data.Category, "NODE_DESCRIPTION": node.Data.Description, "NODE_DATA_JSON": string(nodeDataJSON), "INPUT_ARTIFACTS_PATH": workspace.InputArtifactsPath, "RESULT_OUTPUT_PATH": workspace.ResultPath, "STAGE_MANIFEST_PATH": workspace.StageManifestPath, "OUTPUT_DIR": workspace.OutputDir, "WORKSPACE_ROOT": workspace.RunRoot, } if run.Trigger == domain.RunTriggerScheduled { env["WORKFLOW_CURRENT_RUN_TIMESTAMP"] = runTimestampEnvValue(firstNonEmpty(run.CurrentRunAt, run.CreatedAt)) env["WORKFLOW_PREVIOUS_RUN_TIMESTAMP"] = runTimestampEnvValue(run.PreviousRunAt) } if len(inputArtifactsJSON) <= inputArtifactsInlineLimit { env["INPUT_ARTIFACTS_JSON"] = string(inputArtifactsJSON) } if node.Data.Bucket != "" { env["STORAGE_BUCKET"] = node.Data.Bucket } if node.Data.Prefix != "" { env["STORAGE_PREFIX"] = node.Data.Prefix } if node.Data.Pattern != "" { env["STORAGE_PATTERN"] = node.Data.Pattern } if node.Data.CPU != "" { env["CPU_LIMIT"] = node.Data.CPU } if node.Data.Memory != "" { env["MEMORY_LIMIT"] = node.Data.Memory } if node.Data.GPU != "" { env["GPU_LIMIT"] = node.Data.GPU } if node.Data.Image != "" { env["NODE_IMAGE"] = node.Data.Image } if shouldInjectPlatformMongoEnv(node) { if value := strings.TrimSpace(os.Getenv("MONGO_URI")); value != "" { env["PLATFORM_MONGO_URI"] = value } if value := strings.TrimSpace(os.Getenv("MONGO_DB")); value != "" { env["PLATFORM_MONGO_DB"] = value } } if workflow != nil { applyHyperparamsToEnv(env, workflow.Hyperparams) } applyHyperparamsToEnv(env, node.Data.Hyperparams) applyOperatorCommandPolicyEnv(env) return env, nil } func parseExecutionOutput(stdout string, nodeLabel string) (executionResult, error) { result := executionResult{ Logs: []domain.LogEntryRecord{}, Artifacts: []domain.ExecutionArtifact{}, ResourceSeries: []domain.ResourceSample{}, } lines := strings.Split(strings.ReplaceAll(stdout, "\r\n", "\n"), "\n") structuredPayload := "" structuredPayloadPath := "" rawLogs := make([]domain.LogEntryRecord, 0) for _, line := range lines { trimmed := strings.TrimSpace(line) if trimmed == "" { continue } if strings.HasPrefix(trimmed, resultPrefix) { structuredPayload = strings.TrimPrefix(trimmed, resultPrefix) continue } if strings.HasPrefix(trimmed, resultPathPrefix) { structuredPayloadPath = strings.TrimSpace(strings.TrimPrefix(trimmed, resultPathPrefix)) continue } rawLogs = append(rawLogs, logEntry("INFO", trimmed, nodeLabel)) } if structuredPayloadPath != "" { payloadBytes, err := os.ReadFile(structuredPayloadPath) if err != nil { result.Logs = rawLogs return result, fmt.Errorf("read operator result path: %w", err) } structuredPayload = strings.TrimSpace(string(payloadBytes)) } if structuredPayload == "" { result.Logs = rawLogs return result, fmt.Errorf("operator execution did not emit %s or %s payload", resultPrefix, resultPathPrefix) } var parsed executionResult if err := json.Unmarshal([]byte(structuredPayload), &parsed); err != nil { return result, fmt.Errorf("decode operator result: %w", err) } if len(parsed.Logs) > 0 { result.Logs = append(result.Logs, parsed.Logs...) } else { result.Logs = append(result.Logs, rawLogs...) } result.Artifacts = parsed.Artifacts result.ResourceSeries = parsed.ResourceSeries return result, nil } func loadStageManifestDataRefs(path string) ([]domain.DataRefRecord, error) { trimmed := strings.TrimSpace(path) if trimmed == "" { return nil, nil } payload, err := os.ReadFile(trimmed) if err != nil { return nil, err } var manifest stageManifestRecord if err := json.Unmarshal(payload, &manifest); err != nil { return nil, err } if manifest.DataRefs == nil { return []domain.DataRefRecord{}, nil } return manifest.DataRefs, nil } func (runtime *Runtime) executeNode(ctx context.Context, run *domain.RunRecord, node domain.WorkflowNode, inputArtifacts []domain.ExecutionArtifact) (executionResult, *k8s.JobReference, error) { workspace, err := prepareNodeWorkspace(runtime.workspaceRoot, run.ID, node.ID) if err != nil { return executionResult{}, nil, err } workflow, workflowErr := runtime.store.GetWorkflow(ctx, run.WorkflowID) if workflowErr != nil { workflow = nil } settings, err := runtime.store.GetSettings(ctx) if err != nil { return executionResult{}, nil, err } env, err := buildNodeEnv(run, workflow, node, inputArtifacts, workspace, settings) if err != nil { return executionResult{}, nil, err } nodeTimeoutSeconds := 0 if workflow != nil { nodeTimeoutSeconds = workflow.NodeTimeoutSeconds } launched, err := runtime.launcher.Launch(ctx, k8s.LaunchInput{ Image: chooseImage(node, runtime.operatorRunnerImage), RunID: run.ID, WorkflowID: run.WorkflowID, NodeID: node.ID, NodeLabel: node.Data.Label, TimeoutSeconds: nodeTimeoutSeconds, Env: env, CPURequest: settings.Defaults.CPURequest, CPULimit: firstNonEmpty(node.Data.CPU, settings.Defaults.CPULimit), MemoryRequest: settings.Defaults.MemoryRequest, MemoryLimit: firstNonEmpty(node.Data.Memory, settings.Defaults.MemoryLimit), GPULimit: firstNonEmpty(node.Data.GPU, settings.Defaults.GPULimit), }) if err != nil { return executionResult{}, nil, err } parsed, err := parseExecutionOutput(launched.Output, node.Data.Label) if err == nil { parsed.StageManifestPath = workspace.StageManifestPath parsed.DataRefs, err = loadStageManifestDataRefs(workspace.StageManifestPath) } return parsed, launched.Reference, err }