330 lines
9.6 KiB
Go
330 lines
9.6 KiB
Go
package worker
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"os"
|
|
"os/exec"
|
|
"path/filepath"
|
|
"regexp"
|
|
"slices"
|
|
"strings"
|
|
|
|
"droboticflow/internal/domain"
|
|
"droboticflow/internal/k8s"
|
|
)
|
|
|
|
const resultPrefix = "DROBOTICFLOW_RESULT="
|
|
|
|
type executionResult struct {
|
|
Logs []domain.LogEntryRecord `json:"logs"`
|
|
Artifacts []domain.ExecutionArtifact `json:"artifacts"`
|
|
ResourceSeries []domain.ResourceSample `json:"resourceSeries"`
|
|
}
|
|
|
|
var invalidEnvCharacters = regexp.MustCompile(`[^A-Z0-9_]+`)
|
|
|
|
func sanitizeEnvKey(value string) string {
|
|
normalized := strings.ToUpper(strings.TrimSpace(value))
|
|
if normalized == "" {
|
|
return ""
|
|
}
|
|
normalized = invalidEnvCharacters.ReplaceAllString(normalized, "_")
|
|
normalized = strings.Trim(normalized, "_")
|
|
if normalized == "" {
|
|
return ""
|
|
}
|
|
if normalized[0] >= '0' && normalized[0] <= '9' {
|
|
normalized = "HP_" + normalized
|
|
}
|
|
return normalized
|
|
}
|
|
|
|
func cloneArtifacts(items []domain.ExecutionArtifact) []domain.ExecutionArtifact {
|
|
cloned := make([]domain.ExecutionArtifact, 0, len(items))
|
|
for _, item := range items {
|
|
cloned = append(cloned, item)
|
|
}
|
|
return cloned
|
|
}
|
|
|
|
func gatherInputArtifacts(workflow domain.WorkflowRecord, executionMap map[string]domain.NodeExecutionRecord, nodeID string) []domain.ExecutionArtifact {
|
|
items := make([]domain.ExecutionArtifact, 0)
|
|
seen := map[string]struct{}{}
|
|
for _, edge := range domain.GetIncomingEdges(workflow.Edges, nodeID) {
|
|
execution, ok := executionMap[edge.Source]
|
|
if !ok {
|
|
continue
|
|
}
|
|
requiresHandle := strings.TrimSpace(edge.SourceHandle) != ""
|
|
hasPortAwareArtifacts := false
|
|
for _, artifact := range execution.Artifacts {
|
|
if strings.TrimSpace(artifact.PortID) != "" {
|
|
hasPortAwareArtifacts = true
|
|
break
|
|
}
|
|
}
|
|
for _, artifact := range execution.Artifacts {
|
|
if requiresHandle && hasPortAwareArtifacts && artifact.PortID != edge.SourceHandle {
|
|
continue
|
|
}
|
|
key := artifact.ID
|
|
if key == "" {
|
|
key = artifact.NodeID + ":" + artifact.Name
|
|
}
|
|
if _, exists := seen[key]; exists {
|
|
continue
|
|
}
|
|
seen[key] = struct{}{}
|
|
items = append(items, artifact)
|
|
}
|
|
}
|
|
return items
|
|
}
|
|
|
|
func extractConfidence(artifact domain.ExecutionArtifact) (float64, bool) {
|
|
if artifact.Data == nil {
|
|
return 0, false
|
|
}
|
|
for _, key := range []string{"confidence", "score"} {
|
|
value, ok := artifact.Data[key]
|
|
if !ok {
|
|
continue
|
|
}
|
|
switch typed := value.(type) {
|
|
case float64:
|
|
return typed, true
|
|
case int:
|
|
return float64(typed), true
|
|
}
|
|
}
|
|
return 0, false
|
|
}
|
|
|
|
func extractObjectHints(artifact domain.ExecutionArtifact) []string {
|
|
if artifact.Data == nil {
|
|
return nil
|
|
}
|
|
items := make([]string, 0)
|
|
for _, key := range []string{"detectedObjects", "labels", "entries"} {
|
|
value, ok := artifact.Data[key]
|
|
if !ok {
|
|
continue
|
|
}
|
|
switch typed := value.(type) {
|
|
case []any:
|
|
for _, candidate := range typed {
|
|
text := strings.TrimSpace(fmt.Sprint(candidate))
|
|
if text != "" {
|
|
items = append(items, text)
|
|
}
|
|
}
|
|
case []string:
|
|
for _, candidate := range typed {
|
|
if trimmed := strings.TrimSpace(candidate); trimmed != "" {
|
|
items = append(items, trimmed)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
return items
|
|
}
|
|
|
|
func createAuditPayload(node domain.WorkflowNode, inputArtifacts []domain.ExecutionArtifact) domain.AuditTaskPayload {
|
|
payload := domain.AuditTaskPayload{
|
|
Reason: fmt.Sprintf("%s requires explicit approval before downstream execution", node.Data.Label),
|
|
DetectedObjects: []string{},
|
|
}
|
|
for _, artifact := range inputArtifacts {
|
|
if payload.Image == "" {
|
|
switch {
|
|
case artifact.Metadata != nil && artifact.Metadata.Thumbnail != "":
|
|
payload.Image = artifact.Metadata.Thumbnail
|
|
case artifact.Type == domain.PortTypeImage || artifact.Type == domain.PortTypeVideo:
|
|
payload.Image = artifact.Path
|
|
}
|
|
}
|
|
if payload.Confidence == 0 {
|
|
if confidence, ok := extractConfidence(artifact); ok {
|
|
payload.Confidence = confidence
|
|
}
|
|
}
|
|
payload.DetectedObjects = append(payload.DetectedObjects, extractObjectHints(artifact)...)
|
|
}
|
|
if len(payload.DetectedObjects) == 0 {
|
|
payload.DetectedObjects = []string{fmt.Sprintf("%d upstream artifact(s)", len(inputArtifacts))}
|
|
}
|
|
return payload
|
|
}
|
|
|
|
func prepareOutputDir(runID string, nodeID string) (string, error) {
|
|
outputDir := filepath.Join(os.TempDir(), "droboticflow", runID, nodeID)
|
|
if err := os.MkdirAll(outputDir, 0o755); err != nil {
|
|
return "", err
|
|
}
|
|
return outputDir, nil
|
|
}
|
|
|
|
func envMapToSlice(values map[string]string) []string {
|
|
keys := make([]string, 0, len(values))
|
|
for key := range values {
|
|
keys = append(keys, key)
|
|
}
|
|
slices.Sort(keys)
|
|
items := make([]string, 0, len(keys))
|
|
for _, key := range keys {
|
|
items = append(items, key+"="+values[key])
|
|
}
|
|
return items
|
|
}
|
|
|
|
func buildNodeEnv(run *domain.RunRecord, node domain.WorkflowNode, inputArtifacts []domain.ExecutionArtifact, outputDir string) (map[string]string, error) {
|
|
nodeDataJSON, err := json.Marshal(node.Data)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
inputArtifactsJSON, err := json.Marshal(inputArtifacts)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
env := map[string]string{
|
|
"RUN_ID": run.ID,
|
|
"WORKFLOW_ID": run.WorkflowID,
|
|
"NODE_ID": node.ID,
|
|
"NODE_LABEL": node.Data.Label,
|
|
"NODE_TYPE": node.Type,
|
|
"NODE_CATEGORY": node.Data.Category,
|
|
"NODE_DESCRIPTION": node.Data.Description,
|
|
"NODE_DATA_JSON": string(nodeDataJSON),
|
|
"INPUT_ARTIFACTS_JSON": string(inputArtifactsJSON),
|
|
"OUTPUT_DIR": outputDir,
|
|
}
|
|
if node.Data.Bucket != "" {
|
|
env["STORAGE_BUCKET"] = node.Data.Bucket
|
|
}
|
|
if node.Data.Prefix != "" {
|
|
env["STORAGE_PREFIX"] = node.Data.Prefix
|
|
}
|
|
if node.Data.Pattern != "" {
|
|
env["STORAGE_PATTERN"] = node.Data.Pattern
|
|
}
|
|
if node.Data.CPU != "" {
|
|
env["CPU_LIMIT"] = node.Data.CPU
|
|
}
|
|
if node.Data.Memory != "" {
|
|
env["MEMORY_LIMIT"] = node.Data.Memory
|
|
}
|
|
if node.Data.Image != "" {
|
|
env["NODE_IMAGE"] = node.Data.Image
|
|
}
|
|
for _, item := range node.Data.Hyperparams {
|
|
key := sanitizeEnvKey(item.Key)
|
|
if key == "" {
|
|
continue
|
|
}
|
|
env[key] = item.Value
|
|
}
|
|
return env, nil
|
|
}
|
|
|
|
func parseExecutionOutput(stdout string, nodeLabel string) (executionResult, error) {
|
|
result := executionResult{
|
|
Logs: []domain.LogEntryRecord{},
|
|
Artifacts: []domain.ExecutionArtifact{},
|
|
ResourceSeries: []domain.ResourceSample{},
|
|
}
|
|
lines := strings.Split(strings.ReplaceAll(stdout, "\r\n", "\n"), "\n")
|
|
structuredPayload := ""
|
|
rawLogs := make([]domain.LogEntryRecord, 0)
|
|
for _, line := range lines {
|
|
trimmed := strings.TrimSpace(line)
|
|
if trimmed == "" {
|
|
continue
|
|
}
|
|
if strings.HasPrefix(trimmed, resultPrefix) {
|
|
structuredPayload = strings.TrimPrefix(trimmed, resultPrefix)
|
|
continue
|
|
}
|
|
rawLogs = append(rawLogs, logEntry("INFO", trimmed, nodeLabel))
|
|
}
|
|
if structuredPayload == "" {
|
|
result.Logs = rawLogs
|
|
return result, fmt.Errorf("operator execution did not emit %s payload", resultPrefix)
|
|
}
|
|
var parsed executionResult
|
|
if err := json.Unmarshal([]byte(structuredPayload), &parsed); err != nil {
|
|
return result, fmt.Errorf("decode operator result: %w", err)
|
|
}
|
|
if len(parsed.Logs) > 0 {
|
|
result.Logs = append(result.Logs, parsed.Logs...)
|
|
} else {
|
|
result.Logs = append(result.Logs, rawLogs...)
|
|
}
|
|
result.Artifacts = parsed.Artifacts
|
|
result.ResourceSeries = parsed.ResourceSeries
|
|
return result, nil
|
|
}
|
|
|
|
func (runtime *Runtime) executeLocalProcess(ctx context.Context, command string, env map[string]string) (string, error) {
|
|
cmd := exec.CommandContext(ctx, "sh", "-lc", command)
|
|
cmd.Env = append(os.Environ(), envMapToSlice(env)...)
|
|
output, err := cmd.CombinedOutput()
|
|
return string(output), err
|
|
}
|
|
|
|
func (runtime *Runtime) executeLocalDocker(ctx context.Context, image string, env map[string]string, outputDir string) (string, error) {
|
|
args := []string{"run", "--rm", "-v", outputDir + ":" + outputDir}
|
|
for key, value := range env {
|
|
args = append(args, "-e", key+"="+value)
|
|
}
|
|
args = append(args, image)
|
|
cmd := exec.CommandContext(ctx, "docker", args...)
|
|
output, err := cmd.CombinedOutput()
|
|
return string(output), err
|
|
}
|
|
|
|
func (runtime *Runtime) executeNode(ctx context.Context, run *domain.RunRecord, node domain.WorkflowNode, inputArtifacts []domain.ExecutionArtifact) (executionResult, *k8s.JobReference, error) {
|
|
outputDir, err := prepareOutputDir(run.ID, node.ID)
|
|
if err != nil {
|
|
return executionResult{}, nil, err
|
|
}
|
|
env, err := buildNodeEnv(run, node, inputArtifacts, outputDir)
|
|
if err != nil {
|
|
return executionResult{}, nil, err
|
|
}
|
|
if runtime.launcher.Enabled() {
|
|
nodeTimeoutSeconds := 0
|
|
if workflow, err := runtime.store.GetWorkflow(ctx, run.WorkflowID); err == nil && workflow != nil {
|
|
nodeTimeoutSeconds = workflow.NodeTimeoutSeconds
|
|
}
|
|
launched, err := runtime.launcher.Launch(ctx, k8s.LaunchInput{
|
|
Image: chooseImage(node, runtime.operatorRunnerImage),
|
|
RunID: run.ID,
|
|
WorkflowID: run.WorkflowID,
|
|
NodeID: node.ID,
|
|
NodeLabel: node.Data.Label,
|
|
TimeoutSeconds: nodeTimeoutSeconds,
|
|
Env: env,
|
|
})
|
|
if err != nil {
|
|
return executionResult{}, nil, err
|
|
}
|
|
parsed, err := parseExecutionOutput(launched.Output, node.Data.Label)
|
|
return parsed, launched.Reference, err
|
|
}
|
|
|
|
image := chooseImage(node, runtime.operatorRunnerImage)
|
|
var stdout string
|
|
if node.Data.Image != "" && node.Data.Image != runtime.operatorRunnerImage {
|
|
stdout, err = runtime.executeLocalDocker(ctx, image, env, outputDir)
|
|
} else {
|
|
stdout, err = runtime.executeLocalProcess(ctx, runtime.localRunnerCommand, env)
|
|
}
|
|
if err != nil {
|
|
return executionResult{}, nil, fmt.Errorf("%w: %s", err, stdout)
|
|
}
|
|
parsed, err := parseExecutionOutput(stdout, node.Data.Label)
|
|
return parsed, nil, err
|
|
}
|