forked from longtao.wu/droboticflow
The reconciler used to be the only place that failed a node whose pod sat in ImagePullBackOff/Unschedulable past WORKER_PENDING_FAILURE_SECONDS. Now that it leaves runs with a live execution lease alone (0b365155), such a node would wait for the full launch deadline. Have Launch check the Job's pods (at most every 10s) and, once a pod has been blocked for PendingFailureTimeout, delete the Job and return the blocking reason and message. The executor passes the configured pending threshold. On POC this surfaced builtin validator images that cannot be pulled, which the reconciler race had been masking as missing Jobs. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
910 lines
30 KiB
Go
910 lines
30 KiB
Go
package worker
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"maps"
|
|
"os"
|
|
"path/filepath"
|
|
"regexp"
|
|
"strings"
|
|
"time"
|
|
|
|
"droboticflow/internal/domain"
|
|
"droboticflow/internal/k8s"
|
|
"droboticflow/internal/lightweight"
|
|
)
|
|
|
|
const resultPrefix = "DROBOTICFLOW_RESULT="
|
|
const resultPathPrefix = "DROBOTICFLOW_RESULT_PATH="
|
|
const inputArtifactsInlineLimit = 64 * 1024
|
|
|
|
type nodeWorkspace struct {
|
|
RunRoot string
|
|
Root string
|
|
OutputDir string
|
|
InputArtifactsPath string
|
|
InputBindingsPath string
|
|
ResultPath string
|
|
StageManifestPath string
|
|
}
|
|
|
|
type executionResult struct {
|
|
Logs []domain.LogEntryRecord `json:"logs"`
|
|
Artifacts []domain.ExecutionArtifact `json:"artifacts"`
|
|
DataRefs []domain.DataRefRecord `json:"dataRefs,omitempty"`
|
|
DataMetrics *domain.DataMetricsRecord `json:"dataMetrics,omitempty"`
|
|
StageManifestPath string `json:"stageManifestPath,omitempty"`
|
|
ResourceSeries []domain.ResourceSample `json:"resourceSeries"`
|
|
}
|
|
|
|
type stageManifestRecord struct {
|
|
DataRefs []domain.DataRefRecord `json:"dataRefs"`
|
|
DataMetrics *domain.DataMetricsRecord `json:"dataMetrics,omitempty"`
|
|
}
|
|
|
|
var protectedSystemEnv = map[string]struct{}{
|
|
"WORKFLOW_PREVIOUS_RUN_TIMESTAMP": {},
|
|
"WORKFLOW_CURRENT_RUN_TIMESTAMP": {},
|
|
"PARENT_RUN_ID": {},
|
|
"PARENT_RUN_WORKSPACE_ROOT": {},
|
|
"PARENT_NODE_OUTPUT_DIR": {},
|
|
"OPERATOR_ALLOW_SHELL_COMMANDS": {},
|
|
"OPERATOR_COMMAND_ALLOWLIST_REGEX": {},
|
|
"OPERATOR_COMMAND_DENYLIST_REGEX": {},
|
|
}
|
|
|
|
var operatorCommandPolicyEnvKeys = []string{
|
|
"OPERATOR_ALLOW_SHELL_COMMANDS",
|
|
"OPERATOR_COMMAND_ALLOWLIST_REGEX",
|
|
"OPERATOR_COMMAND_DENYLIST_REGEX",
|
|
}
|
|
|
|
var workflowEnvLocation = loadWorkflowEnvLocation()
|
|
|
|
var invalidEnvCharacters = regexp.MustCompile(`[^A-Z0-9_]+`)
|
|
|
|
func loadWorkflowEnvLocation() *time.Location {
|
|
location, err := time.LoadLocation("Asia/Shanghai")
|
|
if err != nil {
|
|
return time.FixedZone("Asia/Shanghai", 8*60*60)
|
|
}
|
|
return location
|
|
}
|
|
|
|
func sanitizeEnvKey(value string) string {
|
|
normalized := strings.ToUpper(strings.TrimSpace(value))
|
|
if normalized == "" {
|
|
return ""
|
|
}
|
|
normalized = invalidEnvCharacters.ReplaceAllString(normalized, "_")
|
|
normalized = strings.Trim(normalized, "_")
|
|
if normalized == "" {
|
|
return ""
|
|
}
|
|
if normalized[0] >= '0' && normalized[0] <= '9' {
|
|
normalized = "HP_" + normalized
|
|
}
|
|
return normalized
|
|
}
|
|
|
|
func cloneArtifacts(items []domain.ExecutionArtifact) []domain.ExecutionArtifact {
|
|
cloned := make([]domain.ExecutionArtifact, 0, len(items))
|
|
for _, item := range items {
|
|
cloned = append(cloned, cloneArtifact(item))
|
|
}
|
|
return cloned
|
|
}
|
|
|
|
func cloneStringSlice(items []string) []string {
|
|
if items == nil {
|
|
return nil
|
|
}
|
|
cloned := make([]string, len(items))
|
|
copy(cloned, items)
|
|
return cloned
|
|
}
|
|
|
|
func cloneArtifact(item domain.ExecutionArtifact) domain.ExecutionArtifact {
|
|
cloned := item
|
|
if item.Metadata != nil {
|
|
metadata := *item.Metadata
|
|
cloned.Metadata = &metadata
|
|
}
|
|
if item.Data != nil {
|
|
cloned.Data = maps.Clone(item.Data)
|
|
}
|
|
return cloned
|
|
}
|
|
|
|
func cloneInputLineage(items []domain.InputLineageRecord) []domain.InputLineageRecord {
|
|
cloned := make([]domain.InputLineageRecord, 0, len(items))
|
|
for _, item := range items {
|
|
record := item
|
|
record.Artifact = cloneArtifact(item.Artifact)
|
|
cloned = append(cloned, record)
|
|
}
|
|
return cloned
|
|
}
|
|
|
|
func cloneInputBindings(items []domain.NodeInputBinding) []domain.NodeInputBinding {
|
|
if items == nil {
|
|
return nil
|
|
}
|
|
cloned := make([]domain.NodeInputBinding, len(items))
|
|
for index, item := range items {
|
|
cloned[index] = item
|
|
cloned[index].Artifacts = cloneArtifacts(item.Artifacts)
|
|
if item.Payload != nil {
|
|
payload := *item.Payload
|
|
if item.Payload.Ref != nil {
|
|
ref := *item.Payload.Ref
|
|
payload.Ref = &ref
|
|
}
|
|
cloned[index].Payload = &payload
|
|
}
|
|
}
|
|
return cloned
|
|
}
|
|
|
|
func runTimestampEnvValue(value string) string {
|
|
trimmed := strings.TrimSpace(value)
|
|
if trimmed == "" {
|
|
return ""
|
|
}
|
|
parsed, err := time.Parse(time.RFC3339Nano, trimmed)
|
|
if err != nil {
|
|
return ""
|
|
}
|
|
return parsed.In(workflowEnvLocation).Format(time.RFC3339)
|
|
}
|
|
|
|
func firstNonEmpty(values ...string) string {
|
|
for _, value := range values {
|
|
if strings.TrimSpace(value) != "" {
|
|
return value
|
|
}
|
|
}
|
|
return ""
|
|
}
|
|
|
|
func inputArtifactLineageKey(item domain.InputLineageRecord) string {
|
|
artifact := item.Artifact
|
|
key := artifact.ID
|
|
if key == "" {
|
|
key = artifact.NodeID + ":" + artifact.Name
|
|
}
|
|
return fmt.Sprintf("%s:%s:%s:%s:%s", item.WorkflowEdgeID, item.SourceNodeID, item.SourceHandle, item.TargetHandle, key)
|
|
}
|
|
|
|
func gatherInputLineage(workflow domain.WorkflowRecord, executionMap map[string]domain.NodeExecutionRecord, nodeID string) []domain.InputLineageRecord {
|
|
items := make([]domain.InputLineageRecord, 0)
|
|
seen := map[string]struct{}{}
|
|
for _, edge := range domain.GetIncomingEdges(workflow.Edges, nodeID) {
|
|
execution, ok := executionMap[edge.Source]
|
|
if !ok {
|
|
continue
|
|
}
|
|
requiresHandle := strings.TrimSpace(edge.SourceHandle) != ""
|
|
hasPortAwareArtifacts := false
|
|
for _, artifact := range execution.Artifacts {
|
|
if strings.TrimSpace(artifact.PortID) != "" {
|
|
hasPortAwareArtifacts = true
|
|
break
|
|
}
|
|
}
|
|
for _, artifact := range execution.Artifacts {
|
|
if requiresHandle && hasPortAwareArtifacts && artifact.PortID != edge.SourceHandle {
|
|
continue
|
|
}
|
|
item := domain.InputLineageRecord{
|
|
WorkflowEdgeID: edge.ID,
|
|
SourceNodeID: edge.Source,
|
|
SourceHandle: edge.SourceHandle,
|
|
TargetNodeID: edge.Target,
|
|
TargetHandle: edge.TargetHandle,
|
|
Artifact: artifact,
|
|
}
|
|
key := inputArtifactLineageKey(item)
|
|
if _, exists := seen[key]; exists {
|
|
continue
|
|
}
|
|
seen[key] = struct{}{}
|
|
items = append(items, item)
|
|
}
|
|
}
|
|
return items
|
|
}
|
|
|
|
func inputArtifactsFromLineage(lineage []domain.InputLineageRecord) []domain.ExecutionArtifact {
|
|
items := make([]domain.ExecutionArtifact, 0)
|
|
seen := map[string]struct{}{}
|
|
for _, item := range lineage {
|
|
artifact := item.Artifact
|
|
key := artifact.ID
|
|
if key == "" {
|
|
key = artifact.NodeID + ":" + artifact.Name
|
|
}
|
|
if _, exists := seen[key]; exists {
|
|
continue
|
|
}
|
|
seen[key] = struct{}{}
|
|
items = append(items, artifact)
|
|
}
|
|
return items
|
|
}
|
|
|
|
func artifactFromWorkflowInput(run *domain.RunRecord, binding domain.ResolvedInputBinding) domain.ExecutionArtifact {
|
|
payload := binding.Payload
|
|
artifact := domain.ExecutionArtifact{
|
|
ID: run.ID + "-input-" + sanitizeEnvKey(binding.WorkflowInputID) + "-" + sanitizeEnvKey(binding.TargetPortID),
|
|
RunID: run.ID,
|
|
NodeID: binding.NodeID,
|
|
PortID: binding.TargetPortID,
|
|
PortName: binding.TargetPortID,
|
|
Name: binding.WorkflowInputID,
|
|
Type: payload.Type,
|
|
Payload: &payload,
|
|
}
|
|
if payload.Mode == domain.ArtifactTransferModeInline {
|
|
encoded, _ := json.Marshal(payload.Inline)
|
|
artifact.SizeBytes = int64(len(encoded))
|
|
artifact.Size = fmt.Sprintf("%d B", len(encoded))
|
|
if data, ok := payload.Inline.(map[string]any); ok {
|
|
artifact.Data = data
|
|
} else {
|
|
artifact.Data = map[string]any{"value": payload.Inline}
|
|
}
|
|
return artifact
|
|
}
|
|
if payload.Ref != nil {
|
|
ref := payload.Ref
|
|
artifact.SizeBytes = ref.SizeBytes
|
|
artifact.Size = fmt.Sprintf("%d B", ref.SizeBytes)
|
|
if ref.Kind == domain.ArtifactRefKindDirectory {
|
|
artifact.Path = "bos://" + ref.Bucket + "/" + strings.TrimPrefix(ref.Prefix, "/")
|
|
} else {
|
|
artifact.Path = "bos://" + ref.Bucket + "/" + strings.TrimPrefix(ref.Key, "/")
|
|
}
|
|
artifact.Metadata = &domain.ArtifactMetadata{SHA256: ref.SHA256, MimeType: ref.MimeType}
|
|
artifact.Data = map[string]any{"source": map[string]any{"path": artifact.Path}}
|
|
}
|
|
return artifact
|
|
}
|
|
|
|
func workflowInputBindingsForNode(run *domain.RunRecord, nodeID string) ([]domain.NodeInputBinding, []domain.ExecutionArtifact) {
|
|
if run == nil {
|
|
return nil, nil
|
|
}
|
|
bindings := make([]domain.NodeInputBinding, 0)
|
|
artifacts := make([]domain.ExecutionArtifact, 0)
|
|
for _, item := range run.ResolvedInputBindings {
|
|
if item.NodeID != nodeID {
|
|
continue
|
|
}
|
|
artifact := artifactFromWorkflowInput(run, item)
|
|
payload := item.Payload
|
|
bindings = append(bindings, domain.NodeInputBinding{
|
|
TargetPortID: item.TargetPortID,
|
|
WorkflowInputID: item.WorkflowInputID,
|
|
Payload: &payload,
|
|
Artifacts: []domain.ExecutionArtifact{artifact},
|
|
})
|
|
artifacts = append(artifacts, artifact)
|
|
}
|
|
return bindings, artifacts
|
|
}
|
|
|
|
func lineageInputBindings(lineage []domain.InputLineageRecord) []domain.NodeInputBinding {
|
|
items := make([]domain.NodeInputBinding, 0)
|
|
indexByKey := map[string]int{}
|
|
for _, item := range lineage {
|
|
key := item.SourceNodeID + "\x00" + item.SourceHandle + "\x00" + item.TargetHandle
|
|
if index, ok := indexByKey[key]; ok {
|
|
items[index].Artifacts = append(items[index].Artifacts, item.Artifact)
|
|
continue
|
|
}
|
|
indexByKey[key] = len(items)
|
|
items = append(items, domain.NodeInputBinding{
|
|
TargetPortID: item.TargetHandle,
|
|
SourceNodeID: item.SourceNodeID,
|
|
SourcePortID: item.SourceHandle,
|
|
Artifacts: []domain.ExecutionArtifact{item.Artifact},
|
|
})
|
|
}
|
|
return items
|
|
}
|
|
|
|
func validateRequiredNodeInputs(node domain.WorkflowNode, bindings []domain.NodeInputBinding) error {
|
|
for _, port := range node.Data.Inputs {
|
|
if !port.Required {
|
|
continue
|
|
}
|
|
present := false
|
|
for _, binding := range bindings {
|
|
targetPortID := strings.TrimSpace(binding.TargetPortID)
|
|
if targetPortID != strings.TrimSpace(port.ID) && !(targetPortID == "" && len(node.Data.Inputs) == 1) {
|
|
continue
|
|
}
|
|
if binding.Payload != nil || len(binding.Artifacts) > 0 {
|
|
present = true
|
|
break
|
|
}
|
|
}
|
|
if !present {
|
|
return fmt.Errorf("required_input_missing: node %s input %s has no value or upstream artifact", node.ID, port.ID)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func gatherNodeInputs(workflow domain.WorkflowRecord, run *domain.RunRecord, executionMap map[string]domain.NodeExecutionRecord, nodeID string) ([]domain.ExecutionArtifact, []domain.InputLineageRecord, []domain.NodeInputBinding) {
|
|
lineage := gatherInputLineage(workflow, executionMap, nodeID)
|
|
artifacts := inputArtifactsFromLineage(lineage)
|
|
bindings := lineageInputBindings(lineage)
|
|
rootBindings, rootArtifacts := workflowInputBindingsForNode(run, nodeID)
|
|
bindings = append(bindings, rootBindings...)
|
|
artifacts = append(artifacts, rootArtifacts...)
|
|
return artifacts, lineage, bindings
|
|
}
|
|
|
|
func gatherInputArtifacts(workflow domain.WorkflowRecord, executionMap map[string]domain.NodeExecutionRecord, nodeID string) []domain.ExecutionArtifact {
|
|
return inputArtifactsFromLineage(gatherInputLineage(workflow, executionMap, nodeID))
|
|
}
|
|
|
|
func extractConfidence(artifact domain.ExecutionArtifact) (float64, bool) {
|
|
if artifact.Data == nil {
|
|
return 0, false
|
|
}
|
|
for _, key := range []string{"confidence", "score"} {
|
|
value, ok := artifact.Data[key]
|
|
if !ok {
|
|
continue
|
|
}
|
|
switch typed := value.(type) {
|
|
case float64:
|
|
return typed, true
|
|
case int:
|
|
return float64(typed), true
|
|
}
|
|
}
|
|
return 0, false
|
|
}
|
|
|
|
func extractObjectHints(artifact domain.ExecutionArtifact) []string {
|
|
if artifact.Data == nil {
|
|
return nil
|
|
}
|
|
items := make([]string, 0)
|
|
for _, key := range []string{"detectedObjects", "labels", "entries"} {
|
|
value, ok := artifact.Data[key]
|
|
if !ok {
|
|
continue
|
|
}
|
|
switch typed := value.(type) {
|
|
case []any:
|
|
for _, candidate := range typed {
|
|
text := strings.TrimSpace(fmt.Sprint(candidate))
|
|
if text != "" {
|
|
items = append(items, text)
|
|
}
|
|
}
|
|
case []string:
|
|
for _, candidate := range typed {
|
|
if trimmed := strings.TrimSpace(candidate); trimmed != "" {
|
|
items = append(items, trimmed)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
return items
|
|
}
|
|
|
|
func createAuditPayload(node domain.WorkflowNode, inputArtifacts []domain.ExecutionArtifact) domain.AuditTaskPayload {
|
|
payload := domain.AuditTaskPayload{
|
|
Reason: fmt.Sprintf("%s requires explicit approval before downstream execution", node.Data.Label),
|
|
DetectedObjects: []string{},
|
|
}
|
|
for _, artifact := range inputArtifacts {
|
|
if payload.Image == "" {
|
|
switch {
|
|
case artifact.Metadata != nil && artifact.Metadata.Thumbnail != "":
|
|
payload.Image = artifact.Metadata.Thumbnail
|
|
case artifact.Type == domain.PortTypeImage || artifact.Type == domain.PortTypeVideo:
|
|
payload.Image = artifact.Path
|
|
}
|
|
}
|
|
if payload.Confidence == 0 {
|
|
if confidence, ok := extractConfidence(artifact); ok {
|
|
payload.Confidence = confidence
|
|
}
|
|
}
|
|
payload.DetectedObjects = append(payload.DetectedObjects, extractObjectHints(artifact)...)
|
|
}
|
|
if len(payload.DetectedObjects) == 0 {
|
|
payload.DetectedObjects = []string{fmt.Sprintf("%d upstream artifact(s)", len(inputArtifacts))}
|
|
}
|
|
return payload
|
|
}
|
|
|
|
func prepareNodeWorkspace(workspaceRoot string, runID string, nodeID string) (nodeWorkspace, error) {
|
|
root := strings.TrimSpace(workspaceRoot)
|
|
if root == "" {
|
|
root = filepath.Join(os.TempDir(), "droboticflow")
|
|
}
|
|
runRoot := filepath.Join(root, "system", "runs", runID)
|
|
nodeRoot := filepath.Join(runRoot, "nodes", nodeID)
|
|
outputDir := filepath.Join(nodeRoot, "outputs")
|
|
if err := os.MkdirAll(outputDir, 0o755); err != nil {
|
|
return nodeWorkspace{}, err
|
|
}
|
|
return nodeWorkspace{
|
|
RunRoot: runRoot,
|
|
Root: nodeRoot,
|
|
OutputDir: outputDir,
|
|
InputArtifactsPath: filepath.Join(nodeRoot, "input-artifacts.json"),
|
|
InputBindingsPath: filepath.Join(nodeRoot, "input-bindings.json"),
|
|
ResultPath: filepath.Join(nodeRoot, "result.json"),
|
|
StageManifestPath: filepath.Join(nodeRoot, "stage-manifest.json"),
|
|
}, nil
|
|
}
|
|
|
|
func shouldInjectPlatformMongoEnv(node domain.WorkflowNode) bool {
|
|
label := strings.ToLower(strings.TrimSpace(node.Data.Label))
|
|
return strings.Contains(label, "mongodb") || strings.Contains(label, "mongo")
|
|
}
|
|
|
|
func writeJSONFile(path string, value any) error {
|
|
data, err := json.Marshal(value)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
|
return err
|
|
}
|
|
return os.WriteFile(path, data, 0o644)
|
|
}
|
|
|
|
func applyHyperparamsToEnv(env map[string]string, items []domain.HyperParam) {
|
|
for _, item := range items {
|
|
key := sanitizeEnvKey(item.Key)
|
|
if key == "" {
|
|
continue
|
|
}
|
|
if _, blocked := protectedSystemEnv[key]; blocked {
|
|
continue
|
|
}
|
|
env[key] = item.Value
|
|
}
|
|
}
|
|
|
|
func applyOperatorCommandPolicyEnv(env map[string]string) {
|
|
for _, key := range operatorCommandPolicyEnvKeys {
|
|
if value := strings.TrimSpace(os.Getenv(key)); value != "" {
|
|
env[key] = value
|
|
}
|
|
}
|
|
}
|
|
|
|
func buildNodeEnv(run *domain.RunRecord, workflow *domain.WorkflowRecord, node domain.WorkflowNode, inputArtifacts []domain.ExecutionArtifact, workspace nodeWorkspace, settings *domain.SettingsRecord) (map[string]string, error) {
|
|
return buildNodeEnvWithResources(run, workflow, node, inputArtifacts, nil, workspace, effectiveNodeResources{
|
|
CPULimit: node.Data.CPU,
|
|
MemoryLimit: node.Data.Memory,
|
|
GPULimit: node.Data.GPU,
|
|
})
|
|
}
|
|
|
|
type effectiveNodeResources struct {
|
|
CPURequest string
|
|
CPULimit string
|
|
MemoryRequest string
|
|
MemoryLimit string
|
|
GPULimit string
|
|
SharedMemorySize string
|
|
}
|
|
|
|
func buildNodeEnvWithResources(run *domain.RunRecord, workflow *domain.WorkflowRecord, node domain.WorkflowNode, inputArtifacts []domain.ExecutionArtifact, inputBindings []domain.NodeInputBinding, workspace nodeWorkspace, resources effectiveNodeResources) (map[string]string, error) {
|
|
if inputArtifacts == nil {
|
|
inputArtifacts = []domain.ExecutionArtifact{}
|
|
}
|
|
nodeDataJSON, err := json.Marshal(node.Data)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
inputArtifactsJSON, err := json.Marshal(inputArtifacts)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if err := writeJSONFile(workspace.InputArtifactsPath, inputArtifacts); err != nil {
|
|
return nil, err
|
|
}
|
|
if inputBindings == nil {
|
|
inputBindings = []domain.NodeInputBinding{}
|
|
}
|
|
if strings.TrimSpace(workspace.InputBindingsPath) == "" {
|
|
workspace.InputBindingsPath = filepath.Join(workspace.Root, "input-bindings.json")
|
|
}
|
|
inputBindingsJSON, err := json.Marshal(inputBindings)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if err := writeJSONFile(workspace.InputBindingsPath, inputBindings); err != nil {
|
|
return nil, err
|
|
}
|
|
env := map[string]string{
|
|
"RUN_ID": run.ID,
|
|
"WORKFLOW_ID": run.WorkflowID,
|
|
"NODE_ID": node.ID,
|
|
"NODE_LABEL": node.Data.Label,
|
|
"NODE_TYPE": node.Type,
|
|
"NODE_CATEGORY": node.Data.Category,
|
|
"NODE_DESCRIPTION": node.Data.Description,
|
|
"NODE_DATA_JSON": string(nodeDataJSON),
|
|
"INPUT_ARTIFACTS_PATH": workspace.InputArtifactsPath,
|
|
"INPUT_BINDINGS_PATH": workspace.InputBindingsPath,
|
|
"RESULT_OUTPUT_PATH": workspace.ResultPath,
|
|
"STAGE_MANIFEST_PATH": workspace.StageManifestPath,
|
|
"OUTPUT_DIR": workspace.OutputDir,
|
|
"WORKSPACE_ROOT": workspace.RunRoot,
|
|
}
|
|
if parentRunID := strings.TrimSpace(run.ParentRunID); parentRunID != "" {
|
|
parentRunRoot := filepath.Join(filepath.Dir(workspace.RunRoot), filepath.Base(parentRunID))
|
|
env["PARENT_RUN_ID"] = parentRunID
|
|
env["PARENT_RUN_WORKSPACE_ROOT"] = parentRunRoot
|
|
env["PARENT_NODE_OUTPUT_DIR"] = filepath.Join(parentRunRoot, "nodes", node.ID, "outputs")
|
|
}
|
|
if run.Trigger == domain.RunTriggerScheduled {
|
|
env["WORKFLOW_CURRENT_RUN_TIMESTAMP"] = runTimestampEnvValue(firstNonEmpty(run.CurrentRunAt, run.CreatedAt))
|
|
env["WORKFLOW_PREVIOUS_RUN_TIMESTAMP"] = runTimestampEnvValue(run.PreviousRunAt)
|
|
}
|
|
if len(inputArtifactsJSON) <= inputArtifactsInlineLimit {
|
|
env["INPUT_ARTIFACTS_JSON"] = string(inputArtifactsJSON)
|
|
}
|
|
if len(inputBindingsJSON) <= inputArtifactsInlineLimit {
|
|
env["INPUT_BINDINGS_JSON"] = string(inputBindingsJSON)
|
|
}
|
|
if node.Data.Bucket != "" {
|
|
env["STORAGE_BUCKET"] = node.Data.Bucket
|
|
}
|
|
if node.Data.Prefix != "" {
|
|
env["STORAGE_PREFIX"] = node.Data.Prefix
|
|
}
|
|
if node.Data.Pattern != "" {
|
|
env["STORAGE_PATTERN"] = node.Data.Pattern
|
|
}
|
|
if resources.CPULimit != "" {
|
|
env["CPU_LIMIT"] = resources.CPULimit
|
|
}
|
|
if resources.MemoryLimit != "" {
|
|
env["MEMORY_LIMIT"] = resources.MemoryLimit
|
|
}
|
|
if resources.GPULimit != "" {
|
|
env["GPU_LIMIT"] = resources.GPULimit
|
|
}
|
|
if node.Data.Image != "" {
|
|
env["NODE_IMAGE"] = node.Data.Image
|
|
}
|
|
if handler := runtimeContractHandler(node.Data.RuntimeContract); handler != "" {
|
|
env["DROBOTICFLOW_OPERATOR_HANDLER"] = handler
|
|
}
|
|
if shouldInjectPlatformMongoEnv(node) {
|
|
if value := strings.TrimSpace(os.Getenv("MONGO_URI")); value != "" {
|
|
env["PLATFORM_MONGO_URI"] = value
|
|
}
|
|
if value := strings.TrimSpace(os.Getenv("MONGO_DB")); value != "" {
|
|
env["PLATFORM_MONGO_DB"] = value
|
|
}
|
|
}
|
|
if workflow != nil {
|
|
applyHyperparamsToEnv(env, workflow.Hyperparams)
|
|
}
|
|
applyHyperparamsToEnv(env, node.Data.Hyperparams)
|
|
applyOperatorCommandPolicyEnv(env)
|
|
return env, nil
|
|
}
|
|
|
|
func runtimeContractHandler(contract map[string]any) string {
|
|
value, ok := contract["handler"]
|
|
if !ok {
|
|
return ""
|
|
}
|
|
handler := strings.TrimSpace(fmt.Sprint(value))
|
|
return handler
|
|
}
|
|
|
|
func runtimeContractLongRunning(contract map[string]any) bool {
|
|
value, ok := contract["longRunning"].(bool)
|
|
return ok && value
|
|
}
|
|
|
|
func lightweightOperatorMount(contract map[string]any) (string, string) {
|
|
config := lightweight.ConfigFromContract(contract)
|
|
if !config.Enabled {
|
|
return "", ""
|
|
}
|
|
return config.CodeSubPath, config.MountPath
|
|
}
|
|
|
|
func runtimeContractString(items map[string]any, key string) string {
|
|
value, ok := items[key]
|
|
if !ok || value == nil {
|
|
return ""
|
|
}
|
|
return fmt.Sprint(value)
|
|
}
|
|
|
|
func operatorResourceDefaults(ctx context.Context, dataStore interface {
|
|
GetOperator(context.Context, string) (*domain.OperatorRecord, error)
|
|
}, node domain.WorkflowNode) *domain.OperatorResources {
|
|
operatorID := strings.TrimSpace(node.Data.OperatorID)
|
|
if operatorID == "" || dataStore == nil {
|
|
return nil
|
|
}
|
|
operator, err := dataStore.GetOperator(ctx, operatorID)
|
|
if err != nil || operator == nil {
|
|
return nil
|
|
}
|
|
return operator.Resources
|
|
}
|
|
|
|
func resolveEffectiveNodeResources(node domain.WorkflowNode, settings *domain.SettingsRecord, operatorResources *domain.OperatorResources) effectiveNodeResources {
|
|
defaults := domain.DefaultResources{}
|
|
if settings != nil {
|
|
defaults = settings.Defaults
|
|
}
|
|
resources := effectiveNodeResources{
|
|
CPURequest: defaults.CPURequest,
|
|
CPULimit: defaults.CPULimit,
|
|
MemoryRequest: defaults.MemoryRequest,
|
|
MemoryLimit: defaults.MemoryLimit,
|
|
GPULimit: defaults.GPULimit,
|
|
}
|
|
if operatorResources != nil {
|
|
resources.CPURequest = firstNonEmpty(operatorResources.CPURequest, resources.CPURequest)
|
|
resources.CPULimit = firstNonEmpty(operatorResources.CPULimit, resources.CPULimit)
|
|
resources.MemoryRequest = firstNonEmpty(operatorResources.MemoryRequest, resources.MemoryRequest)
|
|
resources.MemoryLimit = firstNonEmpty(operatorResources.MemoryLimit, resources.MemoryLimit)
|
|
resources.GPULimit = firstNonEmpty(operatorResources.GPULimit, resources.GPULimit)
|
|
resources.SharedMemorySize = firstNonEmpty(operatorResources.SharedMemorySize, resources.SharedMemorySize)
|
|
}
|
|
resources.CPULimit = firstNonEmpty(node.Data.CPU, resources.CPULimit)
|
|
resources.MemoryLimit = firstNonEmpty(node.Data.Memory, resources.MemoryLimit)
|
|
resources.GPULimit = firstNonEmpty(node.Data.GPU, resources.GPULimit)
|
|
resources.SharedMemorySize = firstNonEmpty(node.Data.SharedMemorySize, resources.SharedMemorySize)
|
|
return resources
|
|
}
|
|
|
|
func parseExecutionOutput(stdout string, nodeLabel string) (executionResult, error) {
|
|
result := executionResult{
|
|
Logs: []domain.LogEntryRecord{},
|
|
Artifacts: []domain.ExecutionArtifact{},
|
|
ResourceSeries: []domain.ResourceSample{},
|
|
}
|
|
lines := strings.Split(strings.ReplaceAll(stdout, "\r\n", "\n"), "\n")
|
|
structuredPayload := ""
|
|
structuredPayloadPath := ""
|
|
rawLogs := make([]domain.LogEntryRecord, 0)
|
|
for _, line := range lines {
|
|
trimmed := strings.TrimSpace(line)
|
|
if trimmed == "" {
|
|
continue
|
|
}
|
|
if strings.HasPrefix(trimmed, resultPrefix) {
|
|
structuredPayload = strings.TrimPrefix(trimmed, resultPrefix)
|
|
continue
|
|
}
|
|
if strings.HasPrefix(trimmed, resultPathPrefix) {
|
|
structuredPayloadPath = strings.TrimSpace(strings.TrimPrefix(trimmed, resultPathPrefix))
|
|
continue
|
|
}
|
|
rawLogs = append(rawLogs, logEntry("INFO", trimmed, nodeLabel))
|
|
}
|
|
if structuredPayloadPath != "" {
|
|
payloadBytes, err := os.ReadFile(structuredPayloadPath)
|
|
if err != nil {
|
|
result.Logs = rawLogs
|
|
return result, fmt.Errorf("read operator result path: %w", err)
|
|
}
|
|
structuredPayload = strings.TrimSpace(string(payloadBytes))
|
|
}
|
|
if structuredPayload == "" {
|
|
result.Logs = rawLogs
|
|
return result, fmt.Errorf("operator execution did not emit %s or %s payload", resultPrefix, resultPathPrefix)
|
|
}
|
|
var parsed executionResult
|
|
if err := json.Unmarshal([]byte(structuredPayload), &parsed); err != nil {
|
|
return result, fmt.Errorf("decode operator result: %w", err)
|
|
}
|
|
if len(parsed.Logs) > 0 {
|
|
result.Logs = append(result.Logs, parsed.Logs...)
|
|
} else {
|
|
result.Logs = append(result.Logs, rawLogs...)
|
|
}
|
|
result.Artifacts = parsed.Artifacts
|
|
result.ResourceSeries = parsed.ResourceSeries
|
|
return result, nil
|
|
}
|
|
|
|
func loadStageManifestDataRefs(path string) ([]domain.DataRefRecord, error) {
|
|
trimmed := strings.TrimSpace(path)
|
|
if trimmed == "" {
|
|
return nil, nil
|
|
}
|
|
payload, err := os.ReadFile(trimmed)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
var manifest stageManifestRecord
|
|
if err := json.Unmarshal(payload, &manifest); err != nil {
|
|
return nil, err
|
|
}
|
|
if manifest.DataRefs == nil {
|
|
return []domain.DataRefRecord{}, nil
|
|
}
|
|
return manifest.DataRefs, nil
|
|
}
|
|
|
|
func loadStageManifestDataMetrics(path string) (*domain.DataMetricsRecord, error) {
|
|
trimmed := strings.TrimSpace(path)
|
|
if trimmed == "" {
|
|
return nil, nil
|
|
}
|
|
payload, err := os.ReadFile(trimmed)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
var manifest stageManifestRecord
|
|
if err := json.Unmarshal(payload, &manifest); err != nil {
|
|
return nil, err
|
|
}
|
|
return manifest.DataMetrics, nil
|
|
}
|
|
|
|
func (runtime *Runtime) resolveNodeBusinessStage(ctx context.Context, run *domain.RunRecord, node domain.WorkflowNode) domain.WorkflowNode {
|
|
if run != nil && strings.TrimSpace(run.TaskID) != "" {
|
|
if task, err := runtime.store.GetCollectionTask(ctx, run.TaskID); err == nil && task != nil {
|
|
for stage, nodeID := range task.StageBindings {
|
|
if strings.TrimSpace(nodeID) == node.ID {
|
|
node.Data.BusinessStage = stage
|
|
node.Data.MetricBoundary = true
|
|
return node
|
|
}
|
|
}
|
|
}
|
|
}
|
|
if node.Data.BusinessStage != "" {
|
|
node.Data.MetricBoundary = true
|
|
return node
|
|
}
|
|
if strings.TrimSpace(node.Data.OperatorID) != "" {
|
|
if operator, err := runtime.store.GetOperator(ctx, node.Data.OperatorID); err == nil && operator != nil && operator.BusinessStage != "" {
|
|
node.Data.BusinessStage = operator.BusinessStage
|
|
node.Data.MetricBoundary = true
|
|
}
|
|
}
|
|
return node
|
|
}
|
|
|
|
func (runtime *Runtime) executeNode(ctx context.Context, run *domain.RunRecord, node domain.WorkflowNode, inputArtifacts []domain.ExecutionArtifact, inputBindings []domain.NodeInputBinding) (executionResult, *k8s.JobReference, error) {
|
|
if err := validateRequiredNodeInputs(node, inputBindings); err != nil {
|
|
return executionResult{}, nil, err
|
|
}
|
|
node = runtime.resolveNodeBusinessStage(ctx, run, node)
|
|
workspace, err := prepareNodeWorkspace(runtime.workspaceRoot, run.ID, node.ID)
|
|
if err != nil {
|
|
return executionResult{}, nil, err
|
|
}
|
|
workflow, workflowErr := runtime.store.GetWorkflow(ctx, run.WorkflowID)
|
|
if workflowErr != nil {
|
|
workflow = nil
|
|
}
|
|
settings, err := runtime.store.GetSettings(ctx)
|
|
if err != nil {
|
|
return executionResult{}, nil, err
|
|
}
|
|
resources := resolveEffectiveNodeResources(node, settings, operatorResourceDefaults(ctx, runtime.store, node))
|
|
lightweightConfig := lightweight.ConfigFromContract(node.Data.RuntimeContract)
|
|
if lightweightConfig.Enabled {
|
|
if err := lightweight.ValidateRuntimeConfig(lightweight.RuntimeConfig{
|
|
NodeID: node.ID,
|
|
NodeLabel: node.Data.Label,
|
|
Image: node.Data.Image,
|
|
CodeSubPath: lightweightConfig.CodeSubPath,
|
|
MountPath: lightweightConfig.MountPath,
|
|
Command: node.Data.ContainerCommand,
|
|
Args: node.Data.ContainerArgs,
|
|
ArgsConfigured: lightweightConfig.ContainerArgsConfigured,
|
|
WorkspaceRoot: runtime.workspaceRoot,
|
|
Mounts: lightweightConfig.Mounts,
|
|
}); err != nil {
|
|
return executionResult{}, nil, err
|
|
}
|
|
}
|
|
env, err := buildNodeEnvWithResources(run, workflow, node, inputArtifacts, inputBindings, workspace, resources)
|
|
if err != nil {
|
|
return executionResult{}, nil, err
|
|
}
|
|
codeSubPath, codeMountPath := lightweightConfig.CodeSubPath, lightweightConfig.MountPath
|
|
if codeSubPath != "" {
|
|
env["LIGHTWEIGHT_OPERATOR_CODE_SUBPATH"] = codeSubPath
|
|
env["LIGHTWEIGHT_OPERATOR_CODE_DIR"] = codeMountPath
|
|
}
|
|
pvcMounts := make([]k8s.PVCMount, 0, len(lightweightConfig.Mounts))
|
|
for _, item := range lightweightConfig.Mounts {
|
|
pvcMounts = append(pvcMounts, k8s.PVCMount{Name: item.Name, PVCName: item.PVCName, SubPath: item.SubPath, MountPath: item.MountPath, ReadOnly: item.ReadOnly})
|
|
}
|
|
nodeTimeoutSeconds := 0
|
|
if workflow != nil {
|
|
nodeTimeoutSeconds = workflow.NodeTimeoutSeconds
|
|
}
|
|
organization, orgErr := runtime.store.GetOrganization(ctx, run.OrganizationID)
|
|
if orgErr != nil {
|
|
return executionResult{}, nil, orgErr
|
|
}
|
|
scopedWorkspace := organization != nil && organization.StorageRootPrefix != ""
|
|
launched, err := runtime.launcher.Launch(ctx, k8s.LaunchInput{
|
|
Image: chooseImage(node, runtime.operatorRunnerImage),
|
|
ScopedWorkspace: scopedWorkspace,
|
|
Command: cloneStringSlice(node.Data.ContainerCommand),
|
|
Args: cloneStringSlice(node.Data.ContainerArgs),
|
|
RunID: run.ID,
|
|
WorkflowID: run.WorkflowID,
|
|
NodeID: node.ID,
|
|
NodeLabel: node.Data.Label,
|
|
Attempt: run.Attempt,
|
|
TimeoutSeconds: nodeTimeoutSeconds,
|
|
PendingFailureTimeout: runtime.pendingJobFailureThreshold(),
|
|
Env: env,
|
|
CPURequest: resources.CPURequest,
|
|
CPULimit: resources.CPULimit,
|
|
MemoryRequest: resources.MemoryRequest,
|
|
MemoryLimit: resources.MemoryLimit,
|
|
GPULimit: resources.GPULimit,
|
|
SharedMemorySize: resources.SharedMemorySize,
|
|
LongRunning: runtimeContractLongRunning(node.Data.RuntimeContract),
|
|
CodeSubPath: codeSubPath,
|
|
CodeMountPath: codeMountPath,
|
|
CodeReadOnly: lightweightConfig.CodeReadOnly,
|
|
PVCMounts: pvcMounts,
|
|
OperatorRunnerImage: runtime.operatorRunnerImage,
|
|
})
|
|
if err != nil {
|
|
return executionResult{}, nil, err
|
|
}
|
|
if progress, progressErr := latestStructuredProgress(launched.Output); progressErr == nil && progress != nil {
|
|
_ = runtime.store.UpdateRunProgress(ctx, run.ID, progress)
|
|
}
|
|
parsed, err := parseExecutionOutput(launched.Output, node.Data.Label)
|
|
if err == nil {
|
|
parsed.Artifacts, err = runtime.normalizeOutputArtifacts(ctx, workflow, run, node.ID, parsed.Artifacts)
|
|
}
|
|
if err == nil {
|
|
parsed.StageManifestPath = workspace.StageManifestPath
|
|
parsed.DataRefs, err = loadStageManifestDataRefs(workspace.StageManifestPath)
|
|
if err == nil && parsed.DataMetrics == nil {
|
|
parsed.DataMetrics, err = loadStageManifestDataMetrics(workspace.StageManifestPath)
|
|
}
|
|
}
|
|
// Custom-container operators bypass operator-runner's ResourceSampler, so they
|
|
// emit no resourceSeries in result.json. The launcher samples their GPU usage
|
|
// itself (see k8s.Launch); fall back to those platform-collected samples when
|
|
// the operator did not self-report. Builtin operator-runner pods self-report a
|
|
// non-empty series, so their own (more complete, CPU+memory included) samples
|
|
// are preserved.
|
|
if len(parsed.ResourceSeries) == 0 && len(launched.ResourceSeries) > 0 {
|
|
parsed.ResourceSeries = make([]domain.ResourceSample, 0, len(launched.ResourceSeries))
|
|
for _, sample := range launched.ResourceSeries {
|
|
parsed.ResourceSeries = append(parsed.ResourceSeries, domain.ResourceSample{
|
|
Timestamp: sample.Timestamp,
|
|
GPU: sample.GPU,
|
|
GPUMemoryUsedMiB: sample.GPUMemoryUsedMiB,
|
|
GPUMemoryTotalMiB: sample.GPUMemoryTotalMiB,
|
|
GPUName: sample.GPUName,
|
|
GPUCount: sample.GPUCount,
|
|
})
|
|
}
|
|
}
|
|
return parsed, launched.Reference, err
|
|
}
|