forked from longtao.wu/droboticflow
1224 lines
41 KiB
Go
1224 lines
41 KiB
Go
package worker
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"maps"
|
|
"os"
|
|
"path/filepath"
|
|
"regexp"
|
|
"strings"
|
|
"time"
|
|
|
|
"droboticflow/internal/domain"
|
|
"droboticflow/internal/k8s"
|
|
"droboticflow/internal/lightweight"
|
|
)
|
|
|
|
const resultPrefix = "DROBOTICFLOW_RESULT="
|
|
const resultPathPrefix = "DROBOTICFLOW_RESULT_PATH="
|
|
const inputArtifactsInlineLimit = 64 * 1024
|
|
|
|
var inferenceServiceImagePattern = regexp.MustCompile(`^\S+@sha256:[a-f0-9]{64}$`)
|
|
|
|
type nodeWorkspace struct {
|
|
RunRoot string
|
|
Root string
|
|
OutputDir string
|
|
InputArtifactsPath string
|
|
InputBindingsPath string
|
|
ResultPath string
|
|
StageManifestPath string
|
|
}
|
|
|
|
func setNodeHyperparamValue(node *domain.WorkflowNode, key, value string) {
|
|
if node == nil {
|
|
return
|
|
}
|
|
for index := range node.Data.Hyperparams {
|
|
if strings.TrimSpace(node.Data.Hyperparams[index].Key) == key {
|
|
node.Data.Hyperparams[index].Value = value
|
|
return
|
|
}
|
|
}
|
|
node.Data.Hyperparams = append(node.Data.Hyperparams, domain.HyperParam{Key: key, Value: value, Type: "string"})
|
|
}
|
|
|
|
func inferenceSidecarForNode(run *domain.RunRecord, node *domain.WorkflowNode) (sidecar *k8s.InferenceSidecar, err error) {
|
|
defer func() {
|
|
if sidecar != nil && node != nil {
|
|
limits := applyFanyuResourceProfile(*node, effectiveNodeResources{CPURequest: sidecar.CPURequest, CPULimit: sidecar.CPULimit, MemoryRequest: sidecar.MemoryRequest, MemoryLimit: sidecar.MemoryLimit, GPULimit: sidecar.GPULimit})
|
|
sidecar.CPULimit, sidecar.MemoryLimit = limits.CPULimit, limits.MemoryLimit
|
|
}
|
|
}()
|
|
if node == nil {
|
|
return nil, nil
|
|
}
|
|
params := nodeHyperparamsMap(*node)
|
|
switch strings.TrimSpace(node.Data.OperatorID) {
|
|
case "operator-builtin-wilor-hand-pose":
|
|
image := strings.TrimSpace(params["WILOR_SERVICE_IMAGE_ID"])
|
|
if !inferenceServiceImagePattern.MatchString(image) {
|
|
return nil, fmt.Errorf("WiLoR on-demand service requires digest-pinned WILOR_SERVICE_IMAGE_ID")
|
|
}
|
|
setNodeHyperparamValue(node, "WILOR_API_BASE_URL", "http://127.0.0.1:8080")
|
|
return &k8s.InferenceSidecar{Name: "wilor-inference", Image: image, Port: 8080, HealthPath: "/api/health", CPURequest: "2", CPULimit: "4", MemoryRequest: "16Gi", MemoryLimit: "24Gi", GPULimit: "1"}, nil
|
|
case "operator-builtin-head-pose":
|
|
image := strings.TrimSpace(params["HEADPOSE_SERVICE_IMAGE_ID"])
|
|
if !inferenceServiceImagePattern.MatchString(image) {
|
|
return nil, fmt.Errorf("HeadPose on-demand service requires digest-pinned HEADPOSE_SERVICE_IMAGE_ID")
|
|
}
|
|
setNodeHyperparamValue(node, "HEADPOSE_API_BASE_URL", "http://127.0.0.1:8080")
|
|
outputRoot := "/workspace/system/headpose-service"
|
|
if run != nil {
|
|
outputRoot = filepath.Join("/workspace/system/runs", run.ID, "nodes", node.ID, "headpose-service")
|
|
}
|
|
return &k8s.InferenceSidecar{
|
|
Name: "headpose-inference", Image: image, Port: 8080, HealthPath: "/health",
|
|
CPURequest: "1", CPULimit: "2", MemoryRequest: "2Gi", MemoryLimit: "4Gi",
|
|
Env: map[string]string{"HEADPOSE_MODEL_SHA256": params["HEADPOSE_MODEL_SHA256"], "HEADPOSE_LICENSE_ID": params["HEADPOSE_LICENSE_ID"], "HEADPOSE_OUTPUT_ROOT": outputRoot, "HEADPOSE_GPU_ID": "-1"},
|
|
}, nil
|
|
default:
|
|
return nil, nil
|
|
}
|
|
}
|
|
|
|
type executionResult struct {
|
|
Logs []domain.LogEntryRecord `json:"logs"`
|
|
Artifacts []domain.ExecutionArtifact `json:"artifacts"`
|
|
DataRefs []domain.DataRefRecord `json:"dataRefs,omitempty"`
|
|
DataMetrics *domain.DataMetricsRecord `json:"dataMetrics,omitempty"`
|
|
StageManifestPath string `json:"stageManifestPath,omitempty"`
|
|
ResourceSeries []domain.ResourceSample `json:"resourceSeries"`
|
|
}
|
|
|
|
type stageManifestRecord struct {
|
|
DataRefs []domain.DataRefRecord `json:"dataRefs"`
|
|
DataMetrics *domain.DataMetricsRecord `json:"dataMetrics,omitempty"`
|
|
}
|
|
|
|
var protectedSystemEnv = map[string]struct{}{
|
|
"WORKFLOW_PREVIOUS_RUN_TIMESTAMP": {},
|
|
"WORKFLOW_CURRENT_RUN_TIMESTAMP": {},
|
|
"PARENT_RUN_ID": {},
|
|
"PARENT_RUN_WORKSPACE_ROOT": {},
|
|
"PARENT_NODE_OUTPUT_DIR": {},
|
|
"OPERATOR_ALLOW_SHELL_COMMANDS": {},
|
|
"OPERATOR_COMMAND_ALLOWLIST_REGEX": {},
|
|
"OPERATOR_COMMAND_DENYLIST_REGEX": {},
|
|
"DROBOTICFLOW_QWEN_NODE_CACHE_ENABLED": {},
|
|
"DROBOTICFLOW_QWEN_NODE_CACHE_ROOT": {},
|
|
"DROBOTICFLOW_QWEN_NODE_CACHE_STALE_STAGING_SECONDS": {},
|
|
}
|
|
|
|
var operatorCommandPolicyEnvKeys = []string{
|
|
"OPERATOR_ALLOW_SHELL_COMMANDS",
|
|
"OPERATOR_COMMAND_ALLOWLIST_REGEX",
|
|
"OPERATOR_COMMAND_DENYLIST_REGEX",
|
|
}
|
|
|
|
var workflowEnvLocation = loadWorkflowEnvLocation()
|
|
|
|
var invalidEnvCharacters = regexp.MustCompile(`[^A-Z0-9_]+`)
|
|
|
|
func loadWorkflowEnvLocation() *time.Location {
|
|
location, err := time.LoadLocation("Asia/Shanghai")
|
|
if err != nil {
|
|
return time.FixedZone("Asia/Shanghai", 8*60*60)
|
|
}
|
|
return location
|
|
}
|
|
|
|
func sanitizeEnvKey(value string) string {
|
|
normalized := strings.ToUpper(strings.TrimSpace(value))
|
|
if normalized == "" {
|
|
return ""
|
|
}
|
|
normalized = invalidEnvCharacters.ReplaceAllString(normalized, "_")
|
|
normalized = strings.Trim(normalized, "_")
|
|
if normalized == "" {
|
|
return ""
|
|
}
|
|
if normalized[0] >= '0' && normalized[0] <= '9' {
|
|
normalized = "HP_" + normalized
|
|
}
|
|
return normalized
|
|
}
|
|
|
|
func cloneArtifacts(items []domain.ExecutionArtifact) []domain.ExecutionArtifact {
|
|
cloned := make([]domain.ExecutionArtifact, 0, len(items))
|
|
for _, item := range items {
|
|
cloned = append(cloned, cloneArtifact(item))
|
|
}
|
|
return cloned
|
|
}
|
|
|
|
func tagDebugArtifact(run *domain.RunRecord, artifact domain.ExecutionArtifact) domain.ExecutionArtifact {
|
|
if run == nil || run.Mode != domain.RunModeDebug {
|
|
return artifact
|
|
}
|
|
artifact.Mode = domain.RunModeDebug
|
|
artifact.Temporary = true
|
|
if artifact.Metadata == nil {
|
|
artifact.Metadata = &domain.ArtifactMetadata{}
|
|
}
|
|
artifact.Metadata.Temporary = true
|
|
return artifact
|
|
}
|
|
|
|
func tagDebugArtifacts(run *domain.RunRecord, artifacts []domain.ExecutionArtifact) []domain.ExecutionArtifact {
|
|
if run == nil || run.Mode != domain.RunModeDebug {
|
|
return artifacts
|
|
}
|
|
for i := range artifacts {
|
|
artifacts[i] = tagDebugArtifact(run, artifacts[i])
|
|
}
|
|
return artifacts
|
|
}
|
|
|
|
func routeAuditArtifacts(
|
|
node domain.WorkflowNode,
|
|
inputArtifacts []domain.ExecutionArtifact,
|
|
edges []domain.WorkflowEdge,
|
|
) []domain.ExecutionArtifact {
|
|
handles := make([]string, 0, len(edges))
|
|
seenHandles := map[string]struct{}{}
|
|
for _, edge := range edges {
|
|
handle := strings.TrimSpace(edge.SourceHandle)
|
|
if handle == "" {
|
|
continue
|
|
}
|
|
if _, exists := seenHandles[handle]; exists {
|
|
continue
|
|
}
|
|
seenHandles[handle] = struct{}{}
|
|
handles = append(handles, handle)
|
|
}
|
|
if len(handles) == 0 {
|
|
return cloneArtifacts(inputArtifacts)
|
|
}
|
|
|
|
portNames := make(map[string]string, len(node.Data.Outputs))
|
|
for _, output := range node.Data.Outputs {
|
|
portNames[strings.TrimSpace(output.ID)] = strings.TrimSpace(output.Name)
|
|
}
|
|
routed := make([]domain.ExecutionArtifact, 0, len(inputArtifacts)*len(handles))
|
|
for _, handle := range handles {
|
|
portName := portNames[handle]
|
|
if portName == "" {
|
|
portName = handle
|
|
}
|
|
for _, input := range inputArtifacts {
|
|
artifact := cloneArtifact(input)
|
|
artifact.PortID = handle
|
|
artifact.PortName = portName
|
|
routed = append(routed, artifact)
|
|
}
|
|
}
|
|
return routed
|
|
}
|
|
|
|
func cloneStringSlice(items []string) []string {
|
|
if items == nil {
|
|
return nil
|
|
}
|
|
cloned := make([]string, len(items))
|
|
copy(cloned, items)
|
|
return cloned
|
|
}
|
|
|
|
func cloneArtifact(item domain.ExecutionArtifact) domain.ExecutionArtifact {
|
|
cloned := item
|
|
if item.Metadata != nil {
|
|
metadata := *item.Metadata
|
|
cloned.Metadata = &metadata
|
|
}
|
|
if item.Data != nil {
|
|
cloned.Data = maps.Clone(item.Data)
|
|
}
|
|
return cloned
|
|
}
|
|
|
|
func cloneInputLineage(items []domain.InputLineageRecord) []domain.InputLineageRecord {
|
|
cloned := make([]domain.InputLineageRecord, 0, len(items))
|
|
for _, item := range items {
|
|
record := item
|
|
record.Artifact = cloneArtifact(item.Artifact)
|
|
cloned = append(cloned, record)
|
|
}
|
|
return cloned
|
|
}
|
|
|
|
func cloneInputBindings(items []domain.NodeInputBinding) []domain.NodeInputBinding {
|
|
if items == nil {
|
|
return nil
|
|
}
|
|
cloned := make([]domain.NodeInputBinding, len(items))
|
|
for index, item := range items {
|
|
cloned[index] = item
|
|
cloned[index].Artifacts = cloneArtifacts(item.Artifacts)
|
|
if item.Payload != nil {
|
|
payload := *item.Payload
|
|
if item.Payload.Ref != nil {
|
|
ref := *item.Payload.Ref
|
|
payload.Ref = &ref
|
|
}
|
|
cloned[index].Payload = &payload
|
|
}
|
|
}
|
|
return cloned
|
|
}
|
|
|
|
func runTimestampEnvValue(value string) string {
|
|
trimmed := strings.TrimSpace(value)
|
|
if trimmed == "" {
|
|
return ""
|
|
}
|
|
parsed, err := time.Parse(time.RFC3339Nano, trimmed)
|
|
if err != nil {
|
|
return ""
|
|
}
|
|
return parsed.In(workflowEnvLocation).Format(time.RFC3339)
|
|
}
|
|
|
|
func firstNonEmpty(values ...string) string {
|
|
for _, value := range values {
|
|
if strings.TrimSpace(value) != "" {
|
|
return value
|
|
}
|
|
}
|
|
return ""
|
|
}
|
|
|
|
func inputArtifactLineageKey(item domain.InputLineageRecord) string {
|
|
artifact := item.Artifact
|
|
key := artifact.ID
|
|
if key == "" {
|
|
key = artifact.NodeID + ":" + artifact.Name
|
|
}
|
|
return fmt.Sprintf("%s:%s:%s:%s:%s", item.WorkflowEdgeID, item.SourceNodeID, item.SourceHandle, item.TargetHandle, key)
|
|
}
|
|
|
|
func gatherInputLineage(workflow domain.WorkflowRecord, executionMap map[string]domain.NodeExecutionRecord, nodeID string) []domain.InputLineageRecord {
|
|
items := make([]domain.InputLineageRecord, 0)
|
|
seen := map[string]struct{}{}
|
|
for _, edge := range domain.GetIncomingEdges(workflow.Edges, nodeID) {
|
|
execution, ok := executionMap[edge.Source]
|
|
if !ok {
|
|
continue
|
|
}
|
|
requiresHandle := strings.TrimSpace(edge.SourceHandle) != ""
|
|
hasPortAwareArtifacts := false
|
|
for _, artifact := range execution.Artifacts {
|
|
if strings.TrimSpace(artifact.PortID) != "" {
|
|
hasPortAwareArtifacts = true
|
|
break
|
|
}
|
|
}
|
|
for _, artifact := range execution.Artifacts {
|
|
if requiresHandle && hasPortAwareArtifacts && artifact.PortID != edge.SourceHandle {
|
|
continue
|
|
}
|
|
item := domain.InputLineageRecord{
|
|
WorkflowEdgeID: edge.ID,
|
|
SourceNodeID: edge.Source,
|
|
SourceHandle: edge.SourceHandle,
|
|
TargetNodeID: edge.Target,
|
|
TargetHandle: edge.TargetHandle,
|
|
Artifact: artifact,
|
|
}
|
|
key := inputArtifactLineageKey(item)
|
|
if _, exists := seen[key]; exists {
|
|
continue
|
|
}
|
|
seen[key] = struct{}{}
|
|
items = append(items, item)
|
|
}
|
|
}
|
|
return items
|
|
}
|
|
|
|
func inputArtifactsFromLineage(lineage []domain.InputLineageRecord) []domain.ExecutionArtifact {
|
|
items := make([]domain.ExecutionArtifact, 0)
|
|
seen := map[string]struct{}{}
|
|
for _, item := range lineage {
|
|
artifact := item.Artifact
|
|
key := artifact.ID
|
|
if key == "" {
|
|
key = artifact.NodeID + ":" + artifact.Name
|
|
}
|
|
if _, exists := seen[key]; exists {
|
|
continue
|
|
}
|
|
seen[key] = struct{}{}
|
|
items = append(items, artifact)
|
|
}
|
|
return items
|
|
}
|
|
|
|
func artifactFromWorkflowInput(run *domain.RunRecord, binding domain.ResolvedInputBinding) domain.ExecutionArtifact {
|
|
payload := binding.Payload
|
|
artifact := domain.ExecutionArtifact{
|
|
ID: run.ID + "-input-" + sanitizeEnvKey(binding.WorkflowInputID) + "-" + sanitizeEnvKey(binding.TargetPortID),
|
|
RunID: run.ID,
|
|
NodeID: binding.NodeID,
|
|
PortID: binding.TargetPortID,
|
|
PortName: binding.TargetPortID,
|
|
Name: binding.WorkflowInputID,
|
|
Type: payload.Type,
|
|
Payload: &payload,
|
|
}
|
|
if payload.Mode == domain.ArtifactTransferModeInline {
|
|
encoded, _ := json.Marshal(payload.Inline)
|
|
artifact.SizeBytes = int64(len(encoded))
|
|
artifact.Size = fmt.Sprintf("%d B", len(encoded))
|
|
if data, ok := payload.Inline.(map[string]any); ok {
|
|
artifact.Data = data
|
|
} else {
|
|
artifact.Data = map[string]any{"value": payload.Inline}
|
|
}
|
|
return tagDebugArtifact(run, artifact)
|
|
}
|
|
if payload.Ref != nil {
|
|
ref := payload.Ref
|
|
artifact.SizeBytes = ref.SizeBytes
|
|
artifact.Size = fmt.Sprintf("%d B", ref.SizeBytes)
|
|
if ref.Kind == domain.ArtifactRefKindDirectory {
|
|
artifact.Path = "bos://" + ref.Bucket + "/" + strings.TrimPrefix(ref.Prefix, "/")
|
|
} else {
|
|
artifact.Path = "bos://" + ref.Bucket + "/" + strings.TrimPrefix(ref.Key, "/")
|
|
}
|
|
artifact.Metadata = &domain.ArtifactMetadata{SHA256: ref.SHA256, MimeType: ref.MimeType}
|
|
artifact.Data = map[string]any{"source": map[string]any{"path": artifact.Path}}
|
|
}
|
|
return tagDebugArtifact(run, artifact)
|
|
}
|
|
|
|
func workflowInputBindingsForNode(run *domain.RunRecord, nodeID string) ([]domain.NodeInputBinding, []domain.ExecutionArtifact) {
|
|
if run == nil {
|
|
return nil, nil
|
|
}
|
|
bindings := make([]domain.NodeInputBinding, 0)
|
|
artifacts := make([]domain.ExecutionArtifact, 0)
|
|
for _, item := range run.ResolvedInputBindings {
|
|
if item.NodeID != nodeID {
|
|
continue
|
|
}
|
|
artifact := artifactFromWorkflowInput(run, item)
|
|
payload := item.Payload
|
|
bindings = append(bindings, domain.NodeInputBinding{
|
|
TargetPortID: item.TargetPortID,
|
|
WorkflowInputID: item.WorkflowInputID,
|
|
Payload: &payload,
|
|
Artifacts: []domain.ExecutionArtifact{artifact},
|
|
})
|
|
artifacts = append(artifacts, artifact)
|
|
}
|
|
return bindings, artifacts
|
|
}
|
|
|
|
func lineageInputBindings(lineage []domain.InputLineageRecord) []domain.NodeInputBinding {
|
|
items := make([]domain.NodeInputBinding, 0)
|
|
indexByKey := map[string]int{}
|
|
for _, item := range lineage {
|
|
key := item.SourceNodeID + "\x00" + item.SourceHandle + "\x00" + item.TargetHandle
|
|
if index, ok := indexByKey[key]; ok {
|
|
items[index].Artifacts = append(items[index].Artifacts, item.Artifact)
|
|
continue
|
|
}
|
|
indexByKey[key] = len(items)
|
|
items = append(items, domain.NodeInputBinding{
|
|
TargetPortID: item.TargetHandle,
|
|
SourceNodeID: item.SourceNodeID,
|
|
SourcePortID: item.SourceHandle,
|
|
Artifacts: []domain.ExecutionArtifact{item.Artifact},
|
|
})
|
|
}
|
|
return items
|
|
}
|
|
|
|
func validateRequiredNodeInputs(node domain.WorkflowNode, bindings []domain.NodeInputBinding) error {
|
|
for _, port := range node.Data.Inputs {
|
|
if !port.Required {
|
|
continue
|
|
}
|
|
present := false
|
|
for _, binding := range bindings {
|
|
targetPortID := strings.TrimSpace(binding.TargetPortID)
|
|
if targetPortID != strings.TrimSpace(port.ID) && !(targetPortID == "" && len(node.Data.Inputs) == 1) {
|
|
continue
|
|
}
|
|
if binding.Payload != nil || len(binding.Artifacts) > 0 {
|
|
present = true
|
|
break
|
|
}
|
|
}
|
|
if !present {
|
|
return fmt.Errorf("required_input_missing: node %s input %s has no value or upstream artifact", node.ID, port.ID)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func artifactFromNodeOverrideInput(run *domain.RunRecord, nodeID string, portKey string, payload domain.ArtifactPayload) (domain.NodeInputBinding, domain.ExecutionArtifact) {
|
|
runID := ""
|
|
if run != nil {
|
|
runID = run.ID
|
|
}
|
|
artifact := domain.ExecutionArtifact{
|
|
ID: runID + "-override-" + sanitizeEnvKey(nodeID) + "-" + sanitizeEnvKey(portKey),
|
|
RunID: runID,
|
|
NodeID: nodeID,
|
|
PortID: portKey,
|
|
PortName: portKey,
|
|
Name: portKey,
|
|
Type: payload.Type,
|
|
Payload: &payload,
|
|
}
|
|
if payload.Mode == domain.ArtifactTransferModeInline {
|
|
encoded, _ := json.Marshal(payload.Inline)
|
|
artifact.SizeBytes = int64(len(encoded))
|
|
artifact.Size = fmt.Sprintf("%d B", len(encoded))
|
|
if data, ok := payload.Inline.(map[string]any); ok {
|
|
artifact.Data = data
|
|
} else {
|
|
artifact.Data = map[string]any{"value": payload.Inline}
|
|
}
|
|
} else if payload.Ref != nil {
|
|
ref := payload.Ref
|
|
artifact.SizeBytes = ref.SizeBytes
|
|
artifact.Size = fmt.Sprintf("%d B", ref.SizeBytes)
|
|
if ref.Kind == domain.ArtifactRefKindDirectory {
|
|
artifact.Path = "bos://" + ref.Bucket + "/" + strings.TrimPrefix(ref.Prefix, "/")
|
|
} else {
|
|
artifact.Path = "bos://" + ref.Bucket + "/" + strings.TrimPrefix(ref.Key, "/")
|
|
}
|
|
artifact.Metadata = &domain.ArtifactMetadata{SHA256: ref.SHA256, MimeType: ref.MimeType}
|
|
artifact.Data = map[string]any{"source": map[string]any{"path": artifact.Path}}
|
|
}
|
|
binding := domain.NodeInputBinding{
|
|
TargetPortID: portKey,
|
|
Payload: &payload,
|
|
Artifacts: []domain.ExecutionArtifact{artifact},
|
|
}
|
|
return binding, tagDebugArtifact(run, artifact)
|
|
}
|
|
|
|
func applyNodeOverridesToNode(node domain.WorkflowNode, override domain.NodeOverride) domain.WorkflowNode {
|
|
cloned := node
|
|
|
|
// 1. Hyperparams: merge / override by key
|
|
if len(override.Hyperparams) > 0 {
|
|
hpMap := make(map[string]domain.HyperParam, len(node.Data.Hyperparams)+len(override.Hyperparams))
|
|
for _, hp := range node.Data.Hyperparams {
|
|
hpMap[hp.Key] = hp
|
|
}
|
|
for _, hp := range override.Hyperparams {
|
|
hpMap[hp.Key] = hp
|
|
}
|
|
newHyperparams := make([]domain.HyperParam, 0, len(hpMap))
|
|
seen := make(map[string]bool, len(hpMap))
|
|
for _, hp := range node.Data.Hyperparams {
|
|
if updated, ok := hpMap[hp.Key]; ok {
|
|
newHyperparams = append(newHyperparams, updated)
|
|
seen[hp.Key] = true
|
|
}
|
|
}
|
|
for _, hp := range override.Hyperparams {
|
|
if !seen[hp.Key] {
|
|
newHyperparams = append(newHyperparams, hp)
|
|
seen[hp.Key] = true
|
|
}
|
|
}
|
|
cloned.Data.Hyperparams = newHyperparams
|
|
}
|
|
|
|
// 2. Operator image
|
|
if strings.TrimSpace(override.Image) != "" {
|
|
cloned.Data.Image = strings.TrimSpace(override.Image)
|
|
}
|
|
|
|
// 3. Container command and args
|
|
if len(override.ContainerCommand) > 0 {
|
|
cloned.Data.ContainerCommand = cloneStringSlice(override.ContainerCommand)
|
|
}
|
|
if len(override.ContainerArgs) > 0 {
|
|
cloned.Data.ContainerArgs = cloneStringSlice(override.ContainerArgs)
|
|
}
|
|
|
|
return cloned
|
|
}
|
|
|
|
func gatherNodeInputs(workflow domain.WorkflowRecord, run *domain.RunRecord, executionMap map[string]domain.NodeExecutionRecord, nodeID string) ([]domain.ExecutionArtifact, []domain.InputLineageRecord, []domain.NodeInputBinding) {
|
|
lineage := gatherInputLineage(workflow, executionMap, nodeID)
|
|
artifacts := inputArtifactsFromLineage(lineage)
|
|
bindings := lineageInputBindings(lineage)
|
|
rootBindings, rootArtifacts := workflowInputBindingsForNode(run, nodeID)
|
|
bindings = append(bindings, rootBindings...)
|
|
artifacts = append(artifacts, rootArtifacts...)
|
|
if run != nil && len(run.NodeOverrides) > 0 {
|
|
if override, ok := run.NodeOverrides[nodeID]; ok && len(override.Inputs) > 0 {
|
|
for portKey, payload := range override.Inputs {
|
|
overrideBinding, overrideArtifact := artifactFromNodeOverrideInput(run, nodeID, portKey, payload)
|
|
replaced := false
|
|
for i, b := range bindings {
|
|
if strings.TrimSpace(b.TargetPortID) == strings.TrimSpace(portKey) {
|
|
bindings[i] = overrideBinding
|
|
replaced = true
|
|
break
|
|
}
|
|
}
|
|
if !replaced {
|
|
bindings = append(bindings, overrideBinding)
|
|
}
|
|
artifactReplaced := false
|
|
for i, a := range artifacts {
|
|
if strings.TrimSpace(a.PortID) == strings.TrimSpace(portKey) {
|
|
artifacts[i] = overrideArtifact
|
|
artifactReplaced = true
|
|
break
|
|
}
|
|
}
|
|
if !artifactReplaced {
|
|
artifacts = append(artifacts, overrideArtifact)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
return artifacts, lineage, bindings
|
|
}
|
|
|
|
func gatherInputArtifacts(workflow domain.WorkflowRecord, executionMap map[string]domain.NodeExecutionRecord, nodeID string) []domain.ExecutionArtifact {
|
|
return inputArtifactsFromLineage(gatherInputLineage(workflow, executionMap, nodeID))
|
|
}
|
|
|
|
func extractConfidence(artifact domain.ExecutionArtifact) (float64, bool) {
|
|
if artifact.Data == nil {
|
|
return 0, false
|
|
}
|
|
for _, key := range []string{"confidence", "score"} {
|
|
value, ok := artifact.Data[key]
|
|
if !ok {
|
|
continue
|
|
}
|
|
switch typed := value.(type) {
|
|
case float64:
|
|
return typed, true
|
|
case int:
|
|
return float64(typed), true
|
|
}
|
|
}
|
|
return 0, false
|
|
}
|
|
|
|
func extractObjectHints(artifact domain.ExecutionArtifact) []string {
|
|
if artifact.Data == nil {
|
|
return nil
|
|
}
|
|
items := make([]string, 0)
|
|
for _, key := range []string{"detectedObjects", "labels", "entries"} {
|
|
value, ok := artifact.Data[key]
|
|
if !ok {
|
|
continue
|
|
}
|
|
switch typed := value.(type) {
|
|
case []any:
|
|
for _, candidate := range typed {
|
|
text := strings.TrimSpace(fmt.Sprint(candidate))
|
|
if text != "" {
|
|
items = append(items, text)
|
|
}
|
|
}
|
|
case []string:
|
|
for _, candidate := range typed {
|
|
if trimmed := strings.TrimSpace(candidate); trimmed != "" {
|
|
items = append(items, trimmed)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
return items
|
|
}
|
|
|
|
func createAuditPayload(node domain.WorkflowNode, inputArtifacts []domain.ExecutionArtifact) domain.AuditTaskPayload {
|
|
payload := domain.AuditTaskPayload{
|
|
Reason: fmt.Sprintf("%s requires explicit approval before downstream execution", node.Data.Label),
|
|
DetectedObjects: []string{},
|
|
}
|
|
for _, artifact := range inputArtifacts {
|
|
if payload.Image == "" {
|
|
switch {
|
|
case artifact.Metadata != nil && artifact.Metadata.Thumbnail != "":
|
|
payload.Image = artifact.Metadata.Thumbnail
|
|
case artifact.Type == domain.PortTypeImage || artifact.Type == domain.PortTypeVideo:
|
|
payload.Image = artifact.Path
|
|
}
|
|
}
|
|
if payload.Confidence == 0 {
|
|
if confidence, ok := extractConfidence(artifact); ok {
|
|
payload.Confidence = confidence
|
|
}
|
|
}
|
|
payload.DetectedObjects = append(payload.DetectedObjects, extractObjectHints(artifact)...)
|
|
}
|
|
if len(payload.DetectedObjects) == 0 {
|
|
payload.DetectedObjects = []string{fmt.Sprintf("%d upstream artifact(s)", len(inputArtifacts))}
|
|
}
|
|
return payload
|
|
}
|
|
|
|
func prepareNodeWorkspace(workspaceRoot string, runID string, nodeID string) (nodeWorkspace, error) {
|
|
root := strings.TrimSpace(workspaceRoot)
|
|
if root == "" {
|
|
root = filepath.Join(os.TempDir(), "droboticflow")
|
|
}
|
|
runRoot := filepath.Join(root, "system", "runs", runID)
|
|
nodeRoot := filepath.Join(runRoot, "nodes", nodeID)
|
|
outputDir := filepath.Join(nodeRoot, "outputs")
|
|
if err := os.MkdirAll(outputDir, 0o755); err != nil {
|
|
return nodeWorkspace{}, err
|
|
}
|
|
return nodeWorkspace{
|
|
RunRoot: runRoot,
|
|
Root: nodeRoot,
|
|
OutputDir: outputDir,
|
|
InputArtifactsPath: filepath.Join(nodeRoot, "input-artifacts.json"),
|
|
InputBindingsPath: filepath.Join(nodeRoot, "input-bindings.json"),
|
|
ResultPath: filepath.Join(nodeRoot, "result.json"),
|
|
StageManifestPath: filepath.Join(nodeRoot, "stage-manifest.json"),
|
|
}, nil
|
|
}
|
|
|
|
func shouldInjectPlatformMongoEnv(node domain.WorkflowNode) bool {
|
|
label := strings.ToLower(strings.TrimSpace(node.Data.Label))
|
|
return strings.Contains(label, "mongodb") || strings.Contains(label, "mongo")
|
|
}
|
|
|
|
func writeJSONFile(path string, value any) error {
|
|
data, err := json.Marshal(value)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
|
return err
|
|
}
|
|
return os.WriteFile(path, data, 0o644)
|
|
}
|
|
|
|
func applyHyperparamsToEnv(env map[string]string, items []domain.HyperParam) {
|
|
for _, item := range items {
|
|
key := sanitizeEnvKey(item.Key)
|
|
if key == "" {
|
|
continue
|
|
}
|
|
if _, blocked := protectedSystemEnv[key]; blocked {
|
|
continue
|
|
}
|
|
env[key] = item.Value
|
|
}
|
|
}
|
|
|
|
func applyOperatorCommandPolicyEnv(env map[string]string) {
|
|
for _, key := range operatorCommandPolicyEnvKeys {
|
|
if value := strings.TrimSpace(os.Getenv(key)); value != "" {
|
|
env[key] = value
|
|
}
|
|
}
|
|
}
|
|
|
|
func buildNodeEnv(run *domain.RunRecord, workflow *domain.WorkflowRecord, node domain.WorkflowNode, inputArtifacts []domain.ExecutionArtifact, workspace nodeWorkspace, settings *domain.SettingsRecord) (map[string]string, error) {
|
|
return buildNodeEnvWithResources(run, workflow, node, inputArtifacts, nil, workspace, effectiveNodeResources{
|
|
CPULimit: node.Data.CPU,
|
|
MemoryLimit: node.Data.Memory,
|
|
GPULimit: node.Data.GPU,
|
|
})
|
|
}
|
|
|
|
type effectiveNodeResources struct {
|
|
CPURequest string
|
|
CPULimit string
|
|
MemoryRequest string
|
|
MemoryLimit string
|
|
GPULimit string
|
|
SharedMemorySize string
|
|
}
|
|
|
|
func buildNodeEnvWithResources(run *domain.RunRecord, workflow *domain.WorkflowRecord, node domain.WorkflowNode, inputArtifacts []domain.ExecutionArtifact, inputBindings []domain.NodeInputBinding, workspace nodeWorkspace, resources effectiveNodeResources) (map[string]string, error) {
|
|
if inputArtifacts == nil {
|
|
inputArtifacts = []domain.ExecutionArtifact{}
|
|
}
|
|
safeNodeData := node.Data
|
|
safeNodeData.Hyperparams = make([]domain.HyperParam, 0, len(node.Data.Hyperparams))
|
|
for _, hyperparam := range node.Data.Hyperparams {
|
|
if isSensitiveNodeEnvKey(hyperparam.Key) {
|
|
continue
|
|
}
|
|
safeNodeData.Hyperparams = append(safeNodeData.Hyperparams, hyperparam)
|
|
}
|
|
nodeDataJSON, err := json.Marshal(safeNodeData)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
inputArtifactsJSON, err := json.Marshal(inputArtifacts)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if err := writeJSONFile(workspace.InputArtifactsPath, inputArtifacts); err != nil {
|
|
return nil, err
|
|
}
|
|
if inputBindings == nil {
|
|
inputBindings = []domain.NodeInputBinding{}
|
|
}
|
|
if strings.TrimSpace(workspace.InputBindingsPath) == "" {
|
|
workspace.InputBindingsPath = filepath.Join(workspace.Root, "input-bindings.json")
|
|
}
|
|
inputBindingsJSON, err := json.Marshal(inputBindings)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if err := writeJSONFile(workspace.InputBindingsPath, inputBindings); err != nil {
|
|
return nil, err
|
|
}
|
|
env := map[string]string{
|
|
"RUN_ID": run.ID,
|
|
"WORKFLOW_ID": run.WorkflowID,
|
|
"NODE_ID": node.ID,
|
|
"NODE_LABEL": node.Data.Label,
|
|
"NODE_TYPE": node.Type,
|
|
"NODE_CATEGORY": node.Data.Category,
|
|
"NODE_DESCRIPTION": node.Data.Description,
|
|
"NODE_DATA_JSON": string(nodeDataJSON),
|
|
"INPUT_ARTIFACTS_PATH": workspace.InputArtifactsPath,
|
|
"INPUT_BINDINGS_PATH": workspace.InputBindingsPath,
|
|
"RESULT_OUTPUT_PATH": workspace.ResultPath,
|
|
"STAGE_MANIFEST_PATH": workspace.StageManifestPath,
|
|
"OUTPUT_DIR": workspace.OutputDir,
|
|
"WORKSPACE_ROOT": workspace.RunRoot,
|
|
}
|
|
if parentRunID := strings.TrimSpace(run.ParentRunID); parentRunID != "" {
|
|
parentRunRoot := filepath.Join(filepath.Dir(workspace.RunRoot), filepath.Base(parentRunID))
|
|
env["PARENT_RUN_ID"] = parentRunID
|
|
env["PARENT_RUN_WORKSPACE_ROOT"] = parentRunRoot
|
|
env["PARENT_NODE_OUTPUT_DIR"] = filepath.Join(parentRunRoot, "nodes", node.ID, "outputs")
|
|
}
|
|
if run.Trigger == domain.RunTriggerScheduled {
|
|
env["WORKFLOW_CURRENT_RUN_TIMESTAMP"] = runTimestampEnvValue(firstNonEmpty(run.CurrentRunAt, run.CreatedAt))
|
|
env["WORKFLOW_PREVIOUS_RUN_TIMESTAMP"] = runTimestampEnvValue(run.PreviousRunAt)
|
|
}
|
|
if len(inputArtifactsJSON) <= inputArtifactsInlineLimit {
|
|
env["INPUT_ARTIFACTS_JSON"] = string(inputArtifactsJSON)
|
|
}
|
|
if len(inputBindingsJSON) <= inputArtifactsInlineLimit {
|
|
env["INPUT_BINDINGS_JSON"] = string(inputBindingsJSON)
|
|
}
|
|
if node.Data.Bucket != "" {
|
|
env["STORAGE_BUCKET"] = node.Data.Bucket
|
|
}
|
|
if node.Data.Prefix != "" {
|
|
env["STORAGE_PREFIX"] = node.Data.Prefix
|
|
}
|
|
if node.Data.Pattern != "" {
|
|
env["STORAGE_PATTERN"] = node.Data.Pattern
|
|
}
|
|
if resources.CPULimit != "" {
|
|
env["CPU_LIMIT"] = resources.CPULimit
|
|
}
|
|
if resources.MemoryLimit != "" {
|
|
env["MEMORY_LIMIT"] = resources.MemoryLimit
|
|
}
|
|
if resources.GPULimit != "" {
|
|
env["GPU_LIMIT"] = resources.GPULimit
|
|
}
|
|
if node.Data.Image != "" {
|
|
env["NODE_IMAGE"] = node.Data.Image
|
|
}
|
|
if handler := runtimeContractHandler(node.Data.RuntimeContract); handler != "" {
|
|
env["DROBOTICFLOW_OPERATOR_HANDLER"] = handler
|
|
}
|
|
if shouldInjectPlatformMongoEnv(node) {
|
|
if value := strings.TrimSpace(os.Getenv("MONGO_URI")); value != "" {
|
|
env["PLATFORM_MONGO_URI"] = value
|
|
}
|
|
if value := strings.TrimSpace(os.Getenv("MONGO_DB")); value != "" {
|
|
env["PLATFORM_MONGO_DB"] = value
|
|
}
|
|
}
|
|
if workflow != nil {
|
|
applyHyperparamsToEnv(env, workflow.Hyperparams)
|
|
}
|
|
applyHyperparamsToEnv(env, node.Data.Hyperparams)
|
|
applyOperatorCommandPolicyEnv(env)
|
|
return env, nil
|
|
}
|
|
|
|
func isSensitiveNodeEnvKey(key string) bool {
|
|
normalized := strings.ToUpper(strings.TrimSpace(key))
|
|
if normalized == "" {
|
|
return false
|
|
}
|
|
if normalized == "MONGO_URI" || normalized == "PLATFORM_MONGO_URI" {
|
|
return true
|
|
}
|
|
for _, fragment := range []string{"PASSWORD", "SECRET", "PRIVATE_KEY", "ACCESS_KEY_ID", "API_KEY", "AUTH_TOKEN"} {
|
|
if strings.Contains(normalized, fragment) {
|
|
return true
|
|
}
|
|
}
|
|
return strings.HasSuffix(normalized, "_TOKEN") || strings.HasSuffix(normalized, "_CREDENTIALS")
|
|
}
|
|
|
|
func partitionSensitiveNodeEnv(env map[string]string) (map[string]string, map[string]string) {
|
|
public := make(map[string]string, len(env))
|
|
sensitive := make(map[string]string)
|
|
for key, value := range env {
|
|
if isSensitiveNodeEnvKey(key) {
|
|
sensitive[key] = value
|
|
continue
|
|
}
|
|
public[key] = value
|
|
}
|
|
return public, sensitive
|
|
}
|
|
|
|
func runtimeContractHandler(contract map[string]any) string {
|
|
value, ok := contract["handler"]
|
|
if !ok {
|
|
return ""
|
|
}
|
|
handler := strings.TrimSpace(fmt.Sprint(value))
|
|
if strings.HasPrefix(handler, "builtin_") {
|
|
return strings.TrimPrefix(handler, "builtin_")
|
|
}
|
|
return handler
|
|
}
|
|
|
|
func runtimeContractLongRunning(contract map[string]any) bool {
|
|
value, ok := contract["longRunning"].(bool)
|
|
return ok && value
|
|
}
|
|
|
|
func lightweightOperatorMount(contract map[string]any) (string, string) {
|
|
config := lightweight.ConfigFromContract(contract)
|
|
if !config.Enabled {
|
|
return "", ""
|
|
}
|
|
return config.CodeSubPath, config.MountPath
|
|
}
|
|
|
|
func runtimeContractString(items map[string]any, key string) string {
|
|
value, ok := items[key]
|
|
if !ok || value == nil {
|
|
return ""
|
|
}
|
|
return fmt.Sprint(value)
|
|
}
|
|
|
|
func operatorResourceDefaults(ctx context.Context, dataStore interface {
|
|
GetOperator(context.Context, string) (*domain.OperatorRecord, error)
|
|
}, node domain.WorkflowNode) *domain.OperatorResources {
|
|
operatorID := strings.TrimSpace(node.Data.OperatorID)
|
|
if operatorID == "" || dataStore == nil {
|
|
return nil
|
|
}
|
|
operator, err := dataStore.GetOperator(ctx, operatorID)
|
|
if err != nil || operator == nil {
|
|
return nil
|
|
}
|
|
return operator.Resources
|
|
}
|
|
|
|
func resolveEffectiveNodeResources(node domain.WorkflowNode, settings *domain.SettingsRecord, operatorResources *domain.OperatorResources) effectiveNodeResources {
|
|
defaults := domain.DefaultResources{}
|
|
if settings != nil {
|
|
defaults = settings.Defaults
|
|
}
|
|
resources := effectiveNodeResources{
|
|
CPURequest: defaults.CPURequest,
|
|
CPULimit: defaults.CPULimit,
|
|
MemoryRequest: defaults.MemoryRequest,
|
|
MemoryLimit: defaults.MemoryLimit,
|
|
GPULimit: defaults.GPULimit,
|
|
}
|
|
if operatorResources != nil {
|
|
resources.CPURequest = firstNonEmpty(operatorResources.CPURequest, resources.CPURequest)
|
|
resources.CPULimit = firstNonEmpty(operatorResources.CPULimit, resources.CPULimit)
|
|
resources.MemoryRequest = firstNonEmpty(operatorResources.MemoryRequest, resources.MemoryRequest)
|
|
resources.MemoryLimit = firstNonEmpty(operatorResources.MemoryLimit, resources.MemoryLimit)
|
|
resources.GPULimit = firstNonEmpty(operatorResources.GPULimit, resources.GPULimit)
|
|
resources.SharedMemorySize = firstNonEmpty(operatorResources.SharedMemorySize, resources.SharedMemorySize)
|
|
}
|
|
resources.CPULimit = firstNonEmpty(node.Data.CPU, resources.CPULimit)
|
|
resources.MemoryLimit = firstNonEmpty(node.Data.Memory, resources.MemoryLimit)
|
|
resources.GPULimit = firstNonEmpty(node.Data.GPU, resources.GPULimit)
|
|
resources.SharedMemorySize = firstNonEmpty(node.Data.SharedMemorySize, resources.SharedMemorySize)
|
|
return applyFanyuResourceProfile(node, resources)
|
|
}
|
|
|
|
func parseExecutionOutput(stdout string, nodeLabel string) (executionResult, error) {
|
|
result := executionResult{
|
|
Logs: []domain.LogEntryRecord{},
|
|
Artifacts: []domain.ExecutionArtifact{},
|
|
ResourceSeries: []domain.ResourceSample{},
|
|
}
|
|
lines := strings.Split(strings.ReplaceAll(stdout, "\r\n", "\n"), "\n")
|
|
structuredPayload := ""
|
|
structuredPayloadPath := ""
|
|
rawLogs := make([]domain.LogEntryRecord, 0)
|
|
for _, line := range lines {
|
|
trimmed := strings.TrimSpace(line)
|
|
if trimmed == "" {
|
|
continue
|
|
}
|
|
if strings.HasPrefix(trimmed, resultPrefix) {
|
|
structuredPayload = strings.TrimPrefix(trimmed, resultPrefix)
|
|
continue
|
|
}
|
|
if strings.HasPrefix(trimmed, resultPathPrefix) {
|
|
structuredPayloadPath = strings.TrimSpace(strings.TrimPrefix(trimmed, resultPathPrefix))
|
|
continue
|
|
}
|
|
rawLogs = append(rawLogs, logEntry("INFO", trimmed, nodeLabel))
|
|
}
|
|
if structuredPayloadPath != "" {
|
|
payloadBytes, err := os.ReadFile(structuredPayloadPath)
|
|
if err != nil {
|
|
result.Logs = rawLogs
|
|
return result, fmt.Errorf("read operator result path: %w", err)
|
|
}
|
|
structuredPayload = strings.TrimSpace(string(payloadBytes))
|
|
}
|
|
if structuredPayload == "" {
|
|
result.Logs = rawLogs
|
|
return result, fmt.Errorf("operator execution did not emit %s or %s payload", resultPrefix, resultPathPrefix)
|
|
}
|
|
var parsed executionResult
|
|
if err := json.Unmarshal([]byte(structuredPayload), &parsed); err != nil {
|
|
return result, fmt.Errorf("decode operator result: %w", err)
|
|
}
|
|
if len(parsed.Logs) > 0 {
|
|
result.Logs = append(result.Logs, parsed.Logs...)
|
|
} else {
|
|
result.Logs = append(result.Logs, rawLogs...)
|
|
}
|
|
result.Artifacts = parsed.Artifacts
|
|
result.ResourceSeries = parsed.ResourceSeries
|
|
return result, nil
|
|
}
|
|
|
|
func loadStageManifestDataRefs(path string) ([]domain.DataRefRecord, error) {
|
|
trimmed := strings.TrimSpace(path)
|
|
if trimmed == "" {
|
|
return nil, nil
|
|
}
|
|
payload, err := os.ReadFile(trimmed)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
var manifest stageManifestRecord
|
|
if err := json.Unmarshal(payload, &manifest); err != nil {
|
|
return nil, err
|
|
}
|
|
if manifest.DataRefs == nil {
|
|
return []domain.DataRefRecord{}, nil
|
|
}
|
|
return manifest.DataRefs, nil
|
|
}
|
|
|
|
func loadStageManifestDataMetrics(path string) (*domain.DataMetricsRecord, error) {
|
|
trimmed := strings.TrimSpace(path)
|
|
if trimmed == "" {
|
|
return nil, nil
|
|
}
|
|
payload, err := os.ReadFile(trimmed)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
var manifest stageManifestRecord
|
|
if err := json.Unmarshal(payload, &manifest); err != nil {
|
|
return nil, err
|
|
}
|
|
return manifest.DataMetrics, nil
|
|
}
|
|
|
|
func (runtime *Runtime) resolveNodeBusinessStage(ctx context.Context, run *domain.RunRecord, node domain.WorkflowNode) domain.WorkflowNode {
|
|
if run != nil && strings.TrimSpace(run.TaskID) != "" {
|
|
if task, err := runtime.store.GetCollectionTask(ctx, run.TaskID); err == nil && task != nil {
|
|
for stage, nodeID := range task.StageBindings {
|
|
if strings.TrimSpace(nodeID) == node.ID {
|
|
node.Data.BusinessStage = stage
|
|
node.Data.MetricBoundary = true
|
|
return node
|
|
}
|
|
}
|
|
}
|
|
}
|
|
if node.Data.BusinessStage != "" {
|
|
node.Data.MetricBoundary = true
|
|
return node
|
|
}
|
|
if strings.TrimSpace(node.Data.OperatorID) != "" {
|
|
if operator, err := runtime.store.GetOperator(ctx, node.Data.OperatorID); err == nil && operator != nil && operator.BusinessStage != "" {
|
|
node.Data.BusinessStage = operator.BusinessStage
|
|
node.Data.MetricBoundary = true
|
|
}
|
|
}
|
|
return node
|
|
}
|
|
|
|
func (runtime *Runtime) executeNode(ctx context.Context, run *domain.RunRecord, node domain.WorkflowNode, inputArtifacts []domain.ExecutionArtifact, inputBindings []domain.NodeInputBinding) (executionResult, *k8s.JobReference, error) {
|
|
if node.Data.OperatorID == "operator-builtin-package" && strings.TrimSpace(os.Getenv("FANYU_SLICE_DISCARD_PACKAGE_IMAGE")) != "" && discardPackageImageOverride(node) == "" {
|
|
return executionResult{}, nil, fmt.Errorf("FANYU_SLICE_DISCARD_PACKAGE_IMAGE must be a digest-pinned package image")
|
|
}
|
|
if run != nil && len(run.NodeOverrides) > 0 {
|
|
if override, ok := run.NodeOverrides[node.ID]; ok {
|
|
node = applyNodeOverridesToNode(node, override)
|
|
}
|
|
}
|
|
if err := validateRequiredNodeInputs(node, inputBindings); err != nil {
|
|
return executionResult{}, nil, err
|
|
}
|
|
node = runtime.resolveNodeBusinessStage(ctx, run, node)
|
|
inferenceSidecar, err := inferenceSidecarForNode(run, &node)
|
|
if err != nil {
|
|
return executionResult{}, nil, err
|
|
}
|
|
workspace, err := prepareNodeWorkspace(runtime.workspaceRoot, run.ID, node.ID)
|
|
if err != nil {
|
|
return executionResult{}, nil, err
|
|
}
|
|
workflow, workflowErr := runtime.store.GetWorkflow(ctx, run.WorkflowID)
|
|
if workflowErr != nil {
|
|
workflow = nil
|
|
}
|
|
settings, err := runtime.store.GetSettings(ctx)
|
|
if err != nil {
|
|
return executionResult{}, nil, err
|
|
}
|
|
resources := resolveEffectiveNodeResources(node, settings, operatorResourceDefaults(ctx, runtime.store, node))
|
|
lightweightConfig := lightweight.ConfigFromContract(node.Data.RuntimeContract)
|
|
if lightweightConfig.Enabled {
|
|
if err := lightweight.ValidateRuntimeConfig(lightweight.RuntimeConfig{
|
|
NodeID: node.ID,
|
|
NodeLabel: node.Data.Label,
|
|
Image: node.Data.Image,
|
|
CodeSubPath: lightweightConfig.CodeSubPath,
|
|
MountPath: lightweightConfig.MountPath,
|
|
Command: node.Data.ContainerCommand,
|
|
Args: node.Data.ContainerArgs,
|
|
ArgsConfigured: lightweightConfig.ContainerArgsConfigured,
|
|
WorkspaceRoot: runtime.workspaceRoot,
|
|
Mounts: lightweightConfig.Mounts,
|
|
}); err != nil {
|
|
return executionResult{}, nil, err
|
|
}
|
|
}
|
|
env, err := buildNodeEnvWithResources(run, workflow, node, inputArtifacts, inputBindings, workspace, resources)
|
|
if err != nil {
|
|
return executionResult{}, nil, err
|
|
}
|
|
env, sensitiveEnv := partitionSensitiveNodeEnv(env)
|
|
codeSubPath, codeMountPath := lightweightConfig.CodeSubPath, lightweightConfig.MountPath
|
|
if codeSubPath != "" {
|
|
env["LIGHTWEIGHT_OPERATOR_CODE_SUBPATH"] = codeSubPath
|
|
env["LIGHTWEIGHT_OPERATOR_CODE_DIR"] = codeMountPath
|
|
}
|
|
pvcMounts := make([]k8s.PVCMount, 0, len(lightweightConfig.Mounts))
|
|
for _, item := range lightweightConfig.Mounts {
|
|
pvcMounts = append(pvcMounts, k8s.PVCMount{Name: item.Name, PVCName: item.PVCName, SubPath: item.SubPath, MountPath: item.MountPath, ReadOnly: item.ReadOnly})
|
|
}
|
|
gpuNodeSelector := map[string]string(nil)
|
|
env, pvcMounts, gpuNodeSelector, err = runtime.qwenNodeCache.apply(node, env, pvcMounts)
|
|
if err != nil {
|
|
return executionResult{}, nil, err
|
|
}
|
|
nodeTimeoutSeconds := 0
|
|
if workflow != nil {
|
|
nodeTimeoutSeconds = workflow.NodeTimeoutSeconds
|
|
}
|
|
launchGPULimit := resources.GPULimit
|
|
if inferenceSidecar != nil && strings.TrimSpace(inferenceSidecar.GPULimit) != "" {
|
|
launchGPULimit = inferenceSidecar.GPULimit
|
|
}
|
|
if workflow != nil && workflow.ProjectID != "" {
|
|
p, e := runtime.store.GetProject(ctx, workflow.ProjectID)
|
|
if e != nil {
|
|
return executionResult{}, nil, e
|
|
}
|
|
if p == nil || p.DeletionTaskID != "" {
|
|
return executionResult{}, nil, fmt.Errorf("project deletion in progress")
|
|
}
|
|
}
|
|
image, err := sourceIndexPackageImage(node, inputArtifacts, runtime.operatorRunnerImage)
|
|
if err != nil {
|
|
return executionResult{}, nil, err
|
|
}
|
|
launched, err := runtime.launcher.Launch(ctx, k8s.LaunchInput{
|
|
Image: image,
|
|
Command: cloneStringSlice(node.Data.ContainerCommand),
|
|
Args: cloneStringSlice(node.Data.ContainerArgs),
|
|
RunID: run.ID,
|
|
WorkflowID: run.WorkflowID,
|
|
NodeID: node.ID,
|
|
NodeLabel: node.Data.Label,
|
|
Attempt: run.Attempt,
|
|
TimeoutSeconds: nodeTimeoutSeconds,
|
|
Env: env,
|
|
SensitiveEnv: sensitiveEnv,
|
|
CPURequest: resources.CPURequest,
|
|
CPULimit: resources.CPULimit,
|
|
MemoryRequest: resources.MemoryRequest,
|
|
MemoryLimit: resources.MemoryLimit,
|
|
GPULimit: launchGPULimit,
|
|
GPUNodeSelector: gpuNodeSelector,
|
|
SharedMemorySize: resources.SharedMemorySize,
|
|
LongRunning: runtimeContractLongRunning(node.Data.RuntimeContract),
|
|
CodeSubPath: codeSubPath,
|
|
CodeMountPath: codeMountPath,
|
|
CodeReadOnly: lightweightConfig.CodeReadOnly,
|
|
PVCMounts: pvcMounts,
|
|
OperatorRunnerImage: runtime.operatorRunnerImage,
|
|
InferenceSidecar: inferenceSidecar,
|
|
})
|
|
if err != nil {
|
|
return executionResult{}, nil, err
|
|
}
|
|
if progress, progressErr := latestStructuredProgress(launched.Output); progressErr == nil && progress != nil {
|
|
_ = runtime.store.UpdateRunProgress(ctx, run.ID, progress)
|
|
}
|
|
parsed, err := parseExecutionOutput(launched.Output, node.Data.Label)
|
|
if err == nil {
|
|
parsed.Artifacts, err = runtime.normalizeOutputArtifacts(ctx, workflow, run, node.ID, parsed.Artifacts)
|
|
}
|
|
if err == nil {
|
|
parsed.Artifacts = tagDebugArtifacts(run, parsed.Artifacts)
|
|
}
|
|
if err == nil {
|
|
parsed.StageManifestPath = workspace.StageManifestPath
|
|
parsed.DataRefs, err = loadStageManifestDataRefs(workspace.StageManifestPath)
|
|
if err == nil && parsed.DataMetrics == nil {
|
|
parsed.DataMetrics, err = loadStageManifestDataMetrics(workspace.StageManifestPath)
|
|
}
|
|
}
|
|
// Custom-container operators bypass operator-runner's ResourceSampler, so they
|
|
// emit no resourceSeries in result.json. The launcher samples their GPU usage
|
|
// itself (see k8s.Launch); fall back to those platform-collected samples when
|
|
// the operator did not self-report. Builtin operator-runner pods self-report a
|
|
// non-empty series, so their own (more complete, CPU+memory included) samples
|
|
// are preserved.
|
|
if len(parsed.ResourceSeries) == 0 && len(launched.ResourceSeries) > 0 {
|
|
parsed.ResourceSeries = make([]domain.ResourceSample, 0, len(launched.ResourceSeries))
|
|
for _, sample := range launched.ResourceSeries {
|
|
parsed.ResourceSeries = append(parsed.ResourceSeries, domain.ResourceSample{
|
|
Timestamp: sample.Timestamp,
|
|
GPU: sample.GPU,
|
|
GPUMemoryUsedMiB: sample.GPUMemoryUsedMiB,
|
|
GPUMemoryTotalMiB: sample.GPUMemoryTotalMiB,
|
|
GPUName: sample.GPUName,
|
|
GPUCount: sample.GPUCount,
|
|
})
|
|
}
|
|
}
|
|
return parsed, launched.Reference, err
|
|
}
|