eust-w 30d916774c 🐛 Fail launches whose pod stays blocked instead of hanging the run
The reconciler used to be the only place that failed a node whose pod sat
in ImagePullBackOff/Unschedulable past WORKER_PENDING_FAILURE_SECONDS. Now
that it leaves runs with a live execution lease alone (0b365155), such a
node would wait for the full launch deadline.

Have Launch check the Job's pods (at most every 10s) and, once a pod has
been blocked for PendingFailureTimeout, delete the Job and return the
blocking reason and message. The executor passes the configured pending
threshold. On POC this surfaced builtin validator images that cannot be
pulled, which the reconciler race had been masking as missing Jobs.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-27 16:25:24 +08:00

910 lines
30 KiB
Go

package worker
import (
"context"
"encoding/json"
"fmt"
"maps"
"os"
"path/filepath"
"regexp"
"strings"
"time"
"droboticflow/internal/domain"
"droboticflow/internal/k8s"
"droboticflow/internal/lightweight"
)
const resultPrefix = "DROBOTICFLOW_RESULT="
const resultPathPrefix = "DROBOTICFLOW_RESULT_PATH="
const inputArtifactsInlineLimit = 64 * 1024
type nodeWorkspace struct {
RunRoot string
Root string
OutputDir string
InputArtifactsPath string
InputBindingsPath string
ResultPath string
StageManifestPath string
}
type executionResult struct {
Logs []domain.LogEntryRecord `json:"logs"`
Artifacts []domain.ExecutionArtifact `json:"artifacts"`
DataRefs []domain.DataRefRecord `json:"dataRefs,omitempty"`
DataMetrics *domain.DataMetricsRecord `json:"dataMetrics,omitempty"`
StageManifestPath string `json:"stageManifestPath,omitempty"`
ResourceSeries []domain.ResourceSample `json:"resourceSeries"`
}
type stageManifestRecord struct {
DataRefs []domain.DataRefRecord `json:"dataRefs"`
DataMetrics *domain.DataMetricsRecord `json:"dataMetrics,omitempty"`
}
var protectedSystemEnv = map[string]struct{}{
"WORKFLOW_PREVIOUS_RUN_TIMESTAMP": {},
"WORKFLOW_CURRENT_RUN_TIMESTAMP": {},
"PARENT_RUN_ID": {},
"PARENT_RUN_WORKSPACE_ROOT": {},
"PARENT_NODE_OUTPUT_DIR": {},
"OPERATOR_ALLOW_SHELL_COMMANDS": {},
"OPERATOR_COMMAND_ALLOWLIST_REGEX": {},
"OPERATOR_COMMAND_DENYLIST_REGEX": {},
}
var operatorCommandPolicyEnvKeys = []string{
"OPERATOR_ALLOW_SHELL_COMMANDS",
"OPERATOR_COMMAND_ALLOWLIST_REGEX",
"OPERATOR_COMMAND_DENYLIST_REGEX",
}
var workflowEnvLocation = loadWorkflowEnvLocation()
var invalidEnvCharacters = regexp.MustCompile(`[^A-Z0-9_]+`)
func loadWorkflowEnvLocation() *time.Location {
location, err := time.LoadLocation("Asia/Shanghai")
if err != nil {
return time.FixedZone("Asia/Shanghai", 8*60*60)
}
return location
}
func sanitizeEnvKey(value string) string {
normalized := strings.ToUpper(strings.TrimSpace(value))
if normalized == "" {
return ""
}
normalized = invalidEnvCharacters.ReplaceAllString(normalized, "_")
normalized = strings.Trim(normalized, "_")
if normalized == "" {
return ""
}
if normalized[0] >= '0' && normalized[0] <= '9' {
normalized = "HP_" + normalized
}
return normalized
}
func cloneArtifacts(items []domain.ExecutionArtifact) []domain.ExecutionArtifact {
cloned := make([]domain.ExecutionArtifact, 0, len(items))
for _, item := range items {
cloned = append(cloned, cloneArtifact(item))
}
return cloned
}
func cloneStringSlice(items []string) []string {
if items == nil {
return nil
}
cloned := make([]string, len(items))
copy(cloned, items)
return cloned
}
func cloneArtifact(item domain.ExecutionArtifact) domain.ExecutionArtifact {
cloned := item
if item.Metadata != nil {
metadata := *item.Metadata
cloned.Metadata = &metadata
}
if item.Data != nil {
cloned.Data = maps.Clone(item.Data)
}
return cloned
}
func cloneInputLineage(items []domain.InputLineageRecord) []domain.InputLineageRecord {
cloned := make([]domain.InputLineageRecord, 0, len(items))
for _, item := range items {
record := item
record.Artifact = cloneArtifact(item.Artifact)
cloned = append(cloned, record)
}
return cloned
}
func cloneInputBindings(items []domain.NodeInputBinding) []domain.NodeInputBinding {
if items == nil {
return nil
}
cloned := make([]domain.NodeInputBinding, len(items))
for index, item := range items {
cloned[index] = item
cloned[index].Artifacts = cloneArtifacts(item.Artifacts)
if item.Payload != nil {
payload := *item.Payload
if item.Payload.Ref != nil {
ref := *item.Payload.Ref
payload.Ref = &ref
}
cloned[index].Payload = &payload
}
}
return cloned
}
func runTimestampEnvValue(value string) string {
trimmed := strings.TrimSpace(value)
if trimmed == "" {
return ""
}
parsed, err := time.Parse(time.RFC3339Nano, trimmed)
if err != nil {
return ""
}
return parsed.In(workflowEnvLocation).Format(time.RFC3339)
}
func firstNonEmpty(values ...string) string {
for _, value := range values {
if strings.TrimSpace(value) != "" {
return value
}
}
return ""
}
func inputArtifactLineageKey(item domain.InputLineageRecord) string {
artifact := item.Artifact
key := artifact.ID
if key == "" {
key = artifact.NodeID + ":" + artifact.Name
}
return fmt.Sprintf("%s:%s:%s:%s:%s", item.WorkflowEdgeID, item.SourceNodeID, item.SourceHandle, item.TargetHandle, key)
}
func gatherInputLineage(workflow domain.WorkflowRecord, executionMap map[string]domain.NodeExecutionRecord, nodeID string) []domain.InputLineageRecord {
items := make([]domain.InputLineageRecord, 0)
seen := map[string]struct{}{}
for _, edge := range domain.GetIncomingEdges(workflow.Edges, nodeID) {
execution, ok := executionMap[edge.Source]
if !ok {
continue
}
requiresHandle := strings.TrimSpace(edge.SourceHandle) != ""
hasPortAwareArtifacts := false
for _, artifact := range execution.Artifacts {
if strings.TrimSpace(artifact.PortID) != "" {
hasPortAwareArtifacts = true
break
}
}
for _, artifact := range execution.Artifacts {
if requiresHandle && hasPortAwareArtifacts && artifact.PortID != edge.SourceHandle {
continue
}
item := domain.InputLineageRecord{
WorkflowEdgeID: edge.ID,
SourceNodeID: edge.Source,
SourceHandle: edge.SourceHandle,
TargetNodeID: edge.Target,
TargetHandle: edge.TargetHandle,
Artifact: artifact,
}
key := inputArtifactLineageKey(item)
if _, exists := seen[key]; exists {
continue
}
seen[key] = struct{}{}
items = append(items, item)
}
}
return items
}
func inputArtifactsFromLineage(lineage []domain.InputLineageRecord) []domain.ExecutionArtifact {
items := make([]domain.ExecutionArtifact, 0)
seen := map[string]struct{}{}
for _, item := range lineage {
artifact := item.Artifact
key := artifact.ID
if key == "" {
key = artifact.NodeID + ":" + artifact.Name
}
if _, exists := seen[key]; exists {
continue
}
seen[key] = struct{}{}
items = append(items, artifact)
}
return items
}
func artifactFromWorkflowInput(run *domain.RunRecord, binding domain.ResolvedInputBinding) domain.ExecutionArtifact {
payload := binding.Payload
artifact := domain.ExecutionArtifact{
ID: run.ID + "-input-" + sanitizeEnvKey(binding.WorkflowInputID) + "-" + sanitizeEnvKey(binding.TargetPortID),
RunID: run.ID,
NodeID: binding.NodeID,
PortID: binding.TargetPortID,
PortName: binding.TargetPortID,
Name: binding.WorkflowInputID,
Type: payload.Type,
Payload: &payload,
}
if payload.Mode == domain.ArtifactTransferModeInline {
encoded, _ := json.Marshal(payload.Inline)
artifact.SizeBytes = int64(len(encoded))
artifact.Size = fmt.Sprintf("%d B", len(encoded))
if data, ok := payload.Inline.(map[string]any); ok {
artifact.Data = data
} else {
artifact.Data = map[string]any{"value": payload.Inline}
}
return artifact
}
if payload.Ref != nil {
ref := payload.Ref
artifact.SizeBytes = ref.SizeBytes
artifact.Size = fmt.Sprintf("%d B", ref.SizeBytes)
if ref.Kind == domain.ArtifactRefKindDirectory {
artifact.Path = "bos://" + ref.Bucket + "/" + strings.TrimPrefix(ref.Prefix, "/")
} else {
artifact.Path = "bos://" + ref.Bucket + "/" + strings.TrimPrefix(ref.Key, "/")
}
artifact.Metadata = &domain.ArtifactMetadata{SHA256: ref.SHA256, MimeType: ref.MimeType}
artifact.Data = map[string]any{"source": map[string]any{"path": artifact.Path}}
}
return artifact
}
func workflowInputBindingsForNode(run *domain.RunRecord, nodeID string) ([]domain.NodeInputBinding, []domain.ExecutionArtifact) {
if run == nil {
return nil, nil
}
bindings := make([]domain.NodeInputBinding, 0)
artifacts := make([]domain.ExecutionArtifact, 0)
for _, item := range run.ResolvedInputBindings {
if item.NodeID != nodeID {
continue
}
artifact := artifactFromWorkflowInput(run, item)
payload := item.Payload
bindings = append(bindings, domain.NodeInputBinding{
TargetPortID: item.TargetPortID,
WorkflowInputID: item.WorkflowInputID,
Payload: &payload,
Artifacts: []domain.ExecutionArtifact{artifact},
})
artifacts = append(artifacts, artifact)
}
return bindings, artifacts
}
func lineageInputBindings(lineage []domain.InputLineageRecord) []domain.NodeInputBinding {
items := make([]domain.NodeInputBinding, 0)
indexByKey := map[string]int{}
for _, item := range lineage {
key := item.SourceNodeID + "\x00" + item.SourceHandle + "\x00" + item.TargetHandle
if index, ok := indexByKey[key]; ok {
items[index].Artifacts = append(items[index].Artifacts, item.Artifact)
continue
}
indexByKey[key] = len(items)
items = append(items, domain.NodeInputBinding{
TargetPortID: item.TargetHandle,
SourceNodeID: item.SourceNodeID,
SourcePortID: item.SourceHandle,
Artifacts: []domain.ExecutionArtifact{item.Artifact},
})
}
return items
}
func validateRequiredNodeInputs(node domain.WorkflowNode, bindings []domain.NodeInputBinding) error {
for _, port := range node.Data.Inputs {
if !port.Required {
continue
}
present := false
for _, binding := range bindings {
targetPortID := strings.TrimSpace(binding.TargetPortID)
if targetPortID != strings.TrimSpace(port.ID) && !(targetPortID == "" && len(node.Data.Inputs) == 1) {
continue
}
if binding.Payload != nil || len(binding.Artifacts) > 0 {
present = true
break
}
}
if !present {
return fmt.Errorf("required_input_missing: node %s input %s has no value or upstream artifact", node.ID, port.ID)
}
}
return nil
}
func gatherNodeInputs(workflow domain.WorkflowRecord, run *domain.RunRecord, executionMap map[string]domain.NodeExecutionRecord, nodeID string) ([]domain.ExecutionArtifact, []domain.InputLineageRecord, []domain.NodeInputBinding) {
lineage := gatherInputLineage(workflow, executionMap, nodeID)
artifacts := inputArtifactsFromLineage(lineage)
bindings := lineageInputBindings(lineage)
rootBindings, rootArtifacts := workflowInputBindingsForNode(run, nodeID)
bindings = append(bindings, rootBindings...)
artifacts = append(artifacts, rootArtifacts...)
return artifacts, lineage, bindings
}
func gatherInputArtifacts(workflow domain.WorkflowRecord, executionMap map[string]domain.NodeExecutionRecord, nodeID string) []domain.ExecutionArtifact {
return inputArtifactsFromLineage(gatherInputLineage(workflow, executionMap, nodeID))
}
func extractConfidence(artifact domain.ExecutionArtifact) (float64, bool) {
if artifact.Data == nil {
return 0, false
}
for _, key := range []string{"confidence", "score"} {
value, ok := artifact.Data[key]
if !ok {
continue
}
switch typed := value.(type) {
case float64:
return typed, true
case int:
return float64(typed), true
}
}
return 0, false
}
func extractObjectHints(artifact domain.ExecutionArtifact) []string {
if artifact.Data == nil {
return nil
}
items := make([]string, 0)
for _, key := range []string{"detectedObjects", "labels", "entries"} {
value, ok := artifact.Data[key]
if !ok {
continue
}
switch typed := value.(type) {
case []any:
for _, candidate := range typed {
text := strings.TrimSpace(fmt.Sprint(candidate))
if text != "" {
items = append(items, text)
}
}
case []string:
for _, candidate := range typed {
if trimmed := strings.TrimSpace(candidate); trimmed != "" {
items = append(items, trimmed)
}
}
}
}
return items
}
func createAuditPayload(node domain.WorkflowNode, inputArtifacts []domain.ExecutionArtifact) domain.AuditTaskPayload {
payload := domain.AuditTaskPayload{
Reason: fmt.Sprintf("%s requires explicit approval before downstream execution", node.Data.Label),
DetectedObjects: []string{},
}
for _, artifact := range inputArtifacts {
if payload.Image == "" {
switch {
case artifact.Metadata != nil && artifact.Metadata.Thumbnail != "":
payload.Image = artifact.Metadata.Thumbnail
case artifact.Type == domain.PortTypeImage || artifact.Type == domain.PortTypeVideo:
payload.Image = artifact.Path
}
}
if payload.Confidence == 0 {
if confidence, ok := extractConfidence(artifact); ok {
payload.Confidence = confidence
}
}
payload.DetectedObjects = append(payload.DetectedObjects, extractObjectHints(artifact)...)
}
if len(payload.DetectedObjects) == 0 {
payload.DetectedObjects = []string{fmt.Sprintf("%d upstream artifact(s)", len(inputArtifacts))}
}
return payload
}
func prepareNodeWorkspace(workspaceRoot string, runID string, nodeID string) (nodeWorkspace, error) {
root := strings.TrimSpace(workspaceRoot)
if root == "" {
root = filepath.Join(os.TempDir(), "droboticflow")
}
runRoot := filepath.Join(root, "system", "runs", runID)
nodeRoot := filepath.Join(runRoot, "nodes", nodeID)
outputDir := filepath.Join(nodeRoot, "outputs")
if err := os.MkdirAll(outputDir, 0o755); err != nil {
return nodeWorkspace{}, err
}
return nodeWorkspace{
RunRoot: runRoot,
Root: nodeRoot,
OutputDir: outputDir,
InputArtifactsPath: filepath.Join(nodeRoot, "input-artifacts.json"),
InputBindingsPath: filepath.Join(nodeRoot, "input-bindings.json"),
ResultPath: filepath.Join(nodeRoot, "result.json"),
StageManifestPath: filepath.Join(nodeRoot, "stage-manifest.json"),
}, nil
}
func shouldInjectPlatformMongoEnv(node domain.WorkflowNode) bool {
label := strings.ToLower(strings.TrimSpace(node.Data.Label))
return strings.Contains(label, "mongodb") || strings.Contains(label, "mongo")
}
func writeJSONFile(path string, value any) error {
data, err := json.Marshal(value)
if err != nil {
return err
}
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
return err
}
return os.WriteFile(path, data, 0o644)
}
func applyHyperparamsToEnv(env map[string]string, items []domain.HyperParam) {
for _, item := range items {
key := sanitizeEnvKey(item.Key)
if key == "" {
continue
}
if _, blocked := protectedSystemEnv[key]; blocked {
continue
}
env[key] = item.Value
}
}
func applyOperatorCommandPolicyEnv(env map[string]string) {
for _, key := range operatorCommandPolicyEnvKeys {
if value := strings.TrimSpace(os.Getenv(key)); value != "" {
env[key] = value
}
}
}
func buildNodeEnv(run *domain.RunRecord, workflow *domain.WorkflowRecord, node domain.WorkflowNode, inputArtifacts []domain.ExecutionArtifact, workspace nodeWorkspace, settings *domain.SettingsRecord) (map[string]string, error) {
return buildNodeEnvWithResources(run, workflow, node, inputArtifacts, nil, workspace, effectiveNodeResources{
CPULimit: node.Data.CPU,
MemoryLimit: node.Data.Memory,
GPULimit: node.Data.GPU,
})
}
type effectiveNodeResources struct {
CPURequest string
CPULimit string
MemoryRequest string
MemoryLimit string
GPULimit string
SharedMemorySize string
}
func buildNodeEnvWithResources(run *domain.RunRecord, workflow *domain.WorkflowRecord, node domain.WorkflowNode, inputArtifacts []domain.ExecutionArtifact, inputBindings []domain.NodeInputBinding, workspace nodeWorkspace, resources effectiveNodeResources) (map[string]string, error) {
if inputArtifacts == nil {
inputArtifacts = []domain.ExecutionArtifact{}
}
nodeDataJSON, err := json.Marshal(node.Data)
if err != nil {
return nil, err
}
inputArtifactsJSON, err := json.Marshal(inputArtifacts)
if err != nil {
return nil, err
}
if err := writeJSONFile(workspace.InputArtifactsPath, inputArtifacts); err != nil {
return nil, err
}
if inputBindings == nil {
inputBindings = []domain.NodeInputBinding{}
}
if strings.TrimSpace(workspace.InputBindingsPath) == "" {
workspace.InputBindingsPath = filepath.Join(workspace.Root, "input-bindings.json")
}
inputBindingsJSON, err := json.Marshal(inputBindings)
if err != nil {
return nil, err
}
if err := writeJSONFile(workspace.InputBindingsPath, inputBindings); err != nil {
return nil, err
}
env := map[string]string{
"RUN_ID": run.ID,
"WORKFLOW_ID": run.WorkflowID,
"NODE_ID": node.ID,
"NODE_LABEL": node.Data.Label,
"NODE_TYPE": node.Type,
"NODE_CATEGORY": node.Data.Category,
"NODE_DESCRIPTION": node.Data.Description,
"NODE_DATA_JSON": string(nodeDataJSON),
"INPUT_ARTIFACTS_PATH": workspace.InputArtifactsPath,
"INPUT_BINDINGS_PATH": workspace.InputBindingsPath,
"RESULT_OUTPUT_PATH": workspace.ResultPath,
"STAGE_MANIFEST_PATH": workspace.StageManifestPath,
"OUTPUT_DIR": workspace.OutputDir,
"WORKSPACE_ROOT": workspace.RunRoot,
}
if parentRunID := strings.TrimSpace(run.ParentRunID); parentRunID != "" {
parentRunRoot := filepath.Join(filepath.Dir(workspace.RunRoot), filepath.Base(parentRunID))
env["PARENT_RUN_ID"] = parentRunID
env["PARENT_RUN_WORKSPACE_ROOT"] = parentRunRoot
env["PARENT_NODE_OUTPUT_DIR"] = filepath.Join(parentRunRoot, "nodes", node.ID, "outputs")
}
if run.Trigger == domain.RunTriggerScheduled {
env["WORKFLOW_CURRENT_RUN_TIMESTAMP"] = runTimestampEnvValue(firstNonEmpty(run.CurrentRunAt, run.CreatedAt))
env["WORKFLOW_PREVIOUS_RUN_TIMESTAMP"] = runTimestampEnvValue(run.PreviousRunAt)
}
if len(inputArtifactsJSON) <= inputArtifactsInlineLimit {
env["INPUT_ARTIFACTS_JSON"] = string(inputArtifactsJSON)
}
if len(inputBindingsJSON) <= inputArtifactsInlineLimit {
env["INPUT_BINDINGS_JSON"] = string(inputBindingsJSON)
}
if node.Data.Bucket != "" {
env["STORAGE_BUCKET"] = node.Data.Bucket
}
if node.Data.Prefix != "" {
env["STORAGE_PREFIX"] = node.Data.Prefix
}
if node.Data.Pattern != "" {
env["STORAGE_PATTERN"] = node.Data.Pattern
}
if resources.CPULimit != "" {
env["CPU_LIMIT"] = resources.CPULimit
}
if resources.MemoryLimit != "" {
env["MEMORY_LIMIT"] = resources.MemoryLimit
}
if resources.GPULimit != "" {
env["GPU_LIMIT"] = resources.GPULimit
}
if node.Data.Image != "" {
env["NODE_IMAGE"] = node.Data.Image
}
if handler := runtimeContractHandler(node.Data.RuntimeContract); handler != "" {
env["DROBOTICFLOW_OPERATOR_HANDLER"] = handler
}
if shouldInjectPlatformMongoEnv(node) {
if value := strings.TrimSpace(os.Getenv("MONGO_URI")); value != "" {
env["PLATFORM_MONGO_URI"] = value
}
if value := strings.TrimSpace(os.Getenv("MONGO_DB")); value != "" {
env["PLATFORM_MONGO_DB"] = value
}
}
if workflow != nil {
applyHyperparamsToEnv(env, workflow.Hyperparams)
}
applyHyperparamsToEnv(env, node.Data.Hyperparams)
applyOperatorCommandPolicyEnv(env)
return env, nil
}
func runtimeContractHandler(contract map[string]any) string {
value, ok := contract["handler"]
if !ok {
return ""
}
handler := strings.TrimSpace(fmt.Sprint(value))
return handler
}
func runtimeContractLongRunning(contract map[string]any) bool {
value, ok := contract["longRunning"].(bool)
return ok && value
}
func lightweightOperatorMount(contract map[string]any) (string, string) {
config := lightweight.ConfigFromContract(contract)
if !config.Enabled {
return "", ""
}
return config.CodeSubPath, config.MountPath
}
func runtimeContractString(items map[string]any, key string) string {
value, ok := items[key]
if !ok || value == nil {
return ""
}
return fmt.Sprint(value)
}
func operatorResourceDefaults(ctx context.Context, dataStore interface {
GetOperator(context.Context, string) (*domain.OperatorRecord, error)
}, node domain.WorkflowNode) *domain.OperatorResources {
operatorID := strings.TrimSpace(node.Data.OperatorID)
if operatorID == "" || dataStore == nil {
return nil
}
operator, err := dataStore.GetOperator(ctx, operatorID)
if err != nil || operator == nil {
return nil
}
return operator.Resources
}
func resolveEffectiveNodeResources(node domain.WorkflowNode, settings *domain.SettingsRecord, operatorResources *domain.OperatorResources) effectiveNodeResources {
defaults := domain.DefaultResources{}
if settings != nil {
defaults = settings.Defaults
}
resources := effectiveNodeResources{
CPURequest: defaults.CPURequest,
CPULimit: defaults.CPULimit,
MemoryRequest: defaults.MemoryRequest,
MemoryLimit: defaults.MemoryLimit,
GPULimit: defaults.GPULimit,
}
if operatorResources != nil {
resources.CPURequest = firstNonEmpty(operatorResources.CPURequest, resources.CPURequest)
resources.CPULimit = firstNonEmpty(operatorResources.CPULimit, resources.CPULimit)
resources.MemoryRequest = firstNonEmpty(operatorResources.MemoryRequest, resources.MemoryRequest)
resources.MemoryLimit = firstNonEmpty(operatorResources.MemoryLimit, resources.MemoryLimit)
resources.GPULimit = firstNonEmpty(operatorResources.GPULimit, resources.GPULimit)
resources.SharedMemorySize = firstNonEmpty(operatorResources.SharedMemorySize, resources.SharedMemorySize)
}
resources.CPULimit = firstNonEmpty(node.Data.CPU, resources.CPULimit)
resources.MemoryLimit = firstNonEmpty(node.Data.Memory, resources.MemoryLimit)
resources.GPULimit = firstNonEmpty(node.Data.GPU, resources.GPULimit)
resources.SharedMemorySize = firstNonEmpty(node.Data.SharedMemorySize, resources.SharedMemorySize)
return resources
}
func parseExecutionOutput(stdout string, nodeLabel string) (executionResult, error) {
result := executionResult{
Logs: []domain.LogEntryRecord{},
Artifacts: []domain.ExecutionArtifact{},
ResourceSeries: []domain.ResourceSample{},
}
lines := strings.Split(strings.ReplaceAll(stdout, "\r\n", "\n"), "\n")
structuredPayload := ""
structuredPayloadPath := ""
rawLogs := make([]domain.LogEntryRecord, 0)
for _, line := range lines {
trimmed := strings.TrimSpace(line)
if trimmed == "" {
continue
}
if strings.HasPrefix(trimmed, resultPrefix) {
structuredPayload = strings.TrimPrefix(trimmed, resultPrefix)
continue
}
if strings.HasPrefix(trimmed, resultPathPrefix) {
structuredPayloadPath = strings.TrimSpace(strings.TrimPrefix(trimmed, resultPathPrefix))
continue
}
rawLogs = append(rawLogs, logEntry("INFO", trimmed, nodeLabel))
}
if structuredPayloadPath != "" {
payloadBytes, err := os.ReadFile(structuredPayloadPath)
if err != nil {
result.Logs = rawLogs
return result, fmt.Errorf("read operator result path: %w", err)
}
structuredPayload = strings.TrimSpace(string(payloadBytes))
}
if structuredPayload == "" {
result.Logs = rawLogs
return result, fmt.Errorf("operator execution did not emit %s or %s payload", resultPrefix, resultPathPrefix)
}
var parsed executionResult
if err := json.Unmarshal([]byte(structuredPayload), &parsed); err != nil {
return result, fmt.Errorf("decode operator result: %w", err)
}
if len(parsed.Logs) > 0 {
result.Logs = append(result.Logs, parsed.Logs...)
} else {
result.Logs = append(result.Logs, rawLogs...)
}
result.Artifacts = parsed.Artifacts
result.ResourceSeries = parsed.ResourceSeries
return result, nil
}
func loadStageManifestDataRefs(path string) ([]domain.DataRefRecord, error) {
trimmed := strings.TrimSpace(path)
if trimmed == "" {
return nil, nil
}
payload, err := os.ReadFile(trimmed)
if err != nil {
return nil, err
}
var manifest stageManifestRecord
if err := json.Unmarshal(payload, &manifest); err != nil {
return nil, err
}
if manifest.DataRefs == nil {
return []domain.DataRefRecord{}, nil
}
return manifest.DataRefs, nil
}
func loadStageManifestDataMetrics(path string) (*domain.DataMetricsRecord, error) {
trimmed := strings.TrimSpace(path)
if trimmed == "" {
return nil, nil
}
payload, err := os.ReadFile(trimmed)
if err != nil {
return nil, err
}
var manifest stageManifestRecord
if err := json.Unmarshal(payload, &manifest); err != nil {
return nil, err
}
return manifest.DataMetrics, nil
}
func (runtime *Runtime) resolveNodeBusinessStage(ctx context.Context, run *domain.RunRecord, node domain.WorkflowNode) domain.WorkflowNode {
if run != nil && strings.TrimSpace(run.TaskID) != "" {
if task, err := runtime.store.GetCollectionTask(ctx, run.TaskID); err == nil && task != nil {
for stage, nodeID := range task.StageBindings {
if strings.TrimSpace(nodeID) == node.ID {
node.Data.BusinessStage = stage
node.Data.MetricBoundary = true
return node
}
}
}
}
if node.Data.BusinessStage != "" {
node.Data.MetricBoundary = true
return node
}
if strings.TrimSpace(node.Data.OperatorID) != "" {
if operator, err := runtime.store.GetOperator(ctx, node.Data.OperatorID); err == nil && operator != nil && operator.BusinessStage != "" {
node.Data.BusinessStage = operator.BusinessStage
node.Data.MetricBoundary = true
}
}
return node
}
func (runtime *Runtime) executeNode(ctx context.Context, run *domain.RunRecord, node domain.WorkflowNode, inputArtifacts []domain.ExecutionArtifact, inputBindings []domain.NodeInputBinding) (executionResult, *k8s.JobReference, error) {
if err := validateRequiredNodeInputs(node, inputBindings); err != nil {
return executionResult{}, nil, err
}
node = runtime.resolveNodeBusinessStage(ctx, run, node)
workspace, err := prepareNodeWorkspace(runtime.workspaceRoot, run.ID, node.ID)
if err != nil {
return executionResult{}, nil, err
}
workflow, workflowErr := runtime.store.GetWorkflow(ctx, run.WorkflowID)
if workflowErr != nil {
workflow = nil
}
settings, err := runtime.store.GetSettings(ctx)
if err != nil {
return executionResult{}, nil, err
}
resources := resolveEffectiveNodeResources(node, settings, operatorResourceDefaults(ctx, runtime.store, node))
lightweightConfig := lightweight.ConfigFromContract(node.Data.RuntimeContract)
if lightweightConfig.Enabled {
if err := lightweight.ValidateRuntimeConfig(lightweight.RuntimeConfig{
NodeID: node.ID,
NodeLabel: node.Data.Label,
Image: node.Data.Image,
CodeSubPath: lightweightConfig.CodeSubPath,
MountPath: lightweightConfig.MountPath,
Command: node.Data.ContainerCommand,
Args: node.Data.ContainerArgs,
ArgsConfigured: lightweightConfig.ContainerArgsConfigured,
WorkspaceRoot: runtime.workspaceRoot,
Mounts: lightweightConfig.Mounts,
}); err != nil {
return executionResult{}, nil, err
}
}
env, err := buildNodeEnvWithResources(run, workflow, node, inputArtifacts, inputBindings, workspace, resources)
if err != nil {
return executionResult{}, nil, err
}
codeSubPath, codeMountPath := lightweightConfig.CodeSubPath, lightweightConfig.MountPath
if codeSubPath != "" {
env["LIGHTWEIGHT_OPERATOR_CODE_SUBPATH"] = codeSubPath
env["LIGHTWEIGHT_OPERATOR_CODE_DIR"] = codeMountPath
}
pvcMounts := make([]k8s.PVCMount, 0, len(lightweightConfig.Mounts))
for _, item := range lightweightConfig.Mounts {
pvcMounts = append(pvcMounts, k8s.PVCMount{Name: item.Name, PVCName: item.PVCName, SubPath: item.SubPath, MountPath: item.MountPath, ReadOnly: item.ReadOnly})
}
nodeTimeoutSeconds := 0
if workflow != nil {
nodeTimeoutSeconds = workflow.NodeTimeoutSeconds
}
organization, orgErr := runtime.store.GetOrganization(ctx, run.OrganizationID)
if orgErr != nil {
return executionResult{}, nil, orgErr
}
scopedWorkspace := organization != nil && organization.StorageRootPrefix != ""
launched, err := runtime.launcher.Launch(ctx, k8s.LaunchInput{
Image: chooseImage(node, runtime.operatorRunnerImage),
ScopedWorkspace: scopedWorkspace,
Command: cloneStringSlice(node.Data.ContainerCommand),
Args: cloneStringSlice(node.Data.ContainerArgs),
RunID: run.ID,
WorkflowID: run.WorkflowID,
NodeID: node.ID,
NodeLabel: node.Data.Label,
Attempt: run.Attempt,
TimeoutSeconds: nodeTimeoutSeconds,
PendingFailureTimeout: runtime.pendingJobFailureThreshold(),
Env: env,
CPURequest: resources.CPURequest,
CPULimit: resources.CPULimit,
MemoryRequest: resources.MemoryRequest,
MemoryLimit: resources.MemoryLimit,
GPULimit: resources.GPULimit,
SharedMemorySize: resources.SharedMemorySize,
LongRunning: runtimeContractLongRunning(node.Data.RuntimeContract),
CodeSubPath: codeSubPath,
CodeMountPath: codeMountPath,
CodeReadOnly: lightweightConfig.CodeReadOnly,
PVCMounts: pvcMounts,
OperatorRunnerImage: runtime.operatorRunnerImage,
})
if err != nil {
return executionResult{}, nil, err
}
if progress, progressErr := latestStructuredProgress(launched.Output); progressErr == nil && progress != nil {
_ = runtime.store.UpdateRunProgress(ctx, run.ID, progress)
}
parsed, err := parseExecutionOutput(launched.Output, node.Data.Label)
if err == nil {
parsed.Artifacts, err = runtime.normalizeOutputArtifacts(ctx, workflow, run, node.ID, parsed.Artifacts)
}
if err == nil {
parsed.StageManifestPath = workspace.StageManifestPath
parsed.DataRefs, err = loadStageManifestDataRefs(workspace.StageManifestPath)
if err == nil && parsed.DataMetrics == nil {
parsed.DataMetrics, err = loadStageManifestDataMetrics(workspace.StageManifestPath)
}
}
// Custom-container operators bypass operator-runner's ResourceSampler, so they
// emit no resourceSeries in result.json. The launcher samples their GPU usage
// itself (see k8s.Launch); fall back to those platform-collected samples when
// the operator did not self-report. Builtin operator-runner pods self-report a
// non-empty series, so their own (more complete, CPU+memory included) samples
// are preserved.
if len(parsed.ResourceSeries) == 0 && len(launched.ResourceSeries) > 0 {
parsed.ResourceSeries = make([]domain.ResourceSample, 0, len(launched.ResourceSeries))
for _, sample := range launched.ResourceSeries {
parsed.ResourceSeries = append(parsed.ResourceSeries, domain.ResourceSample{
Timestamp: sample.Timestamp,
GPU: sample.GPU,
GPUMemoryUsedMiB: sample.GPUMemoryUsedMiB,
GPUMemoryTotalMiB: sample.GPUMemoryTotalMiB,
GPUName: sample.GPUName,
GPUCount: sample.GPUCount,
})
}
}
return parsed, launched.Reference, err
}