2026-08-18 11:07:09 +08:00

789 lines
25 KiB
Go

package worker
import (
"context"
"encoding/json"
"fmt"
"maps"
"os"
"path/filepath"
"regexp"
"strings"
"time"
"droboticflow/internal/domain"
"droboticflow/internal/k8s"
"droboticflow/internal/lightweight"
)
const resultPrefix = "DROBOTICFLOW_RESULT="
const resultPathPrefix = "DROBOTICFLOW_RESULT_PATH="
const inputArtifactsInlineLimit = 64 * 1024
type nodeWorkspace struct {
RunRoot string
Root string
OutputDir string
InputArtifactsPath string
ResultPath string
StageManifestPath string
}
type executionResult struct {
Logs []domain.LogEntryRecord `json:"logs"`
Artifacts []domain.ExecutionArtifact `json:"artifacts"`
DataRefs []domain.DataRefRecord `json:"dataRefs,omitempty"`
DataMetrics *domain.DataMetricsRecord `json:"dataMetrics,omitempty"`
StageManifestPath string `json:"stageManifestPath,omitempty"`
ResourceSeries []domain.ResourceSample `json:"resourceSeries"`
}
type stageManifestRecord struct {
DataRefs []domain.DataRefRecord `json:"dataRefs"`
DataMetrics *domain.DataMetricsRecord `json:"dataMetrics,omitempty"`
}
var protectedSystemEnv = map[string]struct{}{
"WORKFLOW_PREVIOUS_RUN_TIMESTAMP": {},
"WORKFLOW_CURRENT_RUN_TIMESTAMP": {},
"PARENT_RUN_ID": {},
"PARENT_RUN_WORKSPACE_ROOT": {},
"PARENT_NODE_OUTPUT_DIR": {},
"OPERATOR_ALLOW_SHELL_COMMANDS": {},
"OPERATOR_COMMAND_ALLOWLIST_REGEX": {},
"OPERATOR_COMMAND_DENYLIST_REGEX": {},
}
var operatorCommandPolicyEnvKeys = []string{
"OPERATOR_ALLOW_SHELL_COMMANDS",
"OPERATOR_COMMAND_ALLOWLIST_REGEX",
"OPERATOR_COMMAND_DENYLIST_REGEX",
}
var workflowEnvLocation = loadWorkflowEnvLocation()
var invalidEnvCharacters = regexp.MustCompile(`[^A-Z0-9_]+`)
func loadWorkflowEnvLocation() *time.Location {
location, err := time.LoadLocation("Asia/Shanghai")
if err != nil {
return time.FixedZone("Asia/Shanghai", 8*60*60)
}
return location
}
func sanitizeEnvKey(value string) string {
normalized := strings.ToUpper(strings.TrimSpace(value))
if normalized == "" {
return ""
}
normalized = invalidEnvCharacters.ReplaceAllString(normalized, "_")
normalized = strings.Trim(normalized, "_")
if normalized == "" {
return ""
}
if normalized[0] >= '0' && normalized[0] <= '9' {
normalized = "HP_" + normalized
}
return normalized
}
func cloneArtifacts(items []domain.ExecutionArtifact) []domain.ExecutionArtifact {
cloned := make([]domain.ExecutionArtifact, 0, len(items))
for _, item := range items {
cloned = append(cloned, cloneArtifact(item))
}
return cloned
}
func routeAuditArtifacts(
node domain.WorkflowNode,
inputArtifacts []domain.ExecutionArtifact,
edges []domain.WorkflowEdge,
) []domain.ExecutionArtifact {
handles := make([]string, 0, len(edges))
seenHandles := map[string]struct{}{}
for _, edge := range edges {
handle := strings.TrimSpace(edge.SourceHandle)
if handle == "" {
continue
}
if _, exists := seenHandles[handle]; exists {
continue
}
seenHandles[handle] = struct{}{}
handles = append(handles, handle)
}
if len(handles) == 0 {
return cloneArtifacts(inputArtifacts)
}
portNames := make(map[string]string, len(node.Data.Outputs))
for _, output := range node.Data.Outputs {
portNames[strings.TrimSpace(output.ID)] = strings.TrimSpace(output.Name)
}
routed := make([]domain.ExecutionArtifact, 0, len(inputArtifacts)*len(handles))
for _, handle := range handles {
portName := portNames[handle]
if portName == "" {
portName = handle
}
for _, input := range inputArtifacts {
artifact := cloneArtifact(input)
artifact.PortID = handle
artifact.PortName = portName
routed = append(routed, artifact)
}
}
return routed
}
func cloneStringSlice(items []string) []string {
if items == nil {
return nil
}
cloned := make([]string, len(items))
copy(cloned, items)
return cloned
}
func cloneArtifact(item domain.ExecutionArtifact) domain.ExecutionArtifact {
cloned := item
if item.Metadata != nil {
metadata := *item.Metadata
cloned.Metadata = &metadata
}
if item.Data != nil {
cloned.Data = maps.Clone(item.Data)
}
return cloned
}
func cloneInputLineage(items []domain.InputLineageRecord) []domain.InputLineageRecord {
cloned := make([]domain.InputLineageRecord, 0, len(items))
for _, item := range items {
record := item
record.Artifact = cloneArtifact(item.Artifact)
cloned = append(cloned, record)
}
return cloned
}
func runTimestampEnvValue(value string) string {
trimmed := strings.TrimSpace(value)
if trimmed == "" {
return ""
}
parsed, err := time.Parse(time.RFC3339Nano, trimmed)
if err != nil {
return ""
}
return parsed.In(workflowEnvLocation).Format(time.RFC3339)
}
func firstNonEmpty(values ...string) string {
for _, value := range values {
if strings.TrimSpace(value) != "" {
return value
}
}
return ""
}
func inputArtifactLineageKey(item domain.InputLineageRecord) string {
artifact := item.Artifact
key := artifact.ID
if key == "" {
key = artifact.NodeID + ":" + artifact.Name
}
return fmt.Sprintf("%s:%s:%s:%s:%s", item.WorkflowEdgeID, item.SourceNodeID, item.SourceHandle, item.TargetHandle, key)
}
func gatherInputLineage(workflow domain.WorkflowRecord, executionMap map[string]domain.NodeExecutionRecord, nodeID string) []domain.InputLineageRecord {
items := make([]domain.InputLineageRecord, 0)
seen := map[string]struct{}{}
for _, edge := range domain.GetIncomingEdges(workflow.Edges, nodeID) {
execution, ok := executionMap[edge.Source]
if !ok {
continue
}
requiresHandle := strings.TrimSpace(edge.SourceHandle) != ""
hasPortAwareArtifacts := false
for _, artifact := range execution.Artifacts {
if strings.TrimSpace(artifact.PortID) != "" {
hasPortAwareArtifacts = true
break
}
}
for _, artifact := range execution.Artifacts {
if requiresHandle && hasPortAwareArtifacts && artifact.PortID != edge.SourceHandle {
continue
}
item := domain.InputLineageRecord{
WorkflowEdgeID: edge.ID,
SourceNodeID: edge.Source,
SourceHandle: edge.SourceHandle,
TargetNodeID: edge.Target,
TargetHandle: edge.TargetHandle,
Artifact: artifact,
}
key := inputArtifactLineageKey(item)
if _, exists := seen[key]; exists {
continue
}
seen[key] = struct{}{}
items = append(items, item)
}
}
return items
}
func inputArtifactsFromLineage(lineage []domain.InputLineageRecord) []domain.ExecutionArtifact {
items := make([]domain.ExecutionArtifact, 0)
seen := map[string]struct{}{}
for _, item := range lineage {
artifact := item.Artifact
key := artifact.ID
if key == "" {
key = artifact.NodeID + ":" + artifact.Name
}
if _, exists := seen[key]; exists {
continue
}
seen[key] = struct{}{}
items = append(items, artifact)
}
return items
}
func gatherInputArtifacts(workflow domain.WorkflowRecord, executionMap map[string]domain.NodeExecutionRecord, nodeID string) []domain.ExecutionArtifact {
return inputArtifactsFromLineage(gatherInputLineage(workflow, executionMap, nodeID))
}
func extractConfidence(artifact domain.ExecutionArtifact) (float64, bool) {
if artifact.Data == nil {
return 0, false
}
for _, key := range []string{"confidence", "score"} {
value, ok := artifact.Data[key]
if !ok {
continue
}
switch typed := value.(type) {
case float64:
return typed, true
case int:
return float64(typed), true
}
}
return 0, false
}
func extractObjectHints(artifact domain.ExecutionArtifact) []string {
if artifact.Data == nil {
return nil
}
items := make([]string, 0)
for _, key := range []string{"detectedObjects", "labels", "entries"} {
value, ok := artifact.Data[key]
if !ok {
continue
}
switch typed := value.(type) {
case []any:
for _, candidate := range typed {
text := strings.TrimSpace(fmt.Sprint(candidate))
if text != "" {
items = append(items, text)
}
}
case []string:
for _, candidate := range typed {
if trimmed := strings.TrimSpace(candidate); trimmed != "" {
items = append(items, trimmed)
}
}
}
}
return items
}
func createAuditPayload(node domain.WorkflowNode, inputArtifacts []domain.ExecutionArtifact) domain.AuditTaskPayload {
payload := domain.AuditTaskPayload{
Reason: fmt.Sprintf("%s requires explicit approval before downstream execution", node.Data.Label),
DetectedObjects: []string{},
}
for _, artifact := range inputArtifacts {
if payload.Image == "" {
switch {
case artifact.Metadata != nil && artifact.Metadata.Thumbnail != "":
payload.Image = artifact.Metadata.Thumbnail
case artifact.Type == domain.PortTypeImage || artifact.Type == domain.PortTypeVideo:
payload.Image = artifact.Path
}
}
if payload.Confidence == 0 {
if confidence, ok := extractConfidence(artifact); ok {
payload.Confidence = confidence
}
}
payload.DetectedObjects = append(payload.DetectedObjects, extractObjectHints(artifact)...)
}
if len(payload.DetectedObjects) == 0 {
payload.DetectedObjects = []string{fmt.Sprintf("%d upstream artifact(s)", len(inputArtifacts))}
}
return payload
}
func prepareNodeWorkspace(workspaceRoot string, runID string, nodeID string) (nodeWorkspace, error) {
root := strings.TrimSpace(workspaceRoot)
if root == "" {
root = filepath.Join(os.TempDir(), "droboticflow")
}
runRoot := filepath.Join(root, "system", "runs", runID)
nodeRoot := filepath.Join(runRoot, "nodes", nodeID)
outputDir := filepath.Join(nodeRoot, "outputs")
if err := os.MkdirAll(outputDir, 0o755); err != nil {
return nodeWorkspace{}, err
}
return nodeWorkspace{
RunRoot: runRoot,
Root: nodeRoot,
OutputDir: outputDir,
InputArtifactsPath: filepath.Join(nodeRoot, "input-artifacts.json"),
ResultPath: filepath.Join(nodeRoot, "result.json"),
StageManifestPath: filepath.Join(nodeRoot, "stage-manifest.json"),
}, nil
}
func shouldInjectPlatformMongoEnv(node domain.WorkflowNode) bool {
label := strings.ToLower(strings.TrimSpace(node.Data.Label))
return strings.Contains(label, "mongodb") || strings.Contains(label, "mongo")
}
func writeJSONFile(path string, value any) error {
data, err := json.Marshal(value)
if err != nil {
return err
}
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
return err
}
return os.WriteFile(path, data, 0o644)
}
func applyHyperparamsToEnv(env map[string]string, items []domain.HyperParam) {
for _, item := range items {
key := sanitizeEnvKey(item.Key)
if key == "" {
continue
}
if _, blocked := protectedSystemEnv[key]; blocked {
continue
}
env[key] = item.Value
}
}
func applyOperatorCommandPolicyEnv(env map[string]string) {
for _, key := range operatorCommandPolicyEnvKeys {
if value := strings.TrimSpace(os.Getenv(key)); value != "" {
env[key] = value
}
}
}
func buildNodeEnv(run *domain.RunRecord, workflow *domain.WorkflowRecord, node domain.WorkflowNode, inputArtifacts []domain.ExecutionArtifact, workspace nodeWorkspace, settings *domain.SettingsRecord) (map[string]string, error) {
return buildNodeEnvWithResources(run, workflow, node, inputArtifacts, workspace, effectiveNodeResources{
CPULimit: node.Data.CPU,
MemoryLimit: node.Data.Memory,
GPULimit: node.Data.GPU,
})
}
type effectiveNodeResources struct {
CPURequest string
CPULimit string
MemoryRequest string
MemoryLimit string
GPULimit string
SharedMemorySize string
}
func buildNodeEnvWithResources(run *domain.RunRecord, workflow *domain.WorkflowRecord, node domain.WorkflowNode, inputArtifacts []domain.ExecutionArtifact, workspace nodeWorkspace, resources effectiveNodeResources) (map[string]string, error) {
if inputArtifacts == nil {
inputArtifacts = []domain.ExecutionArtifact{}
}
nodeDataJSON, err := json.Marshal(node.Data)
if err != nil {
return nil, err
}
inputArtifactsJSON, err := json.Marshal(inputArtifacts)
if err != nil {
return nil, err
}
if err := writeJSONFile(workspace.InputArtifactsPath, inputArtifacts); err != nil {
return nil, err
}
env := map[string]string{
"RUN_ID": run.ID,
"WORKFLOW_ID": run.WorkflowID,
"NODE_ID": node.ID,
"NODE_LABEL": node.Data.Label,
"NODE_TYPE": node.Type,
"NODE_CATEGORY": node.Data.Category,
"NODE_DESCRIPTION": node.Data.Description,
"NODE_DATA_JSON": string(nodeDataJSON),
"INPUT_ARTIFACTS_PATH": workspace.InputArtifactsPath,
"RESULT_OUTPUT_PATH": workspace.ResultPath,
"STAGE_MANIFEST_PATH": workspace.StageManifestPath,
"OUTPUT_DIR": workspace.OutputDir,
"WORKSPACE_ROOT": workspace.RunRoot,
}
if parentRunID := strings.TrimSpace(run.ParentRunID); parentRunID != "" {
parentRunRoot := filepath.Join(filepath.Dir(workspace.RunRoot), filepath.Base(parentRunID))
env["PARENT_RUN_ID"] = parentRunID
env["PARENT_RUN_WORKSPACE_ROOT"] = parentRunRoot
env["PARENT_NODE_OUTPUT_DIR"] = filepath.Join(parentRunRoot, "nodes", node.ID, "outputs")
}
if run.Trigger == domain.RunTriggerScheduled {
env["WORKFLOW_CURRENT_RUN_TIMESTAMP"] = runTimestampEnvValue(firstNonEmpty(run.CurrentRunAt, run.CreatedAt))
env["WORKFLOW_PREVIOUS_RUN_TIMESTAMP"] = runTimestampEnvValue(run.PreviousRunAt)
}
if len(inputArtifactsJSON) <= inputArtifactsInlineLimit {
env["INPUT_ARTIFACTS_JSON"] = string(inputArtifactsJSON)
}
if node.Data.Bucket != "" {
env["STORAGE_BUCKET"] = node.Data.Bucket
}
if node.Data.Prefix != "" {
env["STORAGE_PREFIX"] = node.Data.Prefix
}
if node.Data.Pattern != "" {
env["STORAGE_PATTERN"] = node.Data.Pattern
}
if resources.CPULimit != "" {
env["CPU_LIMIT"] = resources.CPULimit
}
if resources.MemoryLimit != "" {
env["MEMORY_LIMIT"] = resources.MemoryLimit
}
if resources.GPULimit != "" {
env["GPU_LIMIT"] = resources.GPULimit
}
if node.Data.Image != "" {
env["NODE_IMAGE"] = node.Data.Image
}
if handler := runtimeContractHandler(node.Data.RuntimeContract); handler != "" {
env["DROBOTICFLOW_OPERATOR_HANDLER"] = handler
}
if shouldInjectPlatformMongoEnv(node) {
if value := strings.TrimSpace(os.Getenv("MONGO_URI")); value != "" {
env["PLATFORM_MONGO_URI"] = value
}
if value := strings.TrimSpace(os.Getenv("MONGO_DB")); value != "" {
env["PLATFORM_MONGO_DB"] = value
}
}
if workflow != nil {
applyHyperparamsToEnv(env, workflow.Hyperparams)
}
applyHyperparamsToEnv(env, node.Data.Hyperparams)
applyOperatorCommandPolicyEnv(env)
return env, nil
}
func runtimeContractHandler(contract map[string]any) string {
value, ok := contract["handler"]
if !ok {
return ""
}
handler := strings.TrimSpace(fmt.Sprint(value))
if strings.HasPrefix(handler, "builtin_") {
return strings.TrimPrefix(handler, "builtin_")
}
return handler
}
func runtimeContractLongRunning(contract map[string]any) bool {
value, ok := contract["longRunning"].(bool)
return ok && value
}
func lightweightOperatorMount(contract map[string]any) (string, string) {
config := lightweight.ConfigFromContract(contract)
if !config.Enabled {
return "", ""
}
return config.CodeSubPath, config.MountPath
}
func runtimeContractString(items map[string]any, key string) string {
value, ok := items[key]
if !ok || value == nil {
return ""
}
return fmt.Sprint(value)
}
func operatorResourceDefaults(ctx context.Context, dataStore interface {
GetOperator(context.Context, string) (*domain.OperatorRecord, error)
}, node domain.WorkflowNode) *domain.OperatorResources {
operatorID := strings.TrimSpace(node.Data.OperatorID)
if operatorID == "" || dataStore == nil {
return nil
}
operator, err := dataStore.GetOperator(ctx, operatorID)
if err != nil || operator == nil {
return nil
}
return operator.Resources
}
func resolveEffectiveNodeResources(node domain.WorkflowNode, settings *domain.SettingsRecord, operatorResources *domain.OperatorResources) effectiveNodeResources {
defaults := domain.DefaultResources{}
if settings != nil {
defaults = settings.Defaults
}
resources := effectiveNodeResources{
CPURequest: defaults.CPURequest,
CPULimit: defaults.CPULimit,
MemoryRequest: defaults.MemoryRequest,
MemoryLimit: defaults.MemoryLimit,
GPULimit: defaults.GPULimit,
}
if operatorResources != nil {
resources.CPURequest = firstNonEmpty(operatorResources.CPURequest, resources.CPURequest)
resources.CPULimit = firstNonEmpty(operatorResources.CPULimit, resources.CPULimit)
resources.MemoryRequest = firstNonEmpty(operatorResources.MemoryRequest, resources.MemoryRequest)
resources.MemoryLimit = firstNonEmpty(operatorResources.MemoryLimit, resources.MemoryLimit)
resources.GPULimit = firstNonEmpty(operatorResources.GPULimit, resources.GPULimit)
resources.SharedMemorySize = firstNonEmpty(operatorResources.SharedMemorySize, resources.SharedMemorySize)
}
resources.CPULimit = firstNonEmpty(node.Data.CPU, resources.CPULimit)
resources.MemoryLimit = firstNonEmpty(node.Data.Memory, resources.MemoryLimit)
resources.GPULimit = firstNonEmpty(node.Data.GPU, resources.GPULimit)
resources.SharedMemorySize = firstNonEmpty(node.Data.SharedMemorySize, resources.SharedMemorySize)
return resources
}
func parseExecutionOutput(stdout string, nodeLabel string) (executionResult, error) {
result := executionResult{
Logs: []domain.LogEntryRecord{},
Artifacts: []domain.ExecutionArtifact{},
ResourceSeries: []domain.ResourceSample{},
}
lines := strings.Split(strings.ReplaceAll(stdout, "\r\n", "\n"), "\n")
structuredPayload := ""
structuredPayloadPath := ""
rawLogs := make([]domain.LogEntryRecord, 0)
for _, line := range lines {
trimmed := strings.TrimSpace(line)
if trimmed == "" {
continue
}
if strings.HasPrefix(trimmed, resultPrefix) {
structuredPayload = strings.TrimPrefix(trimmed, resultPrefix)
continue
}
if strings.HasPrefix(trimmed, resultPathPrefix) {
structuredPayloadPath = strings.TrimSpace(strings.TrimPrefix(trimmed, resultPathPrefix))
continue
}
rawLogs = append(rawLogs, logEntry("INFO", trimmed, nodeLabel))
}
if structuredPayloadPath != "" {
payloadBytes, err := os.ReadFile(structuredPayloadPath)
if err != nil {
result.Logs = rawLogs
return result, fmt.Errorf("read operator result path: %w", err)
}
structuredPayload = strings.TrimSpace(string(payloadBytes))
}
if structuredPayload == "" {
result.Logs = rawLogs
return result, fmt.Errorf("operator execution did not emit %s or %s payload", resultPrefix, resultPathPrefix)
}
var parsed executionResult
if err := json.Unmarshal([]byte(structuredPayload), &parsed); err != nil {
return result, fmt.Errorf("decode operator result: %w", err)
}
if len(parsed.Logs) > 0 {
result.Logs = append(result.Logs, parsed.Logs...)
} else {
result.Logs = append(result.Logs, rawLogs...)
}
result.Artifacts = parsed.Artifacts
result.ResourceSeries = parsed.ResourceSeries
return result, nil
}
func loadStageManifestDataRefs(path string) ([]domain.DataRefRecord, error) {
trimmed := strings.TrimSpace(path)
if trimmed == "" {
return nil, nil
}
payload, err := os.ReadFile(trimmed)
if err != nil {
return nil, err
}
var manifest stageManifestRecord
if err := json.Unmarshal(payload, &manifest); err != nil {
return nil, err
}
if manifest.DataRefs == nil {
return []domain.DataRefRecord{}, nil
}
return manifest.DataRefs, nil
}
func loadStageManifestDataMetrics(path string) (*domain.DataMetricsRecord, error) {
trimmed := strings.TrimSpace(path)
if trimmed == "" {
return nil, nil
}
payload, err := os.ReadFile(trimmed)
if err != nil {
return nil, err
}
var manifest stageManifestRecord
if err := json.Unmarshal(payload, &manifest); err != nil {
return nil, err
}
return manifest.DataMetrics, nil
}
func (runtime *Runtime) resolveNodeBusinessStage(ctx context.Context, run *domain.RunRecord, node domain.WorkflowNode) domain.WorkflowNode {
if run != nil && strings.TrimSpace(run.TaskID) != "" {
if task, err := runtime.store.GetCollectionTask(ctx, run.TaskID); err == nil && task != nil {
for stage, nodeID := range task.StageBindings {
if strings.TrimSpace(nodeID) == node.ID {
node.Data.BusinessStage = stage
node.Data.MetricBoundary = true
return node
}
}
}
}
if node.Data.BusinessStage != "" {
node.Data.MetricBoundary = true
return node
}
if strings.TrimSpace(node.Data.OperatorID) != "" {
if operator, err := runtime.store.GetOperator(ctx, node.Data.OperatorID); err == nil && operator != nil && operator.BusinessStage != "" {
node.Data.BusinessStage = operator.BusinessStage
node.Data.MetricBoundary = true
}
}
return node
}
func (runtime *Runtime) executeNode(ctx context.Context, run *domain.RunRecord, node domain.WorkflowNode, inputArtifacts []domain.ExecutionArtifact) (executionResult, *k8s.JobReference, error) {
node = runtime.resolveNodeBusinessStage(ctx, run, node)
workspace, err := prepareNodeWorkspace(runtime.workspaceRoot, run.ID, node.ID)
if err != nil {
return executionResult{}, nil, err
}
workflow, workflowErr := runtime.store.GetWorkflow(ctx, run.WorkflowID)
if workflowErr != nil {
workflow = nil
}
settings, err := runtime.store.GetSettings(ctx)
if err != nil {
return executionResult{}, nil, err
}
resources := resolveEffectiveNodeResources(node, settings, operatorResourceDefaults(ctx, runtime.store, node))
lightweightConfig := lightweight.ConfigFromContract(node.Data.RuntimeContract)
if lightweightConfig.Enabled {
if err := lightweight.ValidateRuntimeConfig(lightweight.RuntimeConfig{
NodeID: node.ID,
NodeLabel: node.Data.Label,
Image: node.Data.Image,
CodeSubPath: lightweightConfig.CodeSubPath,
MountPath: lightweightConfig.MountPath,
Command: node.Data.ContainerCommand,
Args: node.Data.ContainerArgs,
ArgsConfigured: lightweightConfig.ContainerArgsConfigured,
WorkspaceRoot: runtime.workspaceRoot,
Mounts: lightweightConfig.Mounts,
}); err != nil {
return executionResult{}, nil, err
}
}
env, err := buildNodeEnvWithResources(run, workflow, node, inputArtifacts, workspace, resources)
if err != nil {
return executionResult{}, nil, err
}
codeSubPath, codeMountPath := lightweightConfig.CodeSubPath, lightweightConfig.MountPath
if codeSubPath != "" {
env["LIGHTWEIGHT_OPERATOR_CODE_SUBPATH"] = codeSubPath
env["LIGHTWEIGHT_OPERATOR_CODE_DIR"] = codeMountPath
}
pvcMounts := make([]k8s.PVCMount, 0, len(lightweightConfig.Mounts))
for _, item := range lightweightConfig.Mounts {
pvcMounts = append(pvcMounts, k8s.PVCMount{Name: item.Name, PVCName: item.PVCName, SubPath: item.SubPath, MountPath: item.MountPath, ReadOnly: item.ReadOnly})
}
nodeTimeoutSeconds := 0
if workflow != nil {
nodeTimeoutSeconds = workflow.NodeTimeoutSeconds
}
launched, err := runtime.launcher.Launch(ctx, k8s.LaunchInput{
Image: chooseImage(node, runtime.operatorRunnerImage),
Command: cloneStringSlice(node.Data.ContainerCommand),
Args: cloneStringSlice(node.Data.ContainerArgs),
RunID: run.ID,
WorkflowID: run.WorkflowID,
NodeID: node.ID,
NodeLabel: node.Data.Label,
Attempt: run.Attempt,
TimeoutSeconds: nodeTimeoutSeconds,
Env: env,
CPURequest: resources.CPURequest,
CPULimit: resources.CPULimit,
MemoryRequest: resources.MemoryRequest,
MemoryLimit: resources.MemoryLimit,
GPULimit: resources.GPULimit,
SharedMemorySize: resources.SharedMemorySize,
LongRunning: runtimeContractLongRunning(node.Data.RuntimeContract),
CodeSubPath: codeSubPath,
CodeMountPath: codeMountPath,
CodeReadOnly: lightweightConfig.CodeReadOnly,
PVCMounts: pvcMounts,
OperatorRunnerImage: runtime.operatorRunnerImage,
})
if err != nil {
return executionResult{}, nil, err
}
if progress, progressErr := latestStructuredProgress(launched.Output); progressErr == nil && progress != nil {
_ = runtime.store.UpdateRunProgress(ctx, run.ID, progress)
}
parsed, err := parseExecutionOutput(launched.Output, node.Data.Label)
if err == nil {
parsed.StageManifestPath = workspace.StageManifestPath
parsed.DataRefs, err = loadStageManifestDataRefs(workspace.StageManifestPath)
if err == nil && parsed.DataMetrics == nil {
parsed.DataMetrics, err = loadStageManifestDataMetrics(workspace.StageManifestPath)
}
}
// Custom-container operators bypass operator-runner's ResourceSampler, so they
// emit no resourceSeries in result.json. The launcher samples their GPU usage
// itself (see k8s.Launch); fall back to those platform-collected samples when
// the operator did not self-report. Builtin operator-runner pods self-report a
// non-empty series, so their own (more complete, CPU+memory included) samples
// are preserved.
if len(parsed.ResourceSeries) == 0 && len(launched.ResourceSeries) > 0 {
parsed.ResourceSeries = make([]domain.ResourceSample, 0, len(launched.ResourceSeries))
for _, sample := range launched.ResourceSeries {
parsed.ResourceSeries = append(parsed.ResourceSeries, domain.ResourceSample{
Timestamp: sample.Timestamp,
GPU: sample.GPU,
GPUMemoryUsedMiB: sample.GPUMemoryUsedMiB,
GPUMemoryTotalMiB: sample.GPUMemoryTotalMiB,
GPUName: sample.GPUName,
GPUCount: sample.GPUCount,
})
}
}
return parsed, launched.Reference, err
}