package worker import ( "context" "encoding/json" "errors" "fmt" "math" "os" "reflect" "sort" "strconv" "strings" "sync" "sync/atomic" "time" "droboticflow/internal/dataops" "droboticflow/internal/domain" "droboticflow/internal/k8s" "droboticflow/internal/lightweight" "droboticflow/internal/objectsync" "droboticflow/internal/store" ) type jobLauncher interface { Launch(context.Context, k8s.LaunchInput) (*k8s.LaunchResult, error) Inspect(context.Context, string, string) (*k8s.JobInspection, error) TerminateRun(context.Context, string, string) error // SamplePodGPUForRun samples GPU usage inside the pod backing runID+nodeID. // Used by the worker's progress reporter to stream live GPU metrics for // custom-container operators during execution. Gracefully errors when no // Running pod exists or pods/exec RBAC is unavailable. SamplePodGPUForRun(context.Context, string, string) (k8s.GPUSample, error) } type gpuCapacityProvider interface { GPUCapacity(context.Context) (*k8s.GPUCapacitySnapshot, error) } type orphanRuntimeReconciler interface { ReconcileOrphanedRuntimes(context.Context, map[string]struct{}, time.Time, bool) ([]k8s.OrphanRuntime, error) } type nodePodLogTailer interface { TailNodePodLogs(context.Context, string, string, int64) (string, error) } type BuildInfo struct { BuildCommit string `json:"buildCommit,omitempty"` BuildTime string `json:"buildTime,omitempty"` Image string `json:"image,omitempty"` } type ReadinessSnapshot struct { ReadinessWindowMS int64 `json:"readinessWindowMs,omitempty"` Ready bool `json:"ready"` Status string `json:"status"` Service string `json:"service"` BuildCommit string `json:"buildCommit,omitempty"` BuildTime string `json:"buildTime,omitempty"` Image string `json:"image,omitempty"` RunnerImage string `json:"runnerImage,omitempty"` RunnerBuildCommit string `json:"runnerBuildCommit,omitempty"` RunnerBuildTime string `json:"runnerBuildTime,omitempty"` LastCycleAt string `json:"lastCycleAt,omitempty"` LastSuccessfulCycleAt string `json:"lastSuccessfulCycleAt,omitempty"` CycleDurationMS uint64 `json:"cycleDurationMs"` ConfiguredPollIntervalMS int64 `json:"configuredPollIntervalMs"` EffectivePollIntervalMS int64 `json:"effectivePollIntervalMs"` StuckRunCount uint64 `json:"stuckRunCount"` OrphanRuntimeCount uint64 `json:"orphanRuntimeCount"` OrphanRuntimeCleanedCount uint64 `json:"orphanRuntimeCleanedCount"` OrphanRuntimeCleanupErrorCount uint64 `json:"orphanRuntimeCleanupErrorCount"` Reason string `json:"reason,omitempty"` } type Runtime struct { captureStatusCursor string captureStatusMu sync.Mutex store store.Store operatorRunnerImage string workspaceRoot string globalBOSEndpoint string artifactPublisher objectsync.Publisher qwenNodeCache qwenNodeCacheConfig launcher jobLauncher metrics WorkerMetrics processMu sync.Mutex activeRuns map[string]struct{} dispatchOwner string runLeaseTTL time.Duration buildMu sync.RWMutex buildInfo BuildInfo runnerBuildInfo BuildInfo configuredPollMS atomic.Int64 effectivePollMS atomic.Int64 orphanCleanup bool orphanMinimumAge time.Duration orphanCheckInterval time.Duration lastOrphanCheckNano atomic.Int64 pendingFailureAfter time.Duration missingMaxRecoveries int recoveryReservedSlots int runArchiveMinimumAge time.Duration runArchiveInterval time.Duration lastRunArchiveNano atomic.Int64 materializationCleanup materializationCleanupProgress } type ProcessOnceStatus string const ( ProcessOnceStatusClaimed ProcessOnceStatus = "claimed" ProcessOnceStatusSkippedLeaseActive ProcessOnceStatus = "skipped_lease_active" ProcessOnceStatusNoQueuedRun ProcessOnceStatus = "no_queued_run" ) var errRunLeaseLost = errors.New("run execution lease lost") type ProcessOnceResult struct { Status ProcessOnceStatus `json:"status"` Run *domain.RunRecord `json:"-"` RunRef *ProcessOnceRunSummary `json:"run,omitempty"` } type ProcessOnceRunSummary struct { ID string `json:"id"` Status domain.RunStatus `json:"status"` WorkflowID string `json:"workflowId,omitempty"` WorkflowName string `json:"workflowName,omitempty"` ProjectID string `json:"projectId,omitempty"` ProjectName string `json:"projectName,omitempty"` CurrentNodeID string `json:"currentNodeId,omitempty"` NodeExecutionCount int `json:"nodeExecutionCount"` ArtifactCount int `json:"artifactCount"` Attempt int `json:"attempt,omitempty"` LeaseOwner string `json:"executionLeaseOwner,omitempty"` LeaseExpiresAt string `json:"executionLeaseExpiresAt,omitempty"` UpdatedAt string `json:"updatedAt,omitempty"` } func newProcessOnceResult(status ProcessOnceStatus, run *domain.RunRecord) *ProcessOnceResult { result := &ProcessOnceResult{Status: status, Run: run} if run == nil { return result } artifactCount := len(run.Artifacts) for _, execution := range run.NodeExecutions { artifactCount += len(execution.Artifacts) } result.RunRef = &ProcessOnceRunSummary{ ID: run.ID, Status: run.Status, WorkflowID: run.WorkflowID, WorkflowName: run.WorkflowName, ProjectID: run.ProjectID, ProjectName: run.ProjectName, CurrentNodeID: run.CurrentNodeID, NodeExecutionCount: len(run.NodeExecutions), ArtifactCount: artifactCount, Attempt: run.Attempt, LeaseOwner: run.ExecutionLeaseOwner, LeaseExpiresAt: run.ExecutionLeaseExpiresAt, UpdatedAt: run.UpdatedAt, } return result } var globalActiveDispatchStatuses = map[domain.RunStatus]struct{}{ domain.RunStatusRunning: {}, domain.RunStatusStopping: {}, } var workflowActiveDispatchStatuses = map[domain.RunStatus]struct{}{ domain.RunStatusRunning: {}, domain.RunStatusStopping: {}, } const ( workerQueuedRunLimit = 100 workerReconcileRunLimit = 500 workerWorkflowQueryLimit = 500 ) func NewRuntime(dataStore store.Store, namespace string, imagePullSecretName string, operatorRunnerImage string, workspacePVCName string, workspaceMountPath string, gpuNodeSelector string, gpuTolerationsJSON string, gpuRuntimeClass string, jobTTLSeconds int, globalBOSEndpoint string, qwenNodeCacheEnabled bool, qwenNodeCachePVCName string, qwenNodeCacheMountPath string, qwenNodeCacheNodeSelector string, qwenNodeCacheStaleStagingSeconds int) (*Runtime, error) { launcher, err := k8s.NewLauncher(namespace, imagePullSecretName, workspacePVCName, workspaceMountPath, gpuNodeSelector, gpuTolerationsJSON, gpuRuntimeClass, jobTTLSeconds) if err != nil { return nil, err } qwenNodeCache, err := newQwenNodeCacheConfig(qwenNodeCacheEnabled, qwenNodeCachePVCName, qwenNodeCacheMountPath, qwenNodeCacheNodeSelector, qwenNodeCacheStaleStagingSeconds) if err != nil { return nil, err } trimmedWorkspaceRoot := strings.TrimSpace(workspaceMountPath) if trimmedWorkspaceRoot == "" { trimmedWorkspaceRoot = "/workspace" } return &Runtime{ store: dataStore, operatorRunnerImage: operatorRunnerImage, workspaceRoot: trimmedWorkspaceRoot, globalBOSEndpoint: strings.TrimSpace(globalBOSEndpoint), qwenNodeCache: qwenNodeCache, launcher: launcher, dispatchOwner: defaultDispatchOwner(), runLeaseTTL: defaultRunLeaseTTL(), orphanCleanup: envBool("WORKER_ORPHAN_CLEANUP_ENABLED", false), orphanMinimumAge: envDurationSeconds("WORKER_ORPHAN_MIN_AGE_SECONDS", 24*time.Hour), orphanCheckInterval: envDurationSeconds("WORKER_ORPHAN_CHECK_INTERVAL_SECONDS", 5*time.Minute), pendingFailureAfter: envDurationSeconds("WORKER_PENDING_FAILURE_SECONDS", 10*time.Minute), missingMaxRecoveries: envPositiveInt("WORKER_MISSING_JOB_MAX_RECOVERIES", 3), recoveryReservedSlots: envNonNegativeInt("WORKER_RECOVERY_RESERVED_SLOTS", 1), runArchiveMinimumAge: envDurationSeconds("WORKER_RUN_ARCHIVE_MIN_AGE_SECONDS", 30*24*time.Hour), runArchiveInterval: envDurationSeconds("WORKER_RUN_ARCHIVE_CHECK_INTERVAL_SECONDS", time.Hour), }, nil } func envBool(key string, fallback bool) bool { raw := strings.TrimSpace(os.Getenv(key)) if raw == "" { return fallback } parsed, err := strconv.ParseBool(raw) if err != nil { return fallback } return parsed } func envPositiveInt(key string, fallback int) int { raw := strings.TrimSpace(os.Getenv(key)) if raw == "" { return fallback } parsed, err := strconv.Atoi(raw) if err != nil || parsed <= 0 { return fallback } return parsed } func envNonNegativeInt(key string, fallback int) int { raw := strings.TrimSpace(os.Getenv(key)) if raw == "" { return fallback } parsed, err := strconv.Atoi(raw) if err != nil || parsed < 0 { return fallback } return parsed } func envDurationSeconds(key string, fallback time.Duration) time.Duration { seconds := envPositiveInt(key, int(fallback/time.Second)) return time.Duration(seconds) * time.Second } func (runtime *Runtime) SetBuildInfo(info BuildInfo) { if runtime == nil { return } runtime.buildMu.Lock() runtime.buildInfo = BuildInfo{ BuildCommit: strings.TrimSpace(info.BuildCommit), BuildTime: strings.TrimSpace(info.BuildTime), Image: strings.TrimSpace(info.Image), } runtime.buildMu.Unlock() } func (runtime *Runtime) BuildInfo() BuildInfo { if runtime == nil { return BuildInfo{} } runtime.buildMu.RLock() defer runtime.buildMu.RUnlock() return runtime.buildInfo } func (runtime *Runtime) SetRunnerBuildInfo(info BuildInfo) { if runtime == nil { return } runtime.buildMu.Lock() runtime.runnerBuildInfo = BuildInfo{ BuildCommit: strings.TrimSpace(info.BuildCommit), BuildTime: strings.TrimSpace(info.BuildTime), Image: strings.TrimSpace(info.Image), } runtime.buildMu.Unlock() } func (runtime *Runtime) RunnerBuildInfo() BuildInfo { if runtime == nil { return BuildInfo{} } runtime.buildMu.RLock() defer runtime.buildMu.RUnlock() return runtime.runnerBuildInfo } func runtimeOperatorGeneratedByAI(item domain.OperatorRecord) bool { source := strings.ToLower(strings.TrimSpace(string(item.GenerationSource))) if source == string(domain.OperatorGenerationSourceAI) { return true } if item.RuntimeContract == nil { return false } if sourceValue, ok := item.RuntimeContract["generationSource"].(string); ok && strings.EqualFold(strings.TrimSpace(sourceValue), string(domain.OperatorGenerationSourceAI)) { return true } _, generated := item.RuntimeContract["generatedRepository"] return generated } func runtimeOperatorPublishedStatus(item domain.OperatorRecord) domain.OperatorPublicationStatus { status := domain.OperatorPublicationStatus(strings.ToLower(strings.TrimSpace(string(item.PublicationStatus)))) if status != "" { return status } if runtimeOperatorGeneratedByAI(item) { return domain.OperatorPublicationStatusDraft } return domain.OperatorPublicationStatusPublished } func runtimeNodeGeneratedByAI(node domain.WorkflowNode) bool { return strings.EqualFold(strings.TrimSpace(string(node.Data.GenerationSource)), string(domain.OperatorGenerationSourceAI)) } func runtimeNodePublishedStatus(node domain.WorkflowNode) domain.OperatorPublicationStatus { status := domain.OperatorPublicationStatus(strings.ToLower(strings.TrimSpace(string(node.Data.PublicationStatus)))) if status != "" { return status } if runtimeNodeGeneratedByAI(node) { return domain.OperatorPublicationStatusDraft } return domain.OperatorPublicationStatusPublished } func (runtime *Runtime) workflowPublicationBlocker(ctx context.Context, workflow *domain.WorkflowRecord) string { if workflow == nil || workflow.ExecutionScope == domain.WorkflowExecutionScopeSandbox { return "" } for _, node := range workflow.Nodes { sourceAI := runtimeNodeGeneratedByAI(node) status := runtimeNodePublishedStatus(node) if operatorID := strings.TrimSpace(node.Data.OperatorID); operatorID != "" { operator, err := runtime.store.GetOperator(ctx, operatorID) if err != nil { return "工作流图非法:无法读取算子发布状态。" } if operator != nil { sourceAI = runtimeOperatorGeneratedByAI(*operator) status = runtimeOperatorPublishedStatus(*operator) } } if !sourceAI || status == domain.OperatorPublicationStatusPublished { continue } label := strings.TrimSpace(node.Data.Label) if label == "" { label = node.ID } return "生产工作流不能使用未发布的 AI 生成算子:" + label + "。请在沙盒工作流验证,或发布后再运行。" } return "" } func defaultDispatchOwner() string { hostname, err := os.Hostname() if err != nil || strings.TrimSpace(hostname) == "" { hostname = "unknown-host" } return fmt.Sprintf("%s:%d", hostname, os.Getpid()) } func defaultRunLeaseTTL() time.Duration { raw := strings.TrimSpace(os.Getenv("WORKER_RUN_LEASE_TTL_SECONDS")) if raw == "" { return 15 * time.Minute } seconds, err := strconv.Atoi(raw) if err != nil || seconds <= 0 { return 15 * time.Minute } return time.Duration(seconds) * time.Second } func (runtime *Runtime) leaseOwner() string { if runtime == nil { return "" } if strings.TrimSpace(runtime.dispatchOwner) == "" { runtime.dispatchOwner = defaultDispatchOwner() } return runtime.dispatchOwner } func (runtime *Runtime) leaseTTL() time.Duration { if runtime == nil || runtime.runLeaseTTL <= 0 { return defaultRunLeaseTTL() } return runtime.runLeaseTTL } func (runtime *Runtime) MetricsSnapshot() MetricsSnapshot { return runtime.metrics.Snapshot() } const ( defaultPollIntervalMS int64 = 2000 minimumPollIntervalMS int64 = 250 maximumPollIntervalMS int64 = 60000 ) func normalizedPollIntervalMS(value int64) int64 { if value <= 0 { return defaultPollIntervalMS } if value < minimumPollIntervalMS { return minimumPollIntervalMS } if value > maximumPollIntervalMS { return maximumPollIntervalMS } return value } func (runtime *Runtime) SetConfiguredPollInterval(interval time.Duration) { if runtime == nil { return } configured := normalizedPollIntervalMS(interval.Milliseconds()) runtime.configuredPollMS.Store(configured) if runtime.effectivePollMS.Load() == 0 { runtime.effectivePollMS.Store(configured) } } func (runtime *Runtime) updateEffectivePollInterval(settings *domain.SettingsRecord) { configured := normalizedPollIntervalMS(runtime.configuredPollMS.Load()) effective := configured if settings != nil && settings.Worker.PollIntervalMS > 0 { effective = normalizedPollIntervalMS(int64(settings.Worker.PollIntervalMS)) } runtime.configuredPollMS.Store(configured) runtime.effectivePollMS.Store(effective) } func (runtime *Runtime) EffectivePollInterval() time.Duration { if runtime == nil { return time.Duration(defaultPollIntervalMS) * time.Millisecond } return time.Duration(normalizedPollIntervalMS(runtime.effectivePollMS.Load())) * time.Millisecond } func (runtime *Runtime) RunScheduler(ctx context.Context, configured time.Duration) { if runtime == nil { return } runtime.SetConfiguredPollInterval(configured) ctx, cancel := context.WithCancel(ctx) maintenanceDone := runtime.startMaintenance(ctx) defer func() { cancel(); <-maintenanceDone }() timer := time.NewTimer(0) defer timer.Stop() for { select { case <-ctx.Done(): return case <-timer.C: if err := runtime.ProcessAvailable(ctx); err != nil && !errors.Is(err, context.Canceled) { fmt.Printf("[worker] processAvailable failed: %v\n", err) } timer.Reset(runtime.EffectivePollInterval()) } } } func (runtime *Runtime) statusSnapshot() ReadinessSnapshot { if runtime == nil { return ReadinessSnapshot{Status: "not_ready", Service: "droboticflow-worker", Reason: "worker runtime is not initialized"} } metrics := runtime.MetricsSnapshot() build := runtime.BuildInfo() runnerBuild := runtime.RunnerBuildInfo() configured := normalizedPollIntervalMS(runtime.configuredPollMS.Load()) effective := normalizedPollIntervalMS(runtime.effectivePollMS.Load()) snapshot := ReadinessSnapshot{ Ready: false, Status: "not_ready", Service: "droboticflow-worker", BuildCommit: build.BuildCommit, BuildTime: build.BuildTime, Image: build.Image, RunnerImage: firstNonEmptyRuntimeValue(runnerBuild.Image, runtime.operatorRunnerImage), RunnerBuildCommit: runnerBuild.BuildCommit, RunnerBuildTime: runnerBuild.BuildTime, LastCycleAt: metrics.LastCycleAt, LastSuccessfulCycleAt: metrics.LastSuccessfulCycleAt, CycleDurationMS: metrics.CycleDurationMS, ConfiguredPollIntervalMS: configured, EffectivePollIntervalMS: effective, StuckRunCount: metrics.StuckRunCount, OrphanRuntimeCount: metrics.OrphanRuntimeCount, OrphanRuntimeCleanedCount: metrics.OrphanRuntimeCleanedCount, OrphanRuntimeCleanupErrorCount: metrics.OrphanRuntimeCleanupErrorCount, } return snapshot } func firstNonEmptyRuntimeValue(values ...string) string { for _, value := range values { if trimmed := strings.TrimSpace(value); trimmed != "" { return trimmed } } return "" } func (runtime *Runtime) Readiness(ctx context.Context) ReadinessSnapshot { snapshot := runtime.statusSnapshot() if runtime == nil { return snapshot } if runtime.store == nil || runtime.launcher == nil { snapshot.Reason = "worker runtime is not initialized" return snapshot } if err := runtime.store.Ready(ctx); err != nil { snapshot.Reason = "mongodb is not ready: " + err.Error() return snapshot } lastSuccessful, ok := parseRFC3339Nano(snapshot.LastSuccessfulCycleAt) if !ok { snapshot.Reason = "worker has not completed a successful scheduling cycle" return snapshot } maximumAge := 3 * time.Duration(snapshot.EffectivePollIntervalMS) * time.Millisecond if maximumAge < 30*time.Second { maximumAge = 30 * time.Second } // PollInterval is the idle delay, not the duration of a scheduling cycle. // A healthy large workspace can take longer than 30s per successful cycle. // Use only successful durations and cap this adaptive allowance, so failed // or stalled cycles cannot keep extending their own readiness deadline. observedMS := min(runtime.metrics.lastSuccessfulCycleDurationMS.Load(), uint64(150000)) observedWindow := min(2*time.Duration(observedMS)*time.Millisecond+time.Duration(snapshot.EffectivePollIntervalMS)*time.Millisecond, 5*time.Minute) if observedWindow > maximumAge { maximumAge = observedWindow } snapshot.ReadinessWindowMS = maximumAge.Milliseconds() if time.Since(lastSuccessful) > maximumAge { snapshot.Reason = "last successful scheduling cycle is stale" return snapshot } snapshot.Ready = true snapshot.Status = "ready" return snapshot } func (runtime *Runtime) GPUCapacity(ctx context.Context) (*k8s.GPUCapacitySnapshot, error) { if runtime == nil || runtime.launcher == nil { return nil, fmt.Errorf("worker launcher is not configured") } provider, ok := runtime.launcher.(gpuCapacityProvider) if !ok { return nil, fmt.Errorf("worker launcher does not expose gpu capacity") } return provider.GPUCapacity(ctx) } func nowISO() string { return time.Now().UTC().Format(time.RFC3339Nano) } func logEntry(level string, message string, operator string) domain.LogEntryRecord { return domain.LogEntryRecord{ Timestamp: nowISO(), Level: level, Message: message, Operator: operator, } } func findNodeExecution(run *domain.RunRecord, nodeID string) (int, *domain.NodeExecutionRecord) { for index := range run.NodeExecutions { if run.NodeExecutions[index].NodeID == nodeID { return index, &run.NodeExecutions[index] } } return -1, nil } func appendLog(run *domain.RunRecord, entry domain.LogEntryRecord) { run.Logs = append(run.Logs, entry) } func applyExecution(run *domain.RunRecord, execution domain.NodeExecutionRecord) { index, _ := findNodeExecution(run, execution.NodeID) if index >= 0 { run.NodeExecutions[index] = execution } else { run.NodeExecutions = append(run.NodeExecutions, execution) } if execution.DataMetrics != nil && execution.DataMetrics.Stage == domain.DataMetricStageTrain && len(execution.DataMetrics.TrainingInputs) > 0 { if run.Metadata == nil { run.Metadata = &domain.RunMetadata{} } run.Metadata.TrainingInputs = append([]domain.TrainingInputRecord(nil), execution.DataMetrics.TrainingInputs...) } run.Logs = []domain.LogEntryRecord{} run.ResourceSeries = []domain.ResourceSample{} run.Artifacts = []domain.ExecutionArtifact{} for _, item := range run.NodeExecutions { for _, entry := range item.Logs { appendLog(run, entry) } if len(item.ResourceSeries) > 0 { run.ResourceSeries = append(run.ResourceSeries, item.ResourceSeries...) } if len(item.Artifacts) > 0 { run.Artifacts = append(run.Artifacts, item.Artifacts...) } } } func applyNodeExecutionSnapshot(execution *domain.NodeExecutionRecord, node domain.WorkflowNode, inputArtifacts []domain.ExecutionArtifact, inputLineage []domain.InputLineageRecord, inputBindings []domain.NodeInputBinding) { if execution == nil { return } resolvedImage := chooseImage(node, "") execution.NodeID = node.ID execution.NodeLabel = node.Data.Label execution.OperatorID = strings.TrimSpace(node.Data.OperatorID) execution.OperatorVersion = strings.TrimSpace(node.Data.OperatorVersion) execution.OperatorImage = firstNonEmptyString(resolvedImage, node.Data.Image) execution.OperatorImageID = firstNonEmptyString(node.Data.OperatorImageID, resolvedImage, node.Data.Image) if image := discardPackageImageOverride(node); image != "" { execution.OperatorImageID = image } if execution.Attempt <= 0 { execution.Attempt = 1 } if inputArtifacts != nil { execution.InputArtifacts = cloneArtifacts(inputArtifacts) } if inputLineage != nil { execution.InputLineage = cloneInputLineage(inputLineage) } if inputBindings != nil { execution.InputBindings = cloneInputBindings(inputBindings) } } func globalConcurrencyLimit(settings *domain.SettingsRecord) int { if settings != nil && settings.Worker.MaxConcurrentRuns > 0 { return settings.Worker.MaxConcurrentRuns } return 1 } func effectiveWorkflowConcurrencyLimit(workflow *domain.WorkflowRecord, settings *domain.SettingsRecord) int { if workflow != nil && workflow.MaxConcurrentRuns > 0 { return workflow.MaxConcurrentRuns } return globalConcurrencyLimit(settings) } func parseRFC3339Nano(value string) (time.Time, bool) { trimmed := strings.TrimSpace(value) if trimmed == "" { return time.Time{}, false } parsed, err := time.Parse(time.RFC3339Nano, trimmed) if err != nil { return time.Time{}, false } return parsed, true } func isStaleQueuedNodeExecution(execution domain.NodeExecutionRecord, threshold time.Duration) bool { startedAt, ok := parseRFC3339Nano(execution.StartedAt) if !ok { return false } return time.Since(startedAt) >= threshold } func firstNonEmptyString(values ...string) string { for _, value := range values { if strings.TrimSpace(value) != "" { return value } } return "" } func nextHourlyScheduleTime(schedule *domain.WorkflowSchedule, lastRunAt time.Time, now time.Time) (time.Time, bool) { intervalHours := schedule.IntervalHours if intervalHours <= 0 { return time.Time{}, false } anchorAt, ok := parseRFC3339Nano(schedule.AnchorAt) if !ok { if !lastRunAt.IsZero() { anchorAt = lastRunAt } else { anchorAt = now } } interval := time.Duration(intervalHours) * time.Hour if lastRunAt.IsZero() || lastRunAt.Before(anchorAt) { return anchorAt, true } elapsed := lastRunAt.Sub(anchorAt) slotsElapsed := int(elapsed/interval) + 1 return anchorAt.Add(time.Duration(slotsElapsed) * interval), true } func nextDailyScheduleTime(schedule *domain.WorkflowSchedule, lastRunAt time.Time, now time.Time) (time.Time, bool) { if schedule.DayInterval <= 0 { return time.Time{}, false } parts := strings.Split(strings.TrimSpace(schedule.TimeOfDay), ":") if len(parts) != 2 { return time.Time{}, false } hour, err := strconv.Atoi(parts[0]) if err != nil { return time.Time{}, false } minute, err := strconv.Atoi(parts[1]) if err != nil { return time.Time{}, false } location, err := time.LoadLocation(domain.WorkflowScheduleTimezone(schedule.Timezone)) if err != nil { return time.Time{}, false } anchorAt, ok := parseRFC3339Nano(schedule.AnchorAt) if !ok { if !lastRunAt.IsZero() { anchorAt = lastRunAt } else { anchorAt = now } } anchorLocal := anchorAt.In(location) firstDue := time.Date(anchorLocal.Year(), anchorLocal.Month(), anchorLocal.Day(), hour, minute, 0, 0, location) if anchorLocal.After(firstDue) { firstDue = firstDue.AddDate(0, 0, schedule.DayInterval) } if lastRunAt.IsZero() { return firstDue, true } lastLocal := lastRunAt.In(location) nextDue := firstDue for !nextDue.After(lastLocal) { nextDue = nextDue.AddDate(0, 0, schedule.DayInterval) } return nextDue, true } func shouldEnqueueScheduledRun(workflow domain.WorkflowRecord, now time.Time) bool { if workflow.Type != domain.WorkflowTypeStreaming || !domain.WorkflowScheduleEnabled(workflow.Schedule) { return false } lastRunAt, _ := parseRFC3339Nano(workflow.LastRunAt) switch domain.EffectiveWorkflowScheduleMode(workflow.Schedule) { case domain.WorkflowScheduleModeEveryHours: nextDue, ok := nextHourlyScheduleTime(workflow.Schedule, lastRunAt, now) return ok && !now.Before(nextDue) case domain.WorkflowScheduleModeEveryDaysAtTime: nextDue, ok := nextDailyScheduleTime(workflow.Schedule, lastRunAt, now) return ok && !now.Before(nextDue) case domain.WorkflowScheduleModeIntervalSeconds: if workflow.Schedule.Interval <= 0 { return false } if workflow.LastRunAt == "" { return true } return !lastRunAt.IsZero() && now.Sub(lastRunAt) >= time.Duration(workflow.Schedule.Interval)*time.Second default: return false } } func selectQueuedRunsForDispatch(runs []domain.RunRecord, workflows []domain.WorkflowRecord, settings *domain.SettingsRecord) []string { selected, _, _ := selectQueuedRunsForDispatchWithStats(runs, workflows, settings) return selected } func selectQueuedRunsForDispatchWithStats(runs []domain.RunRecord, workflows []domain.WorkflowRecord, settings *domain.SettingsRecord) ([]string, int, int) { activeByWorkflow := map[string]int{} activeTotal := 0 queued := make([]domain.RunRecord, 0) for _, run := range runs { if _, ok := workflowActiveDispatchStatuses[run.Status]; ok { activeByWorkflow[run.WorkflowID]++ } if _, ok := globalActiveDispatchStatuses[run.Status]; ok { activeTotal++ continue } if run.Status == domain.RunStatusQueued { queued = append(queued, run) } } selected := selectQueuedRunsForDispatchSnapshot(queued, workflows, settings, activeByWorkflow, activeTotal) return selected, len(queued), activeTotal } func selectQueuedRunsForDispatchSnapshot( queued []domain.RunRecord, workflows []domain.WorkflowRecord, settings *domain.SettingsRecord, activeByWorkflow map[string]int, activeTotal int, ) []string { globalLimit := globalConcurrencyLimit(settings) workflowByID := make(map[string]domain.WorkflowRecord, len(workflows)) for _, workflow := range workflows { workflowByID[workflow.ID] = workflow } now := time.Now().UTC() ready := make([]domain.RunRecord, 0, len(queued)) for _, run := range queued { if run.Status == domain.RunStatusQueued && runDispatchReady(run, now) { ready = append(ready, run) } } available := globalLimit - activeTotal if available <= 0 { return nil } sort.Slice(ready, func(left, right int) bool { if ready[left].CreatedAt == ready[right].CreatedAt { return ready[left].ID < ready[right].ID } return ready[left].CreatedAt < ready[right].CreatedAt }) selected := make([]string, 0, available) for _, run := range ready { if available == 0 { break } workflow := workflowByID[run.WorkflowID] workflowLimit := effectiveWorkflowConcurrencyLimit(&workflow, settings) if run.Trigger != domain.RunTriggerHumanRelease && activeByWorkflow[run.WorkflowID] >= workflowLimit { continue } selected = append(selected, run.ID) activeByWorkflow[run.WorkflowID]++ available-- } return selected } func (runtime *Runtime) applyWorkflowRuntimePolicies(ctx context.Context, workflows []domain.WorkflowRecord) { for index := range workflows { workflow := &workflows[index] if workflow.CaptureAutomation == nil || !workflow.CaptureAutomation.Enabled { continue } policy, policyErr := runtime.store.GetWorkflowRuntimePolicy(ctx, workflow.ID) if policyErr != nil { fmt.Printf("[worker] load runtime policy for workflow %s failed: %v\n", workflow.ID, policyErr) continue } limit := domain.DefaultMaxConcurrentCaptureRuns if policy != nil && policy.MaxConcurrentCaptureRuns > 0 { limit = policy.MaxConcurrentCaptureRuns } workflow.MaxConcurrentRuns = limit } } func runRecordsFromSummaries(summaries []domain.RunSummary) []domain.RunRecord { runs := make([]domain.RunRecord, 0, len(summaries)) for _, summary := range summaries { runs = append(runs, domain.RunRecord{ ID: summary.ID, OrganizationID: summary.OrganizationID, WorkflowID: summary.WorkflowID, WorkflowName: summary.WorkflowName, ProjectID: summary.ProjectID, ProjectName: summary.ProjectName, TaskID: summary.TaskID, Trigger: summary.Trigger, Mode: summary.Mode, ReleaseID: summary.ReleaseID, WorkflowVersion: summary.WorkflowVersion, GitCommitSha: summary.GitCommitSha, ImageDigest: summary.ImageDigest, Attempt: summary.Attempt, ParentRunID: summary.ParentRunID, RestartFromNodeID: summary.RestartFromNodeID, AvailableAt: summary.AvailableAt, ExecutionLeaseOwner: summary.ExecutionLeaseOwner, ExecutionLeaseExpiresAt: summary.ExecutionLeaseExpiresAt, Status: summary.Status, StopMode: summary.StopMode, StopRequestedAt: summary.StopRequestedAt, StartedAt: summary.StartedAt, EndedAt: summary.EndedAt, DurationMS: summary.DurationMS, CurrentNodeID: summary.CurrentNodeID, Error: summary.Error, CreatedAt: summary.CreatedAt, UpdatedAt: summary.UpdatedAt, }) } return runs } func (runtime *Runtime) materializeNodeMetricsBestEffort(ctx context.Context, run *domain.RunRecord, execution *domain.NodeExecutionRecord) { if run == nil || execution == nil || execution.DataMetrics == nil || execution.MetricsMaterializedAt != "" { return } if err := dataops.MaterializeNodeMetrics(ctx, runtime.store, run, execution); err != nil { fmt.Printf("[worker] materialize data metrics for run %s node %s failed: %v\n", run.ID, execution.NodeID, err) return } execution.MetricsMaterializedAt = nowISO() applyExecution(run, *execution) } func (runtime *Runtime) processClaimedRun(ctx context.Context, claimed *domain.RunRecord) { if claimed == nil { return } defer runtime.unmarkRunActive(claimed.ID) defer func() { if err := runtime.store.ReleaseRunLease(context.Background(), claimed.ID, runtime.leaseOwner()); err != nil { fmt.Printf("[worker] release run lease %s failed: %v\n", claimed.ID, err) } }() processed, err := runtime.processRun(ctx, claimed) if processed != nil && processed.Status == domain.RunStatusFailed { runtime.bestEffortCleanupRunRuntime(processed.ID) } if err != nil { fmt.Printf("[worker] process run %s failed: %v\n", claimed.ID, err) runtime.bestEffortTerminalizeRun(claimed.ID, err) } } func (runtime *Runtime) bestEffortCleanupRunRuntime(runID string) { if runtime == nil || runtime.launcher == nil || strings.TrimSpace(runID) == "" { return } cleanupCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() if err := runtime.launcher.TerminateRun(cleanupCtx, "", runID); err != nil { fmt.Printf("[worker] cleanup terminal run %s failed: %v\n", runID, err) } } func (runtime *Runtime) bestEffortTerminalizeRun(runID string, cause error) { if strings.TrimSpace(runID) == "" || cause == nil || errors.Is(cause, errRunLeaseLost) || errors.Is(cause, context.Canceled) { return } ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) defer cancel() run, err := runtime.store.GetRun(ctx, runID) if err != nil || run == nil || isTerminalRunStatus(run.Status) { return } message := "Worker failed to process run: " + boundedDiagnostic(cause.Error()) if err := runtime.failReconciledRun(ctx, run, message); err != nil { fmt.Printf("[worker] terminalize run %s failed: %v\n", runID, err) } } func (runtime *Runtime) renewRunLeaseUntil(ctx context.Context, runID string, cancelExecution context.CancelCauseFunc) { ttl := runtime.leaseTTL() interval := ttl / 3 if interval <= 0 || interval > time.Minute { interval = time.Minute } ticker := time.NewTicker(interval) defer ticker.Stop() for { select { case <-ctx.Done(): return case <-ticker.C: renewCtx, cancelRenew := context.WithTimeout(ctx, 10*time.Second) run, readErr := runtime.store.GetRun(renewCtx, runID) if readErr == nil && run != nil && run.ProjectID != "" { p, e := runtime.store.GetProject(renewCtx, run.ProjectID) if e == nil && (p == nil || p.DeletionTaskID != "") { cancelRenew() cancelExecution(context.Canceled) return } } ok, err := runtime.store.RenewRunLease(renewCtx, runID, runtime.leaseOwner(), ttl) cancelRenew() if err != nil { if ctx.Err() != nil { return } fmt.Printf("[worker] renew run lease %s failed: %v\n", runID, err) continue } if !ok { fmt.Printf("[worker] renew run lease %s skipped: owner changed or run missing\n", runID) if cancelExecution != nil { cancelExecution(errRunLeaseLost) } return } } } } func (runtime *Runtime) tryMarkRunActive(runID string) bool { if strings.TrimSpace(runID) == "" { return false } runtime.processMu.Lock() defer runtime.processMu.Unlock() if runtime.activeRuns == nil { runtime.activeRuns = map[string]struct{}{} } if _, ok := runtime.activeRuns[runID]; ok { return false } runtime.activeRuns[runID] = struct{}{} return true } func (runtime *Runtime) unmarkRunActive(runID string) { runtime.processMu.Lock() defer runtime.processMu.Unlock() delete(runtime.activeRuns, runID) } func (runtime *Runtime) orphanMinimumRuntimeAge() time.Duration { if runtime == nil || runtime.orphanMinimumAge <= 0 { return 24 * time.Hour } return runtime.orphanMinimumAge } func (runtime *Runtime) reconcileOrphanedRuntimes(ctx context.Context) { now := time.Now().UTC() interval := runtime.orphanCheckInterval if interval <= 0 { interval = 5 * time.Minute } lastCheck := runtime.lastOrphanCheckNano.Load() if lastCheck > 0 && now.Sub(time.Unix(0, lastCheck)) < interval { return } if !runtime.lastOrphanCheckNano.CompareAndSwap(lastCheck, now.UnixNano()) { return } reconciler, ok := runtime.launcher.(orphanRuntimeReconciler) if !ok { runtime.metrics.RecordOrphanRuntimes(0) return } summaries, err := runtime.store.ListRunSummaries(ctx, "") if err != nil { fmt.Printf("[worker] list run ids for orphan reconciliation failed: %v\n", err) return } knownRunIDs := make(map[string]struct{}, len(summaries)) for _, summary := range summaries { knownRunIDs[summary.ID] = struct{}{} } orphans, err := reconciler.ReconcileOrphanedRuntimes(ctx, knownRunIDs, now.Add(-runtime.orphanMinimumRuntimeAge()), runtime.orphanCleanup) if err != nil { fmt.Printf("[worker] orphan runtime reconciliation failed: %v\n", err) return } cleaned, cleanupErrors := 0, 0 for _, orphan := range orphans { if orphan.Cleaned { cleaned++ } if strings.TrimSpace(orphan.CleanupError) != "" { cleanupErrors++ } } runtime.metrics.RecordOrphanRuntimes(len(orphans) - cleaned) runtime.metrics.RecordOrphanCleanup(cleaned, cleanupErrors) } // archiveTerminalRuns periodically marks old, terminal runs as archived (see // domain.RunRecord.ArchivedAt) so an organization's ever-growing run history // doesn't keep inflating the cost of org-wide scans like // validateDeletionReferences' safety check. Throttled the same way // reconcileOrphanedRuntimes is, since this only needs to run occasionally, // not on every poll tick. func (runtime *Runtime) archiveTerminalRuns(ctx context.Context) { now := time.Now().UTC() interval := runtime.runArchiveInterval if interval <= 0 { interval = time.Hour } lastCheck := runtime.lastRunArchiveNano.Load() if lastCheck > 0 && now.Sub(time.Unix(0, lastCheck)) < interval { return } if !runtime.lastRunArchiveNano.CompareAndSwap(lastCheck, now.UnixNano()) { return } minimumAge := runtime.runArchiveMinimumAge if minimumAge <= 0 { minimumAge = 30 * 24 * time.Hour } archived, err := runtime.store.ArchiveTerminalRuns(ctx, now.Add(-minimumAge)) if err != nil { fmt.Printf("[worker] archive terminal runs failed: %v\n", err) return } if archived > 0 { fmt.Printf("[worker] archived %d terminal run(s) older than %s\n", archived, minimumAge) } } func (runtime *Runtime) ProcessAvailable(ctx context.Context) error { startedAt := time.Now() var processErr error defer func() { runtime.metrics.RecordProcessAvailable(uint64(time.Since(startedAt).Milliseconds()), processErr) }() settings, err := runtime.store.GetSettings(ctx) if err != nil { processErr = err return processErr } runtime.updateEffectivePollInterval(settings) reconcileRuns, err := runtime.store.ListReconcileRunSummaries(ctx, workerReconcileRunLimit) if err != nil { processErr = err return processErr } if err := runtime.reconcileActiveRunsWithSummaries(ctx, reconcileRuns); err != nil { processErr = err return processErr } counts, err := runtime.store.CountWorkerRuns(ctx) if err != nil { processErr = err return processErr } selectedRunIDs, err := runtime.dispatchCandidates(ctx, settings, counts) if err != nil { processErr = err return processErr } runtime.metrics.RecordDispatch(counts.QueuedTotal, counts.ActiveTotal, len(selectedRunIDs)) for _, runID := range selectedRunIDs { if !runtime.tryMarkRunActive(runID) { continue } claimed, err := runtime.store.ClaimQueuedRun(ctx, runID, runtime.leaseOwner(), runtime.leaseTTL()) if err != nil { runtime.unmarkRunActive(runID) fmt.Printf("[worker] claim queued run %s failed: %v\n", runID, err) continue } if claimed == nil { runtime.unmarkRunActive(runID) continue } go runtime.processClaimedRun(ctx, claimed) } return nil } func chooseImage(node domain.WorkflowNode, fallback string) string { if image := discardPackageImageOverride(node); image != "" { return image } if digestPinnedImage(node.Data.OperatorImageID) { return strings.TrimSpace(node.Data.OperatorImageID) } if node.Data.Image != "" { if strings.Contains(node.Data.Image, "droboticflow-operator-runner") && fallback != "" { return fallback } return node.Data.Image } return fallback } // Deployment-scoped, digest-pinned compatible package release. This upgrades // pending package nodes without rewriting historical Runs or workflow releases; // the actually selected digest is recorded in the execution snapshot above. func discardPackageImageOverride(node domain.WorkflowNode) string { if node.Data.OperatorID != "operator-builtin-package" { return "" } image := strings.TrimSpace(os.Getenv("FANYU_SLICE_DISCARD_PACKAGE_IMAGE")) if !digestPinnedImage(image) { return "" } return image } func digestPinnedImage(value string) bool { trimmed := strings.TrimSpace(value) if strings.ContainsAny(trimmed, " \t\r\n") { return false } parts := strings.Split(trimmed, "@sha256:") if len(parts) != 2 || parts[0] == "" || len(parts[1]) != 64 { return false } for _, character := range parts[1] { if (character < '0' || character > '9') && (character < 'a' || character > 'f') && (character < 'A' || character > 'F') { return false } } return true } func chooseValue(value string, fallback string) string { if value == "" { return fallback } return value } func chooseDataSource(value string, fallback string) (string, error) { normalized := strings.ToUpper(strings.TrimSpace(value)) if normalized == "" { return fallback, nil } switch normalized { case "INLINE", "LOCAL", "BOS", "UPSTREAM": return normalized, nil default: return "", fmt.Errorf("DATA_SOURCE must be INLINE, LOCAL, BOS, or UPSTREAM") } } const missingJobRecoveryLogPrefix = "Kubernetes job is missing; recovery attempt " func (runtime *Runtime) missingJobRecoveryLimit() int { if runtime == nil || runtime.missingMaxRecoveries <= 0 { return 3 } return runtime.missingMaxRecoveries } func (runtime *Runtime) pendingJobFailureThreshold() time.Duration { if runtime == nil || runtime.pendingFailureAfter <= 0 { return 10 * time.Minute } return runtime.pendingFailureAfter } func missingJobRecoveryCount(execution *domain.NodeExecutionRecord) int { if execution == nil { return 0 } count := 0 for _, entry := range execution.Logs { if strings.HasPrefix(strings.TrimSpace(entry.Message), missingJobRecoveryLogPrefix) { count++ } } return count } func boundedDiagnostic(value string) string { trimmed := strings.TrimSpace(value) const maximum = 4096 if len(trimmed) <= maximum { return trimmed } return trimmed[:maximum] + "..." } func (runtime *Runtime) failReconciledRun(ctx context.Context, run *domain.RunRecord, message string, structured ...domain.RunFailure) error { if run == nil { return nil } message = boundedDiagnostic(message) if strings.TrimSpace(message) == "" { message = "Kubernetes job failed" } if _, execution := findNodeExecution(run, run.CurrentNodeID); execution != nil { execution.Status = domain.NodeExecutionStatusFailed execution.EndedAt = nowISO() execution.Error = message execution.Logs = append(execution.Logs, logEntry("ERROR", message, execution.NodeLabel)) applyExecution(run, *execution) } run.Status = domain.RunStatusFailed run.EndedAt = nowISO() run.Error = message failure := domain.RunFailure{Code: "worker_execution_failed", Stage: "execution", NodeID: run.CurrentNodeID, Message: message, Retryable: true, Suggestion: "查看节点日志后重试失败子链路"} if len(structured) > 0 { failure = structured[0] failure.Message = message if strings.TrimSpace(failure.NodeID) == "" { failure.NodeID = run.CurrentNodeID } } run.Failure = &failure run.UpdatedAt = nowISO() if err := runtime.store.UpsertRun(ctx, *run); err != nil { return err } workflow, err := runtime.store.GetWorkflow(ctx, run.WorkflowID) if err != nil { return err } if workflow != nil { if err := runtime.store.UpdateWorkflowRunState(ctx, workflow.ID, domain.WorkflowStatusFailed, run.ID, nowISO()); err != nil { return err } } // Persist the terminal state before touching Kubernetes. A failed runtime is // no longer useful to the scheduler, and cleanup is deliberately best-effort // so a transient API error cannot leave the Run active again. if runtime.launcher != nil { cleanupCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second) if cleanupErr := runtime.launcher.TerminateRun(cleanupCtx, "", run.ID); cleanupErr != nil { fmt.Printf("[worker] cleanup failed runtime for terminal run %s: %v\n", run.ID, cleanupErr) } cancel() } return nil } func (runtime *Runtime) recoverMissingJob(ctx context.Context, run *domain.RunRecord, execution *domain.NodeExecutionRecord) error { if run != nil && run.Status == domain.RunStatusStopping { workflow, err := runtime.store.GetWorkflow(ctx, run.WorkflowID) if err != nil { return err } executionMap := make(map[string]domain.NodeExecutionRecord, len(run.NodeExecutions)) for _, item := range run.NodeExecutions { executionMap[item.NodeID] = item } _, err = runtime.finalizeCancelledRun(ctx, workflow, run, executionMap, "用户已停止工作流") return err } if execution == nil { run.Status = domain.RunStatusQueued run.StartedAt = "" run.CurrentNodeID = "" run.AvailableAt = nowISO() run.UpdatedAt = nowISO() return runtime.store.UpsertRun(ctx, *run) } terminateCtx, cancelTerminate := context.WithTimeout(context.Background(), 30*time.Second) err := runtime.launcher.TerminateRun(terminateCtx, "", run.ID) cancelTerminate() if err != nil { return fmt.Errorf("clean up missing job runtime for run %s: %w", run.ID, err) } recoveryAttempt := missingJobRecoveryCount(execution) + 1 if recoveryAttempt > runtime.missingJobRecoveryLimit() { return runtime.failReconciledRun(ctx, run, fmt.Sprintf("Kubernetes job remained missing after %d recovery attempts", runtime.missingJobRecoveryLimit()), domain.RunFailure{ Code: "kubernetes_job_missing", Stage: "execution", Retryable: true, Suggestion: "检查 Job 生命周期与集群事件后从失败节点继续", }) } message := fmt.Sprintf("%s%d/%d; node execution was re-queued", missingJobRecoveryLogPrefix, recoveryAttempt, runtime.missingJobRecoveryLimit()) execution.Status = domain.NodeExecutionStatusFailed execution.EndedAt = nowISO() execution.Error = message execution.Logs = append(execution.Logs, logEntry("WARN", message, execution.NodeLabel)) applyExecution(run, *execution) backoff := time.Duration(15*(1<<(recoveryAttempt-1))) * time.Second run.Status = domain.RunStatusQueued run.StartedAt = "" run.CurrentNodeID = "" run.AvailableAt = time.Now().UTC().Add(backoff).Format(time.RFC3339Nano) run.UpdatedAt = nowISO() return runtime.store.UpsertRun(ctx, *run) } func (runtime *Runtime) recoverSucceededJob(ctx context.Context, run *domain.RunRecord, inspection *k8s.JobInspection) error { workflow, err := runtime.store.GetWorkflow(ctx, run.WorkflowID) if err != nil { return err } if workflow == nil { return runtime.failReconciledRun(ctx, run, fmt.Sprintf("workflow %s was deleted while run %s was active", run.WorkflowID, run.ID), domain.RunFailure{ Code: "workflow_not_found", Stage: "orchestration", Retryable: false, Suggestion: "重新从现有发布版本创建工作流后再运行", }) } workflow, err = deliveryRuntimeWorkflow(workflow, run) if err != nil { return runtime.failReconciledRun(ctx, run, err.Error(), domain.RunFailure{Code: "invalid_delivery_runtime", Stage: "orchestration", Retryable: false}) } node := domain.GetNodeByID(workflow.Nodes, run.CurrentNodeID) if node == nil { return runtime.failReconciledRun(ctx, run, fmt.Sprintf("node %s was deleted while run %s was active", run.CurrentNodeID, run.ID), domain.RunFailure{ Code: "workflow_node_not_found", Stage: "orchestration", NodeID: run.CurrentNodeID, Retryable: false, Suggestion: "从保留该节点的发布版本重新运行", }) } executionMap := map[string]domain.NodeExecutionRecord{} for _, item := range run.NodeExecutions { executionMap[item.NodeID] = item } inputArtifacts, inputLineage, inputBindings := gatherNodeInputs(*workflow, run, executionMap, node.ID) _, existing := findNodeExecution(run, node.ID) execution := domain.NodeExecutionRecord{ NodeID: node.ID, NodeLabel: node.Data.Label, Attempt: runExecutionAttempt(run), Status: domain.NodeExecutionStatusRunning, StartedAt: firstNonEmptyString(run.StartedAt, run.CurrentRunAt, run.CreatedAt), Logs: []domain.LogEntryRecord{}, Artifacts: []domain.ExecutionArtifact{}, DataRefs: []domain.DataRefRecord{}, ResourceSeries: []domain.ResourceSample{}, } if existing != nil { execution = *existing } // A completed Job may be reconciled after a Worker restart. Its deployed // image is authoritative; an older workflow definition must not replace it. recoveredNode := *node recoveredImage := firstNonEmptyString(inspection.Image, execution.OperatorImageID, execution.OperatorImage) if recoveredImage != "" { recoveredNode.Data.Image = recoveredImage recoveredNode.Data.OperatorImageID = recoveredImage } applyNodeExecutionSnapshot(&execution, recoveredNode, inputArtifacts, inputLineage, inputBindings) parsed, err := parseExecutionOutput(inspection.Output, node.Data.Label) if err != nil { return err } execution.Status = domain.NodeExecutionStatusCompleted execution.EndedAt = nowISO() execution.Logs = append(execution.Logs, parsed.Logs...) execution.Artifacts = parsed.Artifacts execution.DataRefs = parsed.DataRefs execution.DataMetrics = parsed.DataMetrics execution.ResourceSeries = parsed.ResourceSeries execution.Error = "" executionMap[node.ID] = execution applyExecution(run, execution) if err := runtime.store.UpsertRun(ctx, *run); err != nil { return err } runtime.materializeNodeMetricsBestEffort(ctx, run, &execution) executionMap[node.ID] = execution if run.Status == domain.RunStatusStopping { _, err := runtime.finalizeCancelledRun(ctx, workflow, run, executionMap, "用户已停止工作流") return err } if allAllowedNodesCompleted(workflow, run, executionMap) { _, err := runtime.finalizeCompletedRun(ctx, workflow, run) return err } run.Status = domain.RunStatusQueued run.StartedAt = "" run.CurrentNodeID = "" run.AvailableAt = nowISO() run.UpdatedAt = nowISO() return runtime.store.UpsertRun(ctx, *run) } func inspectionIsStuck(inspection *k8s.JobInspection, fallbackStartedAt string, threshold time.Duration) bool { if inspection == nil || strings.TrimSpace(inspection.BlockingReason) == "" { return false } startedAt, ok := parseRFC3339Nano(firstNonEmptyString(inspection.CreatedAt, fallbackStartedAt)) return ok && time.Since(startedAt) >= threshold } func nodeExecutionStartedAt(execution *domain.NodeExecutionRecord) string { if execution == nil { return "" } return execution.StartedAt } func runExecutionLeaseActive(run *domain.RunRecord, now time.Time) bool { if run == nil || strings.TrimSpace(run.ExecutionLeaseOwner) == "" || strings.TrimSpace(run.ExecutionLeaseExpiresAt) == "" { return false } expiresAt, ok := parseRFC3339Nano(run.ExecutionLeaseExpiresAt) return ok && expiresAt.After(now) } func (runtime *Runtime) reconcileActiveRun(ctx context.Context, candidate domain.RunSummary) (bool, error) { run, err := runtime.store.GetRun(ctx, candidate.ID) if err != nil || run == nil { return false, err } // The execution coroutine owns Kubernetes Job observation while its lease is // alive. Reconciling the same Run concurrently can observe the short gap // between Job completion/cleanup and the persisted node result, then // incorrectly requeue or terminalize a successful long-running GPU node. // Stopping Runs are exempt so explicit user cancellation remains immediate. if run.Status != domain.RunStatusStopping && runExecutionLeaseActive(run, time.Now().UTC()) { return false, nil } var activeExecution *domain.NodeExecutionRecord if strings.TrimSpace(run.CurrentNodeID) != "" { _, activeExecution = findNodeExecution(run, run.CurrentNodeID) } if run.Status == domain.RunStatusQueued { if activeExecution == nil || activeExecution.Status != domain.NodeExecutionStatusRunning || !isStaleQueuedNodeExecution(*activeExecution, 2*time.Minute) { return false, nil } } if run.Status == domain.RunStatusRunning && activeExecution != nil && activeExecution.Status == domain.NodeExecutionStatusRunning { runtime.syncActiveRunProgress(ctx, run) if !isStaleQueuedNodeExecution(*activeExecution, 2*time.Minute) { return false, nil } } if strings.TrimSpace(run.CurrentNodeID) == "" { if updatedAt, ok := parseRFC3339Nano(run.UpdatedAt); ok && time.Since(updatedAt) < 2*time.Minute { return false, nil } return false, runtime.recoverMissingJob(ctx, run, nil) } inspection, err := runtime.launcher.Inspect(ctx, run.ID, run.CurrentNodeID) if err != nil { return false, err } switch inspection.State { case k8s.JobStateMissing: return true, runtime.recoverMissingJob(ctx, run, activeExecution) case k8s.JobStateSucceeded: return false, runtime.recoverSucceededJob(ctx, run, inspection) case k8s.JobStateFailed: message := "Kubernetes job failed" if output := boundedDiagnostic(inspection.Output); output != "" { message += ": " + output } return true, runtime.failReconciledRun(ctx, run, message, domain.RunFailure{ Code: "kubernetes_job_failed", Stage: "execution", Retryable: true, Suggestion: "查看 Pod 日志和事件后从失败节点继续", }) case k8s.JobStatePending: if inspectionIsStuck(inspection, firstNonEmptyString(nodeExecutionStartedAt(activeExecution), run.StartedAt), runtime.pendingJobFailureThreshold()) { message := fmt.Sprintf("Kubernetes job remained blocked in %s for at least %s", inspection.BlockingReason, runtime.pendingJobFailureThreshold()) if details := boundedDiagnostic(inspection.BlockingMessage); details != "" { message += ": " + details } failureCode := "kubernetes_runtime_blocked" if inspection.BlockingReason == "ImagePullBackOff" || inspection.BlockingReason == "ErrImagePull" || inspection.BlockingReason == "InvalidImageName" { failureCode = "kubernetes_image_pull_blocked" } else if inspection.BlockingReason == "Unschedulable" { failureCode = "kubernetes_unschedulable" } return true, runtime.failReconciledRun(ctx, run, message, domain.RunFailure{ Code: failureCode, Stage: "scheduling", Retryable: true, Suggestion: "修复镜像或调度条件后从失败节点继续", }) } return strings.TrimSpace(inspection.BlockingReason) != "", nil default: return false, nil } } // syncActiveRunProgress keeps structured progress visible after a worker // restart. The original executor-side reporter may have been cancelled during // a rolling deployment while the Kubernetes Job continues to run, so the // reconcile path must also tail the active Pod and persist newer markers. func (runtime *Runtime) syncActiveRunProgress(ctx context.Context, run *domain.RunRecord) { if runtime == nil || runtime.launcher == nil || run == nil || strings.TrimSpace(run.CurrentNodeID) == "" { return } tailer, ok := runtime.launcher.(nodePodLogTailer) if !ok { return } logsCtx, cancel := context.WithTimeout(ctx, 10*time.Second) logs, err := tailer.TailNodePodLogs(logsCtx, run.ID, run.CurrentNodeID, 200) cancel() if err != nil { return } progress, err := latestStructuredProgress(logs) if err != nil || progress == nil || !progressAheadOf(progress, run.Progress) { return } if err := runtime.store.UpdateRunProgress(ctx, run.ID, progress); err != nil { fmt.Printf("[worker] sync progress for run %s failed: %v\n", run.ID, err) } } func progressAheadOf(next *domain.RunProgress, current *domain.RunProgress) bool { if next == nil { return false } if current == nil { return true } if next.NodeID != "" && next.NodeID != current.NodeID { return strings.TrimSpace(next.UpdatedAt) > strings.TrimSpace(current.UpdatedAt) } if next.Epoch < current.Epoch || (next.Epoch == current.Epoch && next.Step < current.Step) { return false } if next.Step != current.Step || next.Epoch != current.Epoch || next.Checkpoint != current.Checkpoint { return true } return strings.TrimSpace(next.UpdatedAt) > strings.TrimSpace(current.UpdatedAt) } func (runtime *Runtime) reconcileActiveRunsWithSummaries(ctx context.Context, runs []domain.RunSummary) error { stuckCount := 0 for _, candidate := range runs { if candidate.Status != domain.RunStatusRunning && candidate.Status != domain.RunStatusStopping && (candidate.Status != domain.RunStatusQueued || strings.TrimSpace(candidate.CurrentNodeID) == "") { continue } stuck, err := runtime.reconcileActiveRun(ctx, candidate) if stuck { stuckCount++ } if err != nil { // A single malformed or temporarily unavailable run must not prevent // unrelated runs from reconciling and the queue from dispatching. fmt.Printf("[worker] reconcile run %s failed: %v\n", candidate.ID, err) } } runtime.metrics.RecordStuckRuns(stuckCount) return nil } func (runtime *Runtime) reconcileActiveRuns(ctx context.Context) error { runs, err := runtime.store.ListReconcileRunSummaries(ctx, workerReconcileRunLimit) if err != nil { return err } return runtime.reconcileActiveRunsWithSummaries(ctx, runs) } func upsertNodeHyperparam(items []domain.HyperParam, key string, value string) []domain.HyperParam { for index := range items { if items[index].Key == key { items[index].Value = value return items } } return append(items, domain.HyperParam{Key: key, Value: value}) } func bosSourcePath(bucket string, prefix string) string { normalizedBucket := strings.TrimSpace(bucket) if normalizedBucket == "" { return "" } sourcePath := "bos://" + normalizedBucket if normalizedPrefix := strings.Trim(strings.TrimSpace(prefix), "/"); normalizedPrefix != "" { sourcePath += "/" + normalizedPrefix } return sourcePath } func (runtime *Runtime) defaultStorageEndpoint() string { return strings.TrimSpace(runtime.globalBOSEndpoint) } func (runtime *Runtime) storageConnectionEndpoint(connection *domain.StorageConnectionRecord) string { if connection == nil { return runtime.defaultStorageEndpoint() } if endpoint := strings.TrimSpace(connection.Endpoint); endpoint != "" { return endpoint } return runtime.defaultStorageEndpoint() } func nodeHyperparamsMap(node domain.WorkflowNode) map[string]string { values := make(map[string]string, len(node.Data.Hyperparams)) for _, item := range node.Data.Hyperparams { key := strings.TrimSpace(item.Key) if key == "" { continue } values[key] = item.Value } return values } func chooseScanMode(node domain.WorkflowNode, params map[string]string) (string, error) { mode := strings.ToUpper(strings.TrimSpace(params["SCAN_MODE"])) switch mode { case "FULL", "INCREMENTAL": return mode, nil case "": if node.Data.Label == "BOS 增量扫描" { return "INCREMENTAL", nil } return "", nil default: return "", fmt.Errorf("SCAN_MODE must be FULL or INCREMENTAL") } } func chooseIncrementalUnit(params map[string]string) (string, error) { switch strings.ToUpper(strings.TrimSpace(params["INCREMENTAL_UNIT"])) { case "": return "OBJECT", nil case "OBJECT": return "OBJECT", nil case "DIRECTORY": return "DIRECTORY", nil default: return "", fmt.Errorf("INCREMENTAL_UNIT must be OBJECT or DIRECTORY") } } func chooseDirectoryDepth(params map[string]string) (int, error) { raw := strings.TrimSpace(params["INCREMENTAL_DIRECTORY_DEPTH"]) if raw == "" { return 1, nil } parsed, err := strconv.Atoi(raw) if err != nil || parsed <= 0 { return 0, fmt.Errorf("INCREMENTAL_DIRECTORY_DEPTH must be a positive integer") } return parsed, nil } func scanWorkflowSourceNode(node domain.WorkflowNode) bool { label := strings.TrimSpace(node.Data.Label) return node.ID == "scan-source" || label == "BOS 增量扫描" } func sourceFingerprint(node domain.WorkflowNode, params map[string]string, dataSource string, scanMode string, incrementalUnit string, directoryDepth int) string { return domain.ComputeSourceFingerprint(domain.SourceFingerprintInput{ ScanMode: scanMode, IncrementalUnit: incrementalUnit, DirectoryDepth: directoryDepth, DataSource: dataSource, SourcePath: params["SOURCE_PATH"], Bucket: node.Data.Bucket, Prefix: node.Data.Prefix, Pattern: node.Data.Pattern, Endpoint: params["BOS_ENDPOINT"], }) } func stringList(value any) []string { if value == nil { return nil } if items, ok := value.([]string); ok { result := make([]string, 0, len(items)) for _, item := range items { trimmed := strings.TrimSpace(item) if trimmed != "" { result = append(result, trimmed) } } return result } items, ok := value.([]any) if ok { result := make([]string, 0, len(items)) for _, item := range items { trimmed := strings.TrimSpace(fmt.Sprint(item)) if trimmed != "" { result = append(result, trimmed) } } return result } reflected := reflect.ValueOf(value) if reflected.Kind() != reflect.Slice && reflected.Kind() != reflect.Array { return nil } result := make([]string, 0, reflected.Len()) for index := 0; index < reflected.Len(); index++ { trimmed := strings.TrimSpace(fmt.Sprint(reflected.Index(index).Interface())) if trimmed != "" { result = append(result, trimmed) } } return result } func inputStorageConnectionID(bindings []domain.NodeInputBinding) (string, error) { connectionID := "" consider := func(payload *domain.ArtifactPayload) error { if payload == nil || payload.Mode != domain.ArtifactTransferModeRef || payload.Ref == nil { return nil } candidate := strings.TrimSpace(payload.Ref.ConnectionID) if candidate == "" { return nil } if connectionID != "" && connectionID != candidate { return fmt.Errorf("input_storage_ambiguous: node inputs reference multiple storage connections") } connectionID = candidate return nil } for index := range bindings { if err := consider(bindings[index].Payload); err != nil { return "", err } for artifactIndex := range bindings[index].Artifacts { if err := consider(bindings[index].Artifacts[artifactIndex].Payload); err != nil { return "", err } } } return connectionID, nil } func (runtime *Runtime) prepareExecutionNode(ctx context.Context, workflow *domain.WorkflowRecord, node domain.WorkflowNode, bindingSets ...[]domain.NodeInputBinding) (domain.WorkflowNode, error) { var inputBindings []domain.NodeInputBinding if len(bindingSets) > 0 { inputBindings = bindingSets[0] } copy, err := runtime.resolveBuiltinNodeRuntime(ctx, node) if err != nil { return node, err } effectiveWorkflow, err := runtime.resolveScanWorkflow(ctx, workflow) if err != nil { return node, err } copy, err = scanExecutionNode(effectiveWorkflow, copy) if err != nil { return node, err } if strings.TrimSpace(copy.Data.ConnectionID) == "" { connectionID, inputErr := inputStorageConnectionID(inputBindings) if inputErr != nil { return node, inputErr } copy.Data.ConnectionID = connectionID } copy, err = runtime.resolveNodeStorageConnection(ctx, copy) if err != nil { return node, err } copy, err = runtime.resolveNodeMongoConnection(ctx, copy) if err != nil { return node, err } params := nodeHyperparamsMap(copy) dataSource, err := chooseDataSource(params["DATA_SOURCE"], "BOS") if err != nil { return node, err } scanMode, err := chooseScanMode(copy, params) if err != nil { return node, err } incrementalUnit, err := chooseIncrementalUnit(params) if err != nil { return node, err } directoryDepth, err := chooseDirectoryDepth(params) if err != nil { return node, err } if scanMode != "INCREMENTAL" || workflow == nil { return copy, nil } fingerprint := sourceFingerprint(copy, params, dataSource, scanMode, incrementalUnit, directoryDepth) state, err := runtime.store.GetSourceState(ctx, workflow.ID, copy.ID, fingerprint) if err != nil { return copy, err } payload := map[string]any{ "sourceFingerprint": fingerprint, "scanMode": scanMode, "incrementalUnit": incrementalUnit, "directoryDepth": directoryDepth, "dataSource": dataSource, "sourcePath": strings.TrimSpace(params["SOURCE_PATH"]), "bucket": strings.TrimSpace(copy.Data.Bucket), "prefix": strings.TrimSpace(copy.Data.Prefix), "pattern": strings.TrimSpace(copy.Data.Pattern), "seenFingerprints": []string{}, } if state != nil { payload["seenFingerprints"] = append([]string(nil), state.SeenFingerprints...) payload["lastRunId"] = state.LastRunID payload["lastListedCount"] = state.LastListedCount payload["lastNewCount"] = state.LastNewCount payload["lastScanCompletedAt"] = state.LastScanCompletedAt } encoded, err := json.Marshal(payload) if err != nil { return copy, err } copy.Data.Hyperparams = append([]domain.HyperParam(nil), copy.Data.Hyperparams...) copy.Data.Hyperparams = upsertNodeHyperparam(copy.Data.Hyperparams, "SCAN_MODE", scanMode) copy.Data.Hyperparams = upsertNodeHyperparam(copy.Data.Hyperparams, "INCREMENTAL_UNIT", incrementalUnit) copy.Data.Hyperparams = upsertNodeHyperparam(copy.Data.Hyperparams, "INCREMENTAL_DIRECTORY_DEPTH", strconv.Itoa(directoryDepth)) copy.Data.Hyperparams = upsertNodeHyperparam(copy.Data.Hyperparams, "SOURCE_FINGERPRINT", fingerprint) copy.Data.Hyperparams = upsertNodeHyperparam(copy.Data.Hyperparams, "SOURCE_STATE_JSON", string(encoded)) return copy, nil } // resolveBuiltinNodeRuntime refreshes platform-owned runtime fields immediately // before execution. Workflow nodes keep their user configuration and graph // shape, while existing workflows automatically consume a newly published // builtin image instead of retaining a stale or deleted registry digest. func (runtime *Runtime) resolveBuiltinNodeRuntime(ctx context.Context, node domain.WorkflowNode) (domain.WorkflowNode, error) { operatorID := strings.TrimSpace(node.Data.OperatorID) if operatorID == "" || runtime == nil || runtime.store == nil { return node, nil } operator, err := runtime.store.GetOperator(ctx, operatorID) if err != nil { return node, fmt.Errorf("resolve builtin operator %s: %w", operatorID, err) } if operator == nil || !strings.EqualFold(strings.TrimSpace(operator.Type), "builtin") { return node, nil } image := strings.TrimSpace(operator.Image) imageID := strings.TrimSpace(operator.ImageID) if image == "" { return node, fmt.Errorf("builtin operator %s has no published image", operatorID) } if imageID == "" { imageID = image } preservePinnedImage := strings.EqualFold(strings.TrimSpace(nodeHyperparamsMap(node)["PIN_OPERATOR_IMAGE"]), "true") && digestPinnedImage(node.Data.OperatorImageID) if !preservePinnedImage { node.Data.Image = image node.Data.OperatorImageID = imageID } node.Data.ContainerCommand = cloneStringSlice(operator.ContainerCommand) node.Data.ContainerArgs = cloneStringSlice(operator.ContainerArgs) node.Data.RuntimeContract = operator.RuntimeContract node.Data.PublicationStatus = operator.PublicationStatus node.Data.GenerationSource = operator.GenerationSource node.Data.PublicationEvidence = operator.PublicationEvidence if strings.TrimSpace(operator.Version) != "" { node.Data.OperatorVersion = strings.TrimSpace(operator.Version) } return node, nil } func (runtime *Runtime) resolveScanWorkflow(ctx context.Context, workflow *domain.WorkflowRecord) (*domain.WorkflowRecord, error) { if workflow == nil || workflow.ScanConfig == nil || strings.TrimSpace(workflow.ScanConfig.ConnectionID) == "" { return workflow, nil } connection, err := runtime.store.GetStorageConnection(ctx, workflow.ScanConfig.ConnectionID) if err != nil { return nil, err } if connection == nil { return nil, fmt.Errorf("storage connection %s not found", workflow.ScanConfig.ConnectionID) } copy := *workflow scanConfig := *workflow.ScanConfig scanConfig.Endpoint = runtime.storageConnectionEndpoint(connection) if strings.TrimSpace(scanConfig.AccessKeyID) == "" { scanConfig.AccessKeyID = connection.AccessKeyID } if strings.TrimSpace(scanConfig.SecretAccessKey) == "" { scanConfig.SecretAccessKey = connection.SecretAccessKey } if strings.TrimSpace(scanConfig.SourcePath) == "" { dataSource, err := chooseDataSource(scanConfig.DataSource, "BOS") if err != nil { return nil, err } if dataSource == "BOS" { scanConfig.SourcePath = bosSourcePath(scanConfig.Bucket, scanConfig.Prefix) } } copy.ScanConfig = &scanConfig return ©, nil } func (runtime *Runtime) resolveNodeStorageConnection(ctx context.Context, node domain.WorkflowNode) (domain.WorkflowNode, error) { connectionID := strings.TrimSpace(node.Data.ConnectionID) if connectionID == "" { return node, nil } connection, err := runtime.store.GetStorageConnection(ctx, connectionID) if err != nil { return node, err } if connection == nil { return node, fmt.Errorf("storage connection %s not found", connectionID) } copy := node copy.Data.Hyperparams = append([]domain.HyperParam(nil), node.Data.Hyperparams...) sourcePath := strings.TrimSpace(nodeHyperparamsMap(copy)["SOURCE_PATH"]) if sourcePath == "" { sourcePath = bosSourcePath(copy.Data.Bucket, copy.Data.Prefix) } copy.Data.Hyperparams = upsertNodeHyperparam(copy.Data.Hyperparams, "DATA_SOURCE", "BOS") copy.Data.Hyperparams = upsertNodeHyperparam(copy.Data.Hyperparams, "SOURCE_PATH", sourcePath) copy.Data.Hyperparams = upsertNodeHyperparam(copy.Data.Hyperparams, "BOS_ENDPOINT", runtime.storageConnectionEndpoint(connection)) copy.Data.Hyperparams = upsertNodeHyperparam(copy.Data.Hyperparams, "BOS_ACCESS_KEY_ID", connection.AccessKeyID) copy.Data.Hyperparams = upsertNodeHyperparam(copy.Data.Hyperparams, "BOS_SECRET_ACCESS_KEY", connection.SecretAccessKey) return copy, nil } func (runtime *Runtime) resolveNodeMongoConnection(ctx context.Context, node domain.WorkflowNode) (domain.WorkflowNode, error) { connectionID := strings.TrimSpace(nodeHyperparamsMap(node)["MONGO_CONNECTION_ID"]) if connectionID == "" { return node, nil } connection, err := runtime.store.GetMongoConnection(ctx, connectionID) if err != nil { return node, err } if connection == nil { return node, fmt.Errorf("mongo connection %s not found", connectionID) } copy := node copy.Data.Hyperparams = append([]domain.HyperParam(nil), node.Data.Hyperparams...) copy.Data.Hyperparams = upsertNodeHyperparam(copy.Data.Hyperparams, "MONGO_URI", strings.TrimSpace(connection.URI)) copy.Data.Hyperparams = upsertNodeHyperparam(copy.Data.Hyperparams, "MONGO_DB", strings.TrimSpace(connection.Database)) copy.Data.Hyperparams = upsertNodeHyperparam(copy.Data.Hyperparams, "MONGO_TARGET", "business") return copy, nil } func checkpointArtifact(execution domain.NodeExecutionRecord) *domain.ExecutionArtifact { for index := range execution.Artifacts { if execution.Artifacts[index].Name == "source-checkpoint.json" { return &execution.Artifacts[index] } } return nil } func sourceCheckpointPayload(artifact *domain.ExecutionArtifact) (map[string]any, error) { if artifact == nil { return nil, nil } path := strings.TrimSpace(artifact.Path) if path != "" { data, err := os.ReadFile(path) if err != nil { return nil, fmt.Errorf("read source checkpoint %s: %w", path, err) } var payload map[string]any if err := json.Unmarshal(data, &payload); err != nil { return nil, fmt.Errorf("parse source checkpoint %s: %w", path, err) } if len(payload) == 0 { return nil, fmt.Errorf("source checkpoint %s must be a non-empty JSON object", path) } return payload, nil } if artifact.Data == nil { return nil, nil } return artifact.Data, nil } func sourceStateFromCheckpoint(workflow *domain.WorkflowRecord, run *domain.RunRecord, execution domain.NodeExecutionRecord, artifact *domain.ExecutionArtifact) (*domain.SourceStateRecord, error) { payload, err := sourceCheckpointPayload(artifact) if err != nil { return nil, err } if workflow == nil || run == nil || payload == nil { return nil, nil } fingerprint := strings.TrimSpace(fmt.Sprint(payload["sourceFingerprint"])) if fingerprint == "" { return nil, fmt.Errorf("source checkpoint for node %s is missing sourceFingerprint", execution.NodeID) } directoryDepth, err := checkpointIntValue(payload, "directoryDepth") if err != nil { return nil, err } if directoryDepth <= 0 { return nil, fmt.Errorf("source checkpoint directoryDepth must be greater than 0") } listedCount, err := checkpointIntValue(payload, "listedCount") if err != nil { return nil, err } if listedCount < 0 { return nil, fmt.Errorf("source checkpoint listedCount must be zero or greater") } newCount, err := checkpointIntValue(payload, "newCount") if err != nil { return nil, err } if newCount < 0 { return nil, fmt.Errorf("source checkpoint newCount must be zero or greater") } return &domain.SourceStateRecord{ ID: workflow.ID + ":" + execution.NodeID + ":" + fingerprint, WorkflowID: workflow.ID, NodeID: execution.NodeID, SourceFingerprint: fingerprint, DataSource: strings.TrimSpace(fmt.Sprint(payload["dataSource"])), SourcePath: strings.TrimSpace(fmt.Sprint(payload["sourcePath"])), Bucket: strings.TrimSpace(fmt.Sprint(payload["bucket"])), Prefix: strings.TrimSpace(fmt.Sprint(payload["prefix"])), Pattern: strings.TrimSpace(fmt.Sprint(payload["pattern"])), ScanMode: strings.TrimSpace(fmt.Sprint(payload["scanMode"])), IncrementalUnit: strings.TrimSpace(fmt.Sprint(payload["incrementalUnit"])), DirectoryDepth: directoryDepth, SeenFingerprints: stringList(payload["seenFingerprints"]), LastRunID: run.ID, LastListedCount: listedCount, LastNewCount: newCount, LastScanStartedAt: strings.TrimSpace(fmt.Sprint(payload["scanStartedAt"])), LastScanCompletedAt: strings.TrimSpace(fmt.Sprint(payload["scanCompletedAt"])), UpdatedAt: nowISO(), }, nil } func checkpointIntValue(payload map[string]any, key string) (int, error) { value, ok := payload[key] if !ok || value == nil { return 0, fmt.Errorf("source checkpoint %s is required", key) } switch typed := value.(type) { case int: return typed, nil case int32: return int(typed), nil case int64: return int(typed), nil case float32: value := float64(typed) if math.IsNaN(value) || math.IsInf(value, 0) || math.Trunc(value) != value { return 0, fmt.Errorf("source checkpoint %s must be an integer", key) } return int(value), nil case float64: if math.IsNaN(typed) || math.IsInf(typed, 0) || math.Trunc(typed) != typed { return 0, fmt.Errorf("source checkpoint %s must be an integer", key) } return int(typed), nil case json.Number: if parsed, err := typed.Int64(); err == nil { return int(parsed), nil } case string: trimmed := strings.TrimSpace(typed) if parsed, err := strconv.Atoi(trimmed); err == nil && strconv.Itoa(parsed) == trimmed { return parsed, nil } } return 0, fmt.Errorf("source checkpoint %s must be an integer", key) } func (runtime *Runtime) commitSourceStates(ctx context.Context, workflow *domain.WorkflowRecord, run *domain.RunRecord) error { if workflow == nil || run == nil { return nil } sourceRun := run persistedRun, err := runtime.store.GetRun(ctx, run.ID) if err != nil { return err } if persistedRun != nil { sourceRun = persistedRun } for _, execution := range sourceRun.NodeExecutions { artifact := checkpointArtifact(execution) if artifact == nil { continue } state, err := sourceStateFromCheckpoint(workflow, sourceRun, execution, artifact) if err != nil { return err } if state == nil { continue } if err := runtime.store.UpsertSourceState(ctx, *state); err != nil { return err } } return nil } func scanExecutionNode(workflow *domain.WorkflowRecord, node domain.WorkflowNode) (domain.WorkflowNode, error) { if workflow == nil || workflow.ScanConfig == nil { return node, nil } if !scanWorkflowSourceNode(node) { return node, nil } dataSource, err := chooseDataSource(workflow.ScanConfig.DataSource, "BOS") if err != nil { return node, err } sourcePath := strings.TrimSpace(workflow.ScanConfig.SourcePath) if sourcePath == "" && dataSource == "BOS" { sourcePath = bosSourcePath(workflow.ScanConfig.Bucket, workflow.ScanConfig.Prefix) } copy := node copy.Data.Bucket = workflow.ScanConfig.Bucket copy.Data.Prefix = workflow.ScanConfig.Prefix copy.Data.Pattern = workflow.ScanConfig.Pattern copy.Data.Hyperparams = append([]domain.HyperParam(nil), node.Data.Hyperparams...) copy.Data.Hyperparams = upsertNodeHyperparam(copy.Data.Hyperparams, "DATA_SOURCE", dataSource) copy.Data.Hyperparams = upsertNodeHyperparam(copy.Data.Hyperparams, "SOURCE_PATH", sourcePath) copy.Data.Hyperparams = upsertNodeHyperparam(copy.Data.Hyperparams, "BOS_ENDPOINT", workflow.ScanConfig.Endpoint) copy.Data.Hyperparams = upsertNodeHyperparam( copy.Data.Hyperparams, "SCAN_MODE", chooseValue(strings.ToUpper(strings.TrimSpace(workflow.ScanConfig.ScanMode)), "INCREMENTAL"), ) bosAccessKeyID := "" bosSecretAccessKey := "" if dataSource == "BOS" { bosAccessKeyID = workflow.ScanConfig.AccessKeyID bosSecretAccessKey = workflow.ScanConfig.SecretAccessKey } copy.Data.Hyperparams = upsertNodeHyperparam(copy.Data.Hyperparams, "BOS_ACCESS_KEY_ID", bosAccessKeyID) copy.Data.Hyperparams = upsertNodeHyperparam(copy.Data.Hyperparams, "BOS_SECRET_ACCESS_KEY", bosSecretAccessKey) return copy, nil } func workflowRunTimeout(workflow *domain.WorkflowRecord) time.Duration { if workflow == nil || workflow.RunTimeoutSeconds <= 0 { return 0 } return time.Duration(workflow.RunTimeoutSeconds) * time.Second } func workflowNodeTimeout(workflow *domain.WorkflowRecord) time.Duration { if workflow == nil || workflow.NodeTimeoutSeconds <= 0 { return 0 } return time.Duration(workflow.NodeTimeoutSeconds) * time.Second } func normalizeRunError(workflow *domain.WorkflowRecord, err error, executionCtx context.Context, nodeTimedOut bool) string { if err == nil { return "" } if workflow != nil && workflow.RunTimeoutSeconds > 0 && (errors.Is(err, context.DeadlineExceeded) || errors.Is(executionCtx.Err(), context.DeadlineExceeded)) { return fmt.Sprintf("工作流执行超时(%d 秒)", workflow.RunTimeoutSeconds) } if workflow != nil && workflow.NodeTimeoutSeconds > 0 && nodeTimedOut { return fmt.Sprintf("节点执行超时(%d 秒)", workflow.NodeTimeoutSeconds) } return err.Error() } func shouldAutoRetry(workflow *domain.WorkflowRecord, run *domain.RunRecord) bool { if workflow == nil || run == nil || workflow.MaxRetryAttempts <= 0 { return false } attempt := run.Attempt if attempt <= 0 { attempt = 1 } return attempt <= workflow.MaxRetryAttempts } func nextRunAttempt(run *domain.RunRecord) int { if run == nil || run.Attempt <= 0 { return 1 } return run.Attempt + 1 } func runExecutionAttempt(run *domain.RunRecord) int { if run == nil || run.Attempt <= 0 { return 1 } return run.Attempt } func nextRunAvailableAt(workflow *domain.WorkflowRecord) string { if workflow == nil || workflow.RetryBackoffSeconds <= 0 { return nowISO() } return time.Now().UTC().Add(time.Duration(workflow.RetryBackoffSeconds) * time.Second).Format(time.RFC3339Nano) } func (runtime *Runtime) enqueueRetryRun(ctx context.Context, workflow *domain.WorkflowRecord, failedRun *domain.RunRecord) (*domain.RunRecord, error) { if workflow == nil || failedRun == nil { return nil, nil } manifests, err := lightweight.GenerateWorkflowCodeManifests(runtime.workspaceRoot, *workflow) if err != nil { return nil, err } inputs := failedRun.Inputs if failedRun.Metadata != nil && failedRun.Metadata.CaptureDeliveryPackage != nil { source, _, e := runtime.originalDeliveryRun(ctx, failedRun) if e != nil { return nil, e } inputs = source.Inputs } retryRun, err := runtime.store.CreateRunWithInputs(ctx, workflow.ID, domain.RunTriggerRetry, inputs, failedRun.TaskID) if err != nil { return nil, err } retryRun.Attempt = nextRunAttempt(failedRun) retryRun.ParentRunID = failedRun.ID retryRun.AvailableAt = nextRunAvailableAt(workflow) lightweight.MergeRunMetadata(retryRun, manifests) if failedRun.Metadata != nil && failedRun.Metadata.CaptureDeliveryPackage != nil { // Preserve the immutable package selection and accepted release baseline. raw, e := json.Marshal(failedRun.Metadata.CaptureDeliveryPackage) if e != nil { return nil, e } var frozen domain.CaptureDeliveryPackageRuntime if e = json.Unmarshal(raw, &frozen); e != nil { return nil, e } if retryRun.Metadata == nil { retryRun.Metadata = &domain.RunMetadata{} } retryRun.Metadata.CaptureDeliveryPackage = &frozen retryRun.Metadata.HumanReleasePartitionID = failedRun.Metadata.HumanReleasePartitionID retryRun.Metadata.HumanReleaseUniqueKey = failedRun.Metadata.HumanReleaseUniqueKey retryRun.RestartFromNodeID = failedRun.RestartFromNodeID for _, execution := range failedRun.NodeExecutions { if execution.NodeID != frozen.Node.ID && execution.Status == domain.NodeExecutionStatusCompleted { applyExecution(retryRun, execution) } } } retryRun.UpdatedAt = nowISO() if err := runtime.store.UpsertRun(ctx, *retryRun); err != nil { return nil, err } return retryRun, nil } type nodeExecutionOutcome struct { nodeID string result executionResult reference *k8s.JobReference err error nodeTimedOut bool } func runAvailableAt(run domain.RunRecord) time.Time { if run.AvailableAt != "" { if parsed, err := time.Parse(time.RFC3339Nano, run.AvailableAt); err == nil { return parsed } } if parsed, err := time.Parse(time.RFC3339Nano, run.CreatedAt); err == nil { return parsed } return time.Time{} } func runDispatchReady(run domain.RunRecord, now time.Time) bool { availableAt := runAvailableAt(run) if availableAt.IsZero() { return true } return !availableAt.After(now) } func syncCurrentNodeID(run *domain.RunRecord) { run.CurrentNodeID = "" for _, status := range []domain.NodeExecutionStatus{ domain.NodeExecutionStatusWaitingForAudit, domain.NodeExecutionStatusWaitingForHuman, domain.NodeExecutionStatusRunning, } { for _, execution := range run.NodeExecutions { if execution.Status == status { run.CurrentNodeID = execution.NodeID return } } } } func isTerminalRunStatus(status domain.RunStatus) bool { return status == domain.RunStatusCompleted || status == domain.RunStatusFailed || status == domain.RunStatusCancelled } func (runtime *Runtime) loadRunControlState(ctx context.Context, runID string) (*domain.RunRecord, error) { if strings.TrimSpace(runID) == "" { return nil, nil } return runtime.store.GetRun(ctx, runID) } func syncStopMetadata(target *domain.RunRecord, source *domain.RunRecord) { if target == nil || source == nil { return } target.StopMode = source.StopMode target.StopRequestedAt = source.StopRequestedAt if source.Progress != nil { progress := *source.Progress target.Progress = &progress } } func finalizeCancelledExecution(execution domain.NodeExecutionRecord, message string) (domain.NodeExecutionRecord, bool) { if execution.Status == domain.NodeExecutionStatusCompleted || execution.Status == domain.NodeExecutionStatusFailed || execution.Status == domain.NodeExecutionStatusCancelled { return execution, false } execution.Status = domain.NodeExecutionStatusCancelled execution.EndedAt = nowISO() execution.Error = message execution.Logs = append(execution.Logs, logEntry("WARN", message, execution.NodeLabel)) return execution, true } func (runtime *Runtime) finalizeCancelledRun(ctx context.Context, workflow *domain.WorkflowRecord, currentRun *domain.RunRecord, executionMap map[string]domain.NodeExecutionRecord, message string) (*domain.RunRecord, error) { for nodeID, execution := range executionMap { updated, changed := finalizeCancelledExecution(execution, message) if changed { executionMap[nodeID] = updated applyExecution(currentRun, updated) } } currentRun.Status = domain.RunStatusCancelled currentRun.EndedAt = nowISO() if strings.TrimSpace(currentRun.Error) == "" { currentRun.Error = message } syncCurrentNodeID(currentRun) currentRun.UpdatedAt = nowISO() if err := runtime.store.UpsertRun(ctx, *currentRun); err != nil { return nil, err } if workflow != nil { workflow.Status = domain.WorkflowStatusCancelled workflow.LastRunID = currentRun.ID workflow.UpdatedAt = nowISO() if err := runtime.store.UpdateWorkflowRunState(ctx, workflow.ID, workflow.Status, workflow.LastRunID, workflow.UpdatedAt); err != nil { return nil, err } } return currentRun, nil } func allAllowedNodesCompleted(workflow *domain.WorkflowRecord, run *domain.RunRecord, executionMap map[string]domain.NodeExecutionRecord) bool { if workflow == nil || run == nil { return false } allowedNodes := rerunAllowedNodeSet(*workflow, run) for _, node := range workflow.Nodes { if !nodeAllowedForExecution(allowedNodes, node.ID) { continue } execution, ok := executionMap[node.ID] if !ok || execution.Status != domain.NodeExecutionStatusCompleted { return false } } return true } func (runtime *Runtime) finalizeCompletedRun(ctx context.Context, workflow *domain.WorkflowRecord, currentRun *domain.RunRecord) (*domain.RunRecord, error) { if err := runtime.commitSourceStates(ctx, workflow, currentRun); err != nil { return nil, err } currentRun.Status = domain.RunStatusCompleted currentRun.EndedAt = nowISO() syncCurrentNodeID(currentRun) currentRun.UpdatedAt = nowISO() if err := runtime.store.UpsertRun(ctx, *currentRun); err != nil { return nil, err } workflow.Status = domain.WorkflowStatusCompleted workflow.LastRunID = currentRun.ID workflow.UpdatedAt = nowISO() if err := runtime.store.UpdateWorkflowRunState(ctx, workflow.ID, workflow.Status, workflow.LastRunID, workflow.UpdatedAt); err != nil { return nil, err } return currentRun, nil } func dependenciesComplete(workflow domain.WorkflowRecord, executionMap map[string]domain.NodeExecutionRecord, nodeID string) bool { for _, candidate := range domain.GetIncomingEdges(workflow.Edges, nodeID) { dependency, ok := executionMap[candidate.Source] if !ok || dependency.Status != domain.NodeExecutionStatusCompleted { return false } } return true } func workflowEdgeIdentity(edge domain.WorkflowEdge) string { if id := strings.TrimSpace(edge.ID); id != "" { return id } return strings.Join([]string{edge.Source, edge.SourceHandle, edge.Target, edge.TargetHandle}, "\x00") } func reachableWorkflowNodes(edges []domain.WorkflowEdge, roots []string) map[string]struct{} { visited := make(map[string]struct{}, len(roots)) queue := append([]string{}, roots...) for len(queue) > 0 { nodeID := strings.TrimSpace(queue[0]) queue = queue[1:] if nodeID == "" { continue } if _, exists := visited[nodeID]; exists { continue } visited[nodeID] = struct{}{} for _, edge := range domain.GetOutgoingEdges(edges, nodeID) { queue = append(queue, edge.Target) } } return visited } func markUnselectedAuditBranchesCompleted( workflow domain.WorkflowRecord, executionMap map[string]domain.NodeExecutionRecord, run *domain.RunRecord, nodeID string, selectedEdges []domain.WorkflowEdge, decision domain.AuditStatus, allowed map[string]struct{}, ) { selectedEdgeIDs := make(map[string]struct{}, len(selectedEdges)) selectedRoots := make([]string, 0, len(selectedEdges)) for _, edge := range selectedEdges { selectedEdgeIDs[workflowEdgeIdentity(edge)] = struct{}{} selectedRoots = append(selectedRoots, edge.Target) } unselectedRoots := make([]string, 0) for _, edge := range domain.GetOutgoingEdges(workflow.Edges, nodeID) { if _, selected := selectedEdgeIDs[workflowEdgeIdentity(edge)]; selected { continue } unselectedRoots = append(unselectedRoots, edge.Target) } if len(unselectedRoots) == 0 { return } selectedReachable := reachableWorkflowNodes(workflow.Edges, selectedRoots) unselectedReachable := reachableWorkflowNodes(workflow.Edges, unselectedRoots) for _, candidate := range workflow.Nodes { if _, unselected := unselectedReachable[candidate.ID]; !unselected { continue } if _, sharedWithSelected := selectedReachable[candidate.ID]; sharedWithSelected { continue } if !nodeAllowedForExecution(allowed, candidate.ID) { continue } if _, exists := executionMap[candidate.ID]; exists { continue } timestamp := nowISO() execution := domain.NodeExecutionRecord{ NodeID: candidate.ID, NodeLabel: candidate.Data.Label, Attempt: runExecutionAttempt(run), Status: domain.NodeExecutionStatusCompleted, StartedAt: timestamp, EndedAt: timestamp, Logs: []domain.LogEntryRecord{logEntry("INFO", fmt.Sprintf("Skipped by audit decision: %s", decision), candidate.Data.Label)}, Artifacts: []domain.ExecutionArtifact{}, InputArtifacts: []domain.ExecutionArtifact{}, InputLineage: []domain.InputLineageRecord{}, DataRefs: []domain.DataRefRecord{}, ResourceSeries: []domain.ResourceSample{}, } executionMap[candidate.ID] = execution applyExecution(run, execution) } } func rerunAllowedNodeSet(workflow domain.WorkflowRecord, run *domain.RunRecord) map[string]struct{} { if run == nil { return nil } if run.DebugTarget != nil && run.DebugTarget.SingleNode && strings.TrimSpace(run.DebugTarget.TargetNodeID) != "" { return map[string]struct{}{ strings.TrimSpace(run.DebugTarget.TargetNodeID): {}, } } frozenDeliveryRetry := run.Trigger == domain.RunTriggerRetry && run.Metadata != nil && run.Metadata.CaptureDeliveryPackage != nil if (run.Trigger != domain.RunTriggerRerunSubtree && run.Trigger != domain.RunTriggerHumanRelease && !frozenDeliveryRetry) || strings.TrimSpace(run.RestartFromNodeID) == "" { return nil } visited := map[string]struct{}{} queue := []string{strings.TrimSpace(run.RestartFromNodeID)} for len(queue) > 0 { nodeID := queue[0] queue = queue[1:] if _, ok := visited[nodeID]; ok { continue } visited[nodeID] = struct{}{} for _, edge := range domain.GetOutgoingEdges(workflow.Edges, nodeID) { target := strings.TrimSpace(edge.Target) if target != "" { queue = append(queue, target) } } } return visited } func nodeAllowedForExecution(allowed map[string]struct{}, nodeID string) bool { if allowed == nil { return true } _, ok := allowed[nodeID] return ok } func firstActiveExecution(executionMap map[string]domain.NodeExecutionRecord) (string, domain.NodeExecutionStatus) { for _, status := range []domain.NodeExecutionStatus{ domain.NodeExecutionStatusWaitingForAudit, domain.NodeExecutionStatusWaitingForHuman, domain.NodeExecutionStatusRunning, domain.NodeExecutionStatusQueued, } { for nodeID, execution := range executionMap { if execution.Status == status { return nodeID, status } } } return "", "" } func queueNodeIfReady(workflow domain.WorkflowRecord, executionMap map[string]domain.NodeExecutionRecord, readyQueue *[]string, readySet map[string]struct{}, allowed map[string]struct{}, nodeID string) { if !nodeAllowedForExecution(allowed, nodeID) { return } if _, queued := readySet[nodeID]; queued { return } if execution, ok := executionMap[nodeID]; ok { if execution.Status == domain.NodeExecutionStatusCompleted || execution.Status == domain.NodeExecutionStatusRunning || execution.Status == domain.NodeExecutionStatusWaitingForAudit || execution.Status == domain.NodeExecutionStatusWaitingForHuman { return } } if !dependenciesComplete(workflow, executionMap, nodeID) { return } *readyQueue = append(*readyQueue, nodeID) readySet[nodeID] = struct{}{} } func enqueueDependents(workflow domain.WorkflowRecord, executionMap map[string]domain.NodeExecutionRecord, readyQueue *[]string, readySet map[string]struct{}, allowed map[string]struct{}, nodeID string) { for _, edge := range domain.GetOutgoingEdges(workflow.Edges, nodeID) { queueNodeIfReady(workflow, executionMap, readyQueue, readySet, allowed, edge.Target) } } func enqueueReadyNodes(workflow domain.WorkflowRecord, executionMap map[string]domain.NodeExecutionRecord, readyQueue *[]string, readySet map[string]struct{}, allowed map[string]struct{}) { for _, node := range workflow.Nodes { queueNodeIfReady(workflow, executionMap, readyQueue, readySet, allowed, node.ID) } } func upstreamBusinessVolume(workflow domain.WorkflowRecord, executionMap map[string]domain.NodeExecutionRecord, nodeID string) domain.DataVolume { for _, edge := range domain.GetIncomingEdges(workflow.Edges, nodeID) { execution, ok := executionMap[edge.Source] if !ok || execution.DataMetrics == nil { continue } if execution.DataMetrics.TrainReady.TrajectoryCount != nil || execution.DataMetrics.TrainReady.DurationSeconds != nil || execution.DataMetrics.TrainReady.Bytes != nil { return execution.DataMetrics.TrainReady } if execution.DataMetrics.TechnicalValid.TrajectoryCount != nil || execution.DataMetrics.TechnicalValid.DurationSeconds != nil || execution.DataMetrics.TechnicalValid.Bytes != nil { return execution.DataMetrics.TechnicalValid } return execution.DataMetrics.Output } return domain.DataVolume{} } func (runtime *Runtime) processRun(ctx context.Context, claimed *domain.RunRecord) (*domain.RunRecord, error) { if claimed.ProjectID != "" { p, err := runtime.store.GetProject(ctx, claimed.ProjectID) if err != nil { return nil, err } if p == nil || p.DeletionTaskID != "" { return claimed, nil } } if _, ok, err := runtime.store.AcquireRunLease(ctx, claimed.ID, runtime.leaseOwner(), runtime.leaseTTL()); err != nil { return nil, err } else if !ok { return claimed, nil } leaseCtx, cancelLeaseExecution := context.WithCancelCause(ctx) defer cancelLeaseExecution(context.Canceled) go runtime.renewRunLeaseUntil(leaseCtx, claimed.ID, cancelLeaseExecution) ctx = leaseCtx workflow, err := runtime.store.GetWorkflow(ctx, claimed.WorkflowID) if err != nil { return nil, err } if workflow == nil { claimed.Status = domain.RunStatusFailed claimed.EndedAt = nowISO() claimed.Error = fmt.Sprintf("workflow %s not found", claimed.WorkflowID) claimed.UpdatedAt = nowISO() return claimed, runtime.store.UpsertRun(ctx, *claimed) } workflow, err = deliveryRuntimeWorkflow(workflow, claimed) if err != nil { claimed.Status = domain.RunStatusFailed claimed.EndedAt = nowISO() claimed.Error = err.Error() claimed.UpdatedAt = nowISO() return claimed, runtime.store.UpsertRun(ctx, *claimed) } if err = runtime.attachFrozenReleaseRetry(ctx, claimed); err != nil { claimed.Status = domain.RunStatusFailed claimed.EndedAt = nowISO() claimed.Error = err.Error() claimed.UpdatedAt = nowISO() return claimed, runtime.store.UpsertRun(ctx, *claimed) } if conflicts := domain.FindInputPortConflicts(workflow.Nodes, workflow.Edges); len(conflicts) > 0 { currentRun := claimed message := "工作流图非法:同一输入端口存在多条上游连线。请拆分输入,或为每个上传结果添加单独的上传算子。" currentRun.Status = domain.RunStatusFailed currentRun.EndedAt = nowISO() currentRun.Error = message currentRun.UpdatedAt = nowISO() appendLog(currentRun, logEntry("ERROR", message, "system")) return currentRun, runtime.store.UpsertRun(ctx, *currentRun) } if graphIssues := domain.ValidateSingleConnectedWorkflowGraph(workflow.Nodes, workflow.Edges); len(graphIssues) > 0 { currentRun := claimed message := "工作流图非法:存在孤立算子或多条互不连接的独立链路。请将所有算子通过 artifact/lineage 连成一个单连通工作流。" currentRun.Status = domain.RunStatusFailed currentRun.EndedAt = nowISO() currentRun.Error = message currentRun.UpdatedAt = nowISO() appendLog(currentRun, logEntry("ERROR", message, "system")) return currentRun, runtime.store.UpsertRun(ctx, *currentRun) } if message := runtime.workflowPublicationBlocker(ctx, workflow); message != "" { currentRun := claimed currentRun.Status = domain.RunStatusFailed currentRun.EndedAt = nowISO() currentRun.Error = message currentRun.UpdatedAt = nowISO() appendLog(currentRun, logEntry("ERROR", message, "system")) return currentRun, runtime.store.UpsertRun(ctx, *currentRun) } currentRun, err := runtime.store.GetRun(ctx, claimed.ID) if err != nil { return nil, err } if currentRun == nil { currentRun = claimed } if isTerminalRunStatus(currentRun.Status) { return currentRun, nil } if err := runtime.resolveDirectoryRunInputs(ctx, currentRun); err != nil { message := err.Error() currentRun.Status = domain.RunStatusFailed currentRun.Error = message currentRun.Failure = &domain.RunFailure{Code: "input_resolution_failed", Stage: "input_resolution", Message: message, Retryable: true, Suggestion: "检查对象存储连接、目录前缀和写入 manifest 的权限"} currentRun.EndedAt = nowISO() currentRun.UpdatedAt = nowISO() _ = runtime.store.UpsertRun(ctx, *currentRun) return currentRun, nil } settings, err := runtime.store.GetSettings(ctx) if err != nil { return nil, err } nodeConcurrencyLimit := effectiveWorkflowConcurrencyLimit(workflow, settings) if nodeConcurrencyLimit <= 0 { nodeConcurrencyLimit = 1 } executionMap := map[string]domain.NodeExecutionRecord{} for _, execution := range currentRun.NodeExecutions { executionMap[execution.NodeID] = execution } allowedNodes := rerunAllowedNodeSet(*workflow, currentRun) readyQueue := make([]string, 0) readySet := make(map[string]struct{}) auditQueue := make([]string, 0) if currentRun.CurrentNodeID != "" { node := domain.GetNodeByID(workflow.Nodes, currentRun.CurrentNodeID) if node != nil && domain.NodeRequiresAnnotation(*node) { resolved, targets, execution, resolveErr := runtime.resolveAcceptedAnnotationGate(ctx, *workflow, currentRun, *node, executionMap, allowedNodes) if resolveErr != nil { return nil, resolveErr } if resolved { for _, target := range targets { queueNodeIfReady(*workflow, executionMap, &readyQueue, readySet, allowedNodes, target) } if err := runtime.store.UpsertRun(ctx, *currentRun); err != nil { return nil, err } if execution != nil { runtime.materializeNodeMetricsBestEffort(ctx, currentRun, execution) executionMap[node.ID] = *execution } } } else { tasks, err := runtime.store.ListAuditTasks(ctx, "all") if err != nil { return nil, err } for _, task := range tasks { if task.RunID == currentRun.ID && task.NodeID == currentRun.CurrentNodeID && task.Status != domain.AuditStatusPending { if node != nil { execution := executionMap[node.ID] execution.NodeID = node.ID execution.NodeLabel = node.Data.Label execution.Status = domain.NodeExecutionStatusCompleted execution.EndedAt = nowISO() execution.Logs = append(execution.Logs, logEntry("INFO", fmt.Sprintf("Audit decision: %s", task.Status), node.Data.Label)) resolvedNode := runtime.resolveNodeBusinessStage(ctx, currentRun, *node) if resolvedNode.Data.BusinessStage == domain.DataMetricStageAudit { volume := upstreamBusinessVolume(*workflow, executionMap, node.ID) execution.DataMetrics = &domain.DataMetricsRecord{ SchemaVersion: 1, Stage: domain.DataMetricStageAudit, Input: volume, Output: volume, Quality: domain.DataMetricQualityExact, OccurredAt: execution.EndedAt, } if task.Status == domain.AuditStatusApproved { execution.DataMetrics.TrainReady = volume } else { execution.DataMetrics.Output = domain.DataVolume{} execution.DataMetrics.Rejected = volume execution.DataMetrics.RejectionReasons = map[string]int64{"人工审核拒绝": 1} } } decisionEdges := domain.GetDecisionOutgoingEdges(workflow.Edges, node.ID, task.Status) execution.Artifacts = routeAuditArtifacts(*node, execution.InputArtifacts, decisionEdges) executionMap[node.ID] = execution applyExecution(currentRun, execution) markUnselectedAuditBranchesCompleted(*workflow, executionMap, currentRun, node.ID, decisionEdges, task.Status, allowedNodes) for _, target := range edgeTargets(decisionEdges) { queueNodeIfReady(*workflow, executionMap, &readyQueue, readySet, allowedNodes, target) } currentRun.Status = domain.RunStatusRunning syncCurrentNodeID(currentRun) currentRun.UpdatedAt = nowISO() if err := runtime.store.UpsertRun(ctx, *currentRun); err != nil { return nil, err } runtime.materializeNodeMetricsBestEffort(ctx, currentRun, &execution) executionMap[node.ID] = execution } break } } } } if activeNodeID, activeStatus := firstActiveExecution(executionMap); activeNodeID != "" { if strings.TrimSpace(currentRun.CurrentNodeID) == "" { currentRun.CurrentNodeID = activeNodeID } switch activeStatus { case domain.NodeExecutionStatusWaitingForAudit: currentRun.Status = domain.RunStatusWaitingForAudit case domain.NodeExecutionStatusWaitingForHuman: currentRun.Status = domain.RunStatusWaitingForHuman default: currentRun.Status = domain.RunStatusRunning } currentRun.UpdatedAt = nowISO() syncCurrentNodeID(currentRun) if err := runtime.store.UpsertRun(ctx, *currentRun); err != nil { return nil, err } workflow.Status = domain.WorkflowStatusRunning workflow.LastRunID = currentRun.ID workflow.UpdatedAt = nowISO() if err := runtime.store.UpdateWorkflowRunState(ctx, workflow.ID, workflow.Status, workflow.LastRunID, workflow.UpdatedAt); err != nil { return nil, err } return currentRun, nil } if len(readyQueue) == 0 && len(executionMap) == 0 { if currentRun.DebugTarget != nil && currentRun.DebugTarget.SingleNode && strings.TrimSpace(currentRun.DebugTarget.TargetNodeID) != "" { targetNodeID := strings.TrimSpace(currentRun.DebugTarget.TargetNodeID) if nodeAllowedForExecution(allowedNodes, targetNodeID) { readyQueue = append(readyQueue, targetNodeID) readySet[targetNodeID] = struct{}{} } } else if allowedNodes != nil && strings.TrimSpace(currentRun.RestartFromNodeID) != "" { queueNodeIfReady(*workflow, executionMap, &readyQueue, readySet, allowedNodes, currentRun.RestartFromNodeID) } else { for _, nodeID := range domain.GetStartNodeIDs(workflow.Nodes, workflow.Edges) { queueNodeIfReady(*workflow, executionMap, &readyQueue, readySet, allowedNodes, nodeID) } } } if len(readyQueue) == 0 && len(executionMap) > 0 && currentRun.CurrentNodeID == "" { for _, execution := range currentRun.NodeExecutions { if execution.Status != domain.NodeExecutionStatusCompleted { continue } enqueueDependents(*workflow, executionMap, &readyQueue, readySet, allowedNodes, execution.NodeID) } for _, node := range workflow.Nodes { queueNodeIfReady(*workflow, executionMap, &readyQueue, readySet, allowedNodes, node.ID) } } executionCtx := ctx cancelExecution := func() {} if timeout := workflowRunTimeout(workflow); timeout > 0 { executionCtx, cancelExecution = context.WithTimeout(ctx, timeout) } else { executionCtx, cancelExecution = context.WithCancel(ctx) } defer cancelExecution() results := make(chan nodeExecutionOutcome, nodeConcurrencyLimit) runningNodes := make(map[string]struct{}) nodeStartedAt := make(map[string]time.Time) var persistedControl *domain.RunRecord gracefulStopRequested := false forceStopRequested := false for { if cause := context.Cause(ctx); cause != nil { return currentRun, cause } persistedControl, err = runtime.loadRunControlState(ctx, currentRun.ID) if err != nil { return nil, err } gracefulStopRequested = false forceStopRequested = false if persistedControl != nil { syncStopMetadata(currentRun, persistedControl) if persistedControl.Status == domain.RunStatusStopping && persistedControl.StopMode == domain.RunStopModeGraceful { gracefulStopRequested = true currentRun.Status = domain.RunStatusStopping } if persistedControl.Status == domain.RunStatusCancelled { forceStopRequested = persistedControl.StopMode == domain.RunStopModeForce currentRun.Status = domain.RunStatusCancelled if currentRun.EndedAt == "" { currentRun.EndedAt = persistedControl.EndedAt } if strings.TrimSpace(currentRun.Error) == "" { currentRun.Error = persistedControl.Error } } } if forceStopRequested { cancelExecution() } launched := false for !gracefulStopRequested && !forceStopRequested && len(runningNodes) < nodeConcurrencyLimit && len(readyQueue) > 0 { nodeID := readyQueue[0] readyQueue = readyQueue[1:] delete(readySet, nodeID) node := domain.GetNodeByID(workflow.Nodes, nodeID) if node == nil { continue } if !nodeAllowedForExecution(allowedNodes, node.ID) { continue } if existing, ok := executionMap[node.ID]; ok { if existing.Status == domain.NodeExecutionStatusCompleted || existing.Status == domain.NodeExecutionStatusRunning || existing.Status == domain.NodeExecutionStatusWaitingForAudit || existing.Status == domain.NodeExecutionStatusWaitingForHuman { continue } } isSingleNodeTarget := currentRun.DebugTarget != nil && currentRun.DebugTarget.SingleNode && strings.TrimSpace(currentRun.DebugTarget.TargetNodeID) == node.ID if !isSingleNodeTarget && !dependenciesComplete(*workflow, executionMap, node.ID) { continue } if domain.NodeRequiresAudit(*node) || domain.NodeRequiresAnnotation(*node) { auditQueue = append(auditQueue, node.ID) continue } inputArtifacts, inputLineage, inputBindings := gatherNodeInputs(*workflow, currentRun, executionMap, node.ID) executionAttempt := runExecutionAttempt(currentRun) executionLogs := []domain.LogEntryRecord{logEntry("INFO", "Starting node execution", node.Data.Label)} if previous, ok := executionMap[node.ID]; ok && missingJobRecoveryCount(&previous) > 0 { if previous.Attempt >= executionAttempt { executionAttempt = previous.Attempt + 1 } executionLogs = append(append([]domain.LogEntryRecord{}, previous.Logs...), executionLogs...) } nodeToPrepare := *node if currentRun.NodeOverrides != nil { if override, ok := currentRun.NodeOverrides[node.ID]; ok { nodeToPrepare = applyNodeOverridesToNode(nodeToPrepare, override) } } execution := domain.NodeExecutionRecord{ NodeID: node.ID, NodeLabel: node.Data.Label, Attempt: executionAttempt, Status: domain.NodeExecutionStatusRunning, StartedAt: nowISO(), Logs: executionLogs, Artifacts: []domain.ExecutionArtifact{}, DataRefs: []domain.DataRefRecord{}, ResourceSeries: []domain.ResourceSample{}, StageManifestPath: "", } applyNodeExecutionSnapshot(&execution, nodeToPrepare, inputArtifacts, inputLineage, inputBindings) executionMap[node.ID] = execution applyExecution(currentRun, execution) runningNodes[node.ID] = struct{}{} nodeStartedAt[node.ID] = time.Now() runtime.metrics.RecordNodeExecutionStarted() launched = true runSnapshot := *currentRun // Missing-Job recovery is a node execution retry inside the same Run. // Use the node attempt for the Kubernetes Job suffix so a recovered // execution cannot collide with the vanished attempt name. runSnapshot.Attempt = executionAttempt nodeSnapshot, err := runtime.prepareExecutionNode(ctx, workflow, nodeToPrepare, inputBindings) if err != nil { results <- nodeExecutionOutcome{ nodeID: node.ID, err: err, } continue } // Persist the runtime-resolved builtin image and contract, not the // stale image snapshot embedded in an older workflow definition. // The Kubernetes Job has always used nodeSnapshot; keeping the Run // snapshot aligned prevents the UI from reporting a different digest. applyNodeExecutionSnapshot(&execution, nodeSnapshot, inputArtifacts, inputLineage, inputBindings) executionMap[node.ID] = execution applyExecution(currentRun, execution) inputSnapshot := cloneArtifacts(inputArtifacts) inputBindingsSnapshot := cloneInputBindings(inputBindings) nodeExecutionCtx := executionCtx cancelNodeExecution := func() {} if timeout := workflowNodeTimeout(workflow); timeout > 0 { nodeExecutionCtx, cancelNodeExecution = context.WithTimeout(executionCtx, timeout) } go func() { defer cancelNodeExecution() // Stream live GPU samples for custom-container nodes during execution so the // resource monitor reflects usage before the node completes (builtin nodes // self-report via operator-runner's ResourceSampler, so they are skipped). progressCtx, progressCancel := context.WithCancel(nodeExecutionCtx) var progressWG sync.WaitGroup if shouldReportNodeProgress(runtime, nodeSnapshot) { sampleGPU := runtime.operatorRunnerImage != "" && chooseImage(nodeSnapshot, runtime.operatorRunnerImage) != runtime.operatorRunnerImage progressWG.Add(1) go runtime.runNodeProgressReporter(progressCtx, runSnapshot.ID, nodeSnapshot.ID, sampleGPU, func() { progressWG.Done() }) } result, reference, err := runtime.executeNode(nodeExecutionCtx, &runSnapshot, nodeSnapshot, inputSnapshot, inputBindingsSnapshot) progressCancel() progressWG.Wait() results <- nodeExecutionOutcome{ nodeID: nodeSnapshot.ID, result: result, reference: reference, err: err, nodeTimedOut: errors.Is(nodeExecutionCtx.Err(), context.DeadlineExceeded), } }() } if launched { currentRun.Status = domain.RunStatusRunning syncCurrentNodeID(currentRun) currentRun.UpdatedAt = nowISO() if err := runtime.store.UpsertRun(ctx, *currentRun); err != nil { return nil, err } } if len(runningNodes) == 0 { if forceStopRequested { return runtime.finalizeCancelledRun(ctx, workflow, currentRun, executionMap, "用户已强制停止工作流") } if gracefulStopRequested || currentRun.Status == domain.RunStatusStopping { return runtime.finalizeCancelledRun(ctx, workflow, currentRun, executionMap, "用户已停止工作流") } if len(readyQueue) == 0 && len(auditQueue) == 0 { enqueueReadyNodes(*workflow, executionMap, &readyQueue, readySet, allowedNodes) } if len(readyQueue) > 0 { continue } if len(auditQueue) > 0 { nodeID := auditQueue[0] auditQueue = auditQueue[1:] node := domain.GetNodeByID(workflow.Nodes, nodeID) if node == nil { continue } inputArtifacts, inputLineage, inputBindings := gatherNodeInputs(*workflow, currentRun, executionMap, node.ID) execution := executionMap[node.ID] if execution.NodeID == "" { execution = domain.NodeExecutionRecord{ NodeID: node.ID, NodeLabel: node.Data.Label, Attempt: runExecutionAttempt(currentRun), Status: domain.NodeExecutionStatusRunning, StartedAt: nowISO(), Logs: []domain.LogEntryRecord{logEntry("INFO", "Starting node execution", node.Data.Label)}, Artifacts: []domain.ExecutionArtifact{}, DataRefs: []domain.DataRefRecord{}, ResourceSeries: []domain.ResourceSample{}, } } applyNodeExecutionSnapshot(&execution, *node, inputArtifacts, inputLineage, inputBindings) waitingMessage := "Node is waiting for human review" if domain.NodeRequiresAnnotation(*node) { tasks, createErr := runtime.createAnnotationGateTasks(ctx, currentRun, *node, inputArtifacts, inputBindings) if createErr != nil { return nil, createErr } waitingMessage = fmt.Sprintf("Node is waiting for %d annotation task(s)", len(tasks)) } else { task := domain.AuditTaskRecord{ ID: domain.NewID("audit"), OrganizationID: currentRun.OrganizationID, RunID: currentRun.ID, WorkflowID: workflow.ID, ProjectID: workflow.ProjectID, NodeID: node.ID, WorkflowName: workflow.Name, ProjectName: currentRun.ProjectName, OperatorName: node.Data.Label, CreatedAt: nowISO(), UpdatedAt: nowISO(), Status: domain.AuditStatusPending, Data: createAuditPayload(*node, inputArtifacts), } if _, err := runtime.store.CreateAuditTask(ctx, task); err != nil { return nil, err } if err := runtime.ensureGenericAuditHumanItem(ctx, currentRun, *node, task, inputArtifacts); err != nil { return nil, err } } if domain.NodeRequiresAnnotation(*node) { execution.Status = domain.NodeExecutionStatusWaitingForHuman } else { execution.Status = domain.NodeExecutionStatusWaitingForAudit } execution.EndedAt = nowISO() execution.Artifacts = cloneArtifacts(inputArtifacts) execution.InputArtifacts = cloneArtifacts(inputArtifacts) execution.InputLineage = cloneInputLineage(inputLineage) execution.Logs = append(execution.Logs, logEntry("WARN", waitingMessage, node.Data.Label)) executionMap[node.ID] = execution applyExecution(currentRun, execution) if domain.NodeRequiresAnnotation(*node) { currentRun.Status = domain.RunStatusWaitingForHuman } else { currentRun.Status = domain.RunStatusWaitingForAudit } syncCurrentNodeID(currentRun) currentRun.UpdatedAt = nowISO() if err := runtime.store.UpsertRun(ctx, *currentRun); err != nil { return nil, err } workflow.Status = domain.WorkflowStatusRunning workflow.UpdatedAt = nowISO() if err := runtime.store.UpdateWorkflowRunState(ctx, workflow.ID, workflow.Status, workflow.LastRunID, workflow.UpdatedAt); err != nil { return nil, err } return currentRun, nil } break } outcome := <-results delete(runningNodes, outcome.nodeID) startedAt := nodeStartedAt[outcome.nodeID] delete(nodeStartedAt, outcome.nodeID) nodeDurationMS := uint64(0) if !startedAt.IsZero() { nodeDurationMS = uint64(time.Since(startedAt).Milliseconds()) } persistedControl, err := runtime.loadRunControlState(ctx, currentRun.ID) if err != nil { return nil, err } gracefulStopRequested := false forceStopRequested := false if persistedControl != nil { syncStopMetadata(currentRun, persistedControl) if persistedControl.Status == domain.RunStatusStopping && persistedControl.StopMode == domain.RunStopModeGraceful { gracefulStopRequested = true } if persistedControl.Status == domain.RunStatusCancelled && persistedControl.StopMode == domain.RunStopModeForce { forceStopRequested = true } } execution := executionMap[outcome.nodeID] if outcome.err != nil { if cause := context.Cause(ctx); cause != nil { return currentRun, cause } if forceStopRequested || (currentRun.StopMode == domain.RunStopModeForce && errors.Is(executionCtx.Err(), context.Canceled)) { cancelledExecution, _ := finalizeCancelledExecution(execution, "用户已强制停止工作流") executionMap[outcome.nodeID] = cancelledExecution applyExecution(currentRun, cancelledExecution) currentRun.Status = domain.RunStatusCancelled currentRun.UpdatedAt = nowISO() if err := runtime.store.UpsertRun(ctx, *currentRun); err != nil { return nil, err } if len(runningNodes) == 0 { return runtime.finalizeCancelledRun(ctx, workflow, currentRun, executionMap, "用户已强制停止工作流") } continue } if k8s.IsJobNotFound(outcome.err) { cancelExecution() executionMap[outcome.nodeID] = execution applyExecution(currentRun, execution) if err := runtime.recoverMissingJob(ctx, currentRun, &execution); err != nil { return nil, err } return currentRun, nil } runtime.metrics.RecordNodeExecutionFailed(nodeDurationMS) cancelExecution() execution.Status = domain.NodeExecutionStatusFailed execution.EndedAt = nowISO() execution.Error = normalizeRunError(workflow, outcome.err, executionCtx, outcome.nodeTimedOut) execution.Logs = append(execution.Logs, logEntry("ERROR", execution.Error, execution.NodeLabel)) executionMap[outcome.nodeID] = execution applyExecution(currentRun, execution) currentRun.Status = domain.RunStatusFailed currentRun.CurrentNodeID = outcome.nodeID currentRun.EndedAt = nowISO() currentRun.Error = execution.Error currentRun.Failure = &domain.RunFailure{ Code: "node_execution_failed", Stage: "execution", NodeID: outcome.nodeID, Message: execution.Error, Retryable: true, Suggestion: "查看节点日志后从失败节点继续", } currentRun.UpdatedAt = nowISO() if err := runtime.store.UpsertRun(ctx, *currentRun); err != nil { return nil, err } workflow.Status = domain.WorkflowStatusFailed workflow.UpdatedAt = nowISO() if err := runtime.store.UpdateWorkflowRunState(ctx, workflow.ID, workflow.Status, workflow.LastRunID, workflow.UpdatedAt); err != nil { return nil, err } if shouldAutoRetry(workflow, currentRun) { retryRun, err := runtime.enqueueRetryRun(ctx, workflow, currentRun) if err != nil { return nil, err } if retryRun != nil { appendLog(currentRun, logEntry("WARN", fmt.Sprintf("已自动创建重试运行,第 %d 次尝试已进入队列", retryRun.Attempt), execution.NodeLabel)) currentRun.UpdatedAt = nowISO() if err := runtime.store.UpsertRun(ctx, *currentRun); err != nil { return nil, err } } } return currentRun, nil } runtime.metrics.RecordNodeExecutionCompleted(nodeDurationMS) execution.Status = domain.NodeExecutionStatusCompleted execution.EndedAt = nowISO() execution.Logs = append(execution.Logs, outcome.result.Logs...) execution.Artifacts = outcome.result.Artifacts execution.DataRefs = outcome.result.DataRefs execution.DataMetrics = outcome.result.DataMetrics execution.StageManifestPath = outcome.result.StageManifestPath execution.ResourceSeries = outcome.result.ResourceSeries executionMap[outcome.nodeID] = execution applyExecution(currentRun, execution) if outcome.reference != nil { currentRun.K8sRefs = append(currentRun.K8sRefs, domain.K8sRef{NodeID: outcome.nodeID, Attempt: execution.Attempt, JobName: outcome.reference.JobName, Namespace: outcome.reference.Namespace}) } if !gracefulStopRequested && !forceStopRequested { enqueueDependents(*workflow, executionMap, &readyQueue, readySet, allowedNodes, outcome.nodeID) } if gracefulStopRequested { currentRun.Status = domain.RunStatusStopping } else if forceStopRequested { currentRun.Status = domain.RunStatusCancelled } else { currentRun.Status = domain.RunStatusRunning } syncCurrentNodeID(currentRun) currentRun.UpdatedAt = nowISO() if err := runtime.store.UpsertRun(ctx, *currentRun); err != nil { return nil, err } runtime.materializeNodeMetricsBestEffort(ctx, currentRun, &execution) executionMap[outcome.nodeID] = execution if forceStopRequested && len(runningNodes) == 0 { return runtime.finalizeCancelledRun(ctx, workflow, currentRun, executionMap, "用户已强制停止工作流") } } if currentRun.Status == domain.RunStatusStopping || currentRun.Status == domain.RunStatusCancelled { message := "用户已停止工作流" if currentRun.StopMode == domain.RunStopModeForce { message = "用户已强制停止工作流" } return runtime.finalizeCancelledRun(ctx, workflow, currentRun, executionMap, message) } return runtime.finalizeCompletedRun(ctx, workflow, currentRun) } // nodeProgressSampleInterval bounds how often the worker reads structured progress markers // from long-running pods and, for custom containers, pushes live GPU samples. const nodeProgressSampleInterval = 15 * time.Second // shouldReportNodeProgress reports whether a node needs live runtime polling. Custom // containers need platform-side GPU samples, while long-running builtin and custom nodes // can emit DROBOTICFLOW_PROGRESS markers that must be persisted before the job finishes. func shouldReportNodeProgress(runtime *Runtime, node domain.WorkflowNode) bool { return runtimeContractLongRunning(node.Data.RuntimeContract) || (runtime.operatorRunnerImage != "" && chooseImage(node, runtime.operatorRunnerImage) != runtime.operatorRunnerImage) } // mapGPUSample maps the launcher's k8s.GPUSample onto the resource monitor's domain // representation — the same field mapping the executor uses when folding the launcher's // final series into the node's resourceSeries. func mapGPUSample(sample k8s.GPUSample) domain.ResourceSample { return domain.ResourceSample{ Timestamp: sample.Timestamp, GPU: sample.GPU, GPUMemoryUsedMiB: sample.GPUMemoryUsedMiB, GPUMemoryTotalMiB: sample.GPUMemoryTotalMiB, GPUName: sample.GPUName, GPUCount: sample.GPUCount, } } // runNodeProgressReporter persists DROBOTICFLOW_PROGRESS markers while a long-running node is // active and optionally samples GPU usage for custom containers. It stops before the final node // result is persisted, preventing a stale live write from racing with completion. const progressMarker = "DROBOTICFLOW_PROGRESS " func latestStructuredProgress(logs string) (*domain.RunProgress, error) { var latest *domain.RunProgress var lastErr error for _, line := range strings.Split(strings.ReplaceAll(logs, "\r\n", "\n"), "\n") { markerIndex := strings.Index(line, progressMarker) if markerIndex < 0 { continue } payload := strings.TrimSpace(line[markerIndex+len(progressMarker):]) if payload == "" { continue } var progress domain.RunProgress if err := json.Unmarshal([]byte(payload), &progress); err != nil { lastErr = fmt.Errorf("decode structured progress marker: %w", err) continue } if strings.TrimSpace(progress.UpdatedAt) == "" { progress.UpdatedAt = nowISO() } copy := progress latest = © } if latest != nil { return latest, nil } return nil, lastErr } func progressFingerprint(progress *domain.RunProgress) string { if progress == nil { return "" } copy := *progress copy.UpdatedAt = "" payload, err := json.Marshal(copy) if err != nil { return "" } return string(payload) } func (runtime *Runtime) runNodeProgressReporter(ctx context.Context, runID string, nodeID string, sampleGPU bool, done func()) { runtime.runNodeProgressReporterWithInterval(ctx, runID, nodeID, sampleGPU, nodeProgressSampleInterval, done) } func (runtime *Runtime) runNodeProgressReporterWithInterval(ctx context.Context, runID string, nodeID string, sampleGPU bool, interval time.Duration, done func()) { defer done() if interval <= 0 { interval = nodeProgressSampleInterval } ticker := time.NewTicker(interval) defer ticker.Stop() lastProgressFingerprint := "" tailer, canTailLogs := runtime.launcher.(nodePodLogTailer) for { select { case <-ctx.Done(): return case <-ticker.C: } if canTailLogs { logsCtx, cancelLogs := context.WithTimeout(ctx, 10*time.Second) logs, err := tailer.TailNodePodLogs(logsCtx, runID, nodeID, 200) cancelLogs() if err == nil { progress, parseErr := latestStructuredProgress(logs) fingerprint := progressFingerprint(progress) if parseErr == nil && progress != nil && fingerprint != "" && fingerprint != lastProgressFingerprint { if err := runtime.store.UpdateRunProgress(ctx, runID, progress); err == nil { lastProgressFingerprint = fingerprint } } } } if !sampleGPU { continue } sampleCtx, cancelSample := context.WithTimeout(ctx, 10*time.Second) sample, err := runtime.launcher.SamplePodGPUForRun(sampleCtx, runID, nodeID) cancelSample() if err == nil && sample.Timestamp != "" { _ = runtime.store.AppendNodeExecutionResourceSample(ctx, runID, nodeID, mapGPUSample(sample)) } } } func edgeTargets(edges []domain.WorkflowEdge) []string { items := make([]string, 0, len(edges)) for _, edge := range edges { items = append(items, edge.Target) } return items } func (runtime *Runtime) ProcessOnce(ctx context.Context) (*ProcessOnceResult, error) { if err := runtime.reconcileActiveRuns(ctx); err != nil { return nil, err } claimed, err := runtime.store.ClaimNextQueuedRun(ctx, runtime.leaseOwner(), runtime.leaseTTL()) if err != nil || claimed == nil { if err != nil { return nil, err } return newProcessOnceResult(ProcessOnceStatusNoQueuedRun, nil), nil } if !runtime.tryMarkRunActive(claimed.ID) { // Another goroutine in this worker is already processing this run // (tryMarkRunActive guards the in-memory activeRuns set). ClaimNextQueuedRun // already set status=Running + claimed the lease for this worker, which is // consistent with the active processor. Release the spurious lease so it // does not block the next ProcessAvailable claim for the lease TTL (~15min), // and leave status=Running rather than reverting to Queued — reverting kept // currentNodeId set, producing an inconsistent "queued+currentNodeId" state // that no other queued-writer emits. Reconcile / the active processor will // advance the run. if releaseErr := runtime.store.ReleaseRunLease(context.Background(), claimed.ID, runtime.leaseOwner()); releaseErr != nil { fmt.Printf("[worker] release spurious run lease %s failed: %v\n", claimed.ID, releaseErr) } return newProcessOnceResult(ProcessOnceStatusSkippedLeaseActive, claimed), nil } defer runtime.unmarkRunActive(claimed.ID) defer func() { if err := runtime.store.ReleaseRunLease(context.Background(), claimed.ID, runtime.leaseOwner()); err != nil { fmt.Printf("[worker] release run lease %s failed: %v\n", claimed.ID, err) } }() run, err := runtime.processRun(ctx, claimed) if err != nil { return nil, err } return newProcessOnceResult(ProcessOnceStatusClaimed, run), nil }