2026-11-08 10:00:31 -08:00

3514 lines
124 KiB
Go

package worker
import (
"context"
"encoding/json"
"errors"
"fmt"
"math"
"os"
"reflect"
"sort"
"strconv"
"strings"
"sync"
"sync/atomic"
"time"
"droboticflow/internal/dataops"
"droboticflow/internal/domain"
"droboticflow/internal/k8s"
"droboticflow/internal/lightweight"
"droboticflow/internal/objectsync"
"droboticflow/internal/store"
)
type jobLauncher interface {
Launch(context.Context, k8s.LaunchInput) (*k8s.LaunchResult, error)
Inspect(context.Context, string, string) (*k8s.JobInspection, error)
TerminateRun(context.Context, string, string) error
// SamplePodGPUForRun samples GPU usage inside the pod backing runID+nodeID.
// Used by the worker's progress reporter to stream live GPU metrics for
// custom-container operators during execution. Gracefully errors when no
// Running pod exists or pods/exec RBAC is unavailable.
SamplePodGPUForRun(context.Context, string, string) (k8s.GPUSample, error)
}
type gpuCapacityProvider interface {
GPUCapacity(context.Context) (*k8s.GPUCapacitySnapshot, error)
}
type orphanRuntimeReconciler interface {
ReconcileOrphanedRuntimes(context.Context, map[string]struct{}, time.Time, bool) ([]k8s.OrphanRuntime, error)
}
type nodePodLogTailer interface {
TailNodePodLogs(context.Context, string, string, int64) (string, error)
}
type BuildInfo struct {
BuildCommit string `json:"buildCommit,omitempty"`
BuildTime string `json:"buildTime,omitempty"`
Image string `json:"image,omitempty"`
}
type ReadinessSnapshot struct {
ReadinessWindowMS int64 `json:"readinessWindowMs,omitempty"`
Ready bool `json:"ready"`
Status string `json:"status"`
Service string `json:"service"`
BuildCommit string `json:"buildCommit,omitempty"`
BuildTime string `json:"buildTime,omitempty"`
Image string `json:"image,omitempty"`
RunnerImage string `json:"runnerImage,omitempty"`
RunnerBuildCommit string `json:"runnerBuildCommit,omitempty"`
RunnerBuildTime string `json:"runnerBuildTime,omitempty"`
LastCycleAt string `json:"lastCycleAt,omitempty"`
LastSuccessfulCycleAt string `json:"lastSuccessfulCycleAt,omitempty"`
CycleDurationMS uint64 `json:"cycleDurationMs"`
ConfiguredPollIntervalMS int64 `json:"configuredPollIntervalMs"`
EffectivePollIntervalMS int64 `json:"effectivePollIntervalMs"`
StuckRunCount uint64 `json:"stuckRunCount"`
OrphanRuntimeCount uint64 `json:"orphanRuntimeCount"`
OrphanRuntimeCleanedCount uint64 `json:"orphanRuntimeCleanedCount"`
OrphanRuntimeCleanupErrorCount uint64 `json:"orphanRuntimeCleanupErrorCount"`
Reason string `json:"reason,omitempty"`
}
type Runtime struct {
captureStatusCursor string
captureStatusMu sync.Mutex
store store.Store
operatorRunnerImage string
workspaceRoot string
globalBOSEndpoint string
artifactPublisher objectsync.Publisher
qwenNodeCache qwenNodeCacheConfig
launcher jobLauncher
metrics WorkerMetrics
processMu sync.Mutex
activeRuns map[string]struct{}
dispatchOwner string
runLeaseTTL time.Duration
buildMu sync.RWMutex
buildInfo BuildInfo
runnerBuildInfo BuildInfo
configuredPollMS atomic.Int64
effectivePollMS atomic.Int64
orphanCleanup bool
orphanMinimumAge time.Duration
orphanCheckInterval time.Duration
lastOrphanCheckNano atomic.Int64
pendingFailureAfter time.Duration
missingMaxRecoveries int
recoveryReservedSlots int
runArchiveMinimumAge time.Duration
runArchiveInterval time.Duration
lastRunArchiveNano atomic.Int64
materializationCleanup materializationCleanupProgress
}
type ProcessOnceStatus string
const (
ProcessOnceStatusClaimed ProcessOnceStatus = "claimed"
ProcessOnceStatusSkippedLeaseActive ProcessOnceStatus = "skipped_lease_active"
ProcessOnceStatusNoQueuedRun ProcessOnceStatus = "no_queued_run"
)
var errRunLeaseLost = errors.New("run execution lease lost")
type ProcessOnceResult struct {
Status ProcessOnceStatus `json:"status"`
Run *domain.RunRecord `json:"-"`
RunRef *ProcessOnceRunSummary `json:"run,omitempty"`
}
type ProcessOnceRunSummary struct {
ID string `json:"id"`
Status domain.RunStatus `json:"status"`
WorkflowID string `json:"workflowId,omitempty"`
WorkflowName string `json:"workflowName,omitempty"`
ProjectID string `json:"projectId,omitempty"`
ProjectName string `json:"projectName,omitempty"`
CurrentNodeID string `json:"currentNodeId,omitempty"`
NodeExecutionCount int `json:"nodeExecutionCount"`
ArtifactCount int `json:"artifactCount"`
Attempt int `json:"attempt,omitempty"`
LeaseOwner string `json:"executionLeaseOwner,omitempty"`
LeaseExpiresAt string `json:"executionLeaseExpiresAt,omitempty"`
UpdatedAt string `json:"updatedAt,omitempty"`
}
func newProcessOnceResult(status ProcessOnceStatus, run *domain.RunRecord) *ProcessOnceResult {
result := &ProcessOnceResult{Status: status, Run: run}
if run == nil {
return result
}
artifactCount := len(run.Artifacts)
for _, execution := range run.NodeExecutions {
artifactCount += len(execution.Artifacts)
}
result.RunRef = &ProcessOnceRunSummary{
ID: run.ID,
Status: run.Status,
WorkflowID: run.WorkflowID,
WorkflowName: run.WorkflowName,
ProjectID: run.ProjectID,
ProjectName: run.ProjectName,
CurrentNodeID: run.CurrentNodeID,
NodeExecutionCount: len(run.NodeExecutions),
ArtifactCount: artifactCount,
Attempt: run.Attempt,
LeaseOwner: run.ExecutionLeaseOwner,
LeaseExpiresAt: run.ExecutionLeaseExpiresAt,
UpdatedAt: run.UpdatedAt,
}
return result
}
var globalActiveDispatchStatuses = map[domain.RunStatus]struct{}{
domain.RunStatusRunning: {},
domain.RunStatusStopping: {},
}
var workflowActiveDispatchStatuses = map[domain.RunStatus]struct{}{
domain.RunStatusRunning: {},
domain.RunStatusStopping: {},
}
const (
workerQueuedRunLimit = 100
workerReconcileRunLimit = 500
workerWorkflowQueryLimit = 500
)
func NewRuntime(dataStore store.Store, namespace string, imagePullSecretName string, operatorRunnerImage string, workspacePVCName string, workspaceMountPath string, gpuNodeSelector string, gpuTolerationsJSON string, gpuRuntimeClass string, jobTTLSeconds int, globalBOSEndpoint string, qwenNodeCacheEnabled bool, qwenNodeCachePVCName string, qwenNodeCacheMountPath string, qwenNodeCacheNodeSelector string, qwenNodeCacheStaleStagingSeconds int) (*Runtime, error) {
launcher, err := k8s.NewLauncher(namespace, imagePullSecretName, workspacePVCName, workspaceMountPath, gpuNodeSelector, gpuTolerationsJSON, gpuRuntimeClass, jobTTLSeconds)
if err != nil {
return nil, err
}
qwenNodeCache, err := newQwenNodeCacheConfig(qwenNodeCacheEnabled, qwenNodeCachePVCName, qwenNodeCacheMountPath, qwenNodeCacheNodeSelector, qwenNodeCacheStaleStagingSeconds)
if err != nil {
return nil, err
}
trimmedWorkspaceRoot := strings.TrimSpace(workspaceMountPath)
if trimmedWorkspaceRoot == "" {
trimmedWorkspaceRoot = "/workspace"
}
return &Runtime{
store: dataStore,
operatorRunnerImage: operatorRunnerImage,
workspaceRoot: trimmedWorkspaceRoot,
globalBOSEndpoint: strings.TrimSpace(globalBOSEndpoint),
qwenNodeCache: qwenNodeCache,
launcher: launcher,
dispatchOwner: defaultDispatchOwner(),
runLeaseTTL: defaultRunLeaseTTL(),
orphanCleanup: envBool("WORKER_ORPHAN_CLEANUP_ENABLED", false),
orphanMinimumAge: envDurationSeconds("WORKER_ORPHAN_MIN_AGE_SECONDS", 24*time.Hour),
orphanCheckInterval: envDurationSeconds("WORKER_ORPHAN_CHECK_INTERVAL_SECONDS", 5*time.Minute),
pendingFailureAfter: envDurationSeconds("WORKER_PENDING_FAILURE_SECONDS", 10*time.Minute),
missingMaxRecoveries: envPositiveInt("WORKER_MISSING_JOB_MAX_RECOVERIES", 3),
recoveryReservedSlots: envNonNegativeInt("WORKER_RECOVERY_RESERVED_SLOTS", 1),
runArchiveMinimumAge: envDurationSeconds("WORKER_RUN_ARCHIVE_MIN_AGE_SECONDS", 30*24*time.Hour),
runArchiveInterval: envDurationSeconds("WORKER_RUN_ARCHIVE_CHECK_INTERVAL_SECONDS", time.Hour),
}, nil
}
func envBool(key string, fallback bool) bool {
raw := strings.TrimSpace(os.Getenv(key))
if raw == "" {
return fallback
}
parsed, err := strconv.ParseBool(raw)
if err != nil {
return fallback
}
return parsed
}
func envPositiveInt(key string, fallback int) int {
raw := strings.TrimSpace(os.Getenv(key))
if raw == "" {
return fallback
}
parsed, err := strconv.Atoi(raw)
if err != nil || parsed <= 0 {
return fallback
}
return parsed
}
func envNonNegativeInt(key string, fallback int) int {
raw := strings.TrimSpace(os.Getenv(key))
if raw == "" {
return fallback
}
parsed, err := strconv.Atoi(raw)
if err != nil || parsed < 0 {
return fallback
}
return parsed
}
func envDurationSeconds(key string, fallback time.Duration) time.Duration {
seconds := envPositiveInt(key, int(fallback/time.Second))
return time.Duration(seconds) * time.Second
}
func (runtime *Runtime) SetBuildInfo(info BuildInfo) {
if runtime == nil {
return
}
runtime.buildMu.Lock()
runtime.buildInfo = BuildInfo{
BuildCommit: strings.TrimSpace(info.BuildCommit),
BuildTime: strings.TrimSpace(info.BuildTime),
Image: strings.TrimSpace(info.Image),
}
runtime.buildMu.Unlock()
}
func (runtime *Runtime) BuildInfo() BuildInfo {
if runtime == nil {
return BuildInfo{}
}
runtime.buildMu.RLock()
defer runtime.buildMu.RUnlock()
return runtime.buildInfo
}
func (runtime *Runtime) SetRunnerBuildInfo(info BuildInfo) {
if runtime == nil {
return
}
runtime.buildMu.Lock()
runtime.runnerBuildInfo = BuildInfo{
BuildCommit: strings.TrimSpace(info.BuildCommit),
BuildTime: strings.TrimSpace(info.BuildTime),
Image: strings.TrimSpace(info.Image),
}
runtime.buildMu.Unlock()
}
func (runtime *Runtime) RunnerBuildInfo() BuildInfo {
if runtime == nil {
return BuildInfo{}
}
runtime.buildMu.RLock()
defer runtime.buildMu.RUnlock()
return runtime.runnerBuildInfo
}
func runtimeOperatorGeneratedByAI(item domain.OperatorRecord) bool {
source := strings.ToLower(strings.TrimSpace(string(item.GenerationSource)))
if source == string(domain.OperatorGenerationSourceAI) {
return true
}
if item.RuntimeContract == nil {
return false
}
if sourceValue, ok := item.RuntimeContract["generationSource"].(string); ok && strings.EqualFold(strings.TrimSpace(sourceValue), string(domain.OperatorGenerationSourceAI)) {
return true
}
_, generated := item.RuntimeContract["generatedRepository"]
return generated
}
func runtimeOperatorPublishedStatus(item domain.OperatorRecord) domain.OperatorPublicationStatus {
status := domain.OperatorPublicationStatus(strings.ToLower(strings.TrimSpace(string(item.PublicationStatus))))
if status != "" {
return status
}
if runtimeOperatorGeneratedByAI(item) {
return domain.OperatorPublicationStatusDraft
}
return domain.OperatorPublicationStatusPublished
}
func runtimeNodeGeneratedByAI(node domain.WorkflowNode) bool {
return strings.EqualFold(strings.TrimSpace(string(node.Data.GenerationSource)), string(domain.OperatorGenerationSourceAI))
}
func runtimeNodePublishedStatus(node domain.WorkflowNode) domain.OperatorPublicationStatus {
status := domain.OperatorPublicationStatus(strings.ToLower(strings.TrimSpace(string(node.Data.PublicationStatus))))
if status != "" {
return status
}
if runtimeNodeGeneratedByAI(node) {
return domain.OperatorPublicationStatusDraft
}
return domain.OperatorPublicationStatusPublished
}
func (runtime *Runtime) workflowPublicationBlocker(ctx context.Context, workflow *domain.WorkflowRecord) string {
if workflow == nil || workflow.ExecutionScope == domain.WorkflowExecutionScopeSandbox {
return ""
}
for _, node := range workflow.Nodes {
sourceAI := runtimeNodeGeneratedByAI(node)
status := runtimeNodePublishedStatus(node)
if operatorID := strings.TrimSpace(node.Data.OperatorID); operatorID != "" {
operator, err := runtime.store.GetOperator(ctx, operatorID)
if err != nil {
return "工作流图非法:无法读取算子发布状态。"
}
if operator != nil {
sourceAI = runtimeOperatorGeneratedByAI(*operator)
status = runtimeOperatorPublishedStatus(*operator)
}
}
if !sourceAI || status == domain.OperatorPublicationStatusPublished {
continue
}
label := strings.TrimSpace(node.Data.Label)
if label == "" {
label = node.ID
}
return "生产工作流不能使用未发布的 AI 生成算子:" + label + "。请在沙盒工作流验证,或发布后再运行。"
}
return ""
}
func defaultDispatchOwner() string {
hostname, err := os.Hostname()
if err != nil || strings.TrimSpace(hostname) == "" {
hostname = "unknown-host"
}
return fmt.Sprintf("%s:%d", hostname, os.Getpid())
}
func defaultRunLeaseTTL() time.Duration {
raw := strings.TrimSpace(os.Getenv("WORKER_RUN_LEASE_TTL_SECONDS"))
if raw == "" {
return 15 * time.Minute
}
seconds, err := strconv.Atoi(raw)
if err != nil || seconds <= 0 {
return 15 * time.Minute
}
return time.Duration(seconds) * time.Second
}
func (runtime *Runtime) leaseOwner() string {
if runtime == nil {
return ""
}
if strings.TrimSpace(runtime.dispatchOwner) == "" {
runtime.dispatchOwner = defaultDispatchOwner()
}
return runtime.dispatchOwner
}
func (runtime *Runtime) leaseTTL() time.Duration {
if runtime == nil || runtime.runLeaseTTL <= 0 {
return defaultRunLeaseTTL()
}
return runtime.runLeaseTTL
}
func (runtime *Runtime) MetricsSnapshot() MetricsSnapshot {
return runtime.metrics.Snapshot()
}
const (
defaultPollIntervalMS int64 = 2000
minimumPollIntervalMS int64 = 250
maximumPollIntervalMS int64 = 60000
)
func normalizedPollIntervalMS(value int64) int64 {
if value <= 0 {
return defaultPollIntervalMS
}
if value < minimumPollIntervalMS {
return minimumPollIntervalMS
}
if value > maximumPollIntervalMS {
return maximumPollIntervalMS
}
return value
}
func (runtime *Runtime) SetConfiguredPollInterval(interval time.Duration) {
if runtime == nil {
return
}
configured := normalizedPollIntervalMS(interval.Milliseconds())
runtime.configuredPollMS.Store(configured)
if runtime.effectivePollMS.Load() == 0 {
runtime.effectivePollMS.Store(configured)
}
}
func (runtime *Runtime) updateEffectivePollInterval(settings *domain.SettingsRecord) {
configured := normalizedPollIntervalMS(runtime.configuredPollMS.Load())
effective := configured
if settings != nil && settings.Worker.PollIntervalMS > 0 {
effective = normalizedPollIntervalMS(int64(settings.Worker.PollIntervalMS))
}
runtime.configuredPollMS.Store(configured)
runtime.effectivePollMS.Store(effective)
}
func (runtime *Runtime) EffectivePollInterval() time.Duration {
if runtime == nil {
return time.Duration(defaultPollIntervalMS) * time.Millisecond
}
return time.Duration(normalizedPollIntervalMS(runtime.effectivePollMS.Load())) * time.Millisecond
}
func (runtime *Runtime) RunScheduler(ctx context.Context, configured time.Duration) {
if runtime == nil {
return
}
runtime.SetConfiguredPollInterval(configured)
ctx, cancel := context.WithCancel(ctx)
maintenanceDone := runtime.startMaintenance(ctx)
defer func() { cancel(); <-maintenanceDone }()
timer := time.NewTimer(0)
defer timer.Stop()
for {
select {
case <-ctx.Done():
return
case <-timer.C:
if err := runtime.ProcessAvailable(ctx); err != nil && !errors.Is(err, context.Canceled) {
fmt.Printf("[worker] processAvailable failed: %v\n", err)
}
timer.Reset(runtime.EffectivePollInterval())
}
}
}
func (runtime *Runtime) statusSnapshot() ReadinessSnapshot {
if runtime == nil {
return ReadinessSnapshot{Status: "not_ready", Service: "droboticflow-worker", Reason: "worker runtime is not initialized"}
}
metrics := runtime.MetricsSnapshot()
build := runtime.BuildInfo()
runnerBuild := runtime.RunnerBuildInfo()
configured := normalizedPollIntervalMS(runtime.configuredPollMS.Load())
effective := normalizedPollIntervalMS(runtime.effectivePollMS.Load())
snapshot := ReadinessSnapshot{
Ready: false,
Status: "not_ready",
Service: "droboticflow-worker",
BuildCommit: build.BuildCommit,
BuildTime: build.BuildTime,
Image: build.Image,
RunnerImage: firstNonEmptyRuntimeValue(runnerBuild.Image, runtime.operatorRunnerImage),
RunnerBuildCommit: runnerBuild.BuildCommit,
RunnerBuildTime: runnerBuild.BuildTime,
LastCycleAt: metrics.LastCycleAt,
LastSuccessfulCycleAt: metrics.LastSuccessfulCycleAt,
CycleDurationMS: metrics.CycleDurationMS,
ConfiguredPollIntervalMS: configured,
EffectivePollIntervalMS: effective,
StuckRunCount: metrics.StuckRunCount,
OrphanRuntimeCount: metrics.OrphanRuntimeCount,
OrphanRuntimeCleanedCount: metrics.OrphanRuntimeCleanedCount,
OrphanRuntimeCleanupErrorCount: metrics.OrphanRuntimeCleanupErrorCount,
}
return snapshot
}
func firstNonEmptyRuntimeValue(values ...string) string {
for _, value := range values {
if trimmed := strings.TrimSpace(value); trimmed != "" {
return trimmed
}
}
return ""
}
func (runtime *Runtime) Readiness(ctx context.Context) ReadinessSnapshot {
snapshot := runtime.statusSnapshot()
if runtime == nil {
return snapshot
}
if runtime.store == nil || runtime.launcher == nil {
snapshot.Reason = "worker runtime is not initialized"
return snapshot
}
if err := runtime.store.Ready(ctx); err != nil {
snapshot.Reason = "mongodb is not ready: " + err.Error()
return snapshot
}
lastSuccessful, ok := parseRFC3339Nano(snapshot.LastSuccessfulCycleAt)
if !ok {
snapshot.Reason = "worker has not completed a successful scheduling cycle"
return snapshot
}
maximumAge := 3 * time.Duration(snapshot.EffectivePollIntervalMS) * time.Millisecond
if maximumAge < 30*time.Second {
maximumAge = 30 * time.Second
}
// PollInterval is the idle delay, not the duration of a scheduling cycle.
// A healthy large workspace can take longer than 30s per successful cycle.
// Use only successful durations and cap this adaptive allowance, so failed
// or stalled cycles cannot keep extending their own readiness deadline.
observedMS := min(runtime.metrics.lastSuccessfulCycleDurationMS.Load(), uint64(150000))
observedWindow := min(2*time.Duration(observedMS)*time.Millisecond+time.Duration(snapshot.EffectivePollIntervalMS)*time.Millisecond, 5*time.Minute)
if observedWindow > maximumAge {
maximumAge = observedWindow
}
snapshot.ReadinessWindowMS = maximumAge.Milliseconds()
if time.Since(lastSuccessful) > maximumAge {
snapshot.Reason = "last successful scheduling cycle is stale"
return snapshot
}
snapshot.Ready = true
snapshot.Status = "ready"
return snapshot
}
func (runtime *Runtime) GPUCapacity(ctx context.Context) (*k8s.GPUCapacitySnapshot, error) {
if runtime == nil || runtime.launcher == nil {
return nil, fmt.Errorf("worker launcher is not configured")
}
provider, ok := runtime.launcher.(gpuCapacityProvider)
if !ok {
return nil, fmt.Errorf("worker launcher does not expose gpu capacity")
}
return provider.GPUCapacity(ctx)
}
func nowISO() string {
return time.Now().UTC().Format(time.RFC3339Nano)
}
func logEntry(level string, message string, operator string) domain.LogEntryRecord {
return domain.LogEntryRecord{
Timestamp: nowISO(),
Level: level,
Message: message,
Operator: operator,
}
}
func findNodeExecution(run *domain.RunRecord, nodeID string) (int, *domain.NodeExecutionRecord) {
for index := range run.NodeExecutions {
if run.NodeExecutions[index].NodeID == nodeID {
return index, &run.NodeExecutions[index]
}
}
return -1, nil
}
func appendLog(run *domain.RunRecord, entry domain.LogEntryRecord) {
run.Logs = append(run.Logs, entry)
}
func applyExecution(run *domain.RunRecord, execution domain.NodeExecutionRecord) {
index, _ := findNodeExecution(run, execution.NodeID)
if index >= 0 {
run.NodeExecutions[index] = execution
} else {
run.NodeExecutions = append(run.NodeExecutions, execution)
}
if execution.DataMetrics != nil && execution.DataMetrics.Stage == domain.DataMetricStageTrain && len(execution.DataMetrics.TrainingInputs) > 0 {
if run.Metadata == nil {
run.Metadata = &domain.RunMetadata{}
}
run.Metadata.TrainingInputs = append([]domain.TrainingInputRecord(nil), execution.DataMetrics.TrainingInputs...)
}
run.Logs = []domain.LogEntryRecord{}
run.ResourceSeries = []domain.ResourceSample{}
run.Artifacts = []domain.ExecutionArtifact{}
for _, item := range run.NodeExecutions {
for _, entry := range item.Logs {
appendLog(run, entry)
}
if len(item.ResourceSeries) > 0 {
run.ResourceSeries = append(run.ResourceSeries, item.ResourceSeries...)
}
if len(item.Artifacts) > 0 {
run.Artifacts = append(run.Artifacts, item.Artifacts...)
}
}
}
func applyNodeExecutionSnapshot(execution *domain.NodeExecutionRecord, node domain.WorkflowNode, inputArtifacts []domain.ExecutionArtifact, inputLineage []domain.InputLineageRecord, inputBindings []domain.NodeInputBinding) {
if execution == nil {
return
}
resolvedImage := chooseImage(node, "")
execution.NodeID = node.ID
execution.NodeLabel = node.Data.Label
execution.OperatorID = strings.TrimSpace(node.Data.OperatorID)
execution.OperatorVersion = strings.TrimSpace(node.Data.OperatorVersion)
execution.OperatorImage = firstNonEmptyString(resolvedImage, node.Data.Image)
execution.OperatorImageID = firstNonEmptyString(node.Data.OperatorImageID, resolvedImage, node.Data.Image)
if image := discardPackageImageOverride(node); image != "" {
execution.OperatorImageID = image
}
if execution.Attempt <= 0 {
execution.Attempt = 1
}
if inputArtifacts != nil {
execution.InputArtifacts = cloneArtifacts(inputArtifacts)
}
if inputLineage != nil {
execution.InputLineage = cloneInputLineage(inputLineage)
}
if inputBindings != nil {
execution.InputBindings = cloneInputBindings(inputBindings)
}
}
func globalConcurrencyLimit(settings *domain.SettingsRecord) int {
if settings != nil && settings.Worker.MaxConcurrentRuns > 0 {
return settings.Worker.MaxConcurrentRuns
}
return 1
}
func effectiveWorkflowConcurrencyLimit(workflow *domain.WorkflowRecord, settings *domain.SettingsRecord) int {
if workflow != nil && workflow.MaxConcurrentRuns > 0 {
return workflow.MaxConcurrentRuns
}
return globalConcurrencyLimit(settings)
}
func parseRFC3339Nano(value string) (time.Time, bool) {
trimmed := strings.TrimSpace(value)
if trimmed == "" {
return time.Time{}, false
}
parsed, err := time.Parse(time.RFC3339Nano, trimmed)
if err != nil {
return time.Time{}, false
}
return parsed, true
}
func isStaleQueuedNodeExecution(execution domain.NodeExecutionRecord, threshold time.Duration) bool {
startedAt, ok := parseRFC3339Nano(execution.StartedAt)
if !ok {
return false
}
return time.Since(startedAt) >= threshold
}
func firstNonEmptyString(values ...string) string {
for _, value := range values {
if strings.TrimSpace(value) != "" {
return value
}
}
return ""
}
func nextHourlyScheduleTime(schedule *domain.WorkflowSchedule, lastRunAt time.Time, now time.Time) (time.Time, bool) {
intervalHours := schedule.IntervalHours
if intervalHours <= 0 {
return time.Time{}, false
}
anchorAt, ok := parseRFC3339Nano(schedule.AnchorAt)
if !ok {
if !lastRunAt.IsZero() {
anchorAt = lastRunAt
} else {
anchorAt = now
}
}
interval := time.Duration(intervalHours) * time.Hour
if lastRunAt.IsZero() || lastRunAt.Before(anchorAt) {
return anchorAt, true
}
elapsed := lastRunAt.Sub(anchorAt)
slotsElapsed := int(elapsed/interval) + 1
return anchorAt.Add(time.Duration(slotsElapsed) * interval), true
}
func nextDailyScheduleTime(schedule *domain.WorkflowSchedule, lastRunAt time.Time, now time.Time) (time.Time, bool) {
if schedule.DayInterval <= 0 {
return time.Time{}, false
}
parts := strings.Split(strings.TrimSpace(schedule.TimeOfDay), ":")
if len(parts) != 2 {
return time.Time{}, false
}
hour, err := strconv.Atoi(parts[0])
if err != nil {
return time.Time{}, false
}
minute, err := strconv.Atoi(parts[1])
if err != nil {
return time.Time{}, false
}
location, err := time.LoadLocation(domain.WorkflowScheduleTimezone(schedule.Timezone))
if err != nil {
return time.Time{}, false
}
anchorAt, ok := parseRFC3339Nano(schedule.AnchorAt)
if !ok {
if !lastRunAt.IsZero() {
anchorAt = lastRunAt
} else {
anchorAt = now
}
}
anchorLocal := anchorAt.In(location)
firstDue := time.Date(anchorLocal.Year(), anchorLocal.Month(), anchorLocal.Day(), hour, minute, 0, 0, location)
if anchorLocal.After(firstDue) {
firstDue = firstDue.AddDate(0, 0, schedule.DayInterval)
}
if lastRunAt.IsZero() {
return firstDue, true
}
lastLocal := lastRunAt.In(location)
nextDue := firstDue
for !nextDue.After(lastLocal) {
nextDue = nextDue.AddDate(0, 0, schedule.DayInterval)
}
return nextDue, true
}
func shouldEnqueueScheduledRun(workflow domain.WorkflowRecord, now time.Time) bool {
if workflow.Type != domain.WorkflowTypeStreaming || !domain.WorkflowScheduleEnabled(workflow.Schedule) {
return false
}
lastRunAt, _ := parseRFC3339Nano(workflow.LastRunAt)
switch domain.EffectiveWorkflowScheduleMode(workflow.Schedule) {
case domain.WorkflowScheduleModeEveryHours:
nextDue, ok := nextHourlyScheduleTime(workflow.Schedule, lastRunAt, now)
return ok && !now.Before(nextDue)
case domain.WorkflowScheduleModeEveryDaysAtTime:
nextDue, ok := nextDailyScheduleTime(workflow.Schedule, lastRunAt, now)
return ok && !now.Before(nextDue)
case domain.WorkflowScheduleModeIntervalSeconds:
if workflow.Schedule.Interval <= 0 {
return false
}
if workflow.LastRunAt == "" {
return true
}
return !lastRunAt.IsZero() && now.Sub(lastRunAt) >= time.Duration(workflow.Schedule.Interval)*time.Second
default:
return false
}
}
func selectQueuedRunsForDispatch(runs []domain.RunRecord, workflows []domain.WorkflowRecord, settings *domain.SettingsRecord) []string {
selected, _, _ := selectQueuedRunsForDispatchWithStats(runs, workflows, settings)
return selected
}
func selectQueuedRunsForDispatchWithStats(runs []domain.RunRecord, workflows []domain.WorkflowRecord, settings *domain.SettingsRecord) ([]string, int, int) {
activeByWorkflow := map[string]int{}
activeTotal := 0
queued := make([]domain.RunRecord, 0)
for _, run := range runs {
if _, ok := workflowActiveDispatchStatuses[run.Status]; ok {
activeByWorkflow[run.WorkflowID]++
}
if _, ok := globalActiveDispatchStatuses[run.Status]; ok {
activeTotal++
continue
}
if run.Status == domain.RunStatusQueued {
queued = append(queued, run)
}
}
selected := selectQueuedRunsForDispatchSnapshot(queued, workflows, settings, activeByWorkflow, activeTotal)
return selected, len(queued), activeTotal
}
func selectQueuedRunsForDispatchSnapshot(
queued []domain.RunRecord,
workflows []domain.WorkflowRecord,
settings *domain.SettingsRecord,
activeByWorkflow map[string]int,
activeTotal int,
) []string {
globalLimit := globalConcurrencyLimit(settings)
workflowByID := make(map[string]domain.WorkflowRecord, len(workflows))
for _, workflow := range workflows {
workflowByID[workflow.ID] = workflow
}
now := time.Now().UTC()
ready := make([]domain.RunRecord, 0, len(queued))
for _, run := range queued {
if run.Status == domain.RunStatusQueued && runDispatchReady(run, now) {
ready = append(ready, run)
}
}
available := globalLimit - activeTotal
if available <= 0 {
return nil
}
sort.Slice(ready, func(left, right int) bool {
if ready[left].CreatedAt == ready[right].CreatedAt {
return ready[left].ID < ready[right].ID
}
return ready[left].CreatedAt < ready[right].CreatedAt
})
selected := make([]string, 0, available)
for _, run := range ready {
if available == 0 {
break
}
workflow := workflowByID[run.WorkflowID]
workflowLimit := effectiveWorkflowConcurrencyLimit(&workflow, settings)
if run.Trigger != domain.RunTriggerHumanRelease && activeByWorkflow[run.WorkflowID] >= workflowLimit {
continue
}
selected = append(selected, run.ID)
activeByWorkflow[run.WorkflowID]++
available--
}
return selected
}
func (runtime *Runtime) applyWorkflowRuntimePolicies(ctx context.Context, workflows []domain.WorkflowRecord) {
for index := range workflows {
workflow := &workflows[index]
if workflow.CaptureAutomation == nil || !workflow.CaptureAutomation.Enabled {
continue
}
policy, policyErr := runtime.store.GetWorkflowRuntimePolicy(ctx, workflow.ID)
if policyErr != nil {
fmt.Printf("[worker] load runtime policy for workflow %s failed: %v\n", workflow.ID, policyErr)
continue
}
limit := domain.DefaultMaxConcurrentCaptureRuns
if policy != nil && policy.MaxConcurrentCaptureRuns > 0 {
limit = policy.MaxConcurrentCaptureRuns
}
workflow.MaxConcurrentRuns = limit
}
}
func runRecordsFromSummaries(summaries []domain.RunSummary) []domain.RunRecord {
runs := make([]domain.RunRecord, 0, len(summaries))
for _, summary := range summaries {
runs = append(runs, domain.RunRecord{
ID: summary.ID,
OrganizationID: summary.OrganizationID,
WorkflowID: summary.WorkflowID,
WorkflowName: summary.WorkflowName,
ProjectID: summary.ProjectID,
ProjectName: summary.ProjectName,
TaskID: summary.TaskID,
Trigger: summary.Trigger,
Mode: summary.Mode,
ReleaseID: summary.ReleaseID,
WorkflowVersion: summary.WorkflowVersion,
GitCommitSha: summary.GitCommitSha,
ImageDigest: summary.ImageDigest,
Attempt: summary.Attempt,
ParentRunID: summary.ParentRunID,
RestartFromNodeID: summary.RestartFromNodeID,
AvailableAt: summary.AvailableAt,
ExecutionLeaseOwner: summary.ExecutionLeaseOwner,
ExecutionLeaseExpiresAt: summary.ExecutionLeaseExpiresAt,
Status: summary.Status,
StopMode: summary.StopMode,
StopRequestedAt: summary.StopRequestedAt,
StartedAt: summary.StartedAt,
EndedAt: summary.EndedAt,
DurationMS: summary.DurationMS,
CurrentNodeID: summary.CurrentNodeID,
Error: summary.Error,
CreatedAt: summary.CreatedAt,
UpdatedAt: summary.UpdatedAt,
})
}
return runs
}
func (runtime *Runtime) materializeNodeMetricsBestEffort(ctx context.Context, run *domain.RunRecord, execution *domain.NodeExecutionRecord) {
if run == nil || execution == nil || execution.DataMetrics == nil || execution.MetricsMaterializedAt != "" {
return
}
if err := dataops.MaterializeNodeMetrics(ctx, runtime.store, run, execution); err != nil {
fmt.Printf("[worker] materialize data metrics for run %s node %s failed: %v\n", run.ID, execution.NodeID, err)
return
}
execution.MetricsMaterializedAt = nowISO()
applyExecution(run, *execution)
}
func (runtime *Runtime) processClaimedRun(ctx context.Context, claimed *domain.RunRecord) {
if claimed == nil {
return
}
defer runtime.unmarkRunActive(claimed.ID)
defer func() {
if err := runtime.store.ReleaseRunLease(context.Background(), claimed.ID, runtime.leaseOwner()); err != nil {
fmt.Printf("[worker] release run lease %s failed: %v\n", claimed.ID, err)
}
}()
processed, err := runtime.processRun(ctx, claimed)
if processed != nil && processed.Status == domain.RunStatusFailed {
runtime.bestEffortCleanupRunRuntime(processed.ID)
}
if err != nil {
fmt.Printf("[worker] process run %s failed: %v\n", claimed.ID, err)
runtime.bestEffortTerminalizeRun(claimed.ID, err)
}
}
func (runtime *Runtime) bestEffortCleanupRunRuntime(runID string) {
if runtime == nil || runtime.launcher == nil || strings.TrimSpace(runID) == "" {
return
}
cleanupCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
if err := runtime.launcher.TerminateRun(cleanupCtx, "", runID); err != nil {
fmt.Printf("[worker] cleanup terminal run %s failed: %v\n", runID, err)
}
}
func (runtime *Runtime) bestEffortTerminalizeRun(runID string, cause error) {
if strings.TrimSpace(runID) == "" || cause == nil || errors.Is(cause, errRunLeaseLost) || errors.Is(cause, context.Canceled) {
return
}
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
defer cancel()
run, err := runtime.store.GetRun(ctx, runID)
if err != nil || run == nil || isTerminalRunStatus(run.Status) {
return
}
message := "Worker failed to process run: " + boundedDiagnostic(cause.Error())
if err := runtime.failReconciledRun(ctx, run, message); err != nil {
fmt.Printf("[worker] terminalize run %s failed: %v\n", runID, err)
}
}
func (runtime *Runtime) renewRunLeaseUntil(ctx context.Context, runID string, cancelExecution context.CancelCauseFunc) {
ttl := runtime.leaseTTL()
interval := ttl / 3
if interval <= 0 || interval > time.Minute {
interval = time.Minute
}
ticker := time.NewTicker(interval)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
renewCtx, cancelRenew := context.WithTimeout(ctx, 10*time.Second)
run, readErr := runtime.store.GetRun(renewCtx, runID)
if readErr == nil && run != nil && run.ProjectID != "" {
p, e := runtime.store.GetProject(renewCtx, run.ProjectID)
if e == nil && (p == nil || p.DeletionTaskID != "") {
cancelRenew()
cancelExecution(context.Canceled)
return
}
}
ok, err := runtime.store.RenewRunLease(renewCtx, runID, runtime.leaseOwner(), ttl)
cancelRenew()
if err != nil {
if ctx.Err() != nil {
return
}
fmt.Printf("[worker] renew run lease %s failed: %v\n", runID, err)
continue
}
if !ok {
fmt.Printf("[worker] renew run lease %s skipped: owner changed or run missing\n", runID)
if cancelExecution != nil {
cancelExecution(errRunLeaseLost)
}
return
}
}
}
}
func (runtime *Runtime) tryMarkRunActive(runID string) bool {
if strings.TrimSpace(runID) == "" {
return false
}
runtime.processMu.Lock()
defer runtime.processMu.Unlock()
if runtime.activeRuns == nil {
runtime.activeRuns = map[string]struct{}{}
}
if _, ok := runtime.activeRuns[runID]; ok {
return false
}
runtime.activeRuns[runID] = struct{}{}
return true
}
func (runtime *Runtime) unmarkRunActive(runID string) {
runtime.processMu.Lock()
defer runtime.processMu.Unlock()
delete(runtime.activeRuns, runID)
}
func (runtime *Runtime) orphanMinimumRuntimeAge() time.Duration {
if runtime == nil || runtime.orphanMinimumAge <= 0 {
return 24 * time.Hour
}
return runtime.orphanMinimumAge
}
func (runtime *Runtime) reconcileOrphanedRuntimes(ctx context.Context) {
now := time.Now().UTC()
interval := runtime.orphanCheckInterval
if interval <= 0 {
interval = 5 * time.Minute
}
lastCheck := runtime.lastOrphanCheckNano.Load()
if lastCheck > 0 && now.Sub(time.Unix(0, lastCheck)) < interval {
return
}
if !runtime.lastOrphanCheckNano.CompareAndSwap(lastCheck, now.UnixNano()) {
return
}
reconciler, ok := runtime.launcher.(orphanRuntimeReconciler)
if !ok {
runtime.metrics.RecordOrphanRuntimes(0)
return
}
summaries, err := runtime.store.ListRunSummaries(ctx, "")
if err != nil {
fmt.Printf("[worker] list run ids for orphan reconciliation failed: %v\n", err)
return
}
knownRunIDs := make(map[string]struct{}, len(summaries))
for _, summary := range summaries {
knownRunIDs[summary.ID] = struct{}{}
}
orphans, err := reconciler.ReconcileOrphanedRuntimes(ctx, knownRunIDs, now.Add(-runtime.orphanMinimumRuntimeAge()), runtime.orphanCleanup)
if err != nil {
fmt.Printf("[worker] orphan runtime reconciliation failed: %v\n", err)
return
}
cleaned, cleanupErrors := 0, 0
for _, orphan := range orphans {
if orphan.Cleaned {
cleaned++
}
if strings.TrimSpace(orphan.CleanupError) != "" {
cleanupErrors++
}
}
runtime.metrics.RecordOrphanRuntimes(len(orphans) - cleaned)
runtime.metrics.RecordOrphanCleanup(cleaned, cleanupErrors)
}
// archiveTerminalRuns periodically marks old, terminal runs as archived (see
// domain.RunRecord.ArchivedAt) so an organization's ever-growing run history
// doesn't keep inflating the cost of org-wide scans like
// validateDeletionReferences' safety check. Throttled the same way
// reconcileOrphanedRuntimes is, since this only needs to run occasionally,
// not on every poll tick.
func (runtime *Runtime) archiveTerminalRuns(ctx context.Context) {
now := time.Now().UTC()
interval := runtime.runArchiveInterval
if interval <= 0 {
interval = time.Hour
}
lastCheck := runtime.lastRunArchiveNano.Load()
if lastCheck > 0 && now.Sub(time.Unix(0, lastCheck)) < interval {
return
}
if !runtime.lastRunArchiveNano.CompareAndSwap(lastCheck, now.UnixNano()) {
return
}
minimumAge := runtime.runArchiveMinimumAge
if minimumAge <= 0 {
minimumAge = 30 * 24 * time.Hour
}
archived, err := runtime.store.ArchiveTerminalRuns(ctx, now.Add(-minimumAge))
if err != nil {
fmt.Printf("[worker] archive terminal runs failed: %v\n", err)
return
}
if archived > 0 {
fmt.Printf("[worker] archived %d terminal run(s) older than %s\n", archived, minimumAge)
}
}
func (runtime *Runtime) ProcessAvailable(ctx context.Context) error {
startedAt := time.Now()
var processErr error
defer func() {
runtime.metrics.RecordProcessAvailable(uint64(time.Since(startedAt).Milliseconds()), processErr)
}()
settings, err := runtime.store.GetSettings(ctx)
if err != nil {
processErr = err
return processErr
}
runtime.updateEffectivePollInterval(settings)
reconcileRuns, err := runtime.store.ListReconcileRunSummaries(ctx, workerReconcileRunLimit)
if err != nil {
processErr = err
return processErr
}
if err := runtime.reconcileActiveRunsWithSummaries(ctx, reconcileRuns); err != nil {
processErr = err
return processErr
}
counts, err := runtime.store.CountWorkerRuns(ctx)
if err != nil {
processErr = err
return processErr
}
selectedRunIDs, err := runtime.dispatchCandidates(ctx, settings, counts)
if err != nil {
processErr = err
return processErr
}
runtime.metrics.RecordDispatch(counts.QueuedTotal, counts.ActiveTotal, len(selectedRunIDs))
for _, runID := range selectedRunIDs {
if !runtime.tryMarkRunActive(runID) {
continue
}
claimed, err := runtime.store.ClaimQueuedRun(ctx, runID, runtime.leaseOwner(), runtime.leaseTTL())
if err != nil {
runtime.unmarkRunActive(runID)
fmt.Printf("[worker] claim queued run %s failed: %v\n", runID, err)
continue
}
if claimed == nil {
runtime.unmarkRunActive(runID)
continue
}
go runtime.processClaimedRun(ctx, claimed)
}
return nil
}
func chooseImage(node domain.WorkflowNode, fallback string) string {
if image := discardPackageImageOverride(node); image != "" {
return image
}
if digestPinnedImage(node.Data.OperatorImageID) {
return strings.TrimSpace(node.Data.OperatorImageID)
}
if node.Data.Image != "" {
if strings.Contains(node.Data.Image, "droboticflow-operator-runner") && fallback != "" {
return fallback
}
return node.Data.Image
}
return fallback
}
// Deployment-scoped, digest-pinned compatible package release. This upgrades
// pending package nodes without rewriting historical Runs or workflow releases;
// the actually selected digest is recorded in the execution snapshot above.
func discardPackageImageOverride(node domain.WorkflowNode) string {
if node.Data.OperatorID != "operator-builtin-package" {
return ""
}
image := strings.TrimSpace(os.Getenv("FANYU_SLICE_DISCARD_PACKAGE_IMAGE"))
if !digestPinnedImage(image) {
return ""
}
return image
}
func digestPinnedImage(value string) bool {
trimmed := strings.TrimSpace(value)
if strings.ContainsAny(trimmed, " \t\r\n") {
return false
}
parts := strings.Split(trimmed, "@sha256:")
if len(parts) != 2 || parts[0] == "" || len(parts[1]) != 64 {
return false
}
for _, character := range parts[1] {
if (character < '0' || character > '9') && (character < 'a' || character > 'f') && (character < 'A' || character > 'F') {
return false
}
}
return true
}
func chooseValue(value string, fallback string) string {
if value == "" {
return fallback
}
return value
}
func chooseDataSource(value string, fallback string) (string, error) {
normalized := strings.ToUpper(strings.TrimSpace(value))
if normalized == "" {
return fallback, nil
}
switch normalized {
case "INLINE", "LOCAL", "BOS", "UPSTREAM":
return normalized, nil
default:
return "", fmt.Errorf("DATA_SOURCE must be INLINE, LOCAL, BOS, or UPSTREAM")
}
}
const missingJobRecoveryLogPrefix = "Kubernetes job is missing; recovery attempt "
func (runtime *Runtime) missingJobRecoveryLimit() int {
if runtime == nil || runtime.missingMaxRecoveries <= 0 {
return 3
}
return runtime.missingMaxRecoveries
}
func (runtime *Runtime) pendingJobFailureThreshold() time.Duration {
if runtime == nil || runtime.pendingFailureAfter <= 0 {
return 10 * time.Minute
}
return runtime.pendingFailureAfter
}
func missingJobRecoveryCount(execution *domain.NodeExecutionRecord) int {
if execution == nil {
return 0
}
count := 0
for _, entry := range execution.Logs {
if strings.HasPrefix(strings.TrimSpace(entry.Message), missingJobRecoveryLogPrefix) {
count++
}
}
return count
}
func boundedDiagnostic(value string) string {
trimmed := strings.TrimSpace(value)
const maximum = 4096
if len(trimmed) <= maximum {
return trimmed
}
return trimmed[:maximum] + "..."
}
func (runtime *Runtime) failReconciledRun(ctx context.Context, run *domain.RunRecord, message string, structured ...domain.RunFailure) error {
if run == nil {
return nil
}
message = boundedDiagnostic(message)
if strings.TrimSpace(message) == "" {
message = "Kubernetes job failed"
}
if _, execution := findNodeExecution(run, run.CurrentNodeID); execution != nil {
execution.Status = domain.NodeExecutionStatusFailed
execution.EndedAt = nowISO()
execution.Error = message
execution.Logs = append(execution.Logs, logEntry("ERROR", message, execution.NodeLabel))
applyExecution(run, *execution)
}
run.Status = domain.RunStatusFailed
run.EndedAt = nowISO()
run.Error = message
failure := domain.RunFailure{Code: "worker_execution_failed", Stage: "execution", NodeID: run.CurrentNodeID, Message: message, Retryable: true, Suggestion: "查看节点日志后重试失败子链路"}
if len(structured) > 0 {
failure = structured[0]
failure.Message = message
if strings.TrimSpace(failure.NodeID) == "" {
failure.NodeID = run.CurrentNodeID
}
}
run.Failure = &failure
run.UpdatedAt = nowISO()
if err := runtime.store.UpsertRun(ctx, *run); err != nil {
return err
}
workflow, err := runtime.store.GetWorkflow(ctx, run.WorkflowID)
if err != nil {
return err
}
if workflow != nil {
if err := runtime.store.UpdateWorkflowRunState(ctx, workflow.ID, domain.WorkflowStatusFailed, run.ID, nowISO()); err != nil {
return err
}
}
// Persist the terminal state before touching Kubernetes. A failed runtime is
// no longer useful to the scheduler, and cleanup is deliberately best-effort
// so a transient API error cannot leave the Run active again.
if runtime.launcher != nil {
cleanupCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
if cleanupErr := runtime.launcher.TerminateRun(cleanupCtx, "", run.ID); cleanupErr != nil {
fmt.Printf("[worker] cleanup failed runtime for terminal run %s: %v\n", run.ID, cleanupErr)
}
cancel()
}
return nil
}
func (runtime *Runtime) recoverMissingJob(ctx context.Context, run *domain.RunRecord, execution *domain.NodeExecutionRecord) error {
if run != nil && run.Status == domain.RunStatusStopping {
workflow, err := runtime.store.GetWorkflow(ctx, run.WorkflowID)
if err != nil {
return err
}
executionMap := make(map[string]domain.NodeExecutionRecord, len(run.NodeExecutions))
for _, item := range run.NodeExecutions {
executionMap[item.NodeID] = item
}
_, err = runtime.finalizeCancelledRun(ctx, workflow, run, executionMap, "用户已停止工作流")
return err
}
if execution == nil {
run.Status = domain.RunStatusQueued
run.StartedAt = ""
run.CurrentNodeID = ""
run.AvailableAt = nowISO()
run.UpdatedAt = nowISO()
return runtime.store.UpsertRun(ctx, *run)
}
terminateCtx, cancelTerminate := context.WithTimeout(context.Background(), 30*time.Second)
err := runtime.launcher.TerminateRun(terminateCtx, "", run.ID)
cancelTerminate()
if err != nil {
return fmt.Errorf("clean up missing job runtime for run %s: %w", run.ID, err)
}
recoveryAttempt := missingJobRecoveryCount(execution) + 1
if recoveryAttempt > runtime.missingJobRecoveryLimit() {
return runtime.failReconciledRun(ctx, run, fmt.Sprintf("Kubernetes job remained missing after %d recovery attempts", runtime.missingJobRecoveryLimit()), domain.RunFailure{
Code: "kubernetes_job_missing", Stage: "execution", Retryable: true, Suggestion: "检查 Job 生命周期与集群事件后从失败节点继续",
})
}
message := fmt.Sprintf("%s%d/%d; node execution was re-queued", missingJobRecoveryLogPrefix, recoveryAttempt, runtime.missingJobRecoveryLimit())
execution.Status = domain.NodeExecutionStatusFailed
execution.EndedAt = nowISO()
execution.Error = message
execution.Logs = append(execution.Logs, logEntry("WARN", message, execution.NodeLabel))
applyExecution(run, *execution)
backoff := time.Duration(15*(1<<(recoveryAttempt-1))) * time.Second
run.Status = domain.RunStatusQueued
run.StartedAt = ""
run.CurrentNodeID = ""
run.AvailableAt = time.Now().UTC().Add(backoff).Format(time.RFC3339Nano)
run.UpdatedAt = nowISO()
return runtime.store.UpsertRun(ctx, *run)
}
func (runtime *Runtime) recoverSucceededJob(ctx context.Context, run *domain.RunRecord, inspection *k8s.JobInspection) error {
workflow, err := runtime.store.GetWorkflow(ctx, run.WorkflowID)
if err != nil {
return err
}
if workflow == nil {
return runtime.failReconciledRun(ctx, run, fmt.Sprintf("workflow %s was deleted while run %s was active", run.WorkflowID, run.ID), domain.RunFailure{
Code: "workflow_not_found", Stage: "orchestration", Retryable: false,
Suggestion: "重新从现有发布版本创建工作流后再运行",
})
}
workflow, err = deliveryRuntimeWorkflow(workflow, run)
if err != nil {
return runtime.failReconciledRun(ctx, run, err.Error(), domain.RunFailure{Code: "invalid_delivery_runtime", Stage: "orchestration", Retryable: false})
}
node := domain.GetNodeByID(workflow.Nodes, run.CurrentNodeID)
if node == nil {
return runtime.failReconciledRun(ctx, run, fmt.Sprintf("node %s was deleted while run %s was active", run.CurrentNodeID, run.ID), domain.RunFailure{
Code: "workflow_node_not_found", Stage: "orchestration", NodeID: run.CurrentNodeID, Retryable: false,
Suggestion: "从保留该节点的发布版本重新运行",
})
}
executionMap := map[string]domain.NodeExecutionRecord{}
for _, item := range run.NodeExecutions {
executionMap[item.NodeID] = item
}
inputArtifacts, inputLineage, inputBindings := gatherNodeInputs(*workflow, run, executionMap, node.ID)
_, existing := findNodeExecution(run, node.ID)
execution := domain.NodeExecutionRecord{
NodeID: node.ID,
NodeLabel: node.Data.Label,
Attempt: runExecutionAttempt(run),
Status: domain.NodeExecutionStatusRunning,
StartedAt: firstNonEmptyString(run.StartedAt, run.CurrentRunAt, run.CreatedAt),
Logs: []domain.LogEntryRecord{},
Artifacts: []domain.ExecutionArtifact{},
DataRefs: []domain.DataRefRecord{},
ResourceSeries: []domain.ResourceSample{},
}
if existing != nil {
execution = *existing
}
// A completed Job may be reconciled after a Worker restart. Its deployed
// image is authoritative; an older workflow definition must not replace it.
recoveredNode := *node
recoveredImage := firstNonEmptyString(inspection.Image, execution.OperatorImageID, execution.OperatorImage)
if recoveredImage != "" {
recoveredNode.Data.Image = recoveredImage
recoveredNode.Data.OperatorImageID = recoveredImage
}
applyNodeExecutionSnapshot(&execution, recoveredNode, inputArtifacts, inputLineage, inputBindings)
parsed, err := parseExecutionOutput(inspection.Output, node.Data.Label)
if err != nil {
return err
}
execution.Status = domain.NodeExecutionStatusCompleted
execution.EndedAt = nowISO()
execution.Logs = append(execution.Logs, parsed.Logs...)
execution.Artifacts = parsed.Artifacts
execution.DataRefs = parsed.DataRefs
execution.DataMetrics = parsed.DataMetrics
execution.ResourceSeries = parsed.ResourceSeries
execution.Error = ""
executionMap[node.ID] = execution
applyExecution(run, execution)
if err := runtime.store.UpsertRun(ctx, *run); err != nil {
return err
}
runtime.materializeNodeMetricsBestEffort(ctx, run, &execution)
executionMap[node.ID] = execution
if run.Status == domain.RunStatusStopping {
_, err := runtime.finalizeCancelledRun(ctx, workflow, run, executionMap, "用户已停止工作流")
return err
}
if allAllowedNodesCompleted(workflow, run, executionMap) {
_, err := runtime.finalizeCompletedRun(ctx, workflow, run)
return err
}
run.Status = domain.RunStatusQueued
run.StartedAt = ""
run.CurrentNodeID = ""
run.AvailableAt = nowISO()
run.UpdatedAt = nowISO()
return runtime.store.UpsertRun(ctx, *run)
}
func inspectionIsStuck(inspection *k8s.JobInspection, fallbackStartedAt string, threshold time.Duration) bool {
if inspection == nil || strings.TrimSpace(inspection.BlockingReason) == "" {
return false
}
startedAt, ok := parseRFC3339Nano(firstNonEmptyString(inspection.CreatedAt, fallbackStartedAt))
return ok && time.Since(startedAt) >= threshold
}
func nodeExecutionStartedAt(execution *domain.NodeExecutionRecord) string {
if execution == nil {
return ""
}
return execution.StartedAt
}
func runExecutionLeaseActive(run *domain.RunRecord, now time.Time) bool {
if run == nil || strings.TrimSpace(run.ExecutionLeaseOwner) == "" || strings.TrimSpace(run.ExecutionLeaseExpiresAt) == "" {
return false
}
expiresAt, ok := parseRFC3339Nano(run.ExecutionLeaseExpiresAt)
return ok && expiresAt.After(now)
}
func (runtime *Runtime) reconcileActiveRun(ctx context.Context, candidate domain.RunSummary) (bool, error) {
run, err := runtime.store.GetRun(ctx, candidate.ID)
if err != nil || run == nil {
return false, err
}
// The execution coroutine owns Kubernetes Job observation while its lease is
// alive. Reconciling the same Run concurrently can observe the short gap
// between Job completion/cleanup and the persisted node result, then
// incorrectly requeue or terminalize a successful long-running GPU node.
// Stopping Runs are exempt so explicit user cancellation remains immediate.
if run.Status != domain.RunStatusStopping && runExecutionLeaseActive(run, time.Now().UTC()) {
return false, nil
}
var activeExecution *domain.NodeExecutionRecord
if strings.TrimSpace(run.CurrentNodeID) != "" {
_, activeExecution = findNodeExecution(run, run.CurrentNodeID)
}
if run.Status == domain.RunStatusQueued {
if activeExecution == nil || activeExecution.Status != domain.NodeExecutionStatusRunning || !isStaleQueuedNodeExecution(*activeExecution, 2*time.Minute) {
return false, nil
}
}
if run.Status == domain.RunStatusRunning && activeExecution != nil && activeExecution.Status == domain.NodeExecutionStatusRunning {
runtime.syncActiveRunProgress(ctx, run)
if !isStaleQueuedNodeExecution(*activeExecution, 2*time.Minute) {
return false, nil
}
}
if strings.TrimSpace(run.CurrentNodeID) == "" {
if updatedAt, ok := parseRFC3339Nano(run.UpdatedAt); ok && time.Since(updatedAt) < 2*time.Minute {
return false, nil
}
return false, runtime.recoverMissingJob(ctx, run, nil)
}
inspection, err := runtime.launcher.Inspect(ctx, run.ID, run.CurrentNodeID)
if err != nil {
return false, err
}
switch inspection.State {
case k8s.JobStateMissing:
return true, runtime.recoverMissingJob(ctx, run, activeExecution)
case k8s.JobStateSucceeded:
return false, runtime.recoverSucceededJob(ctx, run, inspection)
case k8s.JobStateFailed:
message := "Kubernetes job failed"
if output := boundedDiagnostic(inspection.Output); output != "" {
message += ": " + output
}
return true, runtime.failReconciledRun(ctx, run, message, domain.RunFailure{
Code: "kubernetes_job_failed", Stage: "execution", Retryable: true, Suggestion: "查看 Pod 日志和事件后从失败节点继续",
})
case k8s.JobStatePending:
if inspectionIsStuck(inspection, firstNonEmptyString(nodeExecutionStartedAt(activeExecution), run.StartedAt), runtime.pendingJobFailureThreshold()) {
message := fmt.Sprintf("Kubernetes job remained blocked in %s for at least %s", inspection.BlockingReason, runtime.pendingJobFailureThreshold())
if details := boundedDiagnostic(inspection.BlockingMessage); details != "" {
message += ": " + details
}
failureCode := "kubernetes_runtime_blocked"
if inspection.BlockingReason == "ImagePullBackOff" || inspection.BlockingReason == "ErrImagePull" || inspection.BlockingReason == "InvalidImageName" {
failureCode = "kubernetes_image_pull_blocked"
} else if inspection.BlockingReason == "Unschedulable" {
failureCode = "kubernetes_unschedulable"
}
return true, runtime.failReconciledRun(ctx, run, message, domain.RunFailure{
Code: failureCode, Stage: "scheduling", Retryable: true, Suggestion: "修复镜像或调度条件后从失败节点继续",
})
}
return strings.TrimSpace(inspection.BlockingReason) != "", nil
default:
return false, nil
}
}
// syncActiveRunProgress keeps structured progress visible after a worker
// restart. The original executor-side reporter may have been cancelled during
// a rolling deployment while the Kubernetes Job continues to run, so the
// reconcile path must also tail the active Pod and persist newer markers.
func (runtime *Runtime) syncActiveRunProgress(ctx context.Context, run *domain.RunRecord) {
if runtime == nil || runtime.launcher == nil || run == nil || strings.TrimSpace(run.CurrentNodeID) == "" {
return
}
tailer, ok := runtime.launcher.(nodePodLogTailer)
if !ok {
return
}
logsCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
logs, err := tailer.TailNodePodLogs(logsCtx, run.ID, run.CurrentNodeID, 200)
cancel()
if err != nil {
return
}
progress, err := latestStructuredProgress(logs)
if err != nil || progress == nil || !progressAheadOf(progress, run.Progress) {
return
}
if err := runtime.store.UpdateRunProgress(ctx, run.ID, progress); err != nil {
fmt.Printf("[worker] sync progress for run %s failed: %v\n", run.ID, err)
}
}
func progressAheadOf(next *domain.RunProgress, current *domain.RunProgress) bool {
if next == nil {
return false
}
if current == nil {
return true
}
if next.NodeID != "" && next.NodeID != current.NodeID {
return strings.TrimSpace(next.UpdatedAt) > strings.TrimSpace(current.UpdatedAt)
}
if next.Epoch < current.Epoch || (next.Epoch == current.Epoch && next.Step < current.Step) {
return false
}
if next.Step != current.Step || next.Epoch != current.Epoch || next.Checkpoint != current.Checkpoint {
return true
}
return strings.TrimSpace(next.UpdatedAt) > strings.TrimSpace(current.UpdatedAt)
}
func (runtime *Runtime) reconcileActiveRunsWithSummaries(ctx context.Context, runs []domain.RunSummary) error {
stuckCount := 0
for _, candidate := range runs {
if candidate.Status != domain.RunStatusRunning && candidate.Status != domain.RunStatusStopping && (candidate.Status != domain.RunStatusQueued || strings.TrimSpace(candidate.CurrentNodeID) == "") {
continue
}
stuck, err := runtime.reconcileActiveRun(ctx, candidate)
if stuck {
stuckCount++
}
if err != nil {
// A single malformed or temporarily unavailable run must not prevent
// unrelated runs from reconciling and the queue from dispatching.
fmt.Printf("[worker] reconcile run %s failed: %v\n", candidate.ID, err)
}
}
runtime.metrics.RecordStuckRuns(stuckCount)
return nil
}
func (runtime *Runtime) reconcileActiveRuns(ctx context.Context) error {
runs, err := runtime.store.ListReconcileRunSummaries(ctx, workerReconcileRunLimit)
if err != nil {
return err
}
return runtime.reconcileActiveRunsWithSummaries(ctx, runs)
}
func upsertNodeHyperparam(items []domain.HyperParam, key string, value string) []domain.HyperParam {
for index := range items {
if items[index].Key == key {
items[index].Value = value
return items
}
}
return append(items, domain.HyperParam{Key: key, Value: value})
}
func bosSourcePath(bucket string, prefix string) string {
normalizedBucket := strings.TrimSpace(bucket)
if normalizedBucket == "" {
return ""
}
sourcePath := "bos://" + normalizedBucket
if normalizedPrefix := strings.Trim(strings.TrimSpace(prefix), "/"); normalizedPrefix != "" {
sourcePath += "/" + normalizedPrefix
}
return sourcePath
}
func (runtime *Runtime) defaultStorageEndpoint() string {
return strings.TrimSpace(runtime.globalBOSEndpoint)
}
func (runtime *Runtime) storageConnectionEndpoint(connection *domain.StorageConnectionRecord) string {
if connection == nil {
return runtime.defaultStorageEndpoint()
}
if endpoint := strings.TrimSpace(connection.Endpoint); endpoint != "" {
return endpoint
}
return runtime.defaultStorageEndpoint()
}
func nodeHyperparamsMap(node domain.WorkflowNode) map[string]string {
values := make(map[string]string, len(node.Data.Hyperparams))
for _, item := range node.Data.Hyperparams {
key := strings.TrimSpace(item.Key)
if key == "" {
continue
}
values[key] = item.Value
}
return values
}
func chooseScanMode(node domain.WorkflowNode, params map[string]string) (string, error) {
mode := strings.ToUpper(strings.TrimSpace(params["SCAN_MODE"]))
switch mode {
case "FULL", "INCREMENTAL":
return mode, nil
case "":
if node.Data.Label == "BOS 增量扫描" {
return "INCREMENTAL", nil
}
return "", nil
default:
return "", fmt.Errorf("SCAN_MODE must be FULL or INCREMENTAL")
}
}
func chooseIncrementalUnit(params map[string]string) (string, error) {
switch strings.ToUpper(strings.TrimSpace(params["INCREMENTAL_UNIT"])) {
case "":
return "OBJECT", nil
case "OBJECT":
return "OBJECT", nil
case "DIRECTORY":
return "DIRECTORY", nil
default:
return "", fmt.Errorf("INCREMENTAL_UNIT must be OBJECT or DIRECTORY")
}
}
func chooseDirectoryDepth(params map[string]string) (int, error) {
raw := strings.TrimSpace(params["INCREMENTAL_DIRECTORY_DEPTH"])
if raw == "" {
return 1, nil
}
parsed, err := strconv.Atoi(raw)
if err != nil || parsed <= 0 {
return 0, fmt.Errorf("INCREMENTAL_DIRECTORY_DEPTH must be a positive integer")
}
return parsed, nil
}
func scanWorkflowSourceNode(node domain.WorkflowNode) bool {
label := strings.TrimSpace(node.Data.Label)
return node.ID == "scan-source" || label == "BOS 增量扫描"
}
func sourceFingerprint(node domain.WorkflowNode, params map[string]string, dataSource string, scanMode string, incrementalUnit string, directoryDepth int) string {
return domain.ComputeSourceFingerprint(domain.SourceFingerprintInput{
ScanMode: scanMode,
IncrementalUnit: incrementalUnit,
DirectoryDepth: directoryDepth,
DataSource: dataSource,
SourcePath: params["SOURCE_PATH"],
Bucket: node.Data.Bucket,
Prefix: node.Data.Prefix,
Pattern: node.Data.Pattern,
Endpoint: params["BOS_ENDPOINT"],
})
}
func stringList(value any) []string {
if value == nil {
return nil
}
if items, ok := value.([]string); ok {
result := make([]string, 0, len(items))
for _, item := range items {
trimmed := strings.TrimSpace(item)
if trimmed != "" {
result = append(result, trimmed)
}
}
return result
}
items, ok := value.([]any)
if ok {
result := make([]string, 0, len(items))
for _, item := range items {
trimmed := strings.TrimSpace(fmt.Sprint(item))
if trimmed != "" {
result = append(result, trimmed)
}
}
return result
}
reflected := reflect.ValueOf(value)
if reflected.Kind() != reflect.Slice && reflected.Kind() != reflect.Array {
return nil
}
result := make([]string, 0, reflected.Len())
for index := 0; index < reflected.Len(); index++ {
trimmed := strings.TrimSpace(fmt.Sprint(reflected.Index(index).Interface()))
if trimmed != "" {
result = append(result, trimmed)
}
}
return result
}
func inputStorageConnectionID(bindings []domain.NodeInputBinding) (string, error) {
connectionID := ""
consider := func(payload *domain.ArtifactPayload) error {
if payload == nil || payload.Mode != domain.ArtifactTransferModeRef || payload.Ref == nil {
return nil
}
candidate := strings.TrimSpace(payload.Ref.ConnectionID)
if candidate == "" {
return nil
}
if connectionID != "" && connectionID != candidate {
return fmt.Errorf("input_storage_ambiguous: node inputs reference multiple storage connections")
}
connectionID = candidate
return nil
}
for index := range bindings {
if err := consider(bindings[index].Payload); err != nil {
return "", err
}
for artifactIndex := range bindings[index].Artifacts {
if err := consider(bindings[index].Artifacts[artifactIndex].Payload); err != nil {
return "", err
}
}
}
return connectionID, nil
}
func (runtime *Runtime) prepareExecutionNode(ctx context.Context, workflow *domain.WorkflowRecord, node domain.WorkflowNode, bindingSets ...[]domain.NodeInputBinding) (domain.WorkflowNode, error) {
var inputBindings []domain.NodeInputBinding
if len(bindingSets) > 0 {
inputBindings = bindingSets[0]
}
copy, err := runtime.resolveBuiltinNodeRuntime(ctx, node)
if err != nil {
return node, err
}
effectiveWorkflow, err := runtime.resolveScanWorkflow(ctx, workflow)
if err != nil {
return node, err
}
copy, err = scanExecutionNode(effectiveWorkflow, copy)
if err != nil {
return node, err
}
if strings.TrimSpace(copy.Data.ConnectionID) == "" {
connectionID, inputErr := inputStorageConnectionID(inputBindings)
if inputErr != nil {
return node, inputErr
}
copy.Data.ConnectionID = connectionID
}
copy, err = runtime.resolveNodeStorageConnection(ctx, copy)
if err != nil {
return node, err
}
copy, err = runtime.resolveNodeMongoConnection(ctx, copy)
if err != nil {
return node, err
}
params := nodeHyperparamsMap(copy)
dataSource, err := chooseDataSource(params["DATA_SOURCE"], "BOS")
if err != nil {
return node, err
}
scanMode, err := chooseScanMode(copy, params)
if err != nil {
return node, err
}
incrementalUnit, err := chooseIncrementalUnit(params)
if err != nil {
return node, err
}
directoryDepth, err := chooseDirectoryDepth(params)
if err != nil {
return node, err
}
if scanMode != "INCREMENTAL" || workflow == nil {
return copy, nil
}
fingerprint := sourceFingerprint(copy, params, dataSource, scanMode, incrementalUnit, directoryDepth)
state, err := runtime.store.GetSourceState(ctx, workflow.ID, copy.ID, fingerprint)
if err != nil {
return copy, err
}
payload := map[string]any{
"sourceFingerprint": fingerprint,
"scanMode": scanMode,
"incrementalUnit": incrementalUnit,
"directoryDepth": directoryDepth,
"dataSource": dataSource,
"sourcePath": strings.TrimSpace(params["SOURCE_PATH"]),
"bucket": strings.TrimSpace(copy.Data.Bucket),
"prefix": strings.TrimSpace(copy.Data.Prefix),
"pattern": strings.TrimSpace(copy.Data.Pattern),
"seenFingerprints": []string{},
}
if state != nil {
payload["seenFingerprints"] = append([]string(nil), state.SeenFingerprints...)
payload["lastRunId"] = state.LastRunID
payload["lastListedCount"] = state.LastListedCount
payload["lastNewCount"] = state.LastNewCount
payload["lastScanCompletedAt"] = state.LastScanCompletedAt
}
encoded, err := json.Marshal(payload)
if err != nil {
return copy, err
}
copy.Data.Hyperparams = append([]domain.HyperParam(nil), copy.Data.Hyperparams...)
copy.Data.Hyperparams = upsertNodeHyperparam(copy.Data.Hyperparams, "SCAN_MODE", scanMode)
copy.Data.Hyperparams = upsertNodeHyperparam(copy.Data.Hyperparams, "INCREMENTAL_UNIT", incrementalUnit)
copy.Data.Hyperparams = upsertNodeHyperparam(copy.Data.Hyperparams, "INCREMENTAL_DIRECTORY_DEPTH", strconv.Itoa(directoryDepth))
copy.Data.Hyperparams = upsertNodeHyperparam(copy.Data.Hyperparams, "SOURCE_FINGERPRINT", fingerprint)
copy.Data.Hyperparams = upsertNodeHyperparam(copy.Data.Hyperparams, "SOURCE_STATE_JSON", string(encoded))
return copy, nil
}
// resolveBuiltinNodeRuntime refreshes platform-owned runtime fields immediately
// before execution. Workflow nodes keep their user configuration and graph
// shape, while existing workflows automatically consume a newly published
// builtin image instead of retaining a stale or deleted registry digest.
func (runtime *Runtime) resolveBuiltinNodeRuntime(ctx context.Context, node domain.WorkflowNode) (domain.WorkflowNode, error) {
operatorID := strings.TrimSpace(node.Data.OperatorID)
if operatorID == "" || runtime == nil || runtime.store == nil {
return node, nil
}
operator, err := runtime.store.GetOperator(ctx, operatorID)
if err != nil {
return node, fmt.Errorf("resolve builtin operator %s: %w", operatorID, err)
}
if operator == nil || !strings.EqualFold(strings.TrimSpace(operator.Type), "builtin") {
return node, nil
}
image := strings.TrimSpace(operator.Image)
imageID := strings.TrimSpace(operator.ImageID)
if image == "" {
return node, fmt.Errorf("builtin operator %s has no published image", operatorID)
}
if imageID == "" {
imageID = image
}
preservePinnedImage := strings.EqualFold(strings.TrimSpace(nodeHyperparamsMap(node)["PIN_OPERATOR_IMAGE"]), "true") && digestPinnedImage(node.Data.OperatorImageID)
if !preservePinnedImage {
node.Data.Image = image
node.Data.OperatorImageID = imageID
}
node.Data.ContainerCommand = cloneStringSlice(operator.ContainerCommand)
node.Data.ContainerArgs = cloneStringSlice(operator.ContainerArgs)
node.Data.RuntimeContract = operator.RuntimeContract
node.Data.PublicationStatus = operator.PublicationStatus
node.Data.GenerationSource = operator.GenerationSource
node.Data.PublicationEvidence = operator.PublicationEvidence
if strings.TrimSpace(operator.Version) != "" {
node.Data.OperatorVersion = strings.TrimSpace(operator.Version)
}
return node, nil
}
func (runtime *Runtime) resolveScanWorkflow(ctx context.Context, workflow *domain.WorkflowRecord) (*domain.WorkflowRecord, error) {
if workflow == nil || workflow.ScanConfig == nil || strings.TrimSpace(workflow.ScanConfig.ConnectionID) == "" {
return workflow, nil
}
connection, err := runtime.store.GetStorageConnection(ctx, workflow.ScanConfig.ConnectionID)
if err != nil {
return nil, err
}
if connection == nil {
return nil, fmt.Errorf("storage connection %s not found", workflow.ScanConfig.ConnectionID)
}
copy := *workflow
scanConfig := *workflow.ScanConfig
scanConfig.Endpoint = runtime.storageConnectionEndpoint(connection)
if strings.TrimSpace(scanConfig.AccessKeyID) == "" {
scanConfig.AccessKeyID = connection.AccessKeyID
}
if strings.TrimSpace(scanConfig.SecretAccessKey) == "" {
scanConfig.SecretAccessKey = connection.SecretAccessKey
}
if strings.TrimSpace(scanConfig.SourcePath) == "" {
dataSource, err := chooseDataSource(scanConfig.DataSource, "BOS")
if err != nil {
return nil, err
}
if dataSource == "BOS" {
scanConfig.SourcePath = bosSourcePath(scanConfig.Bucket, scanConfig.Prefix)
}
}
copy.ScanConfig = &scanConfig
return &copy, nil
}
func (runtime *Runtime) resolveNodeStorageConnection(ctx context.Context, node domain.WorkflowNode) (domain.WorkflowNode, error) {
connectionID := strings.TrimSpace(node.Data.ConnectionID)
if connectionID == "" {
return node, nil
}
connection, err := runtime.store.GetStorageConnection(ctx, connectionID)
if err != nil {
return node, err
}
if connection == nil {
return node, fmt.Errorf("storage connection %s not found", connectionID)
}
copy := node
copy.Data.Hyperparams = append([]domain.HyperParam(nil), node.Data.Hyperparams...)
sourcePath := strings.TrimSpace(nodeHyperparamsMap(copy)["SOURCE_PATH"])
if sourcePath == "" {
sourcePath = bosSourcePath(copy.Data.Bucket, copy.Data.Prefix)
}
copy.Data.Hyperparams = upsertNodeHyperparam(copy.Data.Hyperparams, "DATA_SOURCE", "BOS")
copy.Data.Hyperparams = upsertNodeHyperparam(copy.Data.Hyperparams, "SOURCE_PATH", sourcePath)
copy.Data.Hyperparams = upsertNodeHyperparam(copy.Data.Hyperparams, "BOS_ENDPOINT", runtime.storageConnectionEndpoint(connection))
copy.Data.Hyperparams = upsertNodeHyperparam(copy.Data.Hyperparams, "BOS_ACCESS_KEY_ID", connection.AccessKeyID)
copy.Data.Hyperparams = upsertNodeHyperparam(copy.Data.Hyperparams, "BOS_SECRET_ACCESS_KEY", connection.SecretAccessKey)
return copy, nil
}
func (runtime *Runtime) resolveNodeMongoConnection(ctx context.Context, node domain.WorkflowNode) (domain.WorkflowNode, error) {
connectionID := strings.TrimSpace(nodeHyperparamsMap(node)["MONGO_CONNECTION_ID"])
if connectionID == "" {
return node, nil
}
connection, err := runtime.store.GetMongoConnection(ctx, connectionID)
if err != nil {
return node, err
}
if connection == nil {
return node, fmt.Errorf("mongo connection %s not found", connectionID)
}
copy := node
copy.Data.Hyperparams = append([]domain.HyperParam(nil), node.Data.Hyperparams...)
copy.Data.Hyperparams = upsertNodeHyperparam(copy.Data.Hyperparams, "MONGO_URI", strings.TrimSpace(connection.URI))
copy.Data.Hyperparams = upsertNodeHyperparam(copy.Data.Hyperparams, "MONGO_DB", strings.TrimSpace(connection.Database))
copy.Data.Hyperparams = upsertNodeHyperparam(copy.Data.Hyperparams, "MONGO_TARGET", "business")
return copy, nil
}
func checkpointArtifact(execution domain.NodeExecutionRecord) *domain.ExecutionArtifact {
for index := range execution.Artifacts {
if execution.Artifacts[index].Name == "source-checkpoint.json" {
return &execution.Artifacts[index]
}
}
return nil
}
func sourceCheckpointPayload(artifact *domain.ExecutionArtifact) (map[string]any, error) {
if artifact == nil {
return nil, nil
}
path := strings.TrimSpace(artifact.Path)
if path != "" {
data, err := os.ReadFile(path)
if err != nil {
return nil, fmt.Errorf("read source checkpoint %s: %w", path, err)
}
var payload map[string]any
if err := json.Unmarshal(data, &payload); err != nil {
return nil, fmt.Errorf("parse source checkpoint %s: %w", path, err)
}
if len(payload) == 0 {
return nil, fmt.Errorf("source checkpoint %s must be a non-empty JSON object", path)
}
return payload, nil
}
if artifact.Data == nil {
return nil, nil
}
return artifact.Data, nil
}
func sourceStateFromCheckpoint(workflow *domain.WorkflowRecord, run *domain.RunRecord, execution domain.NodeExecutionRecord, artifact *domain.ExecutionArtifact) (*domain.SourceStateRecord, error) {
payload, err := sourceCheckpointPayload(artifact)
if err != nil {
return nil, err
}
if workflow == nil || run == nil || payload == nil {
return nil, nil
}
fingerprint := strings.TrimSpace(fmt.Sprint(payload["sourceFingerprint"]))
if fingerprint == "" {
return nil, fmt.Errorf("source checkpoint for node %s is missing sourceFingerprint", execution.NodeID)
}
directoryDepth, err := checkpointIntValue(payload, "directoryDepth")
if err != nil {
return nil, err
}
if directoryDepth <= 0 {
return nil, fmt.Errorf("source checkpoint directoryDepth must be greater than 0")
}
listedCount, err := checkpointIntValue(payload, "listedCount")
if err != nil {
return nil, err
}
if listedCount < 0 {
return nil, fmt.Errorf("source checkpoint listedCount must be zero or greater")
}
newCount, err := checkpointIntValue(payload, "newCount")
if err != nil {
return nil, err
}
if newCount < 0 {
return nil, fmt.Errorf("source checkpoint newCount must be zero or greater")
}
return &domain.SourceStateRecord{
ID: workflow.ID + ":" + execution.NodeID + ":" + fingerprint,
WorkflowID: workflow.ID,
NodeID: execution.NodeID,
SourceFingerprint: fingerprint,
DataSource: strings.TrimSpace(fmt.Sprint(payload["dataSource"])),
SourcePath: strings.TrimSpace(fmt.Sprint(payload["sourcePath"])),
Bucket: strings.TrimSpace(fmt.Sprint(payload["bucket"])),
Prefix: strings.TrimSpace(fmt.Sprint(payload["prefix"])),
Pattern: strings.TrimSpace(fmt.Sprint(payload["pattern"])),
ScanMode: strings.TrimSpace(fmt.Sprint(payload["scanMode"])),
IncrementalUnit: strings.TrimSpace(fmt.Sprint(payload["incrementalUnit"])),
DirectoryDepth: directoryDepth,
SeenFingerprints: stringList(payload["seenFingerprints"]),
LastRunID: run.ID,
LastListedCount: listedCount,
LastNewCount: newCount,
LastScanStartedAt: strings.TrimSpace(fmt.Sprint(payload["scanStartedAt"])),
LastScanCompletedAt: strings.TrimSpace(fmt.Sprint(payload["scanCompletedAt"])),
UpdatedAt: nowISO(),
}, nil
}
func checkpointIntValue(payload map[string]any, key string) (int, error) {
value, ok := payload[key]
if !ok || value == nil {
return 0, fmt.Errorf("source checkpoint %s is required", key)
}
switch typed := value.(type) {
case int:
return typed, nil
case int32:
return int(typed), nil
case int64:
return int(typed), nil
case float32:
value := float64(typed)
if math.IsNaN(value) || math.IsInf(value, 0) || math.Trunc(value) != value {
return 0, fmt.Errorf("source checkpoint %s must be an integer", key)
}
return int(value), nil
case float64:
if math.IsNaN(typed) || math.IsInf(typed, 0) || math.Trunc(typed) != typed {
return 0, fmt.Errorf("source checkpoint %s must be an integer", key)
}
return int(typed), nil
case json.Number:
if parsed, err := typed.Int64(); err == nil {
return int(parsed), nil
}
case string:
trimmed := strings.TrimSpace(typed)
if parsed, err := strconv.Atoi(trimmed); err == nil && strconv.Itoa(parsed) == trimmed {
return parsed, nil
}
}
return 0, fmt.Errorf("source checkpoint %s must be an integer", key)
}
func (runtime *Runtime) commitSourceStates(ctx context.Context, workflow *domain.WorkflowRecord, run *domain.RunRecord) error {
if workflow == nil || run == nil {
return nil
}
sourceRun := run
persistedRun, err := runtime.store.GetRun(ctx, run.ID)
if err != nil {
return err
}
if persistedRun != nil {
sourceRun = persistedRun
}
for _, execution := range sourceRun.NodeExecutions {
artifact := checkpointArtifact(execution)
if artifact == nil {
continue
}
state, err := sourceStateFromCheckpoint(workflow, sourceRun, execution, artifact)
if err != nil {
return err
}
if state == nil {
continue
}
if err := runtime.store.UpsertSourceState(ctx, *state); err != nil {
return err
}
}
return nil
}
func scanExecutionNode(workflow *domain.WorkflowRecord, node domain.WorkflowNode) (domain.WorkflowNode, error) {
if workflow == nil || workflow.ScanConfig == nil {
return node, nil
}
if !scanWorkflowSourceNode(node) {
return node, nil
}
dataSource, err := chooseDataSource(workflow.ScanConfig.DataSource, "BOS")
if err != nil {
return node, err
}
sourcePath := strings.TrimSpace(workflow.ScanConfig.SourcePath)
if sourcePath == "" && dataSource == "BOS" {
sourcePath = bosSourcePath(workflow.ScanConfig.Bucket, workflow.ScanConfig.Prefix)
}
copy := node
copy.Data.Bucket = workflow.ScanConfig.Bucket
copy.Data.Prefix = workflow.ScanConfig.Prefix
copy.Data.Pattern = workflow.ScanConfig.Pattern
copy.Data.Hyperparams = append([]domain.HyperParam(nil), node.Data.Hyperparams...)
copy.Data.Hyperparams = upsertNodeHyperparam(copy.Data.Hyperparams, "DATA_SOURCE", dataSource)
copy.Data.Hyperparams = upsertNodeHyperparam(copy.Data.Hyperparams, "SOURCE_PATH", sourcePath)
copy.Data.Hyperparams = upsertNodeHyperparam(copy.Data.Hyperparams, "BOS_ENDPOINT", workflow.ScanConfig.Endpoint)
copy.Data.Hyperparams = upsertNodeHyperparam(
copy.Data.Hyperparams,
"SCAN_MODE",
chooseValue(strings.ToUpper(strings.TrimSpace(workflow.ScanConfig.ScanMode)), "INCREMENTAL"),
)
bosAccessKeyID := ""
bosSecretAccessKey := ""
if dataSource == "BOS" {
bosAccessKeyID = workflow.ScanConfig.AccessKeyID
bosSecretAccessKey = workflow.ScanConfig.SecretAccessKey
}
copy.Data.Hyperparams = upsertNodeHyperparam(copy.Data.Hyperparams, "BOS_ACCESS_KEY_ID", bosAccessKeyID)
copy.Data.Hyperparams = upsertNodeHyperparam(copy.Data.Hyperparams, "BOS_SECRET_ACCESS_KEY", bosSecretAccessKey)
return copy, nil
}
func workflowRunTimeout(workflow *domain.WorkflowRecord) time.Duration {
if workflow == nil || workflow.RunTimeoutSeconds <= 0 {
return 0
}
return time.Duration(workflow.RunTimeoutSeconds) * time.Second
}
func workflowNodeTimeout(workflow *domain.WorkflowRecord) time.Duration {
if workflow == nil || workflow.NodeTimeoutSeconds <= 0 {
return 0
}
return time.Duration(workflow.NodeTimeoutSeconds) * time.Second
}
func normalizeRunError(workflow *domain.WorkflowRecord, err error, executionCtx context.Context, nodeTimedOut bool) string {
if err == nil {
return ""
}
if workflow != nil && workflow.RunTimeoutSeconds > 0 && (errors.Is(err, context.DeadlineExceeded) || errors.Is(executionCtx.Err(), context.DeadlineExceeded)) {
return fmt.Sprintf("工作流执行超时(%d 秒)", workflow.RunTimeoutSeconds)
}
if workflow != nil && workflow.NodeTimeoutSeconds > 0 && nodeTimedOut {
return fmt.Sprintf("节点执行超时(%d 秒)", workflow.NodeTimeoutSeconds)
}
return err.Error()
}
func shouldAutoRetry(workflow *domain.WorkflowRecord, run *domain.RunRecord) bool {
if workflow == nil || run == nil || workflow.MaxRetryAttempts <= 0 {
return false
}
attempt := run.Attempt
if attempt <= 0 {
attempt = 1
}
return attempt <= workflow.MaxRetryAttempts
}
func nextRunAttempt(run *domain.RunRecord) int {
if run == nil || run.Attempt <= 0 {
return 1
}
return run.Attempt + 1
}
func runExecutionAttempt(run *domain.RunRecord) int {
if run == nil || run.Attempt <= 0 {
return 1
}
return run.Attempt
}
func nextRunAvailableAt(workflow *domain.WorkflowRecord) string {
if workflow == nil || workflow.RetryBackoffSeconds <= 0 {
return nowISO()
}
return time.Now().UTC().Add(time.Duration(workflow.RetryBackoffSeconds) * time.Second).Format(time.RFC3339Nano)
}
func (runtime *Runtime) enqueueRetryRun(ctx context.Context, workflow *domain.WorkflowRecord, failedRun *domain.RunRecord) (*domain.RunRecord, error) {
if workflow == nil || failedRun == nil {
return nil, nil
}
manifests, err := lightweight.GenerateWorkflowCodeManifests(runtime.workspaceRoot, *workflow)
if err != nil {
return nil, err
}
inputs := failedRun.Inputs
if failedRun.Metadata != nil && failedRun.Metadata.CaptureDeliveryPackage != nil {
source, _, e := runtime.originalDeliveryRun(ctx, failedRun)
if e != nil {
return nil, e
}
inputs = source.Inputs
}
retryRun, err := runtime.store.CreateRunWithInputs(ctx, workflow.ID, domain.RunTriggerRetry, inputs, failedRun.TaskID)
if err != nil {
return nil, err
}
retryRun.Attempt = nextRunAttempt(failedRun)
retryRun.ParentRunID = failedRun.ID
retryRun.AvailableAt = nextRunAvailableAt(workflow)
lightweight.MergeRunMetadata(retryRun, manifests)
if failedRun.Metadata != nil && failedRun.Metadata.CaptureDeliveryPackage != nil {
// Preserve the immutable package selection and accepted release baseline.
raw, e := json.Marshal(failedRun.Metadata.CaptureDeliveryPackage)
if e != nil {
return nil, e
}
var frozen domain.CaptureDeliveryPackageRuntime
if e = json.Unmarshal(raw, &frozen); e != nil {
return nil, e
}
if retryRun.Metadata == nil {
retryRun.Metadata = &domain.RunMetadata{}
}
retryRun.Metadata.CaptureDeliveryPackage = &frozen
retryRun.Metadata.HumanReleasePartitionID = failedRun.Metadata.HumanReleasePartitionID
retryRun.Metadata.HumanReleaseUniqueKey = failedRun.Metadata.HumanReleaseUniqueKey
retryRun.RestartFromNodeID = failedRun.RestartFromNodeID
for _, execution := range failedRun.NodeExecutions {
if execution.NodeID != frozen.Node.ID && execution.Status == domain.NodeExecutionStatusCompleted {
applyExecution(retryRun, execution)
}
}
}
retryRun.UpdatedAt = nowISO()
if err := runtime.store.UpsertRun(ctx, *retryRun); err != nil {
return nil, err
}
return retryRun, nil
}
type nodeExecutionOutcome struct {
nodeID string
result executionResult
reference *k8s.JobReference
err error
nodeTimedOut bool
}
func runAvailableAt(run domain.RunRecord) time.Time {
if run.AvailableAt != "" {
if parsed, err := time.Parse(time.RFC3339Nano, run.AvailableAt); err == nil {
return parsed
}
}
if parsed, err := time.Parse(time.RFC3339Nano, run.CreatedAt); err == nil {
return parsed
}
return time.Time{}
}
func runDispatchReady(run domain.RunRecord, now time.Time) bool {
availableAt := runAvailableAt(run)
if availableAt.IsZero() {
return true
}
return !availableAt.After(now)
}
func syncCurrentNodeID(run *domain.RunRecord) {
run.CurrentNodeID = ""
for _, status := range []domain.NodeExecutionStatus{
domain.NodeExecutionStatusWaitingForAudit,
domain.NodeExecutionStatusWaitingForHuman,
domain.NodeExecutionStatusRunning,
} {
for _, execution := range run.NodeExecutions {
if execution.Status == status {
run.CurrentNodeID = execution.NodeID
return
}
}
}
}
func isTerminalRunStatus(status domain.RunStatus) bool {
return status == domain.RunStatusCompleted ||
status == domain.RunStatusFailed ||
status == domain.RunStatusCancelled
}
func (runtime *Runtime) loadRunControlState(ctx context.Context, runID string) (*domain.RunRecord, error) {
if strings.TrimSpace(runID) == "" {
return nil, nil
}
return runtime.store.GetRun(ctx, runID)
}
func syncStopMetadata(target *domain.RunRecord, source *domain.RunRecord) {
if target == nil || source == nil {
return
}
target.StopMode = source.StopMode
target.StopRequestedAt = source.StopRequestedAt
if source.Progress != nil {
progress := *source.Progress
target.Progress = &progress
}
}
func finalizeCancelledExecution(execution domain.NodeExecutionRecord, message string) (domain.NodeExecutionRecord, bool) {
if execution.Status == domain.NodeExecutionStatusCompleted ||
execution.Status == domain.NodeExecutionStatusFailed ||
execution.Status == domain.NodeExecutionStatusCancelled {
return execution, false
}
execution.Status = domain.NodeExecutionStatusCancelled
execution.EndedAt = nowISO()
execution.Error = message
execution.Logs = append(execution.Logs, logEntry("WARN", message, execution.NodeLabel))
return execution, true
}
func (runtime *Runtime) finalizeCancelledRun(ctx context.Context, workflow *domain.WorkflowRecord, currentRun *domain.RunRecord, executionMap map[string]domain.NodeExecutionRecord, message string) (*domain.RunRecord, error) {
for nodeID, execution := range executionMap {
updated, changed := finalizeCancelledExecution(execution, message)
if changed {
executionMap[nodeID] = updated
applyExecution(currentRun, updated)
}
}
currentRun.Status = domain.RunStatusCancelled
currentRun.EndedAt = nowISO()
if strings.TrimSpace(currentRun.Error) == "" {
currentRun.Error = message
}
syncCurrentNodeID(currentRun)
currentRun.UpdatedAt = nowISO()
if err := runtime.store.UpsertRun(ctx, *currentRun); err != nil {
return nil, err
}
if workflow != nil {
workflow.Status = domain.WorkflowStatusCancelled
workflow.LastRunID = currentRun.ID
workflow.UpdatedAt = nowISO()
if err := runtime.store.UpdateWorkflowRunState(ctx, workflow.ID, workflow.Status, workflow.LastRunID, workflow.UpdatedAt); err != nil {
return nil, err
}
}
return currentRun, nil
}
func allAllowedNodesCompleted(workflow *domain.WorkflowRecord, run *domain.RunRecord, executionMap map[string]domain.NodeExecutionRecord) bool {
if workflow == nil || run == nil {
return false
}
allowedNodes := rerunAllowedNodeSet(*workflow, run)
for _, node := range workflow.Nodes {
if !nodeAllowedForExecution(allowedNodes, node.ID) {
continue
}
execution, ok := executionMap[node.ID]
if !ok || execution.Status != domain.NodeExecutionStatusCompleted {
return false
}
}
return true
}
func (runtime *Runtime) finalizeCompletedRun(ctx context.Context, workflow *domain.WorkflowRecord, currentRun *domain.RunRecord) (*domain.RunRecord, error) {
if err := runtime.commitSourceStates(ctx, workflow, currentRun); err != nil {
return nil, err
}
currentRun.Status = domain.RunStatusCompleted
currentRun.EndedAt = nowISO()
syncCurrentNodeID(currentRun)
currentRun.UpdatedAt = nowISO()
if err := runtime.store.UpsertRun(ctx, *currentRun); err != nil {
return nil, err
}
workflow.Status = domain.WorkflowStatusCompleted
workflow.LastRunID = currentRun.ID
workflow.UpdatedAt = nowISO()
if err := runtime.store.UpdateWorkflowRunState(ctx, workflow.ID, workflow.Status, workflow.LastRunID, workflow.UpdatedAt); err != nil {
return nil, err
}
return currentRun, nil
}
func dependenciesComplete(workflow domain.WorkflowRecord, executionMap map[string]domain.NodeExecutionRecord, nodeID string) bool {
for _, candidate := range domain.GetIncomingEdges(workflow.Edges, nodeID) {
dependency, ok := executionMap[candidate.Source]
if !ok || dependency.Status != domain.NodeExecutionStatusCompleted {
return false
}
}
return true
}
func workflowEdgeIdentity(edge domain.WorkflowEdge) string {
if id := strings.TrimSpace(edge.ID); id != "" {
return id
}
return strings.Join([]string{edge.Source, edge.SourceHandle, edge.Target, edge.TargetHandle}, "\x00")
}
func reachableWorkflowNodes(edges []domain.WorkflowEdge, roots []string) map[string]struct{} {
visited := make(map[string]struct{}, len(roots))
queue := append([]string{}, roots...)
for len(queue) > 0 {
nodeID := strings.TrimSpace(queue[0])
queue = queue[1:]
if nodeID == "" {
continue
}
if _, exists := visited[nodeID]; exists {
continue
}
visited[nodeID] = struct{}{}
for _, edge := range domain.GetOutgoingEdges(edges, nodeID) {
queue = append(queue, edge.Target)
}
}
return visited
}
func markUnselectedAuditBranchesCompleted(
workflow domain.WorkflowRecord,
executionMap map[string]domain.NodeExecutionRecord,
run *domain.RunRecord,
nodeID string,
selectedEdges []domain.WorkflowEdge,
decision domain.AuditStatus,
allowed map[string]struct{},
) {
selectedEdgeIDs := make(map[string]struct{}, len(selectedEdges))
selectedRoots := make([]string, 0, len(selectedEdges))
for _, edge := range selectedEdges {
selectedEdgeIDs[workflowEdgeIdentity(edge)] = struct{}{}
selectedRoots = append(selectedRoots, edge.Target)
}
unselectedRoots := make([]string, 0)
for _, edge := range domain.GetOutgoingEdges(workflow.Edges, nodeID) {
if _, selected := selectedEdgeIDs[workflowEdgeIdentity(edge)]; selected {
continue
}
unselectedRoots = append(unselectedRoots, edge.Target)
}
if len(unselectedRoots) == 0 {
return
}
selectedReachable := reachableWorkflowNodes(workflow.Edges, selectedRoots)
unselectedReachable := reachableWorkflowNodes(workflow.Edges, unselectedRoots)
for _, candidate := range workflow.Nodes {
if _, unselected := unselectedReachable[candidate.ID]; !unselected {
continue
}
if _, sharedWithSelected := selectedReachable[candidate.ID]; sharedWithSelected {
continue
}
if !nodeAllowedForExecution(allowed, candidate.ID) {
continue
}
if _, exists := executionMap[candidate.ID]; exists {
continue
}
timestamp := nowISO()
execution := domain.NodeExecutionRecord{
NodeID: candidate.ID,
NodeLabel: candidate.Data.Label,
Attempt: runExecutionAttempt(run),
Status: domain.NodeExecutionStatusCompleted,
StartedAt: timestamp,
EndedAt: timestamp,
Logs: []domain.LogEntryRecord{logEntry("INFO", fmt.Sprintf("Skipped by audit decision: %s", decision), candidate.Data.Label)},
Artifacts: []domain.ExecutionArtifact{},
InputArtifacts: []domain.ExecutionArtifact{},
InputLineage: []domain.InputLineageRecord{},
DataRefs: []domain.DataRefRecord{},
ResourceSeries: []domain.ResourceSample{},
}
executionMap[candidate.ID] = execution
applyExecution(run, execution)
}
}
func rerunAllowedNodeSet(workflow domain.WorkflowRecord, run *domain.RunRecord) map[string]struct{} {
if run == nil {
return nil
}
if run.DebugTarget != nil && run.DebugTarget.SingleNode && strings.TrimSpace(run.DebugTarget.TargetNodeID) != "" {
return map[string]struct{}{
strings.TrimSpace(run.DebugTarget.TargetNodeID): {},
}
}
frozenDeliveryRetry := run.Trigger == domain.RunTriggerRetry && run.Metadata != nil && run.Metadata.CaptureDeliveryPackage != nil
if (run.Trigger != domain.RunTriggerRerunSubtree && run.Trigger != domain.RunTriggerHumanRelease && !frozenDeliveryRetry) || strings.TrimSpace(run.RestartFromNodeID) == "" {
return nil
}
visited := map[string]struct{}{}
queue := []string{strings.TrimSpace(run.RestartFromNodeID)}
for len(queue) > 0 {
nodeID := queue[0]
queue = queue[1:]
if _, ok := visited[nodeID]; ok {
continue
}
visited[nodeID] = struct{}{}
for _, edge := range domain.GetOutgoingEdges(workflow.Edges, nodeID) {
target := strings.TrimSpace(edge.Target)
if target != "" {
queue = append(queue, target)
}
}
}
return visited
}
func nodeAllowedForExecution(allowed map[string]struct{}, nodeID string) bool {
if allowed == nil {
return true
}
_, ok := allowed[nodeID]
return ok
}
func firstActiveExecution(executionMap map[string]domain.NodeExecutionRecord) (string, domain.NodeExecutionStatus) {
for _, status := range []domain.NodeExecutionStatus{
domain.NodeExecutionStatusWaitingForAudit,
domain.NodeExecutionStatusWaitingForHuman,
domain.NodeExecutionStatusRunning,
domain.NodeExecutionStatusQueued,
} {
for nodeID, execution := range executionMap {
if execution.Status == status {
return nodeID, status
}
}
}
return "", ""
}
func queueNodeIfReady(workflow domain.WorkflowRecord, executionMap map[string]domain.NodeExecutionRecord, readyQueue *[]string, readySet map[string]struct{}, allowed map[string]struct{}, nodeID string) {
if !nodeAllowedForExecution(allowed, nodeID) {
return
}
if _, queued := readySet[nodeID]; queued {
return
}
if execution, ok := executionMap[nodeID]; ok {
if execution.Status == domain.NodeExecutionStatusCompleted ||
execution.Status == domain.NodeExecutionStatusRunning ||
execution.Status == domain.NodeExecutionStatusWaitingForAudit ||
execution.Status == domain.NodeExecutionStatusWaitingForHuman {
return
}
}
if !dependenciesComplete(workflow, executionMap, nodeID) {
return
}
*readyQueue = append(*readyQueue, nodeID)
readySet[nodeID] = struct{}{}
}
func enqueueDependents(workflow domain.WorkflowRecord, executionMap map[string]domain.NodeExecutionRecord, readyQueue *[]string, readySet map[string]struct{}, allowed map[string]struct{}, nodeID string) {
for _, edge := range domain.GetOutgoingEdges(workflow.Edges, nodeID) {
queueNodeIfReady(workflow, executionMap, readyQueue, readySet, allowed, edge.Target)
}
}
func enqueueReadyNodes(workflow domain.WorkflowRecord, executionMap map[string]domain.NodeExecutionRecord, readyQueue *[]string, readySet map[string]struct{}, allowed map[string]struct{}) {
for _, node := range workflow.Nodes {
queueNodeIfReady(workflow, executionMap, readyQueue, readySet, allowed, node.ID)
}
}
func upstreamBusinessVolume(workflow domain.WorkflowRecord, executionMap map[string]domain.NodeExecutionRecord, nodeID string) domain.DataVolume {
for _, edge := range domain.GetIncomingEdges(workflow.Edges, nodeID) {
execution, ok := executionMap[edge.Source]
if !ok || execution.DataMetrics == nil {
continue
}
if execution.DataMetrics.TrainReady.TrajectoryCount != nil ||
execution.DataMetrics.TrainReady.DurationSeconds != nil ||
execution.DataMetrics.TrainReady.Bytes != nil {
return execution.DataMetrics.TrainReady
}
if execution.DataMetrics.TechnicalValid.TrajectoryCount != nil ||
execution.DataMetrics.TechnicalValid.DurationSeconds != nil ||
execution.DataMetrics.TechnicalValid.Bytes != nil {
return execution.DataMetrics.TechnicalValid
}
return execution.DataMetrics.Output
}
return domain.DataVolume{}
}
func (runtime *Runtime) processRun(ctx context.Context, claimed *domain.RunRecord) (*domain.RunRecord, error) {
if claimed.ProjectID != "" {
p, err := runtime.store.GetProject(ctx, claimed.ProjectID)
if err != nil {
return nil, err
}
if p == nil || p.DeletionTaskID != "" {
return claimed, nil
}
}
if _, ok, err := runtime.store.AcquireRunLease(ctx, claimed.ID, runtime.leaseOwner(), runtime.leaseTTL()); err != nil {
return nil, err
} else if !ok {
return claimed, nil
}
leaseCtx, cancelLeaseExecution := context.WithCancelCause(ctx)
defer cancelLeaseExecution(context.Canceled)
go runtime.renewRunLeaseUntil(leaseCtx, claimed.ID, cancelLeaseExecution)
ctx = leaseCtx
workflow, err := runtime.store.GetWorkflow(ctx, claimed.WorkflowID)
if err != nil {
return nil, err
}
if workflow == nil {
claimed.Status = domain.RunStatusFailed
claimed.EndedAt = nowISO()
claimed.Error = fmt.Sprintf("workflow %s not found", claimed.WorkflowID)
claimed.UpdatedAt = nowISO()
return claimed, runtime.store.UpsertRun(ctx, *claimed)
}
workflow, err = deliveryRuntimeWorkflow(workflow, claimed)
if err != nil {
claimed.Status = domain.RunStatusFailed
claimed.EndedAt = nowISO()
claimed.Error = err.Error()
claimed.UpdatedAt = nowISO()
return claimed, runtime.store.UpsertRun(ctx, *claimed)
}
if err = runtime.attachFrozenReleaseRetry(ctx, claimed); err != nil {
claimed.Status = domain.RunStatusFailed
claimed.EndedAt = nowISO()
claimed.Error = err.Error()
claimed.UpdatedAt = nowISO()
return claimed, runtime.store.UpsertRun(ctx, *claimed)
}
if conflicts := domain.FindInputPortConflicts(workflow.Nodes, workflow.Edges); len(conflicts) > 0 {
currentRun := claimed
message := "工作流图非法:同一输入端口存在多条上游连线。请拆分输入,或为每个上传结果添加单独的上传算子。"
currentRun.Status = domain.RunStatusFailed
currentRun.EndedAt = nowISO()
currentRun.Error = message
currentRun.UpdatedAt = nowISO()
appendLog(currentRun, logEntry("ERROR", message, "system"))
return currentRun, runtime.store.UpsertRun(ctx, *currentRun)
}
if graphIssues := domain.ValidateSingleConnectedWorkflowGraph(workflow.Nodes, workflow.Edges); len(graphIssues) > 0 {
currentRun := claimed
message := "工作流图非法:存在孤立算子或多条互不连接的独立链路。请将所有算子通过 artifact/lineage 连成一个单连通工作流。"
currentRun.Status = domain.RunStatusFailed
currentRun.EndedAt = nowISO()
currentRun.Error = message
currentRun.UpdatedAt = nowISO()
appendLog(currentRun, logEntry("ERROR", message, "system"))
return currentRun, runtime.store.UpsertRun(ctx, *currentRun)
}
if message := runtime.workflowPublicationBlocker(ctx, workflow); message != "" {
currentRun := claimed
currentRun.Status = domain.RunStatusFailed
currentRun.EndedAt = nowISO()
currentRun.Error = message
currentRun.UpdatedAt = nowISO()
appendLog(currentRun, logEntry("ERROR", message, "system"))
return currentRun, runtime.store.UpsertRun(ctx, *currentRun)
}
currentRun, err := runtime.store.GetRun(ctx, claimed.ID)
if err != nil {
return nil, err
}
if currentRun == nil {
currentRun = claimed
}
if isTerminalRunStatus(currentRun.Status) {
return currentRun, nil
}
if err := runtime.resolveDirectoryRunInputs(ctx, currentRun); err != nil {
message := err.Error()
currentRun.Status = domain.RunStatusFailed
currentRun.Error = message
currentRun.Failure = &domain.RunFailure{Code: "input_resolution_failed", Stage: "input_resolution", Message: message, Retryable: true, Suggestion: "检查对象存储连接、目录前缀和写入 manifest 的权限"}
currentRun.EndedAt = nowISO()
currentRun.UpdatedAt = nowISO()
_ = runtime.store.UpsertRun(ctx, *currentRun)
return currentRun, nil
}
settings, err := runtime.store.GetSettings(ctx)
if err != nil {
return nil, err
}
nodeConcurrencyLimit := effectiveWorkflowConcurrencyLimit(workflow, settings)
if nodeConcurrencyLimit <= 0 {
nodeConcurrencyLimit = 1
}
executionMap := map[string]domain.NodeExecutionRecord{}
for _, execution := range currentRun.NodeExecutions {
executionMap[execution.NodeID] = execution
}
allowedNodes := rerunAllowedNodeSet(*workflow, currentRun)
readyQueue := make([]string, 0)
readySet := make(map[string]struct{})
auditQueue := make([]string, 0)
if currentRun.CurrentNodeID != "" {
node := domain.GetNodeByID(workflow.Nodes, currentRun.CurrentNodeID)
if node != nil && domain.NodeRequiresAnnotation(*node) {
resolved, targets, execution, resolveErr := runtime.resolveAcceptedAnnotationGate(ctx, *workflow, currentRun, *node, executionMap, allowedNodes)
if resolveErr != nil {
return nil, resolveErr
}
if resolved {
for _, target := range targets {
queueNodeIfReady(*workflow, executionMap, &readyQueue, readySet, allowedNodes, target)
}
if err := runtime.store.UpsertRun(ctx, *currentRun); err != nil {
return nil, err
}
if execution != nil {
runtime.materializeNodeMetricsBestEffort(ctx, currentRun, execution)
executionMap[node.ID] = *execution
}
}
} else {
tasks, err := runtime.store.ListAuditTasks(ctx, "all")
if err != nil {
return nil, err
}
for _, task := range tasks {
if task.RunID == currentRun.ID && task.NodeID == currentRun.CurrentNodeID && task.Status != domain.AuditStatusPending {
if node != nil {
execution := executionMap[node.ID]
execution.NodeID = node.ID
execution.NodeLabel = node.Data.Label
execution.Status = domain.NodeExecutionStatusCompleted
execution.EndedAt = nowISO()
execution.Logs = append(execution.Logs, logEntry("INFO", fmt.Sprintf("Audit decision: %s", task.Status), node.Data.Label))
resolvedNode := runtime.resolveNodeBusinessStage(ctx, currentRun, *node)
if resolvedNode.Data.BusinessStage == domain.DataMetricStageAudit {
volume := upstreamBusinessVolume(*workflow, executionMap, node.ID)
execution.DataMetrics = &domain.DataMetricsRecord{
SchemaVersion: 1,
Stage: domain.DataMetricStageAudit,
Input: volume,
Output: volume,
Quality: domain.DataMetricQualityExact,
OccurredAt: execution.EndedAt,
}
if task.Status == domain.AuditStatusApproved {
execution.DataMetrics.TrainReady = volume
} else {
execution.DataMetrics.Output = domain.DataVolume{}
execution.DataMetrics.Rejected = volume
execution.DataMetrics.RejectionReasons = map[string]int64{"人工审核拒绝": 1}
}
}
decisionEdges := domain.GetDecisionOutgoingEdges(workflow.Edges, node.ID, task.Status)
execution.Artifacts = routeAuditArtifacts(*node, execution.InputArtifacts, decisionEdges)
executionMap[node.ID] = execution
applyExecution(currentRun, execution)
markUnselectedAuditBranchesCompleted(*workflow, executionMap, currentRun, node.ID, decisionEdges, task.Status, allowedNodes)
for _, target := range edgeTargets(decisionEdges) {
queueNodeIfReady(*workflow, executionMap, &readyQueue, readySet, allowedNodes, target)
}
currentRun.Status = domain.RunStatusRunning
syncCurrentNodeID(currentRun)
currentRun.UpdatedAt = nowISO()
if err := runtime.store.UpsertRun(ctx, *currentRun); err != nil {
return nil, err
}
runtime.materializeNodeMetricsBestEffort(ctx, currentRun, &execution)
executionMap[node.ID] = execution
}
break
}
}
}
}
if activeNodeID, activeStatus := firstActiveExecution(executionMap); activeNodeID != "" {
if strings.TrimSpace(currentRun.CurrentNodeID) == "" {
currentRun.CurrentNodeID = activeNodeID
}
switch activeStatus {
case domain.NodeExecutionStatusWaitingForAudit:
currentRun.Status = domain.RunStatusWaitingForAudit
case domain.NodeExecutionStatusWaitingForHuman:
currentRun.Status = domain.RunStatusWaitingForHuman
default:
currentRun.Status = domain.RunStatusRunning
}
currentRun.UpdatedAt = nowISO()
syncCurrentNodeID(currentRun)
if err := runtime.store.UpsertRun(ctx, *currentRun); err != nil {
return nil, err
}
workflow.Status = domain.WorkflowStatusRunning
workflow.LastRunID = currentRun.ID
workflow.UpdatedAt = nowISO()
if err := runtime.store.UpdateWorkflowRunState(ctx, workflow.ID, workflow.Status, workflow.LastRunID, workflow.UpdatedAt); err != nil {
return nil, err
}
return currentRun, nil
}
if len(readyQueue) == 0 && len(executionMap) == 0 {
if currentRun.DebugTarget != nil && currentRun.DebugTarget.SingleNode && strings.TrimSpace(currentRun.DebugTarget.TargetNodeID) != "" {
targetNodeID := strings.TrimSpace(currentRun.DebugTarget.TargetNodeID)
if nodeAllowedForExecution(allowedNodes, targetNodeID) {
readyQueue = append(readyQueue, targetNodeID)
readySet[targetNodeID] = struct{}{}
}
} else if allowedNodes != nil && strings.TrimSpace(currentRun.RestartFromNodeID) != "" {
queueNodeIfReady(*workflow, executionMap, &readyQueue, readySet, allowedNodes, currentRun.RestartFromNodeID)
} else {
for _, nodeID := range domain.GetStartNodeIDs(workflow.Nodes, workflow.Edges) {
queueNodeIfReady(*workflow, executionMap, &readyQueue, readySet, allowedNodes, nodeID)
}
}
}
if len(readyQueue) == 0 && len(executionMap) > 0 && currentRun.CurrentNodeID == "" {
for _, execution := range currentRun.NodeExecutions {
if execution.Status != domain.NodeExecutionStatusCompleted {
continue
}
enqueueDependents(*workflow, executionMap, &readyQueue, readySet, allowedNodes, execution.NodeID)
}
for _, node := range workflow.Nodes {
queueNodeIfReady(*workflow, executionMap, &readyQueue, readySet, allowedNodes, node.ID)
}
}
executionCtx := ctx
cancelExecution := func() {}
if timeout := workflowRunTimeout(workflow); timeout > 0 {
executionCtx, cancelExecution = context.WithTimeout(ctx, timeout)
} else {
executionCtx, cancelExecution = context.WithCancel(ctx)
}
defer cancelExecution()
results := make(chan nodeExecutionOutcome, nodeConcurrencyLimit)
runningNodes := make(map[string]struct{})
nodeStartedAt := make(map[string]time.Time)
var persistedControl *domain.RunRecord
gracefulStopRequested := false
forceStopRequested := false
for {
if cause := context.Cause(ctx); cause != nil {
return currentRun, cause
}
persistedControl, err = runtime.loadRunControlState(ctx, currentRun.ID)
if err != nil {
return nil, err
}
gracefulStopRequested = false
forceStopRequested = false
if persistedControl != nil {
syncStopMetadata(currentRun, persistedControl)
if persistedControl.Status == domain.RunStatusStopping && persistedControl.StopMode == domain.RunStopModeGraceful {
gracefulStopRequested = true
currentRun.Status = domain.RunStatusStopping
}
if persistedControl.Status == domain.RunStatusCancelled {
forceStopRequested = persistedControl.StopMode == domain.RunStopModeForce
currentRun.Status = domain.RunStatusCancelled
if currentRun.EndedAt == "" {
currentRun.EndedAt = persistedControl.EndedAt
}
if strings.TrimSpace(currentRun.Error) == "" {
currentRun.Error = persistedControl.Error
}
}
}
if forceStopRequested {
cancelExecution()
}
launched := false
for !gracefulStopRequested && !forceStopRequested && len(runningNodes) < nodeConcurrencyLimit && len(readyQueue) > 0 {
nodeID := readyQueue[0]
readyQueue = readyQueue[1:]
delete(readySet, nodeID)
node := domain.GetNodeByID(workflow.Nodes, nodeID)
if node == nil {
continue
}
if !nodeAllowedForExecution(allowedNodes, node.ID) {
continue
}
if existing, ok := executionMap[node.ID]; ok {
if existing.Status == domain.NodeExecutionStatusCompleted ||
existing.Status == domain.NodeExecutionStatusRunning ||
existing.Status == domain.NodeExecutionStatusWaitingForAudit ||
existing.Status == domain.NodeExecutionStatusWaitingForHuman {
continue
}
}
isSingleNodeTarget := currentRun.DebugTarget != nil && currentRun.DebugTarget.SingleNode && strings.TrimSpace(currentRun.DebugTarget.TargetNodeID) == node.ID
if !isSingleNodeTarget && !dependenciesComplete(*workflow, executionMap, node.ID) {
continue
}
if domain.NodeRequiresAudit(*node) || domain.NodeRequiresAnnotation(*node) {
auditQueue = append(auditQueue, node.ID)
continue
}
inputArtifacts, inputLineage, inputBindings := gatherNodeInputs(*workflow, currentRun, executionMap, node.ID)
executionAttempt := runExecutionAttempt(currentRun)
executionLogs := []domain.LogEntryRecord{logEntry("INFO", "Starting node execution", node.Data.Label)}
if previous, ok := executionMap[node.ID]; ok && missingJobRecoveryCount(&previous) > 0 {
if previous.Attempt >= executionAttempt {
executionAttempt = previous.Attempt + 1
}
executionLogs = append(append([]domain.LogEntryRecord{}, previous.Logs...), executionLogs...)
}
nodeToPrepare := *node
if currentRun.NodeOverrides != nil {
if override, ok := currentRun.NodeOverrides[node.ID]; ok {
nodeToPrepare = applyNodeOverridesToNode(nodeToPrepare, override)
}
}
execution := domain.NodeExecutionRecord{
NodeID: node.ID,
NodeLabel: node.Data.Label,
Attempt: executionAttempt,
Status: domain.NodeExecutionStatusRunning,
StartedAt: nowISO(),
Logs: executionLogs,
Artifacts: []domain.ExecutionArtifact{},
DataRefs: []domain.DataRefRecord{},
ResourceSeries: []domain.ResourceSample{},
StageManifestPath: "",
}
applyNodeExecutionSnapshot(&execution, nodeToPrepare, inputArtifacts, inputLineage, inputBindings)
executionMap[node.ID] = execution
applyExecution(currentRun, execution)
runningNodes[node.ID] = struct{}{}
nodeStartedAt[node.ID] = time.Now()
runtime.metrics.RecordNodeExecutionStarted()
launched = true
runSnapshot := *currentRun
// Missing-Job recovery is a node execution retry inside the same Run.
// Use the node attempt for the Kubernetes Job suffix so a recovered
// execution cannot collide with the vanished attempt name.
runSnapshot.Attempt = executionAttempt
nodeSnapshot, err := runtime.prepareExecutionNode(ctx, workflow, nodeToPrepare, inputBindings)
if err != nil {
results <- nodeExecutionOutcome{
nodeID: node.ID,
err: err,
}
continue
}
// Persist the runtime-resolved builtin image and contract, not the
// stale image snapshot embedded in an older workflow definition.
// The Kubernetes Job has always used nodeSnapshot; keeping the Run
// snapshot aligned prevents the UI from reporting a different digest.
applyNodeExecutionSnapshot(&execution, nodeSnapshot, inputArtifacts, inputLineage, inputBindings)
executionMap[node.ID] = execution
applyExecution(currentRun, execution)
inputSnapshot := cloneArtifacts(inputArtifacts)
inputBindingsSnapshot := cloneInputBindings(inputBindings)
nodeExecutionCtx := executionCtx
cancelNodeExecution := func() {}
if timeout := workflowNodeTimeout(workflow); timeout > 0 {
nodeExecutionCtx, cancelNodeExecution = context.WithTimeout(executionCtx, timeout)
}
go func() {
defer cancelNodeExecution()
// Stream live GPU samples for custom-container nodes during execution so the
// resource monitor reflects usage before the node completes (builtin nodes
// self-report via operator-runner's ResourceSampler, so they are skipped).
progressCtx, progressCancel := context.WithCancel(nodeExecutionCtx)
var progressWG sync.WaitGroup
if shouldReportNodeProgress(runtime, nodeSnapshot) {
sampleGPU := runtime.operatorRunnerImage != "" && chooseImage(nodeSnapshot, runtime.operatorRunnerImage) != runtime.operatorRunnerImage
progressWG.Add(1)
go runtime.runNodeProgressReporter(progressCtx, runSnapshot.ID, nodeSnapshot.ID, sampleGPU, func() { progressWG.Done() })
}
result, reference, err := runtime.executeNode(nodeExecutionCtx, &runSnapshot, nodeSnapshot, inputSnapshot, inputBindingsSnapshot)
progressCancel()
progressWG.Wait()
results <- nodeExecutionOutcome{
nodeID: nodeSnapshot.ID,
result: result,
reference: reference,
err: err,
nodeTimedOut: errors.Is(nodeExecutionCtx.Err(), context.DeadlineExceeded),
}
}()
}
if launched {
currentRun.Status = domain.RunStatusRunning
syncCurrentNodeID(currentRun)
currentRun.UpdatedAt = nowISO()
if err := runtime.store.UpsertRun(ctx, *currentRun); err != nil {
return nil, err
}
}
if len(runningNodes) == 0 {
if forceStopRequested {
return runtime.finalizeCancelledRun(ctx, workflow, currentRun, executionMap, "用户已强制停止工作流")
}
if gracefulStopRequested || currentRun.Status == domain.RunStatusStopping {
return runtime.finalizeCancelledRun(ctx, workflow, currentRun, executionMap, "用户已停止工作流")
}
if len(readyQueue) == 0 && len(auditQueue) == 0 {
enqueueReadyNodes(*workflow, executionMap, &readyQueue, readySet, allowedNodes)
}
if len(readyQueue) > 0 {
continue
}
if len(auditQueue) > 0 {
nodeID := auditQueue[0]
auditQueue = auditQueue[1:]
node := domain.GetNodeByID(workflow.Nodes, nodeID)
if node == nil {
continue
}
inputArtifacts, inputLineage, inputBindings := gatherNodeInputs(*workflow, currentRun, executionMap, node.ID)
execution := executionMap[node.ID]
if execution.NodeID == "" {
execution = domain.NodeExecutionRecord{
NodeID: node.ID,
NodeLabel: node.Data.Label,
Attempt: runExecutionAttempt(currentRun),
Status: domain.NodeExecutionStatusRunning,
StartedAt: nowISO(),
Logs: []domain.LogEntryRecord{logEntry("INFO", "Starting node execution", node.Data.Label)},
Artifacts: []domain.ExecutionArtifact{},
DataRefs: []domain.DataRefRecord{},
ResourceSeries: []domain.ResourceSample{},
}
}
applyNodeExecutionSnapshot(&execution, *node, inputArtifacts, inputLineage, inputBindings)
waitingMessage := "Node is waiting for human review"
if domain.NodeRequiresAnnotation(*node) {
tasks, createErr := runtime.createAnnotationGateTasks(ctx, currentRun, *node, inputArtifacts, inputBindings)
if createErr != nil {
return nil, createErr
}
waitingMessage = fmt.Sprintf("Node is waiting for %d annotation task(s)", len(tasks))
} else {
task := domain.AuditTaskRecord{
ID: domain.NewID("audit"),
OrganizationID: currentRun.OrganizationID,
RunID: currentRun.ID,
WorkflowID: workflow.ID,
ProjectID: workflow.ProjectID,
NodeID: node.ID,
WorkflowName: workflow.Name,
ProjectName: currentRun.ProjectName,
OperatorName: node.Data.Label,
CreatedAt: nowISO(),
UpdatedAt: nowISO(),
Status: domain.AuditStatusPending,
Data: createAuditPayload(*node, inputArtifacts),
}
if _, err := runtime.store.CreateAuditTask(ctx, task); err != nil {
return nil, err
}
if err := runtime.ensureGenericAuditHumanItem(ctx, currentRun, *node, task, inputArtifacts); err != nil {
return nil, err
}
}
if domain.NodeRequiresAnnotation(*node) {
execution.Status = domain.NodeExecutionStatusWaitingForHuman
} else {
execution.Status = domain.NodeExecutionStatusWaitingForAudit
}
execution.EndedAt = nowISO()
execution.Artifacts = cloneArtifacts(inputArtifacts)
execution.InputArtifacts = cloneArtifacts(inputArtifacts)
execution.InputLineage = cloneInputLineage(inputLineage)
execution.Logs = append(execution.Logs, logEntry("WARN", waitingMessage, node.Data.Label))
executionMap[node.ID] = execution
applyExecution(currentRun, execution)
if domain.NodeRequiresAnnotation(*node) {
currentRun.Status = domain.RunStatusWaitingForHuman
} else {
currentRun.Status = domain.RunStatusWaitingForAudit
}
syncCurrentNodeID(currentRun)
currentRun.UpdatedAt = nowISO()
if err := runtime.store.UpsertRun(ctx, *currentRun); err != nil {
return nil, err
}
workflow.Status = domain.WorkflowStatusRunning
workflow.UpdatedAt = nowISO()
if err := runtime.store.UpdateWorkflowRunState(ctx, workflow.ID, workflow.Status, workflow.LastRunID, workflow.UpdatedAt); err != nil {
return nil, err
}
return currentRun, nil
}
break
}
outcome := <-results
delete(runningNodes, outcome.nodeID)
startedAt := nodeStartedAt[outcome.nodeID]
delete(nodeStartedAt, outcome.nodeID)
nodeDurationMS := uint64(0)
if !startedAt.IsZero() {
nodeDurationMS = uint64(time.Since(startedAt).Milliseconds())
}
persistedControl, err := runtime.loadRunControlState(ctx, currentRun.ID)
if err != nil {
return nil, err
}
gracefulStopRequested := false
forceStopRequested := false
if persistedControl != nil {
syncStopMetadata(currentRun, persistedControl)
if persistedControl.Status == domain.RunStatusStopping && persistedControl.StopMode == domain.RunStopModeGraceful {
gracefulStopRequested = true
}
if persistedControl.Status == domain.RunStatusCancelled && persistedControl.StopMode == domain.RunStopModeForce {
forceStopRequested = true
}
}
execution := executionMap[outcome.nodeID]
if outcome.err != nil {
if cause := context.Cause(ctx); cause != nil {
return currentRun, cause
}
if forceStopRequested || (currentRun.StopMode == domain.RunStopModeForce && errors.Is(executionCtx.Err(), context.Canceled)) {
cancelledExecution, _ := finalizeCancelledExecution(execution, "用户已强制停止工作流")
executionMap[outcome.nodeID] = cancelledExecution
applyExecution(currentRun, cancelledExecution)
currentRun.Status = domain.RunStatusCancelled
currentRun.UpdatedAt = nowISO()
if err := runtime.store.UpsertRun(ctx, *currentRun); err != nil {
return nil, err
}
if len(runningNodes) == 0 {
return runtime.finalizeCancelledRun(ctx, workflow, currentRun, executionMap, "用户已强制停止工作流")
}
continue
}
if k8s.IsJobNotFound(outcome.err) {
cancelExecution()
executionMap[outcome.nodeID] = execution
applyExecution(currentRun, execution)
if err := runtime.recoverMissingJob(ctx, currentRun, &execution); err != nil {
return nil, err
}
return currentRun, nil
}
runtime.metrics.RecordNodeExecutionFailed(nodeDurationMS)
cancelExecution()
execution.Status = domain.NodeExecutionStatusFailed
execution.EndedAt = nowISO()
execution.Error = normalizeRunError(workflow, outcome.err, executionCtx, outcome.nodeTimedOut)
execution.Logs = append(execution.Logs, logEntry("ERROR", execution.Error, execution.NodeLabel))
executionMap[outcome.nodeID] = execution
applyExecution(currentRun, execution)
currentRun.Status = domain.RunStatusFailed
currentRun.CurrentNodeID = outcome.nodeID
currentRun.EndedAt = nowISO()
currentRun.Error = execution.Error
currentRun.Failure = &domain.RunFailure{
Code: "node_execution_failed",
Stage: "execution",
NodeID: outcome.nodeID,
Message: execution.Error,
Retryable: true,
Suggestion: "查看节点日志后从失败节点继续",
}
currentRun.UpdatedAt = nowISO()
if err := runtime.store.UpsertRun(ctx, *currentRun); err != nil {
return nil, err
}
workflow.Status = domain.WorkflowStatusFailed
workflow.UpdatedAt = nowISO()
if err := runtime.store.UpdateWorkflowRunState(ctx, workflow.ID, workflow.Status, workflow.LastRunID, workflow.UpdatedAt); err != nil {
return nil, err
}
if shouldAutoRetry(workflow, currentRun) {
retryRun, err := runtime.enqueueRetryRun(ctx, workflow, currentRun)
if err != nil {
return nil, err
}
if retryRun != nil {
appendLog(currentRun, logEntry("WARN", fmt.Sprintf("已自动创建重试运行,第 %d 次尝试已进入队列", retryRun.Attempt), execution.NodeLabel))
currentRun.UpdatedAt = nowISO()
if err := runtime.store.UpsertRun(ctx, *currentRun); err != nil {
return nil, err
}
}
}
return currentRun, nil
}
runtime.metrics.RecordNodeExecutionCompleted(nodeDurationMS)
execution.Status = domain.NodeExecutionStatusCompleted
execution.EndedAt = nowISO()
execution.Logs = append(execution.Logs, outcome.result.Logs...)
execution.Artifacts = outcome.result.Artifacts
execution.DataRefs = outcome.result.DataRefs
execution.DataMetrics = outcome.result.DataMetrics
execution.StageManifestPath = outcome.result.StageManifestPath
execution.ResourceSeries = outcome.result.ResourceSeries
executionMap[outcome.nodeID] = execution
applyExecution(currentRun, execution)
if outcome.reference != nil {
currentRun.K8sRefs = append(currentRun.K8sRefs, domain.K8sRef{NodeID: outcome.nodeID, Attempt: execution.Attempt, JobName: outcome.reference.JobName, Namespace: outcome.reference.Namespace})
}
if !gracefulStopRequested && !forceStopRequested {
enqueueDependents(*workflow, executionMap, &readyQueue, readySet, allowedNodes, outcome.nodeID)
}
if gracefulStopRequested {
currentRun.Status = domain.RunStatusStopping
} else if forceStopRequested {
currentRun.Status = domain.RunStatusCancelled
} else {
currentRun.Status = domain.RunStatusRunning
}
syncCurrentNodeID(currentRun)
currentRun.UpdatedAt = nowISO()
if err := runtime.store.UpsertRun(ctx, *currentRun); err != nil {
return nil, err
}
runtime.materializeNodeMetricsBestEffort(ctx, currentRun, &execution)
executionMap[outcome.nodeID] = execution
if forceStopRequested && len(runningNodes) == 0 {
return runtime.finalizeCancelledRun(ctx, workflow, currentRun, executionMap, "用户已强制停止工作流")
}
}
if currentRun.Status == domain.RunStatusStopping || currentRun.Status == domain.RunStatusCancelled {
message := "用户已停止工作流"
if currentRun.StopMode == domain.RunStopModeForce {
message = "用户已强制停止工作流"
}
return runtime.finalizeCancelledRun(ctx, workflow, currentRun, executionMap, message)
}
return runtime.finalizeCompletedRun(ctx, workflow, currentRun)
}
// nodeProgressSampleInterval bounds how often the worker reads structured progress markers
// from long-running pods and, for custom containers, pushes live GPU samples.
const nodeProgressSampleInterval = 15 * time.Second
// shouldReportNodeProgress reports whether a node needs live runtime polling. Custom
// containers need platform-side GPU samples, while long-running builtin and custom nodes
// can emit DROBOTICFLOW_PROGRESS markers that must be persisted before the job finishes.
func shouldReportNodeProgress(runtime *Runtime, node domain.WorkflowNode) bool {
return runtimeContractLongRunning(node.Data.RuntimeContract) ||
(runtime.operatorRunnerImage != "" && chooseImage(node, runtime.operatorRunnerImage) != runtime.operatorRunnerImage)
}
// mapGPUSample maps the launcher's k8s.GPUSample onto the resource monitor's domain
// representation — the same field mapping the executor uses when folding the launcher's
// final series into the node's resourceSeries.
func mapGPUSample(sample k8s.GPUSample) domain.ResourceSample {
return domain.ResourceSample{
Timestamp: sample.Timestamp,
GPU: sample.GPU,
GPUMemoryUsedMiB: sample.GPUMemoryUsedMiB,
GPUMemoryTotalMiB: sample.GPUMemoryTotalMiB,
GPUName: sample.GPUName,
GPUCount: sample.GPUCount,
}
}
// runNodeProgressReporter persists DROBOTICFLOW_PROGRESS markers while a long-running node is
// active and optionally samples GPU usage for custom containers. It stops before the final node
// result is persisted, preventing a stale live write from racing with completion.
const progressMarker = "DROBOTICFLOW_PROGRESS "
func latestStructuredProgress(logs string) (*domain.RunProgress, error) {
var latest *domain.RunProgress
var lastErr error
for _, line := range strings.Split(strings.ReplaceAll(logs, "\r\n", "\n"), "\n") {
markerIndex := strings.Index(line, progressMarker)
if markerIndex < 0 {
continue
}
payload := strings.TrimSpace(line[markerIndex+len(progressMarker):])
if payload == "" {
continue
}
var progress domain.RunProgress
if err := json.Unmarshal([]byte(payload), &progress); err != nil {
lastErr = fmt.Errorf("decode structured progress marker: %w", err)
continue
}
if strings.TrimSpace(progress.UpdatedAt) == "" {
progress.UpdatedAt = nowISO()
}
copy := progress
latest = &copy
}
if latest != nil {
return latest, nil
}
return nil, lastErr
}
func progressFingerprint(progress *domain.RunProgress) string {
if progress == nil {
return ""
}
copy := *progress
copy.UpdatedAt = ""
payload, err := json.Marshal(copy)
if err != nil {
return ""
}
return string(payload)
}
func (runtime *Runtime) runNodeProgressReporter(ctx context.Context, runID string, nodeID string, sampleGPU bool, done func()) {
runtime.runNodeProgressReporterWithInterval(ctx, runID, nodeID, sampleGPU, nodeProgressSampleInterval, done)
}
func (runtime *Runtime) runNodeProgressReporterWithInterval(ctx context.Context, runID string, nodeID string, sampleGPU bool, interval time.Duration, done func()) {
defer done()
if interval <= 0 {
interval = nodeProgressSampleInterval
}
ticker := time.NewTicker(interval)
defer ticker.Stop()
lastProgressFingerprint := ""
tailer, canTailLogs := runtime.launcher.(nodePodLogTailer)
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
}
if canTailLogs {
logsCtx, cancelLogs := context.WithTimeout(ctx, 10*time.Second)
logs, err := tailer.TailNodePodLogs(logsCtx, runID, nodeID, 200)
cancelLogs()
if err == nil {
progress, parseErr := latestStructuredProgress(logs)
fingerprint := progressFingerprint(progress)
if parseErr == nil && progress != nil && fingerprint != "" && fingerprint != lastProgressFingerprint {
if err := runtime.store.UpdateRunProgress(ctx, runID, progress); err == nil {
lastProgressFingerprint = fingerprint
}
}
}
}
if !sampleGPU {
continue
}
sampleCtx, cancelSample := context.WithTimeout(ctx, 10*time.Second)
sample, err := runtime.launcher.SamplePodGPUForRun(sampleCtx, runID, nodeID)
cancelSample()
if err == nil && sample.Timestamp != "" {
_ = runtime.store.AppendNodeExecutionResourceSample(ctx, runID, nodeID, mapGPUSample(sample))
}
}
}
func edgeTargets(edges []domain.WorkflowEdge) []string {
items := make([]string, 0, len(edges))
for _, edge := range edges {
items = append(items, edge.Target)
}
return items
}
func (runtime *Runtime) ProcessOnce(ctx context.Context) (*ProcessOnceResult, error) {
if err := runtime.reconcileActiveRuns(ctx); err != nil {
return nil, err
}
claimed, err := runtime.store.ClaimNextQueuedRun(ctx, runtime.leaseOwner(), runtime.leaseTTL())
if err != nil || claimed == nil {
if err != nil {
return nil, err
}
return newProcessOnceResult(ProcessOnceStatusNoQueuedRun, nil), nil
}
if !runtime.tryMarkRunActive(claimed.ID) {
// Another goroutine in this worker is already processing this run
// (tryMarkRunActive guards the in-memory activeRuns set). ClaimNextQueuedRun
// already set status=Running + claimed the lease for this worker, which is
// consistent with the active processor. Release the spurious lease so it
// does not block the next ProcessAvailable claim for the lease TTL (~15min),
// and leave status=Running rather than reverting to Queued — reverting kept
// currentNodeId set, producing an inconsistent "queued+currentNodeId" state
// that no other queued-writer emits. Reconcile / the active processor will
// advance the run.
if releaseErr := runtime.store.ReleaseRunLease(context.Background(), claimed.ID, runtime.leaseOwner()); releaseErr != nil {
fmt.Printf("[worker] release spurious run lease %s failed: %v\n", claimed.ID, releaseErr)
}
return newProcessOnceResult(ProcessOnceStatusSkippedLeaseActive, claimed), nil
}
defer runtime.unmarkRunActive(claimed.ID)
defer func() {
if err := runtime.store.ReleaseRunLease(context.Background(), claimed.ID, runtime.leaseOwner()); err != nil {
fmt.Printf("[worker] release run lease %s failed: %v\n", claimed.ID, err)
}
}()
run, err := runtime.processRun(ctx, claimed)
if err != nil {
return nil, err
}
return newProcessOnceResult(ProcessOnceStatusClaimed, run), nil
}