2026-06-25 14:46:50 +08:00

1782 lines
58 KiB
Go

package worker
import (
"context"
"encoding/json"
"errors"
"fmt"
"math"
"os"
"reflect"
"sort"
"strconv"
"strings"
"time"
"droboticflow/internal/domain"
"droboticflow/internal/k8s"
"droboticflow/internal/store"
)
type jobLauncher interface {
Launch(context.Context, k8s.LaunchInput) (*k8s.LaunchResult, error)
Inspect(context.Context, string, string) (*k8s.JobInspection, error)
}
type Runtime struct {
store store.Store
operatorRunnerImage string
workspaceRoot string
globalBOSEndpoint string
launcher jobLauncher
metrics WorkerMetrics
}
var activeDispatchStatuses = map[domain.RunStatus]struct{}{
domain.RunStatusRunning: {},
domain.RunStatusStopping: {},
domain.RunStatusWaitingForAudit: {},
}
var queuedOrActiveStatuses = map[domain.RunStatus]struct{}{
domain.RunStatusQueued: {},
domain.RunStatusRunning: {},
domain.RunStatusStopping: {},
domain.RunStatusWaitingForAudit: {},
}
func NewRuntime(dataStore store.Store, namespace string, imagePullSecretName string, operatorRunnerImage string, workspacePVCName string, workspaceMountPath string, gpuNodeSelector string, gpuTolerationsJSON string, gpuRuntimeClass string, jobTTLSeconds int, globalBOSEndpoint string) (*Runtime, error) {
launcher, err := k8s.NewLauncher(namespace, imagePullSecretName, workspacePVCName, workspaceMountPath, gpuNodeSelector, gpuTolerationsJSON, gpuRuntimeClass, jobTTLSeconds)
if err != nil {
return nil, err
}
trimmedWorkspaceRoot := strings.TrimSpace(workspaceMountPath)
if trimmedWorkspaceRoot == "" {
trimmedWorkspaceRoot = "/workspace"
}
return &Runtime{
store: dataStore,
operatorRunnerImage: operatorRunnerImage,
workspaceRoot: trimmedWorkspaceRoot,
globalBOSEndpoint: strings.TrimSpace(globalBOSEndpoint),
launcher: launcher,
}, nil
}
func (runtime *Runtime) MetricsSnapshot() MetricsSnapshot {
return runtime.metrics.Snapshot()
}
func nowISO() string {
return time.Now().UTC().Format(time.RFC3339Nano)
}
func logEntry(level string, message string, operator string) domain.LogEntryRecord {
return domain.LogEntryRecord{
Timestamp: nowISO(),
Level: level,
Message: message,
Operator: operator,
}
}
func findNodeExecution(run *domain.RunRecord, nodeID string) (int, *domain.NodeExecutionRecord) {
for index := range run.NodeExecutions {
if run.NodeExecutions[index].NodeID == nodeID {
return index, &run.NodeExecutions[index]
}
}
return -1, nil
}
func appendLog(run *domain.RunRecord, entry domain.LogEntryRecord) {
run.Logs = append(run.Logs, entry)
}
func applyExecution(run *domain.RunRecord, execution domain.NodeExecutionRecord) {
index, _ := findNodeExecution(run, execution.NodeID)
if index >= 0 {
run.NodeExecutions[index] = execution
} else {
run.NodeExecutions = append(run.NodeExecutions, execution)
}
run.Logs = []domain.LogEntryRecord{}
run.ResourceSeries = []domain.ResourceSample{}
run.Artifacts = []domain.ExecutionArtifact{}
for _, item := range run.NodeExecutions {
for _, entry := range item.Logs {
appendLog(run, entry)
}
if len(item.ResourceSeries) > 0 {
run.ResourceSeries = append(run.ResourceSeries, item.ResourceSeries...)
}
if len(item.Artifacts) > 0 {
run.Artifacts = append(run.Artifacts, item.Artifacts...)
}
}
}
func applyNodeExecutionSnapshot(execution *domain.NodeExecutionRecord, node domain.WorkflowNode, inputArtifacts []domain.ExecutionArtifact, inputLineage []domain.InputLineageRecord) {
if execution == nil {
return
}
execution.NodeID = node.ID
execution.NodeLabel = node.Data.Label
execution.OperatorID = strings.TrimSpace(node.Data.OperatorID)
execution.OperatorVersion = strings.TrimSpace(node.Data.OperatorVersion)
execution.OperatorImage = strings.TrimSpace(node.Data.Image)
execution.OperatorImageID = firstNonEmptyString(node.Data.OperatorImageID, node.Data.Image)
if inputArtifacts != nil {
execution.InputArtifacts = cloneArtifacts(inputArtifacts)
}
if inputLineage != nil {
execution.InputLineage = cloneInputLineage(inputLineage)
}
}
func globalConcurrencyLimit(settings *domain.SettingsRecord) int {
if settings != nil && settings.Worker.MaxConcurrentRuns > 0 {
return settings.Worker.MaxConcurrentRuns
}
return 1
}
func effectiveWorkflowConcurrencyLimit(workflow *domain.WorkflowRecord, settings *domain.SettingsRecord) int {
if workflow != nil && workflow.MaxConcurrentRuns > 0 {
return workflow.MaxConcurrentRuns
}
return globalConcurrencyLimit(settings)
}
func parseRFC3339Nano(value string) (time.Time, bool) {
trimmed := strings.TrimSpace(value)
if trimmed == "" {
return time.Time{}, false
}
parsed, err := time.Parse(time.RFC3339Nano, trimmed)
if err != nil {
return time.Time{}, false
}
return parsed, true
}
func firstNonEmptyString(values ...string) string {
for _, value := range values {
if strings.TrimSpace(value) != "" {
return value
}
}
return ""
}
func nextHourlyScheduleTime(schedule *domain.WorkflowSchedule, lastRunAt time.Time, now time.Time) (time.Time, bool) {
intervalHours := schedule.IntervalHours
if intervalHours <= 0 {
return time.Time{}, false
}
anchorAt, ok := parseRFC3339Nano(schedule.AnchorAt)
if !ok {
if !lastRunAt.IsZero() {
anchorAt = lastRunAt
} else {
anchorAt = now
}
}
interval := time.Duration(intervalHours) * time.Hour
if lastRunAt.IsZero() || lastRunAt.Before(anchorAt) {
return anchorAt, true
}
elapsed := lastRunAt.Sub(anchorAt)
slotsElapsed := int(elapsed/interval) + 1
return anchorAt.Add(time.Duration(slotsElapsed) * interval), true
}
func nextDailyScheduleTime(schedule *domain.WorkflowSchedule, lastRunAt time.Time, now time.Time) (time.Time, bool) {
if schedule.DayInterval <= 0 {
return time.Time{}, false
}
parts := strings.Split(strings.TrimSpace(schedule.TimeOfDay), ":")
if len(parts) != 2 {
return time.Time{}, false
}
hour, err := strconv.Atoi(parts[0])
if err != nil {
return time.Time{}, false
}
minute, err := strconv.Atoi(parts[1])
if err != nil {
return time.Time{}, false
}
location, err := time.LoadLocation(domain.WorkflowScheduleTimezone(schedule.Timezone))
if err != nil {
return time.Time{}, false
}
anchorAt, ok := parseRFC3339Nano(schedule.AnchorAt)
if !ok {
if !lastRunAt.IsZero() {
anchorAt = lastRunAt
} else {
anchorAt = now
}
}
anchorLocal := anchorAt.In(location)
firstDue := time.Date(anchorLocal.Year(), anchorLocal.Month(), anchorLocal.Day(), hour, minute, 0, 0, location)
if anchorLocal.After(firstDue) {
firstDue = firstDue.AddDate(0, 0, schedule.DayInterval)
}
if lastRunAt.IsZero() {
return firstDue, true
}
lastLocal := lastRunAt.In(location)
nextDue := firstDue
for !nextDue.After(lastLocal) {
nextDue = nextDue.AddDate(0, 0, schedule.DayInterval)
}
return nextDue, true
}
func shouldEnqueueScheduledRun(workflow domain.WorkflowRecord, now time.Time) bool {
if workflow.Type != domain.WorkflowTypeStreaming || !domain.WorkflowScheduleEnabled(workflow.Schedule) {
return false
}
lastRunAt, _ := parseRFC3339Nano(workflow.LastRunAt)
switch domain.EffectiveWorkflowScheduleMode(workflow.Schedule) {
case domain.WorkflowScheduleModeEveryHours:
nextDue, ok := nextHourlyScheduleTime(workflow.Schedule, lastRunAt, now)
return ok && !now.Before(nextDue)
case domain.WorkflowScheduleModeEveryDaysAtTime:
nextDue, ok := nextDailyScheduleTime(workflow.Schedule, lastRunAt, now)
return ok && !now.Before(nextDue)
case domain.WorkflowScheduleModeIntervalSeconds:
if workflow.Schedule.Interval <= 0 {
return false
}
if workflow.LastRunAt == "" {
return true
}
return !lastRunAt.IsZero() && now.Sub(lastRunAt) >= time.Duration(workflow.Schedule.Interval)*time.Second
default:
return false
}
}
func selectQueuedRunsForDispatch(runs []domain.RunRecord, workflows []domain.WorkflowRecord, settings *domain.SettingsRecord) []string {
selected, _, _ := selectQueuedRunsForDispatchWithStats(runs, workflows, settings)
return selected
}
func selectQueuedRunsForDispatchWithStats(runs []domain.RunRecord, workflows []domain.WorkflowRecord, settings *domain.SettingsRecord) ([]string, int, int) {
globalLimit := globalConcurrencyLimit(settings)
workflowByID := make(map[string]domain.WorkflowRecord, len(workflows))
for _, workflow := range workflows {
workflowByID[workflow.ID] = workflow
}
now := time.Now().UTC()
activeByWorkflow := map[string]int{}
activeTotal := 0
queued := make([]domain.RunRecord, 0)
for _, run := range runs {
if _, ok := activeDispatchStatuses[run.Status]; ok {
activeTotal++
activeByWorkflow[run.WorkflowID]++
continue
}
if run.Status == domain.RunStatusQueued {
if !runDispatchReady(run, now) {
continue
}
queued = append(queued, run)
}
}
available := globalLimit - activeTotal
if available <= 0 {
return nil, len(queued), activeTotal
}
sort.Slice(queued, func(left int, right int) bool {
leftAvailableAt := runAvailableAt(queued[left])
rightAvailableAt := runAvailableAt(queued[right])
if leftAvailableAt.Equal(rightAvailableAt) {
return queued[left].CreatedAt < queued[right].CreatedAt
}
return leftAvailableAt.Before(rightAvailableAt)
})
selected := make([]string, 0, available)
for _, run := range queued {
if available == 0 {
break
}
workflow := workflowByID[run.WorkflowID]
workflowLimit := effectiveWorkflowConcurrencyLimit(&workflow, settings)
if activeByWorkflow[run.WorkflowID] >= workflowLimit {
continue
}
selected = append(selected, run.ID)
activeByWorkflow[run.WorkflowID]++
available--
}
return selected, len(queued), activeTotal
}
func (runtime *Runtime) enqueueStreamingWorkflows(ctx context.Context) error {
workflows, err := runtime.store.ListAllWorkflows(ctx)
if err != nil {
return err
}
runs, err := runtime.store.ListRuns(ctx, "")
if err != nil {
return err
}
settings, err := runtime.store.GetSettings(ctx)
if err != nil {
return err
}
now := time.Now().UTC()
for _, workflow := range workflows {
if !shouldEnqueueScheduledRun(workflow, now) {
continue
}
if strings.TrimSpace(workflow.ProjectID) != "" {
project, err := runtime.store.GetProject(ctx, workflow.ProjectID)
if err != nil {
return err
}
if project == nil {
fmt.Printf("[worker] skipping streaming workflow %s because project %s is missing\n", workflow.ID, workflow.ProjectID)
continue
}
}
inflightCount := 0
for _, run := range runs {
if run.WorkflowID == workflow.ID {
if _, ok := queuedOrActiveStatuses[run.Status]; ok {
inflightCount++
}
}
}
if inflightCount >= effectiveWorkflowConcurrencyLimit(&workflow, settings) {
continue
}
if _, err := runtime.store.CreateRun(ctx, workflow.ID, domain.RunTriggerScheduled); err != nil {
return err
}
}
return nil
}
func (runtime *Runtime) processClaimedRun(ctx context.Context, claimed *domain.RunRecord) {
if _, err := runtime.processRun(ctx, claimed); err != nil {
fmt.Printf("[worker] process run %s failed: %v\n", claimed.ID, err)
}
}
func (runtime *Runtime) ProcessAvailable(ctx context.Context) error {
startedAt := time.Now()
var processErr error
defer func() {
runtime.metrics.RecordProcessAvailable(uint64(time.Since(startedAt).Milliseconds()), processErr)
}()
if err := runtime.reconcileActiveRuns(ctx); err != nil {
processErr = err
return processErr
}
if err := runtime.enqueueStreamingWorkflows(ctx); err != nil {
processErr = err
return processErr
}
settings, err := runtime.store.GetSettings(ctx)
if err != nil {
processErr = err
return processErr
}
workflows, err := runtime.store.ListAllWorkflows(ctx)
if err != nil {
processErr = err
return processErr
}
runs, err := runtime.store.ListRuns(ctx, "")
if err != nil {
processErr = err
return processErr
}
selectedRunIDs, queueDepth, runningCount := selectQueuedRunsForDispatchWithStats(runs, workflows, settings)
runtime.metrics.RecordDispatch(queueDepth, runningCount, len(selectedRunIDs))
for _, runID := range selectedRunIDs {
claimed, err := runtime.store.ClaimQueuedRun(ctx, runID)
if err != nil {
processErr = err
return processErr
}
if claimed == nil {
continue
}
go runtime.processClaimedRun(ctx, claimed)
}
return nil
}
func chooseImage(node domain.WorkflowNode, fallback string) string {
if node.Data.Image != "" {
if strings.Contains(node.Data.Image, "droboticflow-operator-runner") && fallback != "" {
return fallback
}
return node.Data.Image
}
return fallback
}
func chooseValue(value string, fallback string) string {
if value == "" {
return fallback
}
return value
}
func chooseDataSource(value string, fallback string) (string, error) {
normalized := strings.ToUpper(strings.TrimSpace(value))
if normalized == "" {
return fallback, nil
}
switch normalized {
case "INLINE", "LOCAL", "BOS", "UPSTREAM":
return normalized, nil
default:
return "", fmt.Errorf("DATA_SOURCE must be INLINE, LOCAL, BOS, or UPSTREAM")
}
}
func (runtime *Runtime) reconcileActiveRuns(ctx context.Context) error {
runs, err := runtime.store.ListRuns(ctx, "")
if err != nil {
return err
}
for _, candidate := range runs {
if candidate.Status != domain.RunStatusRunning {
continue
}
run, err := runtime.store.GetRun(ctx, candidate.ID)
if err != nil {
return err
}
if run == nil {
continue
}
if strings.TrimSpace(run.CurrentNodeID) == "" {
run.Status = domain.RunStatusQueued
run.StartedAt = ""
run.UpdatedAt = nowISO()
if err := runtime.store.UpsertRun(ctx, *run); err != nil {
return err
}
continue
}
inspection, err := runtime.launcher.Inspect(ctx, run.ID, run.CurrentNodeID)
if err != nil {
return err
}
switch inspection.State {
case k8s.JobStateMissing:
run.Status = domain.RunStatusQueued
run.StartedAt = ""
run.UpdatedAt = nowISO()
if err := runtime.store.UpsertRun(ctx, *run); err != nil {
return err
}
case k8s.JobStateSucceeded:
workflow, err := runtime.store.GetWorkflow(ctx, run.WorkflowID)
if err != nil {
return err
}
if workflow == nil {
continue
}
node := domain.GetNodeByID(workflow.Nodes, run.CurrentNodeID)
if node == nil {
continue
}
executionMap := map[string]domain.NodeExecutionRecord{}
for _, item := range run.NodeExecutions {
executionMap[item.NodeID] = item
}
inputLineage := gatherInputLineage(*workflow, executionMap, node.ID)
inputArtifacts := inputArtifactsFromLineage(inputLineage)
_, existing := findNodeExecution(run, node.ID)
execution := domain.NodeExecutionRecord{
NodeID: node.ID,
NodeLabel: node.Data.Label,
Status: domain.NodeExecutionStatusRunning,
StartedAt: firstNonEmptyString(run.StartedAt, run.CurrentRunAt, run.CreatedAt),
Logs: []domain.LogEntryRecord{},
Artifacts: []domain.ExecutionArtifact{},
DataRefs: []domain.DataRefRecord{},
ResourceSeries: []domain.ResourceSample{},
}
if existing != nil {
execution = *existing
}
applyNodeExecutionSnapshot(&execution, *node, inputArtifacts, inputLineage)
parsed, err := parseExecutionOutput(inspection.Output, node.Data.Label)
if err != nil {
return err
}
execution.Status = domain.NodeExecutionStatusCompleted
execution.EndedAt = nowISO()
execution.Logs = parsed.Logs
execution.Artifacts = parsed.Artifacts
execution.DataRefs = parsed.DataRefs
execution.ResourceSeries = parsed.ResourceSeries
execution.Error = ""
applyExecution(run, execution)
run.Status = domain.RunStatusQueued
run.StartedAt = ""
run.CurrentNodeID = ""
run.UpdatedAt = nowISO()
if err := runtime.store.UpsertRun(ctx, *run); err != nil {
return err
}
}
}
return nil
}
func upsertNodeHyperparam(items []domain.HyperParam, key string, value string) []domain.HyperParam {
for index := range items {
if items[index].Key == key {
items[index].Value = value
return items
}
}
return append(items, domain.HyperParam{Key: key, Value: value})
}
func bosSourcePath(bucket string, prefix string) string {
normalizedBucket := strings.TrimSpace(bucket)
if normalizedBucket == "" {
return ""
}
sourcePath := "bos://" + normalizedBucket
if normalizedPrefix := strings.Trim(strings.TrimSpace(prefix), "/"); normalizedPrefix != "" {
sourcePath += "/" + normalizedPrefix
}
return sourcePath
}
func (runtime *Runtime) defaultStorageEndpoint() string {
return strings.TrimSpace(runtime.globalBOSEndpoint)
}
func (runtime *Runtime) storageConnectionEndpoint(connection *domain.StorageConnectionRecord) string {
if connection == nil {
return runtime.defaultStorageEndpoint()
}
if endpoint := strings.TrimSpace(connection.Endpoint); endpoint != "" {
return endpoint
}
return runtime.defaultStorageEndpoint()
}
func nodeHyperparamsMap(node domain.WorkflowNode) map[string]string {
values := make(map[string]string, len(node.Data.Hyperparams))
for _, item := range node.Data.Hyperparams {
key := strings.TrimSpace(item.Key)
if key == "" {
continue
}
values[key] = item.Value
}
return values
}
func chooseScanMode(node domain.WorkflowNode, params map[string]string) (string, error) {
mode := strings.ToUpper(strings.TrimSpace(params["SCAN_MODE"]))
switch mode {
case "FULL", "INCREMENTAL":
return mode, nil
case "":
if node.Data.Label == "BOS 增量扫描" {
return "INCREMENTAL", nil
}
return "", nil
default:
return "", fmt.Errorf("SCAN_MODE must be FULL or INCREMENTAL")
}
}
func chooseIncrementalUnit(params map[string]string) (string, error) {
switch strings.ToUpper(strings.TrimSpace(params["INCREMENTAL_UNIT"])) {
case "":
return "OBJECT", nil
case "OBJECT":
return "OBJECT", nil
case "DIRECTORY":
return "DIRECTORY", nil
default:
return "", fmt.Errorf("INCREMENTAL_UNIT must be OBJECT or DIRECTORY")
}
}
func chooseDirectoryDepth(params map[string]string) (int, error) {
raw := strings.TrimSpace(params["INCREMENTAL_DIRECTORY_DEPTH"])
if raw == "" {
return 1, nil
}
parsed, err := strconv.Atoi(raw)
if err != nil || parsed <= 0 {
return 0, fmt.Errorf("INCREMENTAL_DIRECTORY_DEPTH must be a positive integer")
}
return parsed, nil
}
func scanWorkflowSourceNode(node domain.WorkflowNode) bool {
label := strings.TrimSpace(node.Data.Label)
return node.ID == "scan-source" || label == "BOS 增量扫描"
}
func sourceFingerprint(node domain.WorkflowNode, params map[string]string, dataSource string, scanMode string, incrementalUnit string, directoryDepth int) string {
return domain.ComputeSourceFingerprint(domain.SourceFingerprintInput{
ScanMode: scanMode,
IncrementalUnit: incrementalUnit,
DirectoryDepth: directoryDepth,
DataSource: dataSource,
SourcePath: params["SOURCE_PATH"],
Bucket: node.Data.Bucket,
Prefix: node.Data.Prefix,
Pattern: node.Data.Pattern,
Endpoint: params["BOS_ENDPOINT"],
})
}
func stringList(value any) []string {
if value == nil {
return nil
}
if items, ok := value.([]string); ok {
result := make([]string, 0, len(items))
for _, item := range items {
trimmed := strings.TrimSpace(item)
if trimmed != "" {
result = append(result, trimmed)
}
}
return result
}
items, ok := value.([]any)
if ok {
result := make([]string, 0, len(items))
for _, item := range items {
trimmed := strings.TrimSpace(fmt.Sprint(item))
if trimmed != "" {
result = append(result, trimmed)
}
}
return result
}
reflected := reflect.ValueOf(value)
if reflected.Kind() != reflect.Slice && reflected.Kind() != reflect.Array {
return nil
}
result := make([]string, 0, reflected.Len())
for index := 0; index < reflected.Len(); index++ {
trimmed := strings.TrimSpace(fmt.Sprint(reflected.Index(index).Interface()))
if trimmed != "" {
result = append(result, trimmed)
}
}
return result
}
func (runtime *Runtime) prepareExecutionNode(ctx context.Context, workflow *domain.WorkflowRecord, node domain.WorkflowNode) (domain.WorkflowNode, error) {
effectiveWorkflow, err := runtime.resolveScanWorkflow(ctx, workflow)
if err != nil {
return node, err
}
copy, err := scanExecutionNode(effectiveWorkflow, node)
if err != nil {
return node, err
}
copy, err = runtime.resolveNodeStorageConnection(ctx, copy)
if err != nil {
return node, err
}
copy, err = runtime.resolveNodeMongoConnection(ctx, copy)
if err != nil {
return node, err
}
params := nodeHyperparamsMap(copy)
dataSource, err := chooseDataSource(params["DATA_SOURCE"], "BOS")
if err != nil {
return node, err
}
scanMode, err := chooseScanMode(copy, params)
if err != nil {
return node, err
}
incrementalUnit, err := chooseIncrementalUnit(params)
if err != nil {
return node, err
}
directoryDepth, err := chooseDirectoryDepth(params)
if err != nil {
return node, err
}
if scanMode != "INCREMENTAL" || workflow == nil {
return copy, nil
}
fingerprint := sourceFingerprint(copy, params, dataSource, scanMode, incrementalUnit, directoryDepth)
state, err := runtime.store.GetSourceState(ctx, workflow.ID, copy.ID, fingerprint)
if err != nil {
return copy, err
}
payload := map[string]any{
"sourceFingerprint": fingerprint,
"scanMode": scanMode,
"incrementalUnit": incrementalUnit,
"directoryDepth": directoryDepth,
"dataSource": dataSource,
"sourcePath": strings.TrimSpace(params["SOURCE_PATH"]),
"bucket": strings.TrimSpace(copy.Data.Bucket),
"prefix": strings.TrimSpace(copy.Data.Prefix),
"pattern": strings.TrimSpace(copy.Data.Pattern),
"seenFingerprints": []string{},
}
if state != nil {
payload["seenFingerprints"] = append([]string(nil), state.SeenFingerprints...)
payload["lastRunId"] = state.LastRunID
payload["lastListedCount"] = state.LastListedCount
payload["lastNewCount"] = state.LastNewCount
payload["lastScanCompletedAt"] = state.LastScanCompletedAt
}
encoded, err := json.Marshal(payload)
if err != nil {
return copy, err
}
copy.Data.Hyperparams = append([]domain.HyperParam(nil), copy.Data.Hyperparams...)
copy.Data.Hyperparams = upsertNodeHyperparam(copy.Data.Hyperparams, "SCAN_MODE", scanMode)
copy.Data.Hyperparams = upsertNodeHyperparam(copy.Data.Hyperparams, "INCREMENTAL_UNIT", incrementalUnit)
copy.Data.Hyperparams = upsertNodeHyperparam(copy.Data.Hyperparams, "INCREMENTAL_DIRECTORY_DEPTH", strconv.Itoa(directoryDepth))
copy.Data.Hyperparams = upsertNodeHyperparam(copy.Data.Hyperparams, "SOURCE_FINGERPRINT", fingerprint)
copy.Data.Hyperparams = upsertNodeHyperparam(copy.Data.Hyperparams, "SOURCE_STATE_JSON", string(encoded))
return copy, nil
}
func (runtime *Runtime) resolveScanWorkflow(ctx context.Context, workflow *domain.WorkflowRecord) (*domain.WorkflowRecord, error) {
if workflow == nil || workflow.ScanConfig == nil || strings.TrimSpace(workflow.ScanConfig.ConnectionID) == "" {
return workflow, nil
}
connection, err := runtime.store.GetStorageConnection(ctx, workflow.ScanConfig.ConnectionID)
if err != nil {
return nil, err
}
if connection == nil {
return nil, fmt.Errorf("storage connection %s not found", workflow.ScanConfig.ConnectionID)
}
copy := *workflow
scanConfig := *workflow.ScanConfig
scanConfig.Endpoint = runtime.storageConnectionEndpoint(connection)
if strings.TrimSpace(scanConfig.AccessKeyID) == "" {
scanConfig.AccessKeyID = connection.AccessKeyID
}
if strings.TrimSpace(scanConfig.SecretAccessKey) == "" {
scanConfig.SecretAccessKey = connection.SecretAccessKey
}
if strings.TrimSpace(scanConfig.SourcePath) == "" {
dataSource, err := chooseDataSource(scanConfig.DataSource, "BOS")
if err != nil {
return nil, err
}
if dataSource == "BOS" {
scanConfig.SourcePath = bosSourcePath(scanConfig.Bucket, scanConfig.Prefix)
}
}
copy.ScanConfig = &scanConfig
return &copy, nil
}
func (runtime *Runtime) resolveNodeStorageConnection(ctx context.Context, node domain.WorkflowNode) (domain.WorkflowNode, error) {
connectionID := strings.TrimSpace(node.Data.ConnectionID)
if connectionID == "" {
return node, nil
}
connection, err := runtime.store.GetStorageConnection(ctx, connectionID)
if err != nil {
return node, err
}
if connection == nil {
return node, fmt.Errorf("storage connection %s not found", connectionID)
}
copy := node
copy.Data.Hyperparams = append([]domain.HyperParam(nil), node.Data.Hyperparams...)
sourcePath := strings.TrimSpace(nodeHyperparamsMap(copy)["SOURCE_PATH"])
if sourcePath == "" {
sourcePath = bosSourcePath(copy.Data.Bucket, copy.Data.Prefix)
}
copy.Data.Hyperparams = upsertNodeHyperparam(copy.Data.Hyperparams, "DATA_SOURCE", "BOS")
copy.Data.Hyperparams = upsertNodeHyperparam(copy.Data.Hyperparams, "SOURCE_PATH", sourcePath)
copy.Data.Hyperparams = upsertNodeHyperparam(copy.Data.Hyperparams, "BOS_ENDPOINT", runtime.storageConnectionEndpoint(connection))
copy.Data.Hyperparams = upsertNodeHyperparam(copy.Data.Hyperparams, "BOS_ACCESS_KEY_ID", connection.AccessKeyID)
copy.Data.Hyperparams = upsertNodeHyperparam(copy.Data.Hyperparams, "BOS_SECRET_ACCESS_KEY", connection.SecretAccessKey)
return copy, nil
}
func (runtime *Runtime) resolveNodeMongoConnection(ctx context.Context, node domain.WorkflowNode) (domain.WorkflowNode, error) {
connectionID := strings.TrimSpace(nodeHyperparamsMap(node)["MONGO_CONNECTION_ID"])
if connectionID == "" {
return node, nil
}
connection, err := runtime.store.GetMongoConnection(ctx, connectionID)
if err != nil {
return node, err
}
if connection == nil {
return node, fmt.Errorf("mongo connection %s not found", connectionID)
}
copy := node
copy.Data.Hyperparams = append([]domain.HyperParam(nil), node.Data.Hyperparams...)
copy.Data.Hyperparams = upsertNodeHyperparam(copy.Data.Hyperparams, "MONGO_URI", strings.TrimSpace(connection.URI))
copy.Data.Hyperparams = upsertNodeHyperparam(copy.Data.Hyperparams, "MONGO_DB", strings.TrimSpace(connection.Database))
copy.Data.Hyperparams = upsertNodeHyperparam(copy.Data.Hyperparams, "MONGO_TARGET", "business")
return copy, nil
}
func checkpointArtifact(execution domain.NodeExecutionRecord) *domain.ExecutionArtifact {
for index := range execution.Artifacts {
if execution.Artifacts[index].Name == "source-checkpoint.json" {
return &execution.Artifacts[index]
}
}
return nil
}
func sourceCheckpointPayload(artifact *domain.ExecutionArtifact) (map[string]any, error) {
if artifact == nil {
return nil, nil
}
path := strings.TrimSpace(artifact.Path)
if path != "" {
data, err := os.ReadFile(path)
if err != nil {
return nil, fmt.Errorf("read source checkpoint %s: %w", path, err)
}
var payload map[string]any
if err := json.Unmarshal(data, &payload); err != nil {
return nil, fmt.Errorf("parse source checkpoint %s: %w", path, err)
}
if len(payload) == 0 {
return nil, fmt.Errorf("source checkpoint %s must be a non-empty JSON object", path)
}
return payload, nil
}
if artifact.Data == nil {
return nil, nil
}
return artifact.Data, nil
}
func sourceStateFromCheckpoint(workflow *domain.WorkflowRecord, run *domain.RunRecord, execution domain.NodeExecutionRecord, artifact *domain.ExecutionArtifact) (*domain.SourceStateRecord, error) {
payload, err := sourceCheckpointPayload(artifact)
if err != nil {
return nil, err
}
if workflow == nil || run == nil || payload == nil {
return nil, nil
}
fingerprint := strings.TrimSpace(fmt.Sprint(payload["sourceFingerprint"]))
if fingerprint == "" {
return nil, fmt.Errorf("source checkpoint for node %s is missing sourceFingerprint", execution.NodeID)
}
directoryDepth, err := checkpointIntValue(payload, "directoryDepth")
if err != nil {
return nil, err
}
if directoryDepth <= 0 {
return nil, fmt.Errorf("source checkpoint directoryDepth must be greater than 0")
}
listedCount, err := checkpointIntValue(payload, "listedCount")
if err != nil {
return nil, err
}
if listedCount < 0 {
return nil, fmt.Errorf("source checkpoint listedCount must be zero or greater")
}
newCount, err := checkpointIntValue(payload, "newCount")
if err != nil {
return nil, err
}
if newCount < 0 {
return nil, fmt.Errorf("source checkpoint newCount must be zero or greater")
}
return &domain.SourceStateRecord{
ID: workflow.ID + ":" + execution.NodeID + ":" + fingerprint,
WorkflowID: workflow.ID,
NodeID: execution.NodeID,
SourceFingerprint: fingerprint,
DataSource: strings.TrimSpace(fmt.Sprint(payload["dataSource"])),
SourcePath: strings.TrimSpace(fmt.Sprint(payload["sourcePath"])),
Bucket: strings.TrimSpace(fmt.Sprint(payload["bucket"])),
Prefix: strings.TrimSpace(fmt.Sprint(payload["prefix"])),
Pattern: strings.TrimSpace(fmt.Sprint(payload["pattern"])),
ScanMode: strings.TrimSpace(fmt.Sprint(payload["scanMode"])),
IncrementalUnit: strings.TrimSpace(fmt.Sprint(payload["incrementalUnit"])),
DirectoryDepth: directoryDepth,
SeenFingerprints: stringList(payload["seenFingerprints"]),
LastRunID: run.ID,
LastListedCount: listedCount,
LastNewCount: newCount,
LastScanStartedAt: strings.TrimSpace(fmt.Sprint(payload["scanStartedAt"])),
LastScanCompletedAt: strings.TrimSpace(fmt.Sprint(payload["scanCompletedAt"])),
UpdatedAt: nowISO(),
}, nil
}
func checkpointIntValue(payload map[string]any, key string) (int, error) {
value, ok := payload[key]
if !ok || value == nil {
return 0, fmt.Errorf("source checkpoint %s is required", key)
}
switch typed := value.(type) {
case int:
return typed, nil
case int32:
return int(typed), nil
case int64:
return int(typed), nil
case float32:
value := float64(typed)
if math.IsNaN(value) || math.IsInf(value, 0) || math.Trunc(value) != value {
return 0, fmt.Errorf("source checkpoint %s must be an integer", key)
}
return int(value), nil
case float64:
if math.IsNaN(typed) || math.IsInf(typed, 0) || math.Trunc(typed) != typed {
return 0, fmt.Errorf("source checkpoint %s must be an integer", key)
}
return int(typed), nil
case json.Number:
if parsed, err := typed.Int64(); err == nil {
return int(parsed), nil
}
case string:
trimmed := strings.TrimSpace(typed)
if parsed, err := strconv.Atoi(trimmed); err == nil && strconv.Itoa(parsed) == trimmed {
return parsed, nil
}
}
return 0, fmt.Errorf("source checkpoint %s must be an integer", key)
}
func (runtime *Runtime) commitSourceStates(ctx context.Context, workflow *domain.WorkflowRecord, run *domain.RunRecord) error {
if workflow == nil || run == nil {
return nil
}
sourceRun := run
persistedRun, err := runtime.store.GetRun(ctx, run.ID)
if err != nil {
return err
}
if persistedRun != nil {
sourceRun = persistedRun
}
for _, execution := range sourceRun.NodeExecutions {
artifact := checkpointArtifact(execution)
if artifact == nil {
continue
}
state, err := sourceStateFromCheckpoint(workflow, sourceRun, execution, artifact)
if err != nil {
return err
}
if state == nil {
continue
}
if err := runtime.store.UpsertSourceState(ctx, *state); err != nil {
return err
}
}
return nil
}
func scanExecutionNode(workflow *domain.WorkflowRecord, node domain.WorkflowNode) (domain.WorkflowNode, error) {
if workflow == nil || workflow.ScanConfig == nil {
return node, nil
}
if !scanWorkflowSourceNode(node) {
return node, nil
}
dataSource, err := chooseDataSource(workflow.ScanConfig.DataSource, "BOS")
if err != nil {
return node, err
}
sourcePath := strings.TrimSpace(workflow.ScanConfig.SourcePath)
if sourcePath == "" && dataSource == "BOS" {
sourcePath = bosSourcePath(workflow.ScanConfig.Bucket, workflow.ScanConfig.Prefix)
}
copy := node
copy.Data.Bucket = workflow.ScanConfig.Bucket
copy.Data.Prefix = workflow.ScanConfig.Prefix
copy.Data.Pattern = workflow.ScanConfig.Pattern
copy.Data.Hyperparams = append([]domain.HyperParam(nil), node.Data.Hyperparams...)
copy.Data.Hyperparams = upsertNodeHyperparam(copy.Data.Hyperparams, "DATA_SOURCE", dataSource)
copy.Data.Hyperparams = upsertNodeHyperparam(copy.Data.Hyperparams, "SOURCE_PATH", sourcePath)
copy.Data.Hyperparams = upsertNodeHyperparam(copy.Data.Hyperparams, "BOS_ENDPOINT", workflow.ScanConfig.Endpoint)
copy.Data.Hyperparams = upsertNodeHyperparam(
copy.Data.Hyperparams,
"SCAN_MODE",
chooseValue(strings.ToUpper(strings.TrimSpace(workflow.ScanConfig.ScanMode)), "INCREMENTAL"),
)
bosAccessKeyID := ""
bosSecretAccessKey := ""
if dataSource == "BOS" {
bosAccessKeyID = workflow.ScanConfig.AccessKeyID
bosSecretAccessKey = workflow.ScanConfig.SecretAccessKey
}
copy.Data.Hyperparams = upsertNodeHyperparam(copy.Data.Hyperparams, "BOS_ACCESS_KEY_ID", bosAccessKeyID)
copy.Data.Hyperparams = upsertNodeHyperparam(copy.Data.Hyperparams, "BOS_SECRET_ACCESS_KEY", bosSecretAccessKey)
return copy, nil
}
func workflowRunTimeout(workflow *domain.WorkflowRecord) time.Duration {
if workflow == nil || workflow.RunTimeoutSeconds <= 0 {
return 0
}
return time.Duration(workflow.RunTimeoutSeconds) * time.Second
}
func workflowNodeTimeout(workflow *domain.WorkflowRecord) time.Duration {
if workflow == nil || workflow.NodeTimeoutSeconds <= 0 {
return 0
}
return time.Duration(workflow.NodeTimeoutSeconds) * time.Second
}
func normalizeRunError(workflow *domain.WorkflowRecord, err error, executionCtx context.Context, nodeTimedOut bool) string {
if err == nil {
return ""
}
if workflow != nil && workflow.RunTimeoutSeconds > 0 && (errors.Is(err, context.DeadlineExceeded) || errors.Is(executionCtx.Err(), context.DeadlineExceeded)) {
return fmt.Sprintf("工作流执行超时(%d 秒)", workflow.RunTimeoutSeconds)
}
if workflow != nil && workflow.NodeTimeoutSeconds > 0 && nodeTimedOut {
return fmt.Sprintf("节点执行超时(%d 秒)", workflow.NodeTimeoutSeconds)
}
return err.Error()
}
func shouldAutoRetry(workflow *domain.WorkflowRecord, run *domain.RunRecord) bool {
if workflow == nil || run == nil || workflow.MaxRetryAttempts <= 0 {
return false
}
attempt := run.Attempt
if attempt <= 0 {
attempt = 1
}
return attempt <= workflow.MaxRetryAttempts
}
func nextRunAttempt(run *domain.RunRecord) int {
if run == nil || run.Attempt <= 0 {
return 1
}
return run.Attempt + 1
}
func nextRunAvailableAt(workflow *domain.WorkflowRecord) string {
if workflow == nil || workflow.RetryBackoffSeconds <= 0 {
return nowISO()
}
return time.Now().UTC().Add(time.Duration(workflow.RetryBackoffSeconds) * time.Second).Format(time.RFC3339Nano)
}
func (runtime *Runtime) enqueueRetryRun(ctx context.Context, workflow *domain.WorkflowRecord, failedRun *domain.RunRecord) (*domain.RunRecord, error) {
if workflow == nil || failedRun == nil {
return nil, nil
}
retryRun, err := runtime.store.CreateRun(ctx, workflow.ID, domain.RunTriggerRetry)
if err != nil {
return nil, err
}
retryRun.Attempt = nextRunAttempt(failedRun)
retryRun.ParentRunID = failedRun.ID
retryRun.AvailableAt = nextRunAvailableAt(workflow)
retryRun.UpdatedAt = nowISO()
if err := runtime.store.UpsertRun(ctx, *retryRun); err != nil {
return nil, err
}
return retryRun, nil
}
type nodeExecutionOutcome struct {
nodeID string
result executionResult
reference *k8s.JobReference
err error
nodeTimedOut bool
}
func runAvailableAt(run domain.RunRecord) time.Time {
if run.AvailableAt != "" {
if parsed, err := time.Parse(time.RFC3339Nano, run.AvailableAt); err == nil {
return parsed
}
}
if parsed, err := time.Parse(time.RFC3339Nano, run.CreatedAt); err == nil {
return parsed
}
return time.Time{}
}
func runDispatchReady(run domain.RunRecord, now time.Time) bool {
availableAt := runAvailableAt(run)
if availableAt.IsZero() {
return true
}
return !availableAt.After(now)
}
func syncCurrentNodeID(run *domain.RunRecord) {
run.CurrentNodeID = ""
for _, status := range []domain.NodeExecutionStatus{
domain.NodeExecutionStatusWaitingForAudit,
domain.NodeExecutionStatusRunning,
} {
for _, execution := range run.NodeExecutions {
if execution.Status == status {
run.CurrentNodeID = execution.NodeID
return
}
}
}
}
func isTerminalRunStatus(status domain.RunStatus) bool {
return status == domain.RunStatusCompleted ||
status == domain.RunStatusFailed ||
status == domain.RunStatusCancelled
}
func (runtime *Runtime) loadRunControlState(ctx context.Context, runID string) (*domain.RunRecord, error) {
if strings.TrimSpace(runID) == "" {
return nil, nil
}
return runtime.store.GetRun(ctx, runID)
}
func syncStopMetadata(target *domain.RunRecord, source *domain.RunRecord) {
if target == nil || source == nil {
return
}
target.StopMode = source.StopMode
target.StopRequestedAt = source.StopRequestedAt
}
func finalizeCancelledExecution(execution domain.NodeExecutionRecord, message string) (domain.NodeExecutionRecord, bool) {
if execution.Status == domain.NodeExecutionStatusCompleted ||
execution.Status == domain.NodeExecutionStatusFailed ||
execution.Status == domain.NodeExecutionStatusCancelled {
return execution, false
}
execution.Status = domain.NodeExecutionStatusCancelled
execution.EndedAt = nowISO()
execution.Error = message
execution.Logs = append(execution.Logs, logEntry("WARN", message, execution.NodeLabel))
return execution, true
}
func (runtime *Runtime) finalizeCancelledRun(ctx context.Context, workflow *domain.WorkflowRecord, currentRun *domain.RunRecord, executionMap map[string]domain.NodeExecutionRecord, message string) (*domain.RunRecord, error) {
for nodeID, execution := range executionMap {
updated, changed := finalizeCancelledExecution(execution, message)
if changed {
executionMap[nodeID] = updated
applyExecution(currentRun, updated)
}
}
currentRun.Status = domain.RunStatusCancelled
currentRun.EndedAt = nowISO()
if strings.TrimSpace(currentRun.Error) == "" {
currentRun.Error = message
}
syncCurrentNodeID(currentRun)
currentRun.UpdatedAt = nowISO()
if err := runtime.store.UpsertRun(ctx, *currentRun); err != nil {
return nil, err
}
if workflow != nil {
workflow.Status = domain.WorkflowStatusFailed
workflow.LastRunID = currentRun.ID
workflow.UpdatedAt = nowISO()
if err := runtime.store.UpsertWorkflow(ctx, *workflow); err != nil {
return nil, err
}
}
return currentRun, nil
}
func dependenciesComplete(workflow domain.WorkflowRecord, executionMap map[string]domain.NodeExecutionRecord, nodeID string) bool {
for _, candidate := range domain.GetIncomingEdges(workflow.Edges, nodeID) {
dependency, ok := executionMap[candidate.Source]
if !ok || dependency.Status != domain.NodeExecutionStatusCompleted {
return false
}
}
return true
}
func rerunAllowedNodeSet(workflow domain.WorkflowRecord, run *domain.RunRecord) map[string]struct{} {
if run == nil || run.Trigger != domain.RunTriggerRerunSubtree || strings.TrimSpace(run.RestartFromNodeID) == "" {
return nil
}
visited := map[string]struct{}{}
queue := []string{strings.TrimSpace(run.RestartFromNodeID)}
for len(queue) > 0 {
nodeID := queue[0]
queue = queue[1:]
if _, ok := visited[nodeID]; ok {
continue
}
visited[nodeID] = struct{}{}
for _, edge := range domain.GetOutgoingEdges(workflow.Edges, nodeID) {
target := strings.TrimSpace(edge.Target)
if target != "" {
queue = append(queue, target)
}
}
}
return visited
}
func nodeAllowedForExecution(allowed map[string]struct{}, nodeID string) bool {
if allowed == nil {
return true
}
_, ok := allowed[nodeID]
return ok
}
func firstActiveExecution(executionMap map[string]domain.NodeExecutionRecord) (string, domain.NodeExecutionStatus) {
for _, status := range []domain.NodeExecutionStatus{
domain.NodeExecutionStatusWaitingForAudit,
domain.NodeExecutionStatusRunning,
domain.NodeExecutionStatusQueued,
} {
for nodeID, execution := range executionMap {
if execution.Status == status {
return nodeID, status
}
}
}
return "", ""
}
func queueNodeIfReady(workflow domain.WorkflowRecord, executionMap map[string]domain.NodeExecutionRecord, readyQueue *[]string, readySet map[string]struct{}, allowed map[string]struct{}, nodeID string) {
if !nodeAllowedForExecution(allowed, nodeID) {
return
}
if _, queued := readySet[nodeID]; queued {
return
}
if execution, ok := executionMap[nodeID]; ok {
if execution.Status == domain.NodeExecutionStatusCompleted ||
execution.Status == domain.NodeExecutionStatusRunning ||
execution.Status == domain.NodeExecutionStatusWaitingForAudit {
return
}
}
if !dependenciesComplete(workflow, executionMap, nodeID) {
return
}
*readyQueue = append(*readyQueue, nodeID)
readySet[nodeID] = struct{}{}
}
func enqueueDependents(workflow domain.WorkflowRecord, executionMap map[string]domain.NodeExecutionRecord, readyQueue *[]string, readySet map[string]struct{}, allowed map[string]struct{}, nodeID string) {
for _, edge := range domain.GetOutgoingEdges(workflow.Edges, nodeID) {
queueNodeIfReady(workflow, executionMap, readyQueue, readySet, allowed, edge.Target)
}
}
func (runtime *Runtime) processRun(ctx context.Context, claimed *domain.RunRecord) (*domain.RunRecord, error) {
workflow, err := runtime.store.GetWorkflow(ctx, claimed.WorkflowID)
if err != nil {
return nil, err
}
if workflow == nil {
claimed.Status = domain.RunStatusFailed
claimed.EndedAt = nowISO()
claimed.Error = fmt.Sprintf("workflow %s not found", claimed.WorkflowID)
claimed.UpdatedAt = nowISO()
return claimed, runtime.store.UpsertRun(ctx, *claimed)
}
if conflicts := domain.FindInputPortConflicts(workflow.Nodes, workflow.Edges); len(conflicts) > 0 {
currentRun := claimed
message := "工作流图非法:同一输入端口存在多条上游连线。请拆分输入,或为每个上传结果添加单独的上传算子。"
currentRun.Status = domain.RunStatusFailed
currentRun.EndedAt = nowISO()
currentRun.Error = message
currentRun.UpdatedAt = nowISO()
appendLog(currentRun, logEntry("ERROR", message, "system"))
return currentRun, runtime.store.UpsertRun(ctx, *currentRun)
}
currentRun, err := runtime.store.GetRun(ctx, claimed.ID)
if err != nil {
return nil, err
}
if currentRun == nil {
currentRun = claimed
}
if isTerminalRunStatus(currentRun.Status) {
return currentRun, nil
}
settings, err := runtime.store.GetSettings(ctx)
if err != nil {
return nil, err
}
nodeConcurrencyLimit := effectiveWorkflowConcurrencyLimit(workflow, settings)
if nodeConcurrencyLimit <= 0 {
nodeConcurrencyLimit = 1
}
executionMap := map[string]domain.NodeExecutionRecord{}
for _, execution := range currentRun.NodeExecutions {
executionMap[execution.NodeID] = execution
}
allowedNodes := rerunAllowedNodeSet(*workflow, currentRun)
readyQueue := make([]string, 0)
readySet := make(map[string]struct{})
auditQueue := make([]string, 0)
if currentRun.CurrentNodeID != "" {
tasks, err := runtime.store.ListAuditTasks(ctx, "all")
if err != nil {
return nil, err
}
for _, task := range tasks {
if task.RunID == currentRun.ID && task.NodeID == currentRun.CurrentNodeID && task.Status != domain.AuditStatusPending {
node := domain.GetNodeByID(workflow.Nodes, currentRun.CurrentNodeID)
if node != nil {
execution := executionMap[node.ID]
execution.NodeID = node.ID
execution.NodeLabel = node.Data.Label
execution.Status = domain.NodeExecutionStatusCompleted
execution.EndedAt = nowISO()
execution.Logs = append(execution.Logs, logEntry("INFO", fmt.Sprintf("Audit decision: %s", task.Status), node.Data.Label))
executionMap[node.ID] = execution
applyExecution(currentRun, execution)
for _, target := range edgeTargets(domain.GetDecisionOutgoingEdges(workflow.Edges, node.ID, task.Status)) {
queueNodeIfReady(*workflow, executionMap, &readyQueue, readySet, allowedNodes, target)
}
currentRun.Status = domain.RunStatusRunning
syncCurrentNodeID(currentRun)
currentRun.UpdatedAt = nowISO()
if err := runtime.store.UpsertRun(ctx, *currentRun); err != nil {
return nil, err
}
}
break
}
}
}
if activeNodeID, activeStatus := firstActiveExecution(executionMap); activeNodeID != "" {
if strings.TrimSpace(currentRun.CurrentNodeID) == "" {
currentRun.CurrentNodeID = activeNodeID
}
switch activeStatus {
case domain.NodeExecutionStatusWaitingForAudit:
currentRun.Status = domain.RunStatusWaitingForAudit
default:
currentRun.Status = domain.RunStatusRunning
}
currentRun.UpdatedAt = nowISO()
syncCurrentNodeID(currentRun)
if err := runtime.store.UpsertRun(ctx, *currentRun); err != nil {
return nil, err
}
workflow.Status = domain.WorkflowStatusRunning
workflow.LastRunID = currentRun.ID
workflow.UpdatedAt = nowISO()
if err := runtime.store.UpsertWorkflow(ctx, *workflow); err != nil {
return nil, err
}
return currentRun, nil
}
if len(readyQueue) == 0 && len(executionMap) == 0 {
if allowedNodes != nil && strings.TrimSpace(currentRun.RestartFromNodeID) != "" {
queueNodeIfReady(*workflow, executionMap, &readyQueue, readySet, allowedNodes, currentRun.RestartFromNodeID)
} else {
for _, nodeID := range domain.GetStartNodeIDs(workflow.Nodes, workflow.Edges) {
queueNodeIfReady(*workflow, executionMap, &readyQueue, readySet, allowedNodes, nodeID)
}
}
}
if len(readyQueue) == 0 && len(executionMap) > 0 && currentRun.CurrentNodeID == "" {
for _, execution := range currentRun.NodeExecutions {
if execution.Status != domain.NodeExecutionStatusCompleted {
continue
}
enqueueDependents(*workflow, executionMap, &readyQueue, readySet, allowedNodes, execution.NodeID)
}
}
executionCtx := ctx
cancelExecution := func() {}
if timeout := workflowRunTimeout(workflow); timeout > 0 {
executionCtx, cancelExecution = context.WithTimeout(ctx, timeout)
} else {
executionCtx, cancelExecution = context.WithCancel(ctx)
}
defer cancelExecution()
results := make(chan nodeExecutionOutcome, nodeConcurrencyLimit)
runningNodes := make(map[string]struct{})
nodeStartedAt := make(map[string]time.Time)
var persistedControl *domain.RunRecord
gracefulStopRequested := false
forceStopRequested := false
for {
persistedControl, err = runtime.loadRunControlState(ctx, currentRun.ID)
if err != nil {
return nil, err
}
gracefulStopRequested = false
forceStopRequested = false
if persistedControl != nil {
syncStopMetadata(currentRun, persistedControl)
if persistedControl.Status == domain.RunStatusStopping && persistedControl.StopMode == domain.RunStopModeGraceful {
gracefulStopRequested = true
currentRun.Status = domain.RunStatusStopping
}
if persistedControl.Status == domain.RunStatusCancelled {
forceStopRequested = persistedControl.StopMode == domain.RunStopModeForce
currentRun.Status = domain.RunStatusCancelled
if currentRun.EndedAt == "" {
currentRun.EndedAt = persistedControl.EndedAt
}
if strings.TrimSpace(currentRun.Error) == "" {
currentRun.Error = persistedControl.Error
}
}
}
if forceStopRequested {
cancelExecution()
}
launched := false
for !gracefulStopRequested && !forceStopRequested && len(runningNodes) < nodeConcurrencyLimit && len(readyQueue) > 0 {
nodeID := readyQueue[0]
readyQueue = readyQueue[1:]
delete(readySet, nodeID)
node := domain.GetNodeByID(workflow.Nodes, nodeID)
if node == nil {
continue
}
if !nodeAllowedForExecution(allowedNodes, node.ID) {
continue
}
if existing, ok := executionMap[node.ID]; ok {
if existing.Status == domain.NodeExecutionStatusCompleted ||
existing.Status == domain.NodeExecutionStatusRunning ||
existing.Status == domain.NodeExecutionStatusWaitingForAudit {
continue
}
}
if !dependenciesComplete(*workflow, executionMap, node.ID) {
continue
}
if domain.NodeRequiresAudit(*node) {
auditQueue = append(auditQueue, node.ID)
continue
}
inputLineage := gatherInputLineage(*workflow, executionMap, node.ID)
inputArtifacts := inputArtifactsFromLineage(inputLineage)
execution := domain.NodeExecutionRecord{
NodeID: node.ID,
NodeLabel: node.Data.Label,
Status: domain.NodeExecutionStatusRunning,
StartedAt: nowISO(),
Logs: []domain.LogEntryRecord{logEntry("INFO", "Starting node execution", node.Data.Label)},
Artifacts: []domain.ExecutionArtifact{},
DataRefs: []domain.DataRefRecord{},
ResourceSeries: []domain.ResourceSample{},
StageManifestPath: "",
}
applyNodeExecutionSnapshot(&execution, *node, inputArtifacts, inputLineage)
executionMap[node.ID] = execution
applyExecution(currentRun, execution)
runningNodes[node.ID] = struct{}{}
nodeStartedAt[node.ID] = time.Now()
runtime.metrics.RecordNodeExecutionStarted()
launched = true
runSnapshot := *currentRun
nodeSnapshot, err := runtime.prepareExecutionNode(ctx, workflow, *node)
if err != nil {
results <- nodeExecutionOutcome{
nodeID: node.ID,
err: err,
}
continue
}
inputSnapshot := cloneArtifacts(inputArtifacts)
nodeExecutionCtx := executionCtx
cancelNodeExecution := func() {}
if timeout := workflowNodeTimeout(workflow); timeout > 0 {
nodeExecutionCtx, cancelNodeExecution = context.WithTimeout(executionCtx, timeout)
}
go func() {
defer cancelNodeExecution()
result, reference, err := runtime.executeNode(nodeExecutionCtx, &runSnapshot, nodeSnapshot, inputSnapshot)
results <- nodeExecutionOutcome{
nodeID: nodeSnapshot.ID,
result: result,
reference: reference,
err: err,
nodeTimedOut: errors.Is(nodeExecutionCtx.Err(), context.DeadlineExceeded),
}
}()
}
if launched {
currentRun.Status = domain.RunStatusRunning
syncCurrentNodeID(currentRun)
currentRun.UpdatedAt = nowISO()
if err := runtime.store.UpsertRun(ctx, *currentRun); err != nil {
return nil, err
}
}
if len(runningNodes) == 0 {
if forceStopRequested {
return runtime.finalizeCancelledRun(ctx, workflow, currentRun, executionMap, "用户已强制停止工作流")
}
if gracefulStopRequested || currentRun.Status == domain.RunStatusStopping {
return runtime.finalizeCancelledRun(ctx, workflow, currentRun, executionMap, "用户已停止工作流")
}
if len(readyQueue) > 0 {
continue
}
if len(auditQueue) > 0 {
nodeID := auditQueue[0]
auditQueue = auditQueue[1:]
node := domain.GetNodeByID(workflow.Nodes, nodeID)
if node == nil {
continue
}
inputLineage := gatherInputLineage(*workflow, executionMap, node.ID)
inputArtifacts := inputArtifactsFromLineage(inputLineage)
execution := executionMap[node.ID]
if execution.NodeID == "" {
execution = domain.NodeExecutionRecord{
NodeID: node.ID,
NodeLabel: node.Data.Label,
Status: domain.NodeExecutionStatusRunning,
StartedAt: nowISO(),
Logs: []domain.LogEntryRecord{logEntry("INFO", "Starting node execution", node.Data.Label)},
Artifacts: []domain.ExecutionArtifact{},
DataRefs: []domain.DataRefRecord{},
ResourceSeries: []domain.ResourceSample{},
}
}
applyNodeExecutionSnapshot(&execution, *node, inputArtifacts, inputLineage)
task := domain.AuditTaskRecord{
ID: domain.NewID("audit"),
RunID: currentRun.ID,
WorkflowID: workflow.ID,
ProjectID: workflow.ProjectID,
NodeID: node.ID,
WorkflowName: workflow.Name,
ProjectName: currentRun.ProjectName,
OperatorName: node.Data.Label,
CreatedAt: nowISO(),
UpdatedAt: nowISO(),
Status: domain.AuditStatusPending,
Data: createAuditPayload(*node, inputArtifacts),
}
if _, err := runtime.store.CreateAuditTask(ctx, task); err != nil {
return nil, err
}
execution.Status = domain.NodeExecutionStatusWaitingForAudit
execution.EndedAt = nowISO()
execution.Artifacts = cloneArtifacts(inputArtifacts)
execution.InputArtifacts = cloneArtifacts(inputArtifacts)
execution.InputLineage = cloneInputLineage(inputLineage)
execution.Logs = append(execution.Logs, logEntry("WARN", "Node is waiting for human review", node.Data.Label))
executionMap[node.ID] = execution
applyExecution(currentRun, execution)
currentRun.Status = domain.RunStatusWaitingForAudit
syncCurrentNodeID(currentRun)
currentRun.UpdatedAt = nowISO()
if err := runtime.store.UpsertRun(ctx, *currentRun); err != nil {
return nil, err
}
workflow.Status = domain.WorkflowStatusRunning
workflow.UpdatedAt = nowISO()
if err := runtime.store.UpsertWorkflow(ctx, *workflow); err != nil {
return nil, err
}
return currentRun, nil
}
break
}
outcome := <-results
delete(runningNodes, outcome.nodeID)
startedAt := nodeStartedAt[outcome.nodeID]
delete(nodeStartedAt, outcome.nodeID)
nodeDurationMS := uint64(0)
if !startedAt.IsZero() {
nodeDurationMS = uint64(time.Since(startedAt).Milliseconds())
}
persistedControl, err := runtime.loadRunControlState(ctx, currentRun.ID)
if err != nil {
return nil, err
}
gracefulStopRequested := false
forceStopRequested := false
if persistedControl != nil {
syncStopMetadata(currentRun, persistedControl)
if persistedControl.Status == domain.RunStatusStopping && persistedControl.StopMode == domain.RunStopModeGraceful {
gracefulStopRequested = true
}
if persistedControl.Status == domain.RunStatusCancelled && persistedControl.StopMode == domain.RunStopModeForce {
forceStopRequested = true
}
}
execution := executionMap[outcome.nodeID]
if outcome.err != nil {
if forceStopRequested || (currentRun.StopMode == domain.RunStopModeForce && errors.Is(executionCtx.Err(), context.Canceled)) {
cancelledExecution, _ := finalizeCancelledExecution(execution, "用户已强制停止工作流")
executionMap[outcome.nodeID] = cancelledExecution
applyExecution(currentRun, cancelledExecution)
currentRun.Status = domain.RunStatusCancelled
currentRun.UpdatedAt = nowISO()
if err := runtime.store.UpsertRun(ctx, *currentRun); err != nil {
return nil, err
}
if len(runningNodes) == 0 {
return runtime.finalizeCancelledRun(ctx, workflow, currentRun, executionMap, "用户已强制停止工作流")
}
continue
}
runtime.metrics.RecordNodeExecutionFailed(nodeDurationMS)
cancelExecution()
execution.Status = domain.NodeExecutionStatusFailed
execution.EndedAt = nowISO()
execution.Error = normalizeRunError(workflow, outcome.err, executionCtx, outcome.nodeTimedOut)
execution.Logs = append(execution.Logs, logEntry("ERROR", execution.Error, execution.NodeLabel))
executionMap[outcome.nodeID] = execution
applyExecution(currentRun, execution)
currentRun.Status = domain.RunStatusFailed
currentRun.CurrentNodeID = outcome.nodeID
currentRun.EndedAt = nowISO()
currentRun.Error = execution.Error
currentRun.UpdatedAt = nowISO()
if err := runtime.store.UpsertRun(ctx, *currentRun); err != nil {
return nil, err
}
workflow.Status = domain.WorkflowStatusFailed
workflow.UpdatedAt = nowISO()
if err := runtime.store.UpsertWorkflow(ctx, *workflow); err != nil {
return nil, err
}
if shouldAutoRetry(workflow, currentRun) {
retryRun, err := runtime.enqueueRetryRun(ctx, workflow, currentRun)
if err != nil {
return nil, err
}
if retryRun != nil {
appendLog(currentRun, logEntry("WARN", fmt.Sprintf("已自动创建重试运行,第 %d 次尝试已进入队列", retryRun.Attempt), execution.NodeLabel))
currentRun.UpdatedAt = nowISO()
if err := runtime.store.UpsertRun(ctx, *currentRun); err != nil {
return nil, err
}
}
}
return currentRun, nil
}
runtime.metrics.RecordNodeExecutionCompleted(nodeDurationMS)
execution.Status = domain.NodeExecutionStatusCompleted
execution.EndedAt = nowISO()
execution.Logs = append(execution.Logs, outcome.result.Logs...)
execution.Artifacts = outcome.result.Artifacts
execution.DataRefs = outcome.result.DataRefs
execution.StageManifestPath = outcome.result.StageManifestPath
execution.ResourceSeries = outcome.result.ResourceSeries
executionMap[outcome.nodeID] = execution
applyExecution(currentRun, execution)
if outcome.reference != nil {
currentRun.K8sRefs = append(currentRun.K8sRefs, domain.K8sRef{NodeID: outcome.nodeID, JobName: outcome.reference.JobName, Namespace: outcome.reference.Namespace})
}
if !gracefulStopRequested && !forceStopRequested {
enqueueDependents(*workflow, executionMap, &readyQueue, readySet, allowedNodes, outcome.nodeID)
}
if gracefulStopRequested {
currentRun.Status = domain.RunStatusStopping
} else if forceStopRequested {
currentRun.Status = domain.RunStatusCancelled
} else {
currentRun.Status = domain.RunStatusRunning
}
syncCurrentNodeID(currentRun)
currentRun.UpdatedAt = nowISO()
if err := runtime.store.UpsertRun(ctx, *currentRun); err != nil {
return nil, err
}
if forceStopRequested && len(runningNodes) == 0 {
return runtime.finalizeCancelledRun(ctx, workflow, currentRun, executionMap, "用户已强制停止工作流")
}
}
if currentRun.Status == domain.RunStatusStopping || currentRun.Status == domain.RunStatusCancelled {
message := "用户已停止工作流"
if currentRun.StopMode == domain.RunStopModeForce {
message = "用户已强制停止工作流"
}
return runtime.finalizeCancelledRun(ctx, workflow, currentRun, executionMap, message)
}
if err := runtime.commitSourceStates(ctx, workflow, currentRun); err != nil {
return nil, err
}
currentRun.Status = domain.RunStatusCompleted
currentRun.EndedAt = nowISO()
syncCurrentNodeID(currentRun)
currentRun.UpdatedAt = nowISO()
if err := runtime.store.UpsertRun(ctx, *currentRun); err != nil {
return nil, err
}
workflow.Status = domain.WorkflowStatusCompleted
workflow.LastRunID = currentRun.ID
workflow.UpdatedAt = nowISO()
if err := runtime.store.UpsertWorkflow(ctx, *workflow); err != nil {
return nil, err
}
return currentRun, nil
}
func edgeTargets(edges []domain.WorkflowEdge) []string {
items := make([]string, 0, len(edges))
for _, edge := range edges {
items = append(items, edge.Target)
}
return items
}
func (runtime *Runtime) ProcessOnce(ctx context.Context) (*domain.RunRecord, error) {
if err := runtime.reconcileActiveRuns(ctx); err != nil {
return nil, err
}
if err := runtime.enqueueStreamingWorkflows(ctx); err != nil {
return nil, err
}
claimed, err := runtime.store.ClaimNextQueuedRun(ctx)
if err != nil || claimed == nil {
return claimed, err
}
return runtime.processRun(ctx, claimed)
}