forked from longtao.wu/droboticflow
2185 lines
72 KiB
Go
2185 lines
72 KiB
Go
package worker
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"math"
|
|
"os"
|
|
"reflect"
|
|
"sort"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"droboticflow/internal/domain"
|
|
"droboticflow/internal/k8s"
|
|
"droboticflow/internal/store"
|
|
)
|
|
|
|
type jobLauncher interface {
|
|
Launch(context.Context, k8s.LaunchInput) (*k8s.LaunchResult, error)
|
|
Inspect(context.Context, string, string) (*k8s.JobInspection, error)
|
|
}
|
|
|
|
type gpuCapacityProvider interface {
|
|
GPUCapacity(context.Context) (*k8s.GPUCapacitySnapshot, error)
|
|
}
|
|
|
|
type Runtime struct {
|
|
store store.Store
|
|
operatorRunnerImage string
|
|
workspaceRoot string
|
|
globalBOSEndpoint string
|
|
launcher jobLauncher
|
|
metrics WorkerMetrics
|
|
processMu sync.Mutex
|
|
activeRuns map[string]struct{}
|
|
dispatchOwner string
|
|
runLeaseTTL time.Duration
|
|
}
|
|
|
|
type ProcessOnceStatus string
|
|
|
|
const (
|
|
ProcessOnceStatusClaimed ProcessOnceStatus = "claimed"
|
|
ProcessOnceStatusSkippedLeaseActive ProcessOnceStatus = "skipped_lease_active"
|
|
ProcessOnceStatusNoQueuedRun ProcessOnceStatus = "no_queued_run"
|
|
)
|
|
|
|
type ProcessOnceResult struct {
|
|
Status ProcessOnceStatus `json:"status"`
|
|
Run *domain.RunRecord `json:"-"`
|
|
RunRef *ProcessOnceRunSummary `json:"run,omitempty"`
|
|
}
|
|
|
|
type ProcessOnceRunSummary struct {
|
|
ID string `json:"id"`
|
|
Status domain.RunStatus `json:"status"`
|
|
WorkflowID string `json:"workflowId,omitempty"`
|
|
WorkflowName string `json:"workflowName,omitempty"`
|
|
ProjectID string `json:"projectId,omitempty"`
|
|
ProjectName string `json:"projectName,omitempty"`
|
|
CurrentNodeID string `json:"currentNodeId,omitempty"`
|
|
NodeExecutionCount int `json:"nodeExecutionCount"`
|
|
ArtifactCount int `json:"artifactCount"`
|
|
Attempt int `json:"attempt,omitempty"`
|
|
LeaseOwner string `json:"executionLeaseOwner,omitempty"`
|
|
LeaseExpiresAt string `json:"executionLeaseExpiresAt,omitempty"`
|
|
UpdatedAt string `json:"updatedAt,omitempty"`
|
|
}
|
|
|
|
func newProcessOnceResult(status ProcessOnceStatus, run *domain.RunRecord) *ProcessOnceResult {
|
|
result := &ProcessOnceResult{Status: status, Run: run}
|
|
if run == nil {
|
|
return result
|
|
}
|
|
artifactCount := len(run.Artifacts)
|
|
for _, execution := range run.NodeExecutions {
|
|
artifactCount += len(execution.Artifacts)
|
|
}
|
|
result.RunRef = &ProcessOnceRunSummary{
|
|
ID: run.ID,
|
|
Status: run.Status,
|
|
WorkflowID: run.WorkflowID,
|
|
WorkflowName: run.WorkflowName,
|
|
ProjectID: run.ProjectID,
|
|
ProjectName: run.ProjectName,
|
|
CurrentNodeID: run.CurrentNodeID,
|
|
NodeExecutionCount: len(run.NodeExecutions),
|
|
ArtifactCount: artifactCount,
|
|
Attempt: run.Attempt,
|
|
LeaseOwner: run.ExecutionLeaseOwner,
|
|
LeaseExpiresAt: run.ExecutionLeaseExpiresAt,
|
|
UpdatedAt: run.UpdatedAt,
|
|
}
|
|
return result
|
|
}
|
|
|
|
var globalActiveDispatchStatuses = map[domain.RunStatus]struct{}{
|
|
domain.RunStatusRunning: {},
|
|
domain.RunStatusStopping: {},
|
|
}
|
|
|
|
var workflowActiveDispatchStatuses = map[domain.RunStatus]struct{}{
|
|
domain.RunStatusRunning: {},
|
|
domain.RunStatusStopping: {},
|
|
domain.RunStatusWaitingForAudit: {},
|
|
}
|
|
|
|
var queuedOrActiveStatuses = map[domain.RunStatus]struct{}{
|
|
domain.RunStatusQueued: {},
|
|
domain.RunStatusRunning: {},
|
|
domain.RunStatusStopping: {},
|
|
domain.RunStatusWaitingForAudit: {},
|
|
}
|
|
|
|
func NewRuntime(dataStore store.Store, namespace string, imagePullSecretName string, operatorRunnerImage string, workspacePVCName string, workspaceMountPath string, gpuNodeSelector string, gpuTolerationsJSON string, gpuRuntimeClass string, jobTTLSeconds int, globalBOSEndpoint string) (*Runtime, error) {
|
|
launcher, err := k8s.NewLauncher(namespace, imagePullSecretName, workspacePVCName, workspaceMountPath, gpuNodeSelector, gpuTolerationsJSON, gpuRuntimeClass, jobTTLSeconds)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
trimmedWorkspaceRoot := strings.TrimSpace(workspaceMountPath)
|
|
if trimmedWorkspaceRoot == "" {
|
|
trimmedWorkspaceRoot = "/workspace"
|
|
}
|
|
return &Runtime{
|
|
store: dataStore,
|
|
operatorRunnerImage: operatorRunnerImage,
|
|
workspaceRoot: trimmedWorkspaceRoot,
|
|
globalBOSEndpoint: strings.TrimSpace(globalBOSEndpoint),
|
|
launcher: launcher,
|
|
dispatchOwner: defaultDispatchOwner(),
|
|
runLeaseTTL: defaultRunLeaseTTL(),
|
|
}, nil
|
|
}
|
|
|
|
func runtimeOperatorGeneratedByAI(item domain.OperatorRecord) bool {
|
|
source := strings.ToLower(strings.TrimSpace(string(item.GenerationSource)))
|
|
if source == string(domain.OperatorGenerationSourceAI) {
|
|
return true
|
|
}
|
|
if item.RuntimeContract == nil {
|
|
return false
|
|
}
|
|
if sourceValue, ok := item.RuntimeContract["generationSource"].(string); ok && strings.EqualFold(strings.TrimSpace(sourceValue), string(domain.OperatorGenerationSourceAI)) {
|
|
return true
|
|
}
|
|
_, generated := item.RuntimeContract["generatedRepository"]
|
|
return generated
|
|
}
|
|
|
|
func runtimeOperatorPublishedStatus(item domain.OperatorRecord) domain.OperatorPublicationStatus {
|
|
status := domain.OperatorPublicationStatus(strings.ToLower(strings.TrimSpace(string(item.PublicationStatus))))
|
|
if status != "" {
|
|
return status
|
|
}
|
|
if runtimeOperatorGeneratedByAI(item) {
|
|
return domain.OperatorPublicationStatusDraft
|
|
}
|
|
return domain.OperatorPublicationStatusPublished
|
|
}
|
|
|
|
func runtimeNodeGeneratedByAI(node domain.WorkflowNode) bool {
|
|
return strings.EqualFold(strings.TrimSpace(string(node.Data.GenerationSource)), string(domain.OperatorGenerationSourceAI))
|
|
}
|
|
|
|
func runtimeNodePublishedStatus(node domain.WorkflowNode) domain.OperatorPublicationStatus {
|
|
status := domain.OperatorPublicationStatus(strings.ToLower(strings.TrimSpace(string(node.Data.PublicationStatus))))
|
|
if status != "" {
|
|
return status
|
|
}
|
|
if runtimeNodeGeneratedByAI(node) {
|
|
return domain.OperatorPublicationStatusDraft
|
|
}
|
|
return domain.OperatorPublicationStatusPublished
|
|
}
|
|
|
|
func (runtime *Runtime) workflowPublicationBlocker(ctx context.Context, workflow *domain.WorkflowRecord) string {
|
|
if workflow == nil || workflow.ExecutionScope == domain.WorkflowExecutionScopeSandbox {
|
|
return ""
|
|
}
|
|
for _, node := range workflow.Nodes {
|
|
sourceAI := runtimeNodeGeneratedByAI(node)
|
|
status := runtimeNodePublishedStatus(node)
|
|
if operatorID := strings.TrimSpace(node.Data.OperatorID); operatorID != "" {
|
|
operator, err := runtime.store.GetOperator(ctx, operatorID)
|
|
if err != nil {
|
|
return "工作流图非法:无法读取算子发布状态。"
|
|
}
|
|
if operator != nil {
|
|
sourceAI = runtimeOperatorGeneratedByAI(*operator)
|
|
status = runtimeOperatorPublishedStatus(*operator)
|
|
}
|
|
}
|
|
if !sourceAI || status == domain.OperatorPublicationStatusPublished {
|
|
continue
|
|
}
|
|
label := strings.TrimSpace(node.Data.Label)
|
|
if label == "" {
|
|
label = node.ID
|
|
}
|
|
return "生产工作流不能使用未发布的 AI 生成算子:" + label + "。请在沙盒工作流验证,或发布后再运行。"
|
|
}
|
|
return ""
|
|
}
|
|
|
|
func defaultDispatchOwner() string {
|
|
hostname, err := os.Hostname()
|
|
if err != nil || strings.TrimSpace(hostname) == "" {
|
|
hostname = "unknown-host"
|
|
}
|
|
return fmt.Sprintf("%s:%d", hostname, os.Getpid())
|
|
}
|
|
|
|
func defaultRunLeaseTTL() time.Duration {
|
|
raw := strings.TrimSpace(os.Getenv("WORKER_RUN_LEASE_TTL_SECONDS"))
|
|
if raw == "" {
|
|
return 15 * time.Minute
|
|
}
|
|
seconds, err := strconv.Atoi(raw)
|
|
if err != nil || seconds <= 0 {
|
|
return 15 * time.Minute
|
|
}
|
|
return time.Duration(seconds) * time.Second
|
|
}
|
|
|
|
func (runtime *Runtime) leaseOwner() string {
|
|
if runtime == nil {
|
|
return ""
|
|
}
|
|
if strings.TrimSpace(runtime.dispatchOwner) == "" {
|
|
runtime.dispatchOwner = defaultDispatchOwner()
|
|
}
|
|
return runtime.dispatchOwner
|
|
}
|
|
|
|
func (runtime *Runtime) leaseTTL() time.Duration {
|
|
if runtime == nil || runtime.runLeaseTTL <= 0 {
|
|
return defaultRunLeaseTTL()
|
|
}
|
|
return runtime.runLeaseTTL
|
|
}
|
|
|
|
func (runtime *Runtime) MetricsSnapshot() MetricsSnapshot {
|
|
return runtime.metrics.Snapshot()
|
|
}
|
|
|
|
func (runtime *Runtime) GPUCapacity(ctx context.Context) (*k8s.GPUCapacitySnapshot, error) {
|
|
if runtime == nil || runtime.launcher == nil {
|
|
return nil, fmt.Errorf("worker launcher is not configured")
|
|
}
|
|
provider, ok := runtime.launcher.(gpuCapacityProvider)
|
|
if !ok {
|
|
return nil, fmt.Errorf("worker launcher does not expose gpu capacity")
|
|
}
|
|
return provider.GPUCapacity(ctx)
|
|
}
|
|
|
|
func nowISO() string {
|
|
return time.Now().UTC().Format(time.RFC3339Nano)
|
|
}
|
|
|
|
func logEntry(level string, message string, operator string) domain.LogEntryRecord {
|
|
return domain.LogEntryRecord{
|
|
Timestamp: nowISO(),
|
|
Level: level,
|
|
Message: message,
|
|
Operator: operator,
|
|
}
|
|
}
|
|
|
|
func findNodeExecution(run *domain.RunRecord, nodeID string) (int, *domain.NodeExecutionRecord) {
|
|
for index := range run.NodeExecutions {
|
|
if run.NodeExecutions[index].NodeID == nodeID {
|
|
return index, &run.NodeExecutions[index]
|
|
}
|
|
}
|
|
return -1, nil
|
|
}
|
|
|
|
func appendLog(run *domain.RunRecord, entry domain.LogEntryRecord) {
|
|
run.Logs = append(run.Logs, entry)
|
|
}
|
|
|
|
func applyExecution(run *domain.RunRecord, execution domain.NodeExecutionRecord) {
|
|
index, _ := findNodeExecution(run, execution.NodeID)
|
|
if index >= 0 {
|
|
run.NodeExecutions[index] = execution
|
|
} else {
|
|
run.NodeExecutions = append(run.NodeExecutions, execution)
|
|
}
|
|
run.Logs = []domain.LogEntryRecord{}
|
|
run.ResourceSeries = []domain.ResourceSample{}
|
|
run.Artifacts = []domain.ExecutionArtifact{}
|
|
for _, item := range run.NodeExecutions {
|
|
for _, entry := range item.Logs {
|
|
appendLog(run, entry)
|
|
}
|
|
if len(item.ResourceSeries) > 0 {
|
|
run.ResourceSeries = append(run.ResourceSeries, item.ResourceSeries...)
|
|
}
|
|
if len(item.Artifacts) > 0 {
|
|
run.Artifacts = append(run.Artifacts, item.Artifacts...)
|
|
}
|
|
}
|
|
}
|
|
|
|
func applyNodeExecutionSnapshot(execution *domain.NodeExecutionRecord, node domain.WorkflowNode, inputArtifacts []domain.ExecutionArtifact, inputLineage []domain.InputLineageRecord) {
|
|
if execution == nil {
|
|
return
|
|
}
|
|
resolvedImage := chooseImage(node, "")
|
|
execution.NodeID = node.ID
|
|
execution.NodeLabel = node.Data.Label
|
|
execution.OperatorID = strings.TrimSpace(node.Data.OperatorID)
|
|
execution.OperatorVersion = strings.TrimSpace(node.Data.OperatorVersion)
|
|
execution.OperatorImage = firstNonEmptyString(resolvedImage, node.Data.Image)
|
|
execution.OperatorImageID = firstNonEmptyString(node.Data.OperatorImageID, resolvedImage, node.Data.Image)
|
|
if execution.Attempt <= 0 {
|
|
execution.Attempt = 1
|
|
}
|
|
if inputArtifacts != nil {
|
|
execution.InputArtifacts = cloneArtifacts(inputArtifacts)
|
|
}
|
|
if inputLineage != nil {
|
|
execution.InputLineage = cloneInputLineage(inputLineage)
|
|
}
|
|
}
|
|
|
|
func globalConcurrencyLimit(settings *domain.SettingsRecord) int {
|
|
if settings != nil && settings.Worker.MaxConcurrentRuns > 0 {
|
|
return settings.Worker.MaxConcurrentRuns
|
|
}
|
|
return 1
|
|
}
|
|
|
|
func effectiveWorkflowConcurrencyLimit(workflow *domain.WorkflowRecord, settings *domain.SettingsRecord) int {
|
|
if workflow != nil && workflow.MaxConcurrentRuns > 0 {
|
|
return workflow.MaxConcurrentRuns
|
|
}
|
|
return globalConcurrencyLimit(settings)
|
|
}
|
|
|
|
func parseRFC3339Nano(value string) (time.Time, bool) {
|
|
trimmed := strings.TrimSpace(value)
|
|
if trimmed == "" {
|
|
return time.Time{}, false
|
|
}
|
|
parsed, err := time.Parse(time.RFC3339Nano, trimmed)
|
|
if err != nil {
|
|
return time.Time{}, false
|
|
}
|
|
return parsed, true
|
|
}
|
|
|
|
func isStaleQueuedNodeExecution(execution domain.NodeExecutionRecord, threshold time.Duration) bool {
|
|
startedAt, ok := parseRFC3339Nano(execution.StartedAt)
|
|
if !ok {
|
|
return false
|
|
}
|
|
return time.Since(startedAt) >= threshold
|
|
}
|
|
|
|
func firstNonEmptyString(values ...string) string {
|
|
for _, value := range values {
|
|
if strings.TrimSpace(value) != "" {
|
|
return value
|
|
}
|
|
}
|
|
return ""
|
|
}
|
|
|
|
func nextHourlyScheduleTime(schedule *domain.WorkflowSchedule, lastRunAt time.Time, now time.Time) (time.Time, bool) {
|
|
intervalHours := schedule.IntervalHours
|
|
if intervalHours <= 0 {
|
|
return time.Time{}, false
|
|
}
|
|
anchorAt, ok := parseRFC3339Nano(schedule.AnchorAt)
|
|
if !ok {
|
|
if !lastRunAt.IsZero() {
|
|
anchorAt = lastRunAt
|
|
} else {
|
|
anchorAt = now
|
|
}
|
|
}
|
|
interval := time.Duration(intervalHours) * time.Hour
|
|
if lastRunAt.IsZero() || lastRunAt.Before(anchorAt) {
|
|
return anchorAt, true
|
|
}
|
|
elapsed := lastRunAt.Sub(anchorAt)
|
|
slotsElapsed := int(elapsed/interval) + 1
|
|
return anchorAt.Add(time.Duration(slotsElapsed) * interval), true
|
|
}
|
|
|
|
func nextDailyScheduleTime(schedule *domain.WorkflowSchedule, lastRunAt time.Time, now time.Time) (time.Time, bool) {
|
|
if schedule.DayInterval <= 0 {
|
|
return time.Time{}, false
|
|
}
|
|
parts := strings.Split(strings.TrimSpace(schedule.TimeOfDay), ":")
|
|
if len(parts) != 2 {
|
|
return time.Time{}, false
|
|
}
|
|
hour, err := strconv.Atoi(parts[0])
|
|
if err != nil {
|
|
return time.Time{}, false
|
|
}
|
|
minute, err := strconv.Atoi(parts[1])
|
|
if err != nil {
|
|
return time.Time{}, false
|
|
}
|
|
location, err := time.LoadLocation(domain.WorkflowScheduleTimezone(schedule.Timezone))
|
|
if err != nil {
|
|
return time.Time{}, false
|
|
}
|
|
anchorAt, ok := parseRFC3339Nano(schedule.AnchorAt)
|
|
if !ok {
|
|
if !lastRunAt.IsZero() {
|
|
anchorAt = lastRunAt
|
|
} else {
|
|
anchorAt = now
|
|
}
|
|
}
|
|
anchorLocal := anchorAt.In(location)
|
|
firstDue := time.Date(anchorLocal.Year(), anchorLocal.Month(), anchorLocal.Day(), hour, minute, 0, 0, location)
|
|
if anchorLocal.After(firstDue) {
|
|
firstDue = firstDue.AddDate(0, 0, schedule.DayInterval)
|
|
}
|
|
if lastRunAt.IsZero() {
|
|
return firstDue, true
|
|
}
|
|
lastLocal := lastRunAt.In(location)
|
|
nextDue := firstDue
|
|
for !nextDue.After(lastLocal) {
|
|
nextDue = nextDue.AddDate(0, 0, schedule.DayInterval)
|
|
}
|
|
return nextDue, true
|
|
}
|
|
|
|
func shouldEnqueueScheduledRun(workflow domain.WorkflowRecord, now time.Time) bool {
|
|
if workflow.Type != domain.WorkflowTypeStreaming || !domain.WorkflowScheduleEnabled(workflow.Schedule) {
|
|
return false
|
|
}
|
|
lastRunAt, _ := parseRFC3339Nano(workflow.LastRunAt)
|
|
switch domain.EffectiveWorkflowScheduleMode(workflow.Schedule) {
|
|
case domain.WorkflowScheduleModeEveryHours:
|
|
nextDue, ok := nextHourlyScheduleTime(workflow.Schedule, lastRunAt, now)
|
|
return ok && !now.Before(nextDue)
|
|
case domain.WorkflowScheduleModeEveryDaysAtTime:
|
|
nextDue, ok := nextDailyScheduleTime(workflow.Schedule, lastRunAt, now)
|
|
return ok && !now.Before(nextDue)
|
|
case domain.WorkflowScheduleModeIntervalSeconds:
|
|
if workflow.Schedule.Interval <= 0 {
|
|
return false
|
|
}
|
|
if workflow.LastRunAt == "" {
|
|
return true
|
|
}
|
|
return !lastRunAt.IsZero() && now.Sub(lastRunAt) >= time.Duration(workflow.Schedule.Interval)*time.Second
|
|
default:
|
|
return false
|
|
}
|
|
}
|
|
|
|
func selectQueuedRunsForDispatch(runs []domain.RunRecord, workflows []domain.WorkflowRecord, settings *domain.SettingsRecord) []string {
|
|
selected, _, _ := selectQueuedRunsForDispatchWithStats(runs, workflows, settings)
|
|
return selected
|
|
}
|
|
|
|
func selectQueuedRunsForDispatchWithStats(runs []domain.RunRecord, workflows []domain.WorkflowRecord, settings *domain.SettingsRecord) ([]string, int, int) {
|
|
globalLimit := globalConcurrencyLimit(settings)
|
|
workflowByID := make(map[string]domain.WorkflowRecord, len(workflows))
|
|
for _, workflow := range workflows {
|
|
workflowByID[workflow.ID] = workflow
|
|
}
|
|
now := time.Now().UTC()
|
|
|
|
activeByWorkflow := map[string]int{}
|
|
activeTotal := 0
|
|
queued := make([]domain.RunRecord, 0)
|
|
for _, run := range runs {
|
|
if _, ok := workflowActiveDispatchStatuses[run.Status]; ok {
|
|
activeByWorkflow[run.WorkflowID]++
|
|
}
|
|
if _, ok := globalActiveDispatchStatuses[run.Status]; ok {
|
|
activeTotal++
|
|
continue
|
|
}
|
|
if run.Status == domain.RunStatusQueued {
|
|
if !runDispatchReady(run, now) {
|
|
continue
|
|
}
|
|
queued = append(queued, run)
|
|
}
|
|
}
|
|
|
|
available := globalLimit - activeTotal
|
|
if available <= 0 {
|
|
return nil, len(queued), activeTotal
|
|
}
|
|
|
|
sort.Slice(queued, func(left int, right int) bool {
|
|
leftAvailableAt := runAvailableAt(queued[left])
|
|
rightAvailableAt := runAvailableAt(queued[right])
|
|
if leftAvailableAt.Equal(rightAvailableAt) {
|
|
return queued[left].CreatedAt < queued[right].CreatedAt
|
|
}
|
|
return leftAvailableAt.Before(rightAvailableAt)
|
|
})
|
|
|
|
selected := make([]string, 0, available)
|
|
for _, run := range queued {
|
|
if available == 0 {
|
|
break
|
|
}
|
|
workflow := workflowByID[run.WorkflowID]
|
|
workflowLimit := effectiveWorkflowConcurrencyLimit(&workflow, settings)
|
|
if activeByWorkflow[run.WorkflowID] >= workflowLimit {
|
|
continue
|
|
}
|
|
selected = append(selected, run.ID)
|
|
activeByWorkflow[run.WorkflowID]++
|
|
available--
|
|
}
|
|
return selected, len(queued), activeTotal
|
|
}
|
|
|
|
func (runtime *Runtime) enqueueStreamingWorkflows(ctx context.Context) error {
|
|
workflows, err := runtime.store.ListAllWorkflows(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
runs, err := runtime.store.ListRuns(ctx, "")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
settings, err := runtime.store.GetSettings(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
now := time.Now().UTC()
|
|
for _, workflow := range workflows {
|
|
if !shouldEnqueueScheduledRun(workflow, now) {
|
|
continue
|
|
}
|
|
if strings.TrimSpace(workflow.ProjectID) != "" {
|
|
project, err := runtime.store.GetProject(ctx, workflow.ProjectID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if project == nil {
|
|
fmt.Printf("[worker] skipping streaming workflow %s because project %s is missing\n", workflow.ID, workflow.ProjectID)
|
|
continue
|
|
}
|
|
}
|
|
inflightCount := 0
|
|
for _, run := range runs {
|
|
if run.WorkflowID == workflow.ID {
|
|
if _, ok := queuedOrActiveStatuses[run.Status]; ok {
|
|
inflightCount++
|
|
}
|
|
}
|
|
}
|
|
if inflightCount >= effectiveWorkflowConcurrencyLimit(&workflow, settings) {
|
|
continue
|
|
}
|
|
if _, err := runtime.store.CreateRun(ctx, workflow.ID, domain.RunTriggerScheduled); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (runtime *Runtime) processClaimedRun(ctx context.Context, claimed *domain.RunRecord) {
|
|
if claimed == nil {
|
|
return
|
|
}
|
|
defer runtime.unmarkRunActive(claimed.ID)
|
|
defer func() {
|
|
if err := runtime.store.ReleaseRunLease(context.Background(), claimed.ID, runtime.leaseOwner()); err != nil {
|
|
fmt.Printf("[worker] release run lease %s failed: %v\n", claimed.ID, err)
|
|
}
|
|
}()
|
|
if _, err := runtime.processRun(ctx, claimed); err != nil {
|
|
fmt.Printf("[worker] process run %s failed: %v\n", claimed.ID, err)
|
|
}
|
|
}
|
|
|
|
func (runtime *Runtime) renewRunLeaseUntil(ctx context.Context, runID string) {
|
|
ttl := runtime.leaseTTL()
|
|
interval := ttl / 3
|
|
if interval <= 0 || interval > time.Minute {
|
|
interval = time.Minute
|
|
}
|
|
ticker := time.NewTicker(interval)
|
|
defer ticker.Stop()
|
|
for {
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case <-ticker.C:
|
|
ok, err := runtime.store.RenewRunLease(context.Background(), runID, runtime.leaseOwner(), ttl)
|
|
if err != nil {
|
|
fmt.Printf("[worker] renew run lease %s failed: %v\n", runID, err)
|
|
continue
|
|
}
|
|
if !ok {
|
|
fmt.Printf("[worker] renew run lease %s skipped: owner changed or run missing\n", runID)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func (runtime *Runtime) tryMarkRunActive(runID string) bool {
|
|
if strings.TrimSpace(runID) == "" {
|
|
return false
|
|
}
|
|
runtime.processMu.Lock()
|
|
defer runtime.processMu.Unlock()
|
|
if runtime.activeRuns == nil {
|
|
runtime.activeRuns = map[string]struct{}{}
|
|
}
|
|
if _, ok := runtime.activeRuns[runID]; ok {
|
|
return false
|
|
}
|
|
runtime.activeRuns[runID] = struct{}{}
|
|
return true
|
|
}
|
|
|
|
func (runtime *Runtime) unmarkRunActive(runID string) {
|
|
runtime.processMu.Lock()
|
|
defer runtime.processMu.Unlock()
|
|
delete(runtime.activeRuns, runID)
|
|
}
|
|
|
|
func (runtime *Runtime) ProcessAvailable(ctx context.Context) error {
|
|
startedAt := time.Now()
|
|
var processErr error
|
|
defer func() {
|
|
runtime.metrics.RecordProcessAvailable(uint64(time.Since(startedAt).Milliseconds()), processErr)
|
|
}()
|
|
if err := runtime.reconcileActiveRuns(ctx); err != nil {
|
|
processErr = err
|
|
return processErr
|
|
}
|
|
if err := runtime.enqueueStreamingWorkflows(ctx); err != nil {
|
|
processErr = err
|
|
return processErr
|
|
}
|
|
settings, err := runtime.store.GetSettings(ctx)
|
|
if err != nil {
|
|
processErr = err
|
|
return processErr
|
|
}
|
|
workflows, err := runtime.store.ListAllWorkflows(ctx)
|
|
if err != nil {
|
|
processErr = err
|
|
return processErr
|
|
}
|
|
runs, err := runtime.store.ListRuns(ctx, "")
|
|
if err != nil {
|
|
processErr = err
|
|
return processErr
|
|
}
|
|
selectedRunIDs, queueDepth, runningCount := selectQueuedRunsForDispatchWithStats(runs, workflows, settings)
|
|
runtime.metrics.RecordDispatch(queueDepth, runningCount, len(selectedRunIDs))
|
|
for _, runID := range selectedRunIDs {
|
|
if !runtime.tryMarkRunActive(runID) {
|
|
continue
|
|
}
|
|
claimed, err := runtime.store.ClaimQueuedRun(ctx, runID, runtime.leaseOwner(), runtime.leaseTTL())
|
|
if err != nil {
|
|
runtime.unmarkRunActive(runID)
|
|
processErr = err
|
|
return processErr
|
|
}
|
|
if claimed == nil {
|
|
runtime.unmarkRunActive(runID)
|
|
continue
|
|
}
|
|
go runtime.processClaimedRun(ctx, claimed)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func chooseImage(node domain.WorkflowNode, fallback string) string {
|
|
if digestPinnedImage(node.Data.OperatorImageID) {
|
|
return strings.TrimSpace(node.Data.OperatorImageID)
|
|
}
|
|
if node.Data.Image != "" {
|
|
if strings.Contains(node.Data.Image, "droboticflow-operator-runner") && fallback != "" {
|
|
return fallback
|
|
}
|
|
return node.Data.Image
|
|
}
|
|
return fallback
|
|
}
|
|
|
|
func digestPinnedImage(value string) bool {
|
|
trimmed := strings.TrimSpace(value)
|
|
if strings.ContainsAny(trimmed, " \t\r\n") {
|
|
return false
|
|
}
|
|
parts := strings.Split(trimmed, "@sha256:")
|
|
if len(parts) != 2 || parts[0] == "" || len(parts[1]) != 64 {
|
|
return false
|
|
}
|
|
for _, character := range parts[1] {
|
|
if (character < '0' || character > '9') && (character < 'a' || character > 'f') && (character < 'A' || character > 'F') {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
|
|
func chooseValue(value string, fallback string) string {
|
|
if value == "" {
|
|
return fallback
|
|
}
|
|
return value
|
|
}
|
|
|
|
func chooseDataSource(value string, fallback string) (string, error) {
|
|
normalized := strings.ToUpper(strings.TrimSpace(value))
|
|
if normalized == "" {
|
|
return fallback, nil
|
|
}
|
|
switch normalized {
|
|
case "INLINE", "LOCAL", "BOS", "UPSTREAM":
|
|
return normalized, nil
|
|
default:
|
|
return "", fmt.Errorf("DATA_SOURCE must be INLINE, LOCAL, BOS, or UPSTREAM")
|
|
}
|
|
}
|
|
|
|
func (runtime *Runtime) reconcileActiveRuns(ctx context.Context) error {
|
|
runs, err := runtime.store.ListRuns(ctx, "")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
for _, candidate := range runs {
|
|
if candidate.Status != domain.RunStatusRunning && (candidate.Status != domain.RunStatusQueued || strings.TrimSpace(candidate.CurrentNodeID) == "") {
|
|
continue
|
|
}
|
|
run, err := runtime.store.GetRun(ctx, candidate.ID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if run == nil {
|
|
continue
|
|
}
|
|
var activeExecution *domain.NodeExecutionRecord
|
|
if strings.TrimSpace(run.CurrentNodeID) != "" {
|
|
_, activeExecution = findNodeExecution(run, run.CurrentNodeID)
|
|
}
|
|
if run.Status == domain.RunStatusQueued {
|
|
if activeExecution == nil || activeExecution.Status != domain.NodeExecutionStatusRunning || !isStaleQueuedNodeExecution(*activeExecution, 2*time.Minute) {
|
|
continue
|
|
}
|
|
}
|
|
if strings.TrimSpace(run.CurrentNodeID) == "" {
|
|
run.Status = domain.RunStatusQueued
|
|
run.StartedAt = ""
|
|
run.UpdatedAt = nowISO()
|
|
if err := runtime.store.UpsertRun(ctx, *run); err != nil {
|
|
return err
|
|
}
|
|
continue
|
|
}
|
|
inspection, err := runtime.launcher.Inspect(ctx, run.ID, run.CurrentNodeID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
switch inspection.State {
|
|
case k8s.JobStateMissing:
|
|
if strings.TrimSpace(run.CurrentNodeID) != "" {
|
|
if index, execution := findNodeExecution(run, run.CurrentNodeID); execution != nil && execution.Status == domain.NodeExecutionStatusRunning {
|
|
message := "Kubernetes job is missing; node execution was re-queued"
|
|
run.Logs = append(run.Logs, logEntry("WARN", message, execution.NodeLabel))
|
|
run.NodeExecutions = append(run.NodeExecutions[:index], run.NodeExecutions[index+1:]...)
|
|
}
|
|
}
|
|
run.Status = domain.RunStatusQueued
|
|
run.StartedAt = ""
|
|
run.CurrentNodeID = ""
|
|
run.UpdatedAt = nowISO()
|
|
if err := runtime.store.UpsertRun(ctx, *run); err != nil {
|
|
return err
|
|
}
|
|
case k8s.JobStateSucceeded:
|
|
workflow, err := runtime.store.GetWorkflow(ctx, run.WorkflowID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if workflow == nil {
|
|
continue
|
|
}
|
|
node := domain.GetNodeByID(workflow.Nodes, run.CurrentNodeID)
|
|
if node == nil {
|
|
continue
|
|
}
|
|
executionMap := map[string]domain.NodeExecutionRecord{}
|
|
for _, item := range run.NodeExecutions {
|
|
executionMap[item.NodeID] = item
|
|
}
|
|
inputLineage := gatherInputLineage(*workflow, executionMap, node.ID)
|
|
inputArtifacts := inputArtifactsFromLineage(inputLineage)
|
|
_, existing := findNodeExecution(run, node.ID)
|
|
execution := domain.NodeExecutionRecord{
|
|
NodeID: node.ID,
|
|
NodeLabel: node.Data.Label,
|
|
Attempt: runExecutionAttempt(run),
|
|
Status: domain.NodeExecutionStatusRunning,
|
|
StartedAt: firstNonEmptyString(run.StartedAt, run.CurrentRunAt, run.CreatedAt),
|
|
Logs: []domain.LogEntryRecord{},
|
|
Artifacts: []domain.ExecutionArtifact{},
|
|
DataRefs: []domain.DataRefRecord{},
|
|
ResourceSeries: []domain.ResourceSample{},
|
|
}
|
|
if existing != nil {
|
|
execution = *existing
|
|
}
|
|
applyNodeExecutionSnapshot(&execution, *node, inputArtifacts, inputLineage)
|
|
parsed, err := parseExecutionOutput(inspection.Output, node.Data.Label)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
execution.Status = domain.NodeExecutionStatusCompleted
|
|
execution.EndedAt = nowISO()
|
|
execution.Logs = parsed.Logs
|
|
execution.Artifacts = parsed.Artifacts
|
|
execution.DataRefs = parsed.DataRefs
|
|
execution.ResourceSeries = parsed.ResourceSeries
|
|
execution.Error = ""
|
|
executionMap[node.ID] = execution
|
|
applyExecution(run, execution)
|
|
if allAllowedNodesCompleted(workflow, run, executionMap) {
|
|
if _, err := runtime.finalizeCompletedRun(ctx, workflow, run); err != nil {
|
|
return err
|
|
}
|
|
continue
|
|
}
|
|
run.Status = domain.RunStatusQueued
|
|
run.StartedAt = ""
|
|
run.CurrentNodeID = ""
|
|
run.UpdatedAt = nowISO()
|
|
if err := runtime.store.UpsertRun(ctx, *run); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func upsertNodeHyperparam(items []domain.HyperParam, key string, value string) []domain.HyperParam {
|
|
for index := range items {
|
|
if items[index].Key == key {
|
|
items[index].Value = value
|
|
return items
|
|
}
|
|
}
|
|
return append(items, domain.HyperParam{Key: key, Value: value})
|
|
}
|
|
|
|
func bosSourcePath(bucket string, prefix string) string {
|
|
normalizedBucket := strings.TrimSpace(bucket)
|
|
if normalizedBucket == "" {
|
|
return ""
|
|
}
|
|
sourcePath := "bos://" + normalizedBucket
|
|
if normalizedPrefix := strings.Trim(strings.TrimSpace(prefix), "/"); normalizedPrefix != "" {
|
|
sourcePath += "/" + normalizedPrefix
|
|
}
|
|
return sourcePath
|
|
}
|
|
|
|
func (runtime *Runtime) defaultStorageEndpoint() string {
|
|
return strings.TrimSpace(runtime.globalBOSEndpoint)
|
|
}
|
|
|
|
func (runtime *Runtime) storageConnectionEndpoint(connection *domain.StorageConnectionRecord) string {
|
|
if connection == nil {
|
|
return runtime.defaultStorageEndpoint()
|
|
}
|
|
if endpoint := strings.TrimSpace(connection.Endpoint); endpoint != "" {
|
|
return endpoint
|
|
}
|
|
return runtime.defaultStorageEndpoint()
|
|
}
|
|
|
|
func nodeHyperparamsMap(node domain.WorkflowNode) map[string]string {
|
|
values := make(map[string]string, len(node.Data.Hyperparams))
|
|
for _, item := range node.Data.Hyperparams {
|
|
key := strings.TrimSpace(item.Key)
|
|
if key == "" {
|
|
continue
|
|
}
|
|
values[key] = item.Value
|
|
}
|
|
return values
|
|
}
|
|
|
|
func chooseScanMode(node domain.WorkflowNode, params map[string]string) (string, error) {
|
|
mode := strings.ToUpper(strings.TrimSpace(params["SCAN_MODE"]))
|
|
switch mode {
|
|
case "FULL", "INCREMENTAL":
|
|
return mode, nil
|
|
case "":
|
|
if node.Data.Label == "BOS 增量扫描" {
|
|
return "INCREMENTAL", nil
|
|
}
|
|
return "", nil
|
|
default:
|
|
return "", fmt.Errorf("SCAN_MODE must be FULL or INCREMENTAL")
|
|
}
|
|
}
|
|
|
|
func chooseIncrementalUnit(params map[string]string) (string, error) {
|
|
switch strings.ToUpper(strings.TrimSpace(params["INCREMENTAL_UNIT"])) {
|
|
case "":
|
|
return "OBJECT", nil
|
|
case "OBJECT":
|
|
return "OBJECT", nil
|
|
case "DIRECTORY":
|
|
return "DIRECTORY", nil
|
|
default:
|
|
return "", fmt.Errorf("INCREMENTAL_UNIT must be OBJECT or DIRECTORY")
|
|
}
|
|
}
|
|
|
|
func chooseDirectoryDepth(params map[string]string) (int, error) {
|
|
raw := strings.TrimSpace(params["INCREMENTAL_DIRECTORY_DEPTH"])
|
|
if raw == "" {
|
|
return 1, nil
|
|
}
|
|
parsed, err := strconv.Atoi(raw)
|
|
if err != nil || parsed <= 0 {
|
|
return 0, fmt.Errorf("INCREMENTAL_DIRECTORY_DEPTH must be a positive integer")
|
|
}
|
|
return parsed, nil
|
|
}
|
|
|
|
func scanWorkflowSourceNode(node domain.WorkflowNode) bool {
|
|
label := strings.TrimSpace(node.Data.Label)
|
|
return node.ID == "scan-source" || label == "BOS 增量扫描"
|
|
}
|
|
|
|
func sourceFingerprint(node domain.WorkflowNode, params map[string]string, dataSource string, scanMode string, incrementalUnit string, directoryDepth int) string {
|
|
return domain.ComputeSourceFingerprint(domain.SourceFingerprintInput{
|
|
ScanMode: scanMode,
|
|
IncrementalUnit: incrementalUnit,
|
|
DirectoryDepth: directoryDepth,
|
|
DataSource: dataSource,
|
|
SourcePath: params["SOURCE_PATH"],
|
|
Bucket: node.Data.Bucket,
|
|
Prefix: node.Data.Prefix,
|
|
Pattern: node.Data.Pattern,
|
|
Endpoint: params["BOS_ENDPOINT"],
|
|
})
|
|
}
|
|
|
|
func stringList(value any) []string {
|
|
if value == nil {
|
|
return nil
|
|
}
|
|
if items, ok := value.([]string); ok {
|
|
result := make([]string, 0, len(items))
|
|
for _, item := range items {
|
|
trimmed := strings.TrimSpace(item)
|
|
if trimmed != "" {
|
|
result = append(result, trimmed)
|
|
}
|
|
}
|
|
return result
|
|
}
|
|
items, ok := value.([]any)
|
|
if ok {
|
|
result := make([]string, 0, len(items))
|
|
for _, item := range items {
|
|
trimmed := strings.TrimSpace(fmt.Sprint(item))
|
|
if trimmed != "" {
|
|
result = append(result, trimmed)
|
|
}
|
|
}
|
|
return result
|
|
}
|
|
reflected := reflect.ValueOf(value)
|
|
if reflected.Kind() != reflect.Slice && reflected.Kind() != reflect.Array {
|
|
return nil
|
|
}
|
|
result := make([]string, 0, reflected.Len())
|
|
for index := 0; index < reflected.Len(); index++ {
|
|
trimmed := strings.TrimSpace(fmt.Sprint(reflected.Index(index).Interface()))
|
|
if trimmed != "" {
|
|
result = append(result, trimmed)
|
|
}
|
|
}
|
|
return result
|
|
}
|
|
|
|
func (runtime *Runtime) prepareExecutionNode(ctx context.Context, workflow *domain.WorkflowRecord, node domain.WorkflowNode) (domain.WorkflowNode, error) {
|
|
effectiveWorkflow, err := runtime.resolveScanWorkflow(ctx, workflow)
|
|
if err != nil {
|
|
return node, err
|
|
}
|
|
copy, err := scanExecutionNode(effectiveWorkflow, node)
|
|
if err != nil {
|
|
return node, err
|
|
}
|
|
copy, err = runtime.resolveNodeStorageConnection(ctx, copy)
|
|
if err != nil {
|
|
return node, err
|
|
}
|
|
copy, err = runtime.resolveNodeMongoConnection(ctx, copy)
|
|
if err != nil {
|
|
return node, err
|
|
}
|
|
params := nodeHyperparamsMap(copy)
|
|
dataSource, err := chooseDataSource(params["DATA_SOURCE"], "BOS")
|
|
if err != nil {
|
|
return node, err
|
|
}
|
|
scanMode, err := chooseScanMode(copy, params)
|
|
if err != nil {
|
|
return node, err
|
|
}
|
|
incrementalUnit, err := chooseIncrementalUnit(params)
|
|
if err != nil {
|
|
return node, err
|
|
}
|
|
directoryDepth, err := chooseDirectoryDepth(params)
|
|
if err != nil {
|
|
return node, err
|
|
}
|
|
if scanMode != "INCREMENTAL" || workflow == nil {
|
|
return copy, nil
|
|
}
|
|
fingerprint := sourceFingerprint(copy, params, dataSource, scanMode, incrementalUnit, directoryDepth)
|
|
state, err := runtime.store.GetSourceState(ctx, workflow.ID, copy.ID, fingerprint)
|
|
if err != nil {
|
|
return copy, err
|
|
}
|
|
payload := map[string]any{
|
|
"sourceFingerprint": fingerprint,
|
|
"scanMode": scanMode,
|
|
"incrementalUnit": incrementalUnit,
|
|
"directoryDepth": directoryDepth,
|
|
"dataSource": dataSource,
|
|
"sourcePath": strings.TrimSpace(params["SOURCE_PATH"]),
|
|
"bucket": strings.TrimSpace(copy.Data.Bucket),
|
|
"prefix": strings.TrimSpace(copy.Data.Prefix),
|
|
"pattern": strings.TrimSpace(copy.Data.Pattern),
|
|
"seenFingerprints": []string{},
|
|
}
|
|
if state != nil {
|
|
payload["seenFingerprints"] = append([]string(nil), state.SeenFingerprints...)
|
|
payload["lastRunId"] = state.LastRunID
|
|
payload["lastListedCount"] = state.LastListedCount
|
|
payload["lastNewCount"] = state.LastNewCount
|
|
payload["lastScanCompletedAt"] = state.LastScanCompletedAt
|
|
}
|
|
encoded, err := json.Marshal(payload)
|
|
if err != nil {
|
|
return copy, err
|
|
}
|
|
copy.Data.Hyperparams = append([]domain.HyperParam(nil), copy.Data.Hyperparams...)
|
|
copy.Data.Hyperparams = upsertNodeHyperparam(copy.Data.Hyperparams, "SCAN_MODE", scanMode)
|
|
copy.Data.Hyperparams = upsertNodeHyperparam(copy.Data.Hyperparams, "INCREMENTAL_UNIT", incrementalUnit)
|
|
copy.Data.Hyperparams = upsertNodeHyperparam(copy.Data.Hyperparams, "INCREMENTAL_DIRECTORY_DEPTH", strconv.Itoa(directoryDepth))
|
|
copy.Data.Hyperparams = upsertNodeHyperparam(copy.Data.Hyperparams, "SOURCE_FINGERPRINT", fingerprint)
|
|
copy.Data.Hyperparams = upsertNodeHyperparam(copy.Data.Hyperparams, "SOURCE_STATE_JSON", string(encoded))
|
|
return copy, nil
|
|
}
|
|
|
|
func (runtime *Runtime) resolveScanWorkflow(ctx context.Context, workflow *domain.WorkflowRecord) (*domain.WorkflowRecord, error) {
|
|
if workflow == nil || workflow.ScanConfig == nil || strings.TrimSpace(workflow.ScanConfig.ConnectionID) == "" {
|
|
return workflow, nil
|
|
}
|
|
connection, err := runtime.store.GetStorageConnection(ctx, workflow.ScanConfig.ConnectionID)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if connection == nil {
|
|
return nil, fmt.Errorf("storage connection %s not found", workflow.ScanConfig.ConnectionID)
|
|
}
|
|
copy := *workflow
|
|
scanConfig := *workflow.ScanConfig
|
|
scanConfig.Endpoint = runtime.storageConnectionEndpoint(connection)
|
|
if strings.TrimSpace(scanConfig.AccessKeyID) == "" {
|
|
scanConfig.AccessKeyID = connection.AccessKeyID
|
|
}
|
|
if strings.TrimSpace(scanConfig.SecretAccessKey) == "" {
|
|
scanConfig.SecretAccessKey = connection.SecretAccessKey
|
|
}
|
|
if strings.TrimSpace(scanConfig.SourcePath) == "" {
|
|
dataSource, err := chooseDataSource(scanConfig.DataSource, "BOS")
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if dataSource == "BOS" {
|
|
scanConfig.SourcePath = bosSourcePath(scanConfig.Bucket, scanConfig.Prefix)
|
|
}
|
|
}
|
|
copy.ScanConfig = &scanConfig
|
|
return ©, nil
|
|
}
|
|
|
|
func (runtime *Runtime) resolveNodeStorageConnection(ctx context.Context, node domain.WorkflowNode) (domain.WorkflowNode, error) {
|
|
connectionID := strings.TrimSpace(node.Data.ConnectionID)
|
|
if connectionID == "" {
|
|
return node, nil
|
|
}
|
|
connection, err := runtime.store.GetStorageConnection(ctx, connectionID)
|
|
if err != nil {
|
|
return node, err
|
|
}
|
|
if connection == nil {
|
|
return node, fmt.Errorf("storage connection %s not found", connectionID)
|
|
}
|
|
copy := node
|
|
copy.Data.Hyperparams = append([]domain.HyperParam(nil), node.Data.Hyperparams...)
|
|
sourcePath := strings.TrimSpace(nodeHyperparamsMap(copy)["SOURCE_PATH"])
|
|
if sourcePath == "" {
|
|
sourcePath = bosSourcePath(copy.Data.Bucket, copy.Data.Prefix)
|
|
}
|
|
copy.Data.Hyperparams = upsertNodeHyperparam(copy.Data.Hyperparams, "DATA_SOURCE", "BOS")
|
|
copy.Data.Hyperparams = upsertNodeHyperparam(copy.Data.Hyperparams, "SOURCE_PATH", sourcePath)
|
|
copy.Data.Hyperparams = upsertNodeHyperparam(copy.Data.Hyperparams, "BOS_ENDPOINT", runtime.storageConnectionEndpoint(connection))
|
|
copy.Data.Hyperparams = upsertNodeHyperparam(copy.Data.Hyperparams, "BOS_ACCESS_KEY_ID", connection.AccessKeyID)
|
|
copy.Data.Hyperparams = upsertNodeHyperparam(copy.Data.Hyperparams, "BOS_SECRET_ACCESS_KEY", connection.SecretAccessKey)
|
|
return copy, nil
|
|
}
|
|
|
|
func (runtime *Runtime) resolveNodeMongoConnection(ctx context.Context, node domain.WorkflowNode) (domain.WorkflowNode, error) {
|
|
connectionID := strings.TrimSpace(nodeHyperparamsMap(node)["MONGO_CONNECTION_ID"])
|
|
if connectionID == "" {
|
|
return node, nil
|
|
}
|
|
connection, err := runtime.store.GetMongoConnection(ctx, connectionID)
|
|
if err != nil {
|
|
return node, err
|
|
}
|
|
if connection == nil {
|
|
return node, fmt.Errorf("mongo connection %s not found", connectionID)
|
|
}
|
|
copy := node
|
|
copy.Data.Hyperparams = append([]domain.HyperParam(nil), node.Data.Hyperparams...)
|
|
copy.Data.Hyperparams = upsertNodeHyperparam(copy.Data.Hyperparams, "MONGO_URI", strings.TrimSpace(connection.URI))
|
|
copy.Data.Hyperparams = upsertNodeHyperparam(copy.Data.Hyperparams, "MONGO_DB", strings.TrimSpace(connection.Database))
|
|
copy.Data.Hyperparams = upsertNodeHyperparam(copy.Data.Hyperparams, "MONGO_TARGET", "business")
|
|
return copy, nil
|
|
}
|
|
|
|
func checkpointArtifact(execution domain.NodeExecutionRecord) *domain.ExecutionArtifact {
|
|
for index := range execution.Artifacts {
|
|
if execution.Artifacts[index].Name == "source-checkpoint.json" {
|
|
return &execution.Artifacts[index]
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func sourceCheckpointPayload(artifact *domain.ExecutionArtifact) (map[string]any, error) {
|
|
if artifact == nil {
|
|
return nil, nil
|
|
}
|
|
path := strings.TrimSpace(artifact.Path)
|
|
if path != "" {
|
|
data, err := os.ReadFile(path)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("read source checkpoint %s: %w", path, err)
|
|
}
|
|
var payload map[string]any
|
|
if err := json.Unmarshal(data, &payload); err != nil {
|
|
return nil, fmt.Errorf("parse source checkpoint %s: %w", path, err)
|
|
}
|
|
if len(payload) == 0 {
|
|
return nil, fmt.Errorf("source checkpoint %s must be a non-empty JSON object", path)
|
|
}
|
|
return payload, nil
|
|
}
|
|
if artifact.Data == nil {
|
|
return nil, nil
|
|
}
|
|
return artifact.Data, nil
|
|
}
|
|
|
|
func sourceStateFromCheckpoint(workflow *domain.WorkflowRecord, run *domain.RunRecord, execution domain.NodeExecutionRecord, artifact *domain.ExecutionArtifact) (*domain.SourceStateRecord, error) {
|
|
payload, err := sourceCheckpointPayload(artifact)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if workflow == nil || run == nil || payload == nil {
|
|
return nil, nil
|
|
}
|
|
fingerprint := strings.TrimSpace(fmt.Sprint(payload["sourceFingerprint"]))
|
|
if fingerprint == "" {
|
|
return nil, fmt.Errorf("source checkpoint for node %s is missing sourceFingerprint", execution.NodeID)
|
|
}
|
|
directoryDepth, err := checkpointIntValue(payload, "directoryDepth")
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if directoryDepth <= 0 {
|
|
return nil, fmt.Errorf("source checkpoint directoryDepth must be greater than 0")
|
|
}
|
|
listedCount, err := checkpointIntValue(payload, "listedCount")
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if listedCount < 0 {
|
|
return nil, fmt.Errorf("source checkpoint listedCount must be zero or greater")
|
|
}
|
|
newCount, err := checkpointIntValue(payload, "newCount")
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if newCount < 0 {
|
|
return nil, fmt.Errorf("source checkpoint newCount must be zero or greater")
|
|
}
|
|
return &domain.SourceStateRecord{
|
|
ID: workflow.ID + ":" + execution.NodeID + ":" + fingerprint,
|
|
WorkflowID: workflow.ID,
|
|
NodeID: execution.NodeID,
|
|
SourceFingerprint: fingerprint,
|
|
DataSource: strings.TrimSpace(fmt.Sprint(payload["dataSource"])),
|
|
SourcePath: strings.TrimSpace(fmt.Sprint(payload["sourcePath"])),
|
|
Bucket: strings.TrimSpace(fmt.Sprint(payload["bucket"])),
|
|
Prefix: strings.TrimSpace(fmt.Sprint(payload["prefix"])),
|
|
Pattern: strings.TrimSpace(fmt.Sprint(payload["pattern"])),
|
|
ScanMode: strings.TrimSpace(fmt.Sprint(payload["scanMode"])),
|
|
IncrementalUnit: strings.TrimSpace(fmt.Sprint(payload["incrementalUnit"])),
|
|
DirectoryDepth: directoryDepth,
|
|
SeenFingerprints: stringList(payload["seenFingerprints"]),
|
|
LastRunID: run.ID,
|
|
LastListedCount: listedCount,
|
|
LastNewCount: newCount,
|
|
LastScanStartedAt: strings.TrimSpace(fmt.Sprint(payload["scanStartedAt"])),
|
|
LastScanCompletedAt: strings.TrimSpace(fmt.Sprint(payload["scanCompletedAt"])),
|
|
UpdatedAt: nowISO(),
|
|
}, nil
|
|
}
|
|
|
|
func checkpointIntValue(payload map[string]any, key string) (int, error) {
|
|
value, ok := payload[key]
|
|
if !ok || value == nil {
|
|
return 0, fmt.Errorf("source checkpoint %s is required", key)
|
|
}
|
|
switch typed := value.(type) {
|
|
case int:
|
|
return typed, nil
|
|
case int32:
|
|
return int(typed), nil
|
|
case int64:
|
|
return int(typed), nil
|
|
case float32:
|
|
value := float64(typed)
|
|
if math.IsNaN(value) || math.IsInf(value, 0) || math.Trunc(value) != value {
|
|
return 0, fmt.Errorf("source checkpoint %s must be an integer", key)
|
|
}
|
|
return int(value), nil
|
|
case float64:
|
|
if math.IsNaN(typed) || math.IsInf(typed, 0) || math.Trunc(typed) != typed {
|
|
return 0, fmt.Errorf("source checkpoint %s must be an integer", key)
|
|
}
|
|
return int(typed), nil
|
|
case json.Number:
|
|
if parsed, err := typed.Int64(); err == nil {
|
|
return int(parsed), nil
|
|
}
|
|
case string:
|
|
trimmed := strings.TrimSpace(typed)
|
|
if parsed, err := strconv.Atoi(trimmed); err == nil && strconv.Itoa(parsed) == trimmed {
|
|
return parsed, nil
|
|
}
|
|
}
|
|
return 0, fmt.Errorf("source checkpoint %s must be an integer", key)
|
|
}
|
|
|
|
func (runtime *Runtime) commitSourceStates(ctx context.Context, workflow *domain.WorkflowRecord, run *domain.RunRecord) error {
|
|
if workflow == nil || run == nil {
|
|
return nil
|
|
}
|
|
sourceRun := run
|
|
persistedRun, err := runtime.store.GetRun(ctx, run.ID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if persistedRun != nil {
|
|
sourceRun = persistedRun
|
|
}
|
|
for _, execution := range sourceRun.NodeExecutions {
|
|
artifact := checkpointArtifact(execution)
|
|
if artifact == nil {
|
|
continue
|
|
}
|
|
state, err := sourceStateFromCheckpoint(workflow, sourceRun, execution, artifact)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if state == nil {
|
|
continue
|
|
}
|
|
if err := runtime.store.UpsertSourceState(ctx, *state); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func scanExecutionNode(workflow *domain.WorkflowRecord, node domain.WorkflowNode) (domain.WorkflowNode, error) {
|
|
if workflow == nil || workflow.ScanConfig == nil {
|
|
return node, nil
|
|
}
|
|
if !scanWorkflowSourceNode(node) {
|
|
return node, nil
|
|
}
|
|
dataSource, err := chooseDataSource(workflow.ScanConfig.DataSource, "BOS")
|
|
if err != nil {
|
|
return node, err
|
|
}
|
|
sourcePath := strings.TrimSpace(workflow.ScanConfig.SourcePath)
|
|
if sourcePath == "" && dataSource == "BOS" {
|
|
sourcePath = bosSourcePath(workflow.ScanConfig.Bucket, workflow.ScanConfig.Prefix)
|
|
}
|
|
copy := node
|
|
copy.Data.Bucket = workflow.ScanConfig.Bucket
|
|
copy.Data.Prefix = workflow.ScanConfig.Prefix
|
|
copy.Data.Pattern = workflow.ScanConfig.Pattern
|
|
copy.Data.Hyperparams = append([]domain.HyperParam(nil), node.Data.Hyperparams...)
|
|
copy.Data.Hyperparams = upsertNodeHyperparam(copy.Data.Hyperparams, "DATA_SOURCE", dataSource)
|
|
copy.Data.Hyperparams = upsertNodeHyperparam(copy.Data.Hyperparams, "SOURCE_PATH", sourcePath)
|
|
copy.Data.Hyperparams = upsertNodeHyperparam(copy.Data.Hyperparams, "BOS_ENDPOINT", workflow.ScanConfig.Endpoint)
|
|
copy.Data.Hyperparams = upsertNodeHyperparam(
|
|
copy.Data.Hyperparams,
|
|
"SCAN_MODE",
|
|
chooseValue(strings.ToUpper(strings.TrimSpace(workflow.ScanConfig.ScanMode)), "INCREMENTAL"),
|
|
)
|
|
bosAccessKeyID := ""
|
|
bosSecretAccessKey := ""
|
|
if dataSource == "BOS" {
|
|
bosAccessKeyID = workflow.ScanConfig.AccessKeyID
|
|
bosSecretAccessKey = workflow.ScanConfig.SecretAccessKey
|
|
}
|
|
copy.Data.Hyperparams = upsertNodeHyperparam(copy.Data.Hyperparams, "BOS_ACCESS_KEY_ID", bosAccessKeyID)
|
|
copy.Data.Hyperparams = upsertNodeHyperparam(copy.Data.Hyperparams, "BOS_SECRET_ACCESS_KEY", bosSecretAccessKey)
|
|
return copy, nil
|
|
}
|
|
|
|
func workflowRunTimeout(workflow *domain.WorkflowRecord) time.Duration {
|
|
if workflow == nil || workflow.RunTimeoutSeconds <= 0 {
|
|
return 0
|
|
}
|
|
return time.Duration(workflow.RunTimeoutSeconds) * time.Second
|
|
}
|
|
|
|
func workflowNodeTimeout(workflow *domain.WorkflowRecord) time.Duration {
|
|
if workflow == nil || workflow.NodeTimeoutSeconds <= 0 {
|
|
return 0
|
|
}
|
|
return time.Duration(workflow.NodeTimeoutSeconds) * time.Second
|
|
}
|
|
|
|
func normalizeRunError(workflow *domain.WorkflowRecord, err error, executionCtx context.Context, nodeTimedOut bool) string {
|
|
if err == nil {
|
|
return ""
|
|
}
|
|
if workflow != nil && workflow.RunTimeoutSeconds > 0 && (errors.Is(err, context.DeadlineExceeded) || errors.Is(executionCtx.Err(), context.DeadlineExceeded)) {
|
|
return fmt.Sprintf("工作流执行超时(%d 秒)", workflow.RunTimeoutSeconds)
|
|
}
|
|
if workflow != nil && workflow.NodeTimeoutSeconds > 0 && nodeTimedOut {
|
|
return fmt.Sprintf("节点执行超时(%d 秒)", workflow.NodeTimeoutSeconds)
|
|
}
|
|
return err.Error()
|
|
}
|
|
|
|
func shouldAutoRetry(workflow *domain.WorkflowRecord, run *domain.RunRecord) bool {
|
|
if workflow == nil || run == nil || workflow.MaxRetryAttempts <= 0 {
|
|
return false
|
|
}
|
|
attempt := run.Attempt
|
|
if attempt <= 0 {
|
|
attempt = 1
|
|
}
|
|
return attempt <= workflow.MaxRetryAttempts
|
|
}
|
|
|
|
func nextRunAttempt(run *domain.RunRecord) int {
|
|
if run == nil || run.Attempt <= 0 {
|
|
return 1
|
|
}
|
|
return run.Attempt + 1
|
|
}
|
|
|
|
func runExecutionAttempt(run *domain.RunRecord) int {
|
|
if run == nil || run.Attempt <= 0 {
|
|
return 1
|
|
}
|
|
return run.Attempt
|
|
}
|
|
|
|
func nextRunAvailableAt(workflow *domain.WorkflowRecord) string {
|
|
if workflow == nil || workflow.RetryBackoffSeconds <= 0 {
|
|
return nowISO()
|
|
}
|
|
return time.Now().UTC().Add(time.Duration(workflow.RetryBackoffSeconds) * time.Second).Format(time.RFC3339Nano)
|
|
}
|
|
|
|
func (runtime *Runtime) enqueueRetryRun(ctx context.Context, workflow *domain.WorkflowRecord, failedRun *domain.RunRecord) (*domain.RunRecord, error) {
|
|
if workflow == nil || failedRun == nil {
|
|
return nil, nil
|
|
}
|
|
retryRun, err := runtime.store.CreateRun(ctx, workflow.ID, domain.RunTriggerRetry)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
retryRun.Attempt = nextRunAttempt(failedRun)
|
|
retryRun.ParentRunID = failedRun.ID
|
|
retryRun.AvailableAt = nextRunAvailableAt(workflow)
|
|
retryRun.UpdatedAt = nowISO()
|
|
if err := runtime.store.UpsertRun(ctx, *retryRun); err != nil {
|
|
return nil, err
|
|
}
|
|
return retryRun, nil
|
|
}
|
|
|
|
type nodeExecutionOutcome struct {
|
|
nodeID string
|
|
result executionResult
|
|
reference *k8s.JobReference
|
|
err error
|
|
nodeTimedOut bool
|
|
}
|
|
|
|
func runAvailableAt(run domain.RunRecord) time.Time {
|
|
if run.AvailableAt != "" {
|
|
if parsed, err := time.Parse(time.RFC3339Nano, run.AvailableAt); err == nil {
|
|
return parsed
|
|
}
|
|
}
|
|
if parsed, err := time.Parse(time.RFC3339Nano, run.CreatedAt); err == nil {
|
|
return parsed
|
|
}
|
|
return time.Time{}
|
|
}
|
|
|
|
func runDispatchReady(run domain.RunRecord, now time.Time) bool {
|
|
availableAt := runAvailableAt(run)
|
|
if availableAt.IsZero() {
|
|
return true
|
|
}
|
|
return !availableAt.After(now)
|
|
}
|
|
|
|
func syncCurrentNodeID(run *domain.RunRecord) {
|
|
run.CurrentNodeID = ""
|
|
for _, status := range []domain.NodeExecutionStatus{
|
|
domain.NodeExecutionStatusWaitingForAudit,
|
|
domain.NodeExecutionStatusRunning,
|
|
} {
|
|
for _, execution := range run.NodeExecutions {
|
|
if execution.Status == status {
|
|
run.CurrentNodeID = execution.NodeID
|
|
return
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func isTerminalRunStatus(status domain.RunStatus) bool {
|
|
return status == domain.RunStatusCompleted ||
|
|
status == domain.RunStatusFailed ||
|
|
status == domain.RunStatusCancelled
|
|
}
|
|
|
|
func (runtime *Runtime) loadRunControlState(ctx context.Context, runID string) (*domain.RunRecord, error) {
|
|
if strings.TrimSpace(runID) == "" {
|
|
return nil, nil
|
|
}
|
|
return runtime.store.GetRun(ctx, runID)
|
|
}
|
|
|
|
func syncStopMetadata(target *domain.RunRecord, source *domain.RunRecord) {
|
|
if target == nil || source == nil {
|
|
return
|
|
}
|
|
target.StopMode = source.StopMode
|
|
target.StopRequestedAt = source.StopRequestedAt
|
|
}
|
|
|
|
func finalizeCancelledExecution(execution domain.NodeExecutionRecord, message string) (domain.NodeExecutionRecord, bool) {
|
|
if execution.Status == domain.NodeExecutionStatusCompleted ||
|
|
execution.Status == domain.NodeExecutionStatusFailed ||
|
|
execution.Status == domain.NodeExecutionStatusCancelled {
|
|
return execution, false
|
|
}
|
|
execution.Status = domain.NodeExecutionStatusCancelled
|
|
execution.EndedAt = nowISO()
|
|
execution.Error = message
|
|
execution.Logs = append(execution.Logs, logEntry("WARN", message, execution.NodeLabel))
|
|
return execution, true
|
|
}
|
|
|
|
func (runtime *Runtime) finalizeCancelledRun(ctx context.Context, workflow *domain.WorkflowRecord, currentRun *domain.RunRecord, executionMap map[string]domain.NodeExecutionRecord, message string) (*domain.RunRecord, error) {
|
|
for nodeID, execution := range executionMap {
|
|
updated, changed := finalizeCancelledExecution(execution, message)
|
|
if changed {
|
|
executionMap[nodeID] = updated
|
|
applyExecution(currentRun, updated)
|
|
}
|
|
}
|
|
currentRun.Status = domain.RunStatusCancelled
|
|
currentRun.EndedAt = nowISO()
|
|
if strings.TrimSpace(currentRun.Error) == "" {
|
|
currentRun.Error = message
|
|
}
|
|
syncCurrentNodeID(currentRun)
|
|
currentRun.UpdatedAt = nowISO()
|
|
if err := runtime.store.UpsertRun(ctx, *currentRun); err != nil {
|
|
return nil, err
|
|
}
|
|
if workflow != nil {
|
|
workflow.Status = domain.WorkflowStatusFailed
|
|
workflow.LastRunID = currentRun.ID
|
|
workflow.UpdatedAt = nowISO()
|
|
if err := runtime.store.UpsertWorkflow(ctx, *workflow); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
return currentRun, nil
|
|
}
|
|
|
|
func allAllowedNodesCompleted(workflow *domain.WorkflowRecord, run *domain.RunRecord, executionMap map[string]domain.NodeExecutionRecord) bool {
|
|
if workflow == nil || run == nil {
|
|
return false
|
|
}
|
|
allowedNodes := rerunAllowedNodeSet(*workflow, run)
|
|
for _, node := range workflow.Nodes {
|
|
if !nodeAllowedForExecution(allowedNodes, node.ID) {
|
|
continue
|
|
}
|
|
execution, ok := executionMap[node.ID]
|
|
if !ok || execution.Status != domain.NodeExecutionStatusCompleted {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
|
|
func (runtime *Runtime) finalizeCompletedRun(ctx context.Context, workflow *domain.WorkflowRecord, currentRun *domain.RunRecord) (*domain.RunRecord, error) {
|
|
if err := runtime.commitSourceStates(ctx, workflow, currentRun); err != nil {
|
|
return nil, err
|
|
}
|
|
currentRun.Status = domain.RunStatusCompleted
|
|
currentRun.EndedAt = nowISO()
|
|
syncCurrentNodeID(currentRun)
|
|
currentRun.UpdatedAt = nowISO()
|
|
if err := runtime.store.UpsertRun(ctx, *currentRun); err != nil {
|
|
return nil, err
|
|
}
|
|
workflow.Status = domain.WorkflowStatusCompleted
|
|
workflow.LastRunID = currentRun.ID
|
|
workflow.UpdatedAt = nowISO()
|
|
if err := runtime.store.UpsertWorkflow(ctx, *workflow); err != nil {
|
|
return nil, err
|
|
}
|
|
return currentRun, nil
|
|
}
|
|
|
|
func dependenciesComplete(workflow domain.WorkflowRecord, executionMap map[string]domain.NodeExecutionRecord, nodeID string) bool {
|
|
for _, candidate := range domain.GetIncomingEdges(workflow.Edges, nodeID) {
|
|
dependency, ok := executionMap[candidate.Source]
|
|
if !ok || dependency.Status != domain.NodeExecutionStatusCompleted {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
|
|
func rerunAllowedNodeSet(workflow domain.WorkflowRecord, run *domain.RunRecord) map[string]struct{} {
|
|
if run == nil || run.Trigger != domain.RunTriggerRerunSubtree || strings.TrimSpace(run.RestartFromNodeID) == "" {
|
|
return nil
|
|
}
|
|
visited := map[string]struct{}{}
|
|
queue := []string{strings.TrimSpace(run.RestartFromNodeID)}
|
|
for len(queue) > 0 {
|
|
nodeID := queue[0]
|
|
queue = queue[1:]
|
|
if _, ok := visited[nodeID]; ok {
|
|
continue
|
|
}
|
|
visited[nodeID] = struct{}{}
|
|
for _, edge := range domain.GetOutgoingEdges(workflow.Edges, nodeID) {
|
|
target := strings.TrimSpace(edge.Target)
|
|
if target != "" {
|
|
queue = append(queue, target)
|
|
}
|
|
}
|
|
}
|
|
return visited
|
|
}
|
|
|
|
func nodeAllowedForExecution(allowed map[string]struct{}, nodeID string) bool {
|
|
if allowed == nil {
|
|
return true
|
|
}
|
|
_, ok := allowed[nodeID]
|
|
return ok
|
|
}
|
|
|
|
func firstActiveExecution(executionMap map[string]domain.NodeExecutionRecord) (string, domain.NodeExecutionStatus) {
|
|
for _, status := range []domain.NodeExecutionStatus{
|
|
domain.NodeExecutionStatusWaitingForAudit,
|
|
domain.NodeExecutionStatusRunning,
|
|
domain.NodeExecutionStatusQueued,
|
|
} {
|
|
for nodeID, execution := range executionMap {
|
|
if execution.Status == status {
|
|
return nodeID, status
|
|
}
|
|
}
|
|
}
|
|
return "", ""
|
|
}
|
|
|
|
func queueNodeIfReady(workflow domain.WorkflowRecord, executionMap map[string]domain.NodeExecutionRecord, readyQueue *[]string, readySet map[string]struct{}, allowed map[string]struct{}, nodeID string) {
|
|
if !nodeAllowedForExecution(allowed, nodeID) {
|
|
return
|
|
}
|
|
if _, queued := readySet[nodeID]; queued {
|
|
return
|
|
}
|
|
if execution, ok := executionMap[nodeID]; ok {
|
|
if execution.Status == domain.NodeExecutionStatusCompleted ||
|
|
execution.Status == domain.NodeExecutionStatusRunning ||
|
|
execution.Status == domain.NodeExecutionStatusWaitingForAudit {
|
|
return
|
|
}
|
|
}
|
|
if !dependenciesComplete(workflow, executionMap, nodeID) {
|
|
return
|
|
}
|
|
*readyQueue = append(*readyQueue, nodeID)
|
|
readySet[nodeID] = struct{}{}
|
|
}
|
|
|
|
func enqueueDependents(workflow domain.WorkflowRecord, executionMap map[string]domain.NodeExecutionRecord, readyQueue *[]string, readySet map[string]struct{}, allowed map[string]struct{}, nodeID string) {
|
|
for _, edge := range domain.GetOutgoingEdges(workflow.Edges, nodeID) {
|
|
queueNodeIfReady(workflow, executionMap, readyQueue, readySet, allowed, edge.Target)
|
|
}
|
|
}
|
|
|
|
func enqueueReadyNodes(workflow domain.WorkflowRecord, executionMap map[string]domain.NodeExecutionRecord, readyQueue *[]string, readySet map[string]struct{}, allowed map[string]struct{}) {
|
|
for _, node := range workflow.Nodes {
|
|
queueNodeIfReady(workflow, executionMap, readyQueue, readySet, allowed, node.ID)
|
|
}
|
|
}
|
|
|
|
func (runtime *Runtime) processRun(ctx context.Context, claimed *domain.RunRecord) (*domain.RunRecord, error) {
|
|
if _, ok, err := runtime.store.AcquireRunLease(ctx, claimed.ID, runtime.leaseOwner(), runtime.leaseTTL()); err != nil {
|
|
return nil, err
|
|
} else if !ok {
|
|
return claimed, nil
|
|
}
|
|
leaseCtx, stopLeaseRenewal := context.WithCancel(context.Background())
|
|
defer stopLeaseRenewal()
|
|
go runtime.renewRunLeaseUntil(leaseCtx, claimed.ID)
|
|
workflow, err := runtime.store.GetWorkflow(ctx, claimed.WorkflowID)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if workflow == nil {
|
|
claimed.Status = domain.RunStatusFailed
|
|
claimed.EndedAt = nowISO()
|
|
claimed.Error = fmt.Sprintf("workflow %s not found", claimed.WorkflowID)
|
|
claimed.UpdatedAt = nowISO()
|
|
return claimed, runtime.store.UpsertRun(ctx, *claimed)
|
|
}
|
|
if conflicts := domain.FindInputPortConflicts(workflow.Nodes, workflow.Edges); len(conflicts) > 0 {
|
|
currentRun := claimed
|
|
message := "工作流图非法:同一输入端口存在多条上游连线。请拆分输入,或为每个上传结果添加单独的上传算子。"
|
|
currentRun.Status = domain.RunStatusFailed
|
|
currentRun.EndedAt = nowISO()
|
|
currentRun.Error = message
|
|
currentRun.UpdatedAt = nowISO()
|
|
appendLog(currentRun, logEntry("ERROR", message, "system"))
|
|
return currentRun, runtime.store.UpsertRun(ctx, *currentRun)
|
|
}
|
|
if graphIssues := domain.ValidateSingleConnectedWorkflowGraph(workflow.Nodes, workflow.Edges); len(graphIssues) > 0 {
|
|
currentRun := claimed
|
|
message := "工作流图非法:存在孤立算子或多条互不连接的独立链路。请将所有算子通过 artifact/lineage 连成一个单连通工作流。"
|
|
currentRun.Status = domain.RunStatusFailed
|
|
currentRun.EndedAt = nowISO()
|
|
currentRun.Error = message
|
|
currentRun.UpdatedAt = nowISO()
|
|
appendLog(currentRun, logEntry("ERROR", message, "system"))
|
|
return currentRun, runtime.store.UpsertRun(ctx, *currentRun)
|
|
}
|
|
if message := runtime.workflowPublicationBlocker(ctx, workflow); message != "" {
|
|
currentRun := claimed
|
|
currentRun.Status = domain.RunStatusFailed
|
|
currentRun.EndedAt = nowISO()
|
|
currentRun.Error = message
|
|
currentRun.UpdatedAt = nowISO()
|
|
appendLog(currentRun, logEntry("ERROR", message, "system"))
|
|
return currentRun, runtime.store.UpsertRun(ctx, *currentRun)
|
|
}
|
|
|
|
currentRun, err := runtime.store.GetRun(ctx, claimed.ID)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if currentRun == nil {
|
|
currentRun = claimed
|
|
}
|
|
if isTerminalRunStatus(currentRun.Status) {
|
|
return currentRun, nil
|
|
}
|
|
settings, err := runtime.store.GetSettings(ctx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
nodeConcurrencyLimit := effectiveWorkflowConcurrencyLimit(workflow, settings)
|
|
if nodeConcurrencyLimit <= 0 {
|
|
nodeConcurrencyLimit = 1
|
|
}
|
|
|
|
executionMap := map[string]domain.NodeExecutionRecord{}
|
|
for _, execution := range currentRun.NodeExecutions {
|
|
executionMap[execution.NodeID] = execution
|
|
}
|
|
allowedNodes := rerunAllowedNodeSet(*workflow, currentRun)
|
|
readyQueue := make([]string, 0)
|
|
readySet := make(map[string]struct{})
|
|
auditQueue := make([]string, 0)
|
|
|
|
if currentRun.CurrentNodeID != "" {
|
|
tasks, err := runtime.store.ListAuditTasks(ctx, "all")
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
for _, task := range tasks {
|
|
if task.RunID == currentRun.ID && task.NodeID == currentRun.CurrentNodeID && task.Status != domain.AuditStatusPending {
|
|
node := domain.GetNodeByID(workflow.Nodes, currentRun.CurrentNodeID)
|
|
if node != nil {
|
|
execution := executionMap[node.ID]
|
|
execution.NodeID = node.ID
|
|
execution.NodeLabel = node.Data.Label
|
|
execution.Status = domain.NodeExecutionStatusCompleted
|
|
execution.EndedAt = nowISO()
|
|
execution.Logs = append(execution.Logs, logEntry("INFO", fmt.Sprintf("Audit decision: %s", task.Status), node.Data.Label))
|
|
executionMap[node.ID] = execution
|
|
applyExecution(currentRun, execution)
|
|
for _, target := range edgeTargets(domain.GetDecisionOutgoingEdges(workflow.Edges, node.ID, task.Status)) {
|
|
queueNodeIfReady(*workflow, executionMap, &readyQueue, readySet, allowedNodes, target)
|
|
}
|
|
currentRun.Status = domain.RunStatusRunning
|
|
syncCurrentNodeID(currentRun)
|
|
currentRun.UpdatedAt = nowISO()
|
|
if err := runtime.store.UpsertRun(ctx, *currentRun); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
break
|
|
}
|
|
}
|
|
}
|
|
|
|
if activeNodeID, activeStatus := firstActiveExecution(executionMap); activeNodeID != "" {
|
|
if strings.TrimSpace(currentRun.CurrentNodeID) == "" {
|
|
currentRun.CurrentNodeID = activeNodeID
|
|
}
|
|
switch activeStatus {
|
|
case domain.NodeExecutionStatusWaitingForAudit:
|
|
currentRun.Status = domain.RunStatusWaitingForAudit
|
|
default:
|
|
currentRun.Status = domain.RunStatusRunning
|
|
}
|
|
currentRun.UpdatedAt = nowISO()
|
|
syncCurrentNodeID(currentRun)
|
|
if err := runtime.store.UpsertRun(ctx, *currentRun); err != nil {
|
|
return nil, err
|
|
}
|
|
workflow.Status = domain.WorkflowStatusRunning
|
|
workflow.LastRunID = currentRun.ID
|
|
workflow.UpdatedAt = nowISO()
|
|
if err := runtime.store.UpsertWorkflow(ctx, *workflow); err != nil {
|
|
return nil, err
|
|
}
|
|
return currentRun, nil
|
|
}
|
|
|
|
if len(readyQueue) == 0 && len(executionMap) == 0 {
|
|
if allowedNodes != nil && strings.TrimSpace(currentRun.RestartFromNodeID) != "" {
|
|
queueNodeIfReady(*workflow, executionMap, &readyQueue, readySet, allowedNodes, currentRun.RestartFromNodeID)
|
|
} else {
|
|
for _, nodeID := range domain.GetStartNodeIDs(workflow.Nodes, workflow.Edges) {
|
|
queueNodeIfReady(*workflow, executionMap, &readyQueue, readySet, allowedNodes, nodeID)
|
|
}
|
|
}
|
|
}
|
|
if len(readyQueue) == 0 && len(executionMap) > 0 && currentRun.CurrentNodeID == "" {
|
|
for _, execution := range currentRun.NodeExecutions {
|
|
if execution.Status != domain.NodeExecutionStatusCompleted {
|
|
continue
|
|
}
|
|
enqueueDependents(*workflow, executionMap, &readyQueue, readySet, allowedNodes, execution.NodeID)
|
|
}
|
|
for _, node := range workflow.Nodes {
|
|
queueNodeIfReady(*workflow, executionMap, &readyQueue, readySet, allowedNodes, node.ID)
|
|
}
|
|
}
|
|
|
|
executionCtx := ctx
|
|
cancelExecution := func() {}
|
|
if timeout := workflowRunTimeout(workflow); timeout > 0 {
|
|
executionCtx, cancelExecution = context.WithTimeout(ctx, timeout)
|
|
} else {
|
|
executionCtx, cancelExecution = context.WithCancel(ctx)
|
|
}
|
|
defer cancelExecution()
|
|
results := make(chan nodeExecutionOutcome, nodeConcurrencyLimit)
|
|
runningNodes := make(map[string]struct{})
|
|
nodeStartedAt := make(map[string]time.Time)
|
|
var persistedControl *domain.RunRecord
|
|
gracefulStopRequested := false
|
|
forceStopRequested := false
|
|
for {
|
|
persistedControl, err = runtime.loadRunControlState(ctx, currentRun.ID)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
gracefulStopRequested = false
|
|
forceStopRequested = false
|
|
if persistedControl != nil {
|
|
syncStopMetadata(currentRun, persistedControl)
|
|
if persistedControl.Status == domain.RunStatusStopping && persistedControl.StopMode == domain.RunStopModeGraceful {
|
|
gracefulStopRequested = true
|
|
currentRun.Status = domain.RunStatusStopping
|
|
}
|
|
if persistedControl.Status == domain.RunStatusCancelled {
|
|
forceStopRequested = persistedControl.StopMode == domain.RunStopModeForce
|
|
currentRun.Status = domain.RunStatusCancelled
|
|
if currentRun.EndedAt == "" {
|
|
currentRun.EndedAt = persistedControl.EndedAt
|
|
}
|
|
if strings.TrimSpace(currentRun.Error) == "" {
|
|
currentRun.Error = persistedControl.Error
|
|
}
|
|
}
|
|
}
|
|
if forceStopRequested {
|
|
cancelExecution()
|
|
}
|
|
|
|
launched := false
|
|
for !gracefulStopRequested && !forceStopRequested && len(runningNodes) < nodeConcurrencyLimit && len(readyQueue) > 0 {
|
|
nodeID := readyQueue[0]
|
|
readyQueue = readyQueue[1:]
|
|
delete(readySet, nodeID)
|
|
|
|
node := domain.GetNodeByID(workflow.Nodes, nodeID)
|
|
if node == nil {
|
|
continue
|
|
}
|
|
if !nodeAllowedForExecution(allowedNodes, node.ID) {
|
|
continue
|
|
}
|
|
if existing, ok := executionMap[node.ID]; ok {
|
|
if existing.Status == domain.NodeExecutionStatusCompleted ||
|
|
existing.Status == domain.NodeExecutionStatusRunning ||
|
|
existing.Status == domain.NodeExecutionStatusWaitingForAudit {
|
|
continue
|
|
}
|
|
}
|
|
if !dependenciesComplete(*workflow, executionMap, node.ID) {
|
|
continue
|
|
}
|
|
if domain.NodeRequiresAudit(*node) {
|
|
auditQueue = append(auditQueue, node.ID)
|
|
continue
|
|
}
|
|
|
|
inputLineage := gatherInputLineage(*workflow, executionMap, node.ID)
|
|
inputArtifacts := inputArtifactsFromLineage(inputLineage)
|
|
execution := domain.NodeExecutionRecord{
|
|
NodeID: node.ID,
|
|
NodeLabel: node.Data.Label,
|
|
Attempt: runExecutionAttempt(currentRun),
|
|
Status: domain.NodeExecutionStatusRunning,
|
|
StartedAt: nowISO(),
|
|
Logs: []domain.LogEntryRecord{logEntry("INFO", "Starting node execution", node.Data.Label)},
|
|
Artifacts: []domain.ExecutionArtifact{},
|
|
DataRefs: []domain.DataRefRecord{},
|
|
ResourceSeries: []domain.ResourceSample{},
|
|
StageManifestPath: "",
|
|
}
|
|
applyNodeExecutionSnapshot(&execution, *node, inputArtifacts, inputLineage)
|
|
executionMap[node.ID] = execution
|
|
applyExecution(currentRun, execution)
|
|
runningNodes[node.ID] = struct{}{}
|
|
nodeStartedAt[node.ID] = time.Now()
|
|
runtime.metrics.RecordNodeExecutionStarted()
|
|
launched = true
|
|
|
|
runSnapshot := *currentRun
|
|
nodeSnapshot, err := runtime.prepareExecutionNode(ctx, workflow, *node)
|
|
if err != nil {
|
|
results <- nodeExecutionOutcome{
|
|
nodeID: node.ID,
|
|
err: err,
|
|
}
|
|
continue
|
|
}
|
|
inputSnapshot := cloneArtifacts(inputArtifacts)
|
|
nodeExecutionCtx := executionCtx
|
|
cancelNodeExecution := func() {}
|
|
if timeout := workflowNodeTimeout(workflow); timeout > 0 {
|
|
nodeExecutionCtx, cancelNodeExecution = context.WithTimeout(executionCtx, timeout)
|
|
}
|
|
go func() {
|
|
defer cancelNodeExecution()
|
|
result, reference, err := runtime.executeNode(nodeExecutionCtx, &runSnapshot, nodeSnapshot, inputSnapshot)
|
|
results <- nodeExecutionOutcome{
|
|
nodeID: nodeSnapshot.ID,
|
|
result: result,
|
|
reference: reference,
|
|
err: err,
|
|
nodeTimedOut: errors.Is(nodeExecutionCtx.Err(), context.DeadlineExceeded),
|
|
}
|
|
}()
|
|
}
|
|
|
|
if launched {
|
|
currentRun.Status = domain.RunStatusRunning
|
|
syncCurrentNodeID(currentRun)
|
|
currentRun.UpdatedAt = nowISO()
|
|
if err := runtime.store.UpsertRun(ctx, *currentRun); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
|
|
if len(runningNodes) == 0 {
|
|
if forceStopRequested {
|
|
return runtime.finalizeCancelledRun(ctx, workflow, currentRun, executionMap, "用户已强制停止工作流")
|
|
}
|
|
if gracefulStopRequested || currentRun.Status == domain.RunStatusStopping {
|
|
return runtime.finalizeCancelledRun(ctx, workflow, currentRun, executionMap, "用户已停止工作流")
|
|
}
|
|
if len(readyQueue) == 0 && len(auditQueue) == 0 {
|
|
enqueueReadyNodes(*workflow, executionMap, &readyQueue, readySet, allowedNodes)
|
|
}
|
|
if len(readyQueue) > 0 {
|
|
continue
|
|
}
|
|
if len(auditQueue) > 0 {
|
|
nodeID := auditQueue[0]
|
|
auditQueue = auditQueue[1:]
|
|
node := domain.GetNodeByID(workflow.Nodes, nodeID)
|
|
if node == nil {
|
|
continue
|
|
}
|
|
inputLineage := gatherInputLineage(*workflow, executionMap, node.ID)
|
|
inputArtifacts := inputArtifactsFromLineage(inputLineage)
|
|
execution := executionMap[node.ID]
|
|
if execution.NodeID == "" {
|
|
execution = domain.NodeExecutionRecord{
|
|
NodeID: node.ID,
|
|
NodeLabel: node.Data.Label,
|
|
Attempt: runExecutionAttempt(currentRun),
|
|
Status: domain.NodeExecutionStatusRunning,
|
|
StartedAt: nowISO(),
|
|
Logs: []domain.LogEntryRecord{logEntry("INFO", "Starting node execution", node.Data.Label)},
|
|
Artifacts: []domain.ExecutionArtifact{},
|
|
DataRefs: []domain.DataRefRecord{},
|
|
ResourceSeries: []domain.ResourceSample{},
|
|
}
|
|
}
|
|
applyNodeExecutionSnapshot(&execution, *node, inputArtifacts, inputLineage)
|
|
task := domain.AuditTaskRecord{
|
|
ID: domain.NewID("audit"),
|
|
OrganizationID: currentRun.OrganizationID,
|
|
RunID: currentRun.ID,
|
|
WorkflowID: workflow.ID,
|
|
ProjectID: workflow.ProjectID,
|
|
NodeID: node.ID,
|
|
WorkflowName: workflow.Name,
|
|
ProjectName: currentRun.ProjectName,
|
|
OperatorName: node.Data.Label,
|
|
CreatedAt: nowISO(),
|
|
UpdatedAt: nowISO(),
|
|
Status: domain.AuditStatusPending,
|
|
Data: createAuditPayload(*node, inputArtifacts),
|
|
}
|
|
if _, err := runtime.store.CreateAuditTask(ctx, task); err != nil {
|
|
return nil, err
|
|
}
|
|
execution.Status = domain.NodeExecutionStatusWaitingForAudit
|
|
execution.EndedAt = nowISO()
|
|
execution.Artifacts = cloneArtifacts(inputArtifacts)
|
|
execution.InputArtifacts = cloneArtifacts(inputArtifacts)
|
|
execution.InputLineage = cloneInputLineage(inputLineage)
|
|
execution.Logs = append(execution.Logs, logEntry("WARN", "Node is waiting for human review", node.Data.Label))
|
|
executionMap[node.ID] = execution
|
|
applyExecution(currentRun, execution)
|
|
currentRun.Status = domain.RunStatusWaitingForAudit
|
|
syncCurrentNodeID(currentRun)
|
|
currentRun.UpdatedAt = nowISO()
|
|
if err := runtime.store.UpsertRun(ctx, *currentRun); err != nil {
|
|
return nil, err
|
|
}
|
|
workflow.Status = domain.WorkflowStatusRunning
|
|
workflow.UpdatedAt = nowISO()
|
|
if err := runtime.store.UpsertWorkflow(ctx, *workflow); err != nil {
|
|
return nil, err
|
|
}
|
|
return currentRun, nil
|
|
}
|
|
break
|
|
}
|
|
|
|
outcome := <-results
|
|
delete(runningNodes, outcome.nodeID)
|
|
startedAt := nodeStartedAt[outcome.nodeID]
|
|
delete(nodeStartedAt, outcome.nodeID)
|
|
nodeDurationMS := uint64(0)
|
|
if !startedAt.IsZero() {
|
|
nodeDurationMS = uint64(time.Since(startedAt).Milliseconds())
|
|
}
|
|
persistedControl, err := runtime.loadRunControlState(ctx, currentRun.ID)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
gracefulStopRequested := false
|
|
forceStopRequested := false
|
|
if persistedControl != nil {
|
|
syncStopMetadata(currentRun, persistedControl)
|
|
if persistedControl.Status == domain.RunStatusStopping && persistedControl.StopMode == domain.RunStopModeGraceful {
|
|
gracefulStopRequested = true
|
|
}
|
|
if persistedControl.Status == domain.RunStatusCancelled && persistedControl.StopMode == domain.RunStopModeForce {
|
|
forceStopRequested = true
|
|
}
|
|
}
|
|
execution := executionMap[outcome.nodeID]
|
|
if outcome.err != nil {
|
|
if forceStopRequested || (currentRun.StopMode == domain.RunStopModeForce && errors.Is(executionCtx.Err(), context.Canceled)) {
|
|
cancelledExecution, _ := finalizeCancelledExecution(execution, "用户已强制停止工作流")
|
|
executionMap[outcome.nodeID] = cancelledExecution
|
|
applyExecution(currentRun, cancelledExecution)
|
|
currentRun.Status = domain.RunStatusCancelled
|
|
currentRun.UpdatedAt = nowISO()
|
|
if err := runtime.store.UpsertRun(ctx, *currentRun); err != nil {
|
|
return nil, err
|
|
}
|
|
if len(runningNodes) == 0 {
|
|
return runtime.finalizeCancelledRun(ctx, workflow, currentRun, executionMap, "用户已强制停止工作流")
|
|
}
|
|
continue
|
|
}
|
|
runtime.metrics.RecordNodeExecutionFailed(nodeDurationMS)
|
|
cancelExecution()
|
|
execution.Status = domain.NodeExecutionStatusFailed
|
|
execution.EndedAt = nowISO()
|
|
execution.Error = normalizeRunError(workflow, outcome.err, executionCtx, outcome.nodeTimedOut)
|
|
execution.Logs = append(execution.Logs, logEntry("ERROR", execution.Error, execution.NodeLabel))
|
|
executionMap[outcome.nodeID] = execution
|
|
applyExecution(currentRun, execution)
|
|
currentRun.Status = domain.RunStatusFailed
|
|
currentRun.CurrentNodeID = outcome.nodeID
|
|
currentRun.EndedAt = nowISO()
|
|
currentRun.Error = execution.Error
|
|
currentRun.UpdatedAt = nowISO()
|
|
if err := runtime.store.UpsertRun(ctx, *currentRun); err != nil {
|
|
return nil, err
|
|
}
|
|
workflow.Status = domain.WorkflowStatusFailed
|
|
workflow.UpdatedAt = nowISO()
|
|
if err := runtime.store.UpsertWorkflow(ctx, *workflow); err != nil {
|
|
return nil, err
|
|
}
|
|
if shouldAutoRetry(workflow, currentRun) {
|
|
retryRun, err := runtime.enqueueRetryRun(ctx, workflow, currentRun)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if retryRun != nil {
|
|
appendLog(currentRun, logEntry("WARN", fmt.Sprintf("已自动创建重试运行,第 %d 次尝试已进入队列", retryRun.Attempt), execution.NodeLabel))
|
|
currentRun.UpdatedAt = nowISO()
|
|
if err := runtime.store.UpsertRun(ctx, *currentRun); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
}
|
|
return currentRun, nil
|
|
}
|
|
|
|
runtime.metrics.RecordNodeExecutionCompleted(nodeDurationMS)
|
|
execution.Status = domain.NodeExecutionStatusCompleted
|
|
execution.EndedAt = nowISO()
|
|
execution.Logs = append(execution.Logs, outcome.result.Logs...)
|
|
execution.Artifacts = outcome.result.Artifacts
|
|
execution.DataRefs = outcome.result.DataRefs
|
|
execution.StageManifestPath = outcome.result.StageManifestPath
|
|
execution.ResourceSeries = outcome.result.ResourceSeries
|
|
executionMap[outcome.nodeID] = execution
|
|
applyExecution(currentRun, execution)
|
|
if outcome.reference != nil {
|
|
currentRun.K8sRefs = append(currentRun.K8sRefs, domain.K8sRef{NodeID: outcome.nodeID, Attempt: runExecutionAttempt(currentRun), JobName: outcome.reference.JobName, Namespace: outcome.reference.Namespace})
|
|
}
|
|
if !gracefulStopRequested && !forceStopRequested {
|
|
enqueueDependents(*workflow, executionMap, &readyQueue, readySet, allowedNodes, outcome.nodeID)
|
|
}
|
|
if gracefulStopRequested {
|
|
currentRun.Status = domain.RunStatusStopping
|
|
} else if forceStopRequested {
|
|
currentRun.Status = domain.RunStatusCancelled
|
|
} else {
|
|
currentRun.Status = domain.RunStatusRunning
|
|
}
|
|
syncCurrentNodeID(currentRun)
|
|
currentRun.UpdatedAt = nowISO()
|
|
if err := runtime.store.UpsertRun(ctx, *currentRun); err != nil {
|
|
return nil, err
|
|
}
|
|
if forceStopRequested && len(runningNodes) == 0 {
|
|
return runtime.finalizeCancelledRun(ctx, workflow, currentRun, executionMap, "用户已强制停止工作流")
|
|
}
|
|
}
|
|
|
|
if currentRun.Status == domain.RunStatusStopping || currentRun.Status == domain.RunStatusCancelled {
|
|
message := "用户已停止工作流"
|
|
if currentRun.StopMode == domain.RunStopModeForce {
|
|
message = "用户已强制停止工作流"
|
|
}
|
|
return runtime.finalizeCancelledRun(ctx, workflow, currentRun, executionMap, message)
|
|
}
|
|
return runtime.finalizeCompletedRun(ctx, workflow, currentRun)
|
|
}
|
|
|
|
func edgeTargets(edges []domain.WorkflowEdge) []string {
|
|
items := make([]string, 0, len(edges))
|
|
for _, edge := range edges {
|
|
items = append(items, edge.Target)
|
|
}
|
|
return items
|
|
}
|
|
|
|
func (runtime *Runtime) ProcessOnce(ctx context.Context) (*ProcessOnceResult, error) {
|
|
if err := runtime.reconcileActiveRuns(ctx); err != nil {
|
|
return nil, err
|
|
}
|
|
if err := runtime.enqueueStreamingWorkflows(ctx); err != nil {
|
|
return nil, err
|
|
}
|
|
claimed, err := runtime.store.ClaimNextQueuedRun(ctx, runtime.leaseOwner(), runtime.leaseTTL())
|
|
if err != nil || claimed == nil {
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return newProcessOnceResult(ProcessOnceStatusNoQueuedRun, nil), nil
|
|
}
|
|
if !runtime.tryMarkRunActive(claimed.ID) {
|
|
claimed.Status = domain.RunStatusQueued
|
|
claimed.StartedAt = ""
|
|
claimed.UpdatedAt = nowISO()
|
|
if updateErr := runtime.store.UpsertRun(ctx, *claimed); updateErr != nil {
|
|
return nil, updateErr
|
|
}
|
|
return newProcessOnceResult(ProcessOnceStatusSkippedLeaseActive, claimed), nil
|
|
}
|
|
defer runtime.unmarkRunActive(claimed.ID)
|
|
defer func() {
|
|
if err := runtime.store.ReleaseRunLease(context.Background(), claimed.ID, runtime.leaseOwner()); err != nil {
|
|
fmt.Printf("[worker] release run lease %s failed: %v\n", claimed.ID, err)
|
|
}
|
|
}()
|
|
run, err := runtime.processRun(ctx, claimed)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return newProcessOnceResult(ProcessOnceStatusClaimed, run), nil
|
|
}
|