725 lines
22 KiB
Go

package worker
import (
"context"
"errors"
"fmt"
"sort"
"time"
"droboticflow/internal/domain"
"droboticflow/internal/k8s"
"droboticflow/internal/store"
)
type Runtime struct {
store store.Store
mode string
namespace string
imagePullSecretName string
operatorRunnerImage string
localRunnerCommand string
launcher *k8s.Launcher
}
var activeDispatchStatuses = map[domain.RunStatus]struct{}{
domain.RunStatusRunning: {},
domain.RunStatusWaitingForAudit: {},
}
var queuedOrActiveStatuses = map[domain.RunStatus]struct{}{
domain.RunStatusQueued: {},
domain.RunStatusRunning: {},
domain.RunStatusWaitingForAudit: {},
}
func NewRuntime(dataStore store.Store, mode string, namespace string, imagePullSecretName string, operatorRunnerImage string, localRunnerCommand string) (*Runtime, error) {
launcher, err := k8s.NewLauncher(mode == "k8s", namespace, imagePullSecretName)
if err != nil {
return nil, err
}
return &Runtime{
store: dataStore,
mode: mode,
namespace: namespace,
imagePullSecretName: imagePullSecretName,
operatorRunnerImage: operatorRunnerImage,
localRunnerCommand: localRunnerCommand,
launcher: launcher,
}, nil
}
func nowISO() string {
return time.Now().UTC().Format(time.RFC3339Nano)
}
func logEntry(level string, message string, operator string) domain.LogEntryRecord {
return domain.LogEntryRecord{
Timestamp: nowISO(),
Level: level,
Message: message,
Operator: operator,
}
}
func findNodeExecution(run *domain.RunRecord, nodeID string) (int, *domain.NodeExecutionRecord) {
for index := range run.NodeExecutions {
if run.NodeExecutions[index].NodeID == nodeID {
return index, &run.NodeExecutions[index]
}
}
return -1, nil
}
func appendLog(run *domain.RunRecord, entry domain.LogEntryRecord) {
run.Logs = append(run.Logs, entry)
}
func applyExecution(run *domain.RunRecord, execution domain.NodeExecutionRecord) {
index, _ := findNodeExecution(run, execution.NodeID)
if index >= 0 {
run.NodeExecutions[index] = execution
} else {
run.NodeExecutions = append(run.NodeExecutions, execution)
}
run.Logs = []domain.LogEntryRecord{}
run.ResourceSeries = []domain.ResourceSample{}
run.Artifacts = []domain.ExecutionArtifact{}
for _, item := range run.NodeExecutions {
for _, entry := range item.Logs {
appendLog(run, entry)
}
if len(item.ResourceSeries) > 0 {
run.ResourceSeries = append(run.ResourceSeries, item.ResourceSeries...)
}
if len(item.Artifacts) > 0 {
run.Artifacts = append(run.Artifacts, item.Artifacts...)
}
}
}
func globalConcurrencyLimit(settings *domain.SettingsRecord) int {
if settings != nil && settings.Worker.MaxConcurrentRuns > 0 {
return settings.Worker.MaxConcurrentRuns
}
return 1
}
func effectiveWorkflowConcurrencyLimit(workflow *domain.WorkflowRecord, settings *domain.SettingsRecord) int {
if workflow != nil && workflow.MaxConcurrentRuns > 0 {
return workflow.MaxConcurrentRuns
}
return globalConcurrencyLimit(settings)
}
func selectQueuedRunsForDispatch(runs []domain.RunRecord, workflows []domain.WorkflowRecord, settings *domain.SettingsRecord) []string {
globalLimit := globalConcurrencyLimit(settings)
workflowByID := make(map[string]domain.WorkflowRecord, len(workflows))
for _, workflow := range workflows {
workflowByID[workflow.ID] = workflow
}
now := time.Now().UTC()
activeByWorkflow := map[string]int{}
activeTotal := 0
queued := make([]domain.RunRecord, 0)
for _, run := range runs {
if _, ok := activeDispatchStatuses[run.Status]; ok {
activeTotal++
activeByWorkflow[run.WorkflowID]++
continue
}
if run.Status == domain.RunStatusQueued {
if !runDispatchReady(run, now) {
continue
}
queued = append(queued, run)
}
}
available := globalLimit - activeTotal
if available <= 0 {
return nil
}
sort.Slice(queued, func(left int, right int) bool {
leftAvailableAt := runAvailableAt(queued[left])
rightAvailableAt := runAvailableAt(queued[right])
if leftAvailableAt.Equal(rightAvailableAt) {
return queued[left].CreatedAt < queued[right].CreatedAt
}
return leftAvailableAt.Before(rightAvailableAt)
})
selected := make([]string, 0, available)
for _, run := range queued {
if available == 0 {
break
}
workflow := workflowByID[run.WorkflowID]
workflowLimit := effectiveWorkflowConcurrencyLimit(&workflow, settings)
if activeByWorkflow[run.WorkflowID] >= workflowLimit {
continue
}
selected = append(selected, run.ID)
activeByWorkflow[run.WorkflowID]++
available--
}
return selected
}
func (runtime *Runtime) enqueueStreamingWorkflows(ctx context.Context) error {
workflows, err := runtime.store.ListAllWorkflows(ctx)
if err != nil {
return err
}
runs, err := runtime.store.ListRuns(ctx, "")
if err != nil {
return err
}
settings, err := runtime.store.GetSettings(ctx)
if err != nil {
return err
}
now := time.Now().UTC()
for _, workflow := range workflows {
if workflow.Type != domain.WorkflowTypeStreaming || workflow.Schedule == nil || workflow.Schedule.Interval <= 0 {
continue
}
inflightCount := 0
for _, run := range runs {
if run.WorkflowID == workflow.ID {
if _, ok := queuedOrActiveStatuses[run.Status]; ok {
inflightCount++
}
}
}
if inflightCount >= effectiveWorkflowConcurrencyLimit(&workflow, settings) {
continue
}
lastRunAt := time.Time{}
if workflow.LastRunAt != "" {
parsed, err := time.Parse(time.RFC3339Nano, workflow.LastRunAt)
if err == nil {
lastRunAt = parsed
}
}
if workflow.LastRunAt == "" || now.Sub(lastRunAt) >= time.Duration(workflow.Schedule.Interval)*time.Second {
if _, err := runtime.store.CreateRun(ctx, workflow.ID, domain.RunTriggerScheduled); err != nil {
return err
}
}
}
return nil
}
func (runtime *Runtime) processClaimedRun(claimed *domain.RunRecord) {
if _, err := runtime.processRun(context.Background(), claimed); err != nil {
fmt.Printf("[worker] process run %s failed: %v\n", claimed.ID, err)
}
}
func (runtime *Runtime) ProcessAvailable(ctx context.Context) error {
if err := runtime.enqueueStreamingWorkflows(ctx); err != nil {
return err
}
settings, err := runtime.store.GetSettings(ctx)
if err != nil {
return err
}
workflows, err := runtime.store.ListAllWorkflows(ctx)
if err != nil {
return err
}
runs, err := runtime.store.ListRuns(ctx, "")
if err != nil {
return err
}
for _, runID := range selectQueuedRunsForDispatch(runs, workflows, settings) {
claimed, err := runtime.store.ClaimQueuedRun(ctx, runID)
if err != nil {
return err
}
if claimed == nil {
continue
}
go runtime.processClaimedRun(claimed)
}
return nil
}
func chooseImage(node domain.WorkflowNode, fallback string) string {
if node.Data.Image != "" {
return node.Data.Image
}
return fallback
}
func chooseValue(value string, fallback string) string {
if value == "" {
return fallback
}
return value
}
func workflowRunTimeout(workflow *domain.WorkflowRecord) time.Duration {
if workflow == nil || workflow.RunTimeoutSeconds <= 0 {
return 0
}
return time.Duration(workflow.RunTimeoutSeconds) * time.Second
}
func workflowNodeTimeout(workflow *domain.WorkflowRecord) time.Duration {
if workflow == nil || workflow.NodeTimeoutSeconds <= 0 {
return 0
}
return time.Duration(workflow.NodeTimeoutSeconds) * time.Second
}
func normalizeRunError(workflow *domain.WorkflowRecord, err error, executionCtx context.Context, nodeTimedOut bool) string {
if err == nil {
return ""
}
if workflow != nil && workflow.RunTimeoutSeconds > 0 && (errors.Is(err, context.DeadlineExceeded) || errors.Is(executionCtx.Err(), context.DeadlineExceeded)) {
return fmt.Sprintf("工作流执行超时(%d 秒)", workflow.RunTimeoutSeconds)
}
if workflow != nil && workflow.NodeTimeoutSeconds > 0 && nodeTimedOut {
return fmt.Sprintf("节点执行超时(%d 秒)", workflow.NodeTimeoutSeconds)
}
return err.Error()
}
func shouldAutoRetry(workflow *domain.WorkflowRecord, run *domain.RunRecord) bool {
if workflow == nil || run == nil || workflow.MaxRetryAttempts <= 0 {
return false
}
attempt := run.Attempt
if attempt <= 0 {
attempt = 1
}
return attempt <= workflow.MaxRetryAttempts
}
func nextRunAttempt(run *domain.RunRecord) int {
if run == nil || run.Attempt <= 0 {
return 1
}
return run.Attempt + 1
}
func nextRunAvailableAt(workflow *domain.WorkflowRecord) string {
if workflow == nil || workflow.RetryBackoffSeconds <= 0 {
return nowISO()
}
return time.Now().UTC().Add(time.Duration(workflow.RetryBackoffSeconds) * time.Second).Format(time.RFC3339Nano)
}
func (runtime *Runtime) enqueueRetryRun(ctx context.Context, workflow *domain.WorkflowRecord, failedRun *domain.RunRecord) (*domain.RunRecord, error) {
if workflow == nil || failedRun == nil {
return nil, nil
}
retryRun, err := runtime.store.CreateRun(ctx, workflow.ID, domain.RunTriggerRetry)
if err != nil {
return nil, err
}
retryRun.Attempt = nextRunAttempt(failedRun)
retryRun.ParentRunID = failedRun.ID
retryRun.AvailableAt = nextRunAvailableAt(workflow)
retryRun.UpdatedAt = nowISO()
if err := runtime.store.UpsertRun(ctx, *retryRun); err != nil {
return nil, err
}
return retryRun, nil
}
type nodeExecutionOutcome struct {
nodeID string
result executionResult
reference *k8s.JobReference
err error
nodeTimedOut bool
}
func runAvailableAt(run domain.RunRecord) time.Time {
if run.AvailableAt != "" {
if parsed, err := time.Parse(time.RFC3339Nano, run.AvailableAt); err == nil {
return parsed
}
}
if parsed, err := time.Parse(time.RFC3339Nano, run.CreatedAt); err == nil {
return parsed
}
return time.Time{}
}
func runDispatchReady(run domain.RunRecord, now time.Time) bool {
availableAt := runAvailableAt(run)
if availableAt.IsZero() {
return true
}
return !availableAt.After(now)
}
func syncCurrentNodeID(run *domain.RunRecord) {
run.CurrentNodeID = ""
for _, status := range []domain.NodeExecutionStatus{
domain.NodeExecutionStatusWaitingForAudit,
domain.NodeExecutionStatusRunning,
} {
for _, execution := range run.NodeExecutions {
if execution.Status == status {
run.CurrentNodeID = execution.NodeID
return
}
}
}
}
func dependenciesComplete(workflow domain.WorkflowRecord, executionMap map[string]domain.NodeExecutionRecord, nodeID string) bool {
for _, candidate := range domain.GetIncomingEdges(workflow.Edges, nodeID) {
dependency, ok := executionMap[candidate.Source]
if !ok || dependency.Status != domain.NodeExecutionStatusCompleted {
return false
}
}
return true
}
func queueNodeIfReady(workflow domain.WorkflowRecord, executionMap map[string]domain.NodeExecutionRecord, readyQueue *[]string, readySet map[string]struct{}, nodeID string) {
if _, queued := readySet[nodeID]; queued {
return
}
if execution, ok := executionMap[nodeID]; ok {
if execution.Status == domain.NodeExecutionStatusCompleted ||
execution.Status == domain.NodeExecutionStatusRunning ||
execution.Status == domain.NodeExecutionStatusWaitingForAudit {
return
}
}
if !dependenciesComplete(workflow, executionMap, nodeID) {
return
}
*readyQueue = append(*readyQueue, nodeID)
readySet[nodeID] = struct{}{}
}
func enqueueDependents(workflow domain.WorkflowRecord, executionMap map[string]domain.NodeExecutionRecord, readyQueue *[]string, readySet map[string]struct{}, nodeID string) {
for _, edge := range domain.GetOutgoingEdges(workflow.Edges, nodeID) {
queueNodeIfReady(workflow, executionMap, readyQueue, readySet, edge.Target)
}
}
func (runtime *Runtime) processRun(ctx context.Context, claimed *domain.RunRecord) (*domain.RunRecord, error) {
workflow, err := runtime.store.GetWorkflow(ctx, claimed.WorkflowID)
if err != nil {
return nil, err
}
if workflow == nil {
claimed.Status = domain.RunStatusFailed
claimed.EndedAt = nowISO()
claimed.Error = fmt.Sprintf("workflow %s not found", claimed.WorkflowID)
claimed.UpdatedAt = nowISO()
return claimed, runtime.store.UpsertRun(ctx, *claimed)
}
currentRun, err := runtime.store.GetRun(ctx, claimed.ID)
if err != nil {
return nil, err
}
if currentRun == nil {
currentRun = claimed
}
settings, err := runtime.store.GetSettings(ctx)
if err != nil {
return nil, err
}
nodeConcurrencyLimit := effectiveWorkflowConcurrencyLimit(workflow, settings)
if nodeConcurrencyLimit <= 0 {
nodeConcurrencyLimit = 1
}
executionMap := map[string]domain.NodeExecutionRecord{}
for _, execution := range currentRun.NodeExecutions {
executionMap[execution.NodeID] = execution
}
readyQueue := make([]string, 0)
readySet := make(map[string]struct{})
auditQueue := make([]string, 0)
if currentRun.CurrentNodeID != "" {
tasks, err := runtime.store.ListAuditTasks(ctx, "all")
if err != nil {
return nil, err
}
for _, task := range tasks {
if task.RunID == currentRun.ID && task.NodeID == currentRun.CurrentNodeID && task.Status != domain.AuditStatusPending {
node := domain.GetNodeByID(workflow.Nodes, currentRun.CurrentNodeID)
if node != nil {
execution := executionMap[node.ID]
execution.NodeID = node.ID
execution.NodeLabel = node.Data.Label
execution.Status = domain.NodeExecutionStatusCompleted
execution.EndedAt = nowISO()
execution.Logs = append(execution.Logs, logEntry("INFO", fmt.Sprintf("Audit decision: %s", task.Status), node.Data.Label))
executionMap[node.ID] = execution
applyExecution(currentRun, execution)
for _, target := range edgeTargets(domain.GetDecisionOutgoingEdges(workflow.Edges, node.ID, task.Status)) {
queueNodeIfReady(*workflow, executionMap, &readyQueue, readySet, target)
}
currentRun.Status = domain.RunStatusRunning
syncCurrentNodeID(currentRun)
currentRun.UpdatedAt = nowISO()
if err := runtime.store.UpsertRun(ctx, *currentRun); err != nil {
return nil, err
}
}
break
}
}
}
if len(readyQueue) == 0 && len(executionMap) == 0 {
for _, nodeID := range domain.GetStartNodeIDs(workflow.Nodes, workflow.Edges) {
queueNodeIfReady(*workflow, executionMap, &readyQueue, readySet, nodeID)
}
}
executionCtx := ctx
cancelExecution := func() {}
if timeout := workflowRunTimeout(workflow); timeout > 0 {
executionCtx, cancelExecution = context.WithTimeout(ctx, timeout)
} else {
executionCtx, cancelExecution = context.WithCancel(ctx)
}
defer cancelExecution()
results := make(chan nodeExecutionOutcome, nodeConcurrencyLimit)
runningNodes := make(map[string]struct{})
for {
launched := false
for len(runningNodes) < nodeConcurrencyLimit && len(readyQueue) > 0 {
nodeID := readyQueue[0]
readyQueue = readyQueue[1:]
delete(readySet, nodeID)
node := domain.GetNodeByID(workflow.Nodes, nodeID)
if node == nil {
continue
}
if existing, ok := executionMap[node.ID]; ok {
if existing.Status == domain.NodeExecutionStatusCompleted ||
existing.Status == domain.NodeExecutionStatusRunning ||
existing.Status == domain.NodeExecutionStatusWaitingForAudit {
continue
}
}
if !dependenciesComplete(*workflow, executionMap, node.ID) {
continue
}
if domain.NodeRequiresAudit(*node) {
auditQueue = append(auditQueue, node.ID)
continue
}
inputArtifacts := gatherInputArtifacts(*workflow, executionMap, node.ID)
execution := domain.NodeExecutionRecord{
NodeID: node.ID,
NodeLabel: node.Data.Label,
Status: domain.NodeExecutionStatusRunning,
StartedAt: nowISO(),
Logs: []domain.LogEntryRecord{logEntry("INFO", "Starting node execution", node.Data.Label)},
Artifacts: []domain.ExecutionArtifact{},
ResourceSeries: []domain.ResourceSample{},
}
executionMap[node.ID] = execution
applyExecution(currentRun, execution)
runningNodes[node.ID] = struct{}{}
launched = true
runSnapshot := *currentRun
nodeSnapshot := *node
inputSnapshot := cloneArtifacts(inputArtifacts)
nodeExecutionCtx := executionCtx
cancelNodeExecution := func() {}
if timeout := workflowNodeTimeout(workflow); timeout > 0 {
nodeExecutionCtx, cancelNodeExecution = context.WithTimeout(executionCtx, timeout)
}
go func() {
defer cancelNodeExecution()
result, reference, err := runtime.executeNode(nodeExecutionCtx, &runSnapshot, nodeSnapshot, inputSnapshot)
results <- nodeExecutionOutcome{
nodeID: nodeSnapshot.ID,
result: result,
reference: reference,
err: err,
nodeTimedOut: errors.Is(nodeExecutionCtx.Err(), context.DeadlineExceeded),
}
}()
}
if launched {
currentRun.Status = domain.RunStatusRunning
syncCurrentNodeID(currentRun)
currentRun.UpdatedAt = nowISO()
if err := runtime.store.UpsertRun(ctx, *currentRun); err != nil {
return nil, err
}
}
if len(runningNodes) == 0 {
if len(readyQueue) > 0 {
continue
}
if len(auditQueue) > 0 {
nodeID := auditQueue[0]
auditQueue = auditQueue[1:]
node := domain.GetNodeByID(workflow.Nodes, nodeID)
if node == nil {
continue
}
inputArtifacts := gatherInputArtifacts(*workflow, executionMap, node.ID)
execution := executionMap[node.ID]
if execution.NodeID == "" {
execution = domain.NodeExecutionRecord{
NodeID: node.ID,
NodeLabel: node.Data.Label,
Status: domain.NodeExecutionStatusRunning,
StartedAt: nowISO(),
Logs: []domain.LogEntryRecord{logEntry("INFO", "Starting node execution", node.Data.Label)},
Artifacts: []domain.ExecutionArtifact{},
ResourceSeries: []domain.ResourceSample{},
}
}
task := domain.AuditTaskRecord{
ID: domain.NewID("audit"),
RunID: currentRun.ID,
WorkflowID: workflow.ID,
ProjectID: workflow.ProjectID,
NodeID: node.ID,
WorkflowName: workflow.Name,
ProjectName: currentRun.ProjectName,
OperatorName: node.Data.Label,
CreatedAt: nowISO(),
UpdatedAt: nowISO(),
Status: domain.AuditStatusPending,
Data: createAuditPayload(*node, inputArtifacts),
}
if _, err := runtime.store.CreateAuditTask(ctx, task); err != nil {
return nil, err
}
execution.Status = domain.NodeExecutionStatusWaitingForAudit
execution.EndedAt = nowISO()
execution.Artifacts = cloneArtifacts(inputArtifacts)
execution.Logs = append(execution.Logs, logEntry("WARN", "Node is waiting for human review", node.Data.Label))
executionMap[node.ID] = execution
applyExecution(currentRun, execution)
currentRun.Status = domain.RunStatusWaitingForAudit
syncCurrentNodeID(currentRun)
currentRun.UpdatedAt = nowISO()
if err := runtime.store.UpsertRun(ctx, *currentRun); err != nil {
return nil, err
}
workflow.Status = domain.WorkflowStatusRunning
workflow.UpdatedAt = nowISO()
if err := runtime.store.UpsertWorkflow(ctx, *workflow); err != nil {
return nil, err
}
return currentRun, nil
}
break
}
outcome := <-results
delete(runningNodes, outcome.nodeID)
execution := executionMap[outcome.nodeID]
if outcome.err != nil {
cancelExecution()
execution.Status = domain.NodeExecutionStatusFailed
execution.EndedAt = nowISO()
execution.Error = normalizeRunError(workflow, outcome.err, executionCtx, outcome.nodeTimedOut)
execution.Logs = append(execution.Logs, logEntry("ERROR", execution.Error, execution.NodeLabel))
executionMap[outcome.nodeID] = execution
applyExecution(currentRun, execution)
currentRun.Status = domain.RunStatusFailed
currentRun.CurrentNodeID = outcome.nodeID
currentRun.EndedAt = nowISO()
currentRun.Error = execution.Error
currentRun.UpdatedAt = nowISO()
if err := runtime.store.UpsertRun(ctx, *currentRun); err != nil {
return nil, err
}
workflow.Status = domain.WorkflowStatusFailed
workflow.UpdatedAt = nowISO()
if err := runtime.store.UpsertWorkflow(ctx, *workflow); err != nil {
return nil, err
}
if shouldAutoRetry(workflow, currentRun) {
retryRun, err := runtime.enqueueRetryRun(ctx, workflow, currentRun)
if err != nil {
return nil, err
}
if retryRun != nil {
appendLog(currentRun, logEntry("WARN", fmt.Sprintf("已自动创建重试运行,第 %d 次尝试已进入队列", retryRun.Attempt), execution.NodeLabel))
currentRun.UpdatedAt = nowISO()
if err := runtime.store.UpsertRun(ctx, *currentRun); err != nil {
return nil, err
}
}
}
return currentRun, nil
}
execution.Status = domain.NodeExecutionStatusCompleted
execution.EndedAt = nowISO()
execution.Logs = append(execution.Logs, outcome.result.Logs...)
execution.Artifacts = outcome.result.Artifacts
execution.ResourceSeries = outcome.result.ResourceSeries
executionMap[outcome.nodeID] = execution
applyExecution(currentRun, execution)
if outcome.reference != nil {
currentRun.K8sRefs = append(currentRun.K8sRefs, domain.K8sRef{NodeID: outcome.nodeID, JobName: outcome.reference.JobName, Namespace: outcome.reference.Namespace})
}
enqueueDependents(*workflow, executionMap, &readyQueue, readySet, outcome.nodeID)
currentRun.Status = domain.RunStatusRunning
syncCurrentNodeID(currentRun)
currentRun.UpdatedAt = nowISO()
if err := runtime.store.UpsertRun(ctx, *currentRun); err != nil {
return nil, err
}
}
currentRun.Status = domain.RunStatusCompleted
currentRun.EndedAt = nowISO()
syncCurrentNodeID(currentRun)
currentRun.UpdatedAt = nowISO()
if err := runtime.store.UpsertRun(ctx, *currentRun); err != nil {
return nil, err
}
workflow.Status = domain.WorkflowStatusCompleted
workflow.LastRunID = currentRun.ID
workflow.LastRunAt = nowISO()
workflow.UpdatedAt = nowISO()
if err := runtime.store.UpsertWorkflow(ctx, *workflow); err != nil {
return nil, err
}
return currentRun, nil
}
func edgeTargets(edges []domain.WorkflowEdge) []string {
items := make([]string, 0, len(edges))
for _, edge := range edges {
items = append(items, edge.Target)
}
return items
}
func (runtime *Runtime) ProcessOnce(ctx context.Context) (*domain.RunRecord, error) {
if err := runtime.enqueueStreamingWorkflows(ctx); err != nil {
return nil, err
}
claimed, err := runtime.store.ClaimNextQueuedRun(ctx)
if err != nil || claimed == nil {
return claimed, err
}
return runtime.processRun(ctx, claimed)
}