725 lines
22 KiB
Go
725 lines
22 KiB
Go
package worker
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"sort"
|
|
"time"
|
|
|
|
"droboticflow/internal/domain"
|
|
"droboticflow/internal/k8s"
|
|
"droboticflow/internal/store"
|
|
)
|
|
|
|
type Runtime struct {
|
|
store store.Store
|
|
mode string
|
|
namespace string
|
|
imagePullSecretName string
|
|
operatorRunnerImage string
|
|
localRunnerCommand string
|
|
launcher *k8s.Launcher
|
|
}
|
|
|
|
var activeDispatchStatuses = map[domain.RunStatus]struct{}{
|
|
domain.RunStatusRunning: {},
|
|
domain.RunStatusWaitingForAudit: {},
|
|
}
|
|
|
|
var queuedOrActiveStatuses = map[domain.RunStatus]struct{}{
|
|
domain.RunStatusQueued: {},
|
|
domain.RunStatusRunning: {},
|
|
domain.RunStatusWaitingForAudit: {},
|
|
}
|
|
|
|
func NewRuntime(dataStore store.Store, mode string, namespace string, imagePullSecretName string, operatorRunnerImage string, localRunnerCommand string) (*Runtime, error) {
|
|
launcher, err := k8s.NewLauncher(mode == "k8s", namespace, imagePullSecretName)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return &Runtime{
|
|
store: dataStore,
|
|
mode: mode,
|
|
namespace: namespace,
|
|
imagePullSecretName: imagePullSecretName,
|
|
operatorRunnerImage: operatorRunnerImage,
|
|
localRunnerCommand: localRunnerCommand,
|
|
launcher: launcher,
|
|
}, nil
|
|
}
|
|
|
|
func nowISO() string {
|
|
return time.Now().UTC().Format(time.RFC3339Nano)
|
|
}
|
|
|
|
func logEntry(level string, message string, operator string) domain.LogEntryRecord {
|
|
return domain.LogEntryRecord{
|
|
Timestamp: nowISO(),
|
|
Level: level,
|
|
Message: message,
|
|
Operator: operator,
|
|
}
|
|
}
|
|
|
|
func findNodeExecution(run *domain.RunRecord, nodeID string) (int, *domain.NodeExecutionRecord) {
|
|
for index := range run.NodeExecutions {
|
|
if run.NodeExecutions[index].NodeID == nodeID {
|
|
return index, &run.NodeExecutions[index]
|
|
}
|
|
}
|
|
return -1, nil
|
|
}
|
|
|
|
func appendLog(run *domain.RunRecord, entry domain.LogEntryRecord) {
|
|
run.Logs = append(run.Logs, entry)
|
|
}
|
|
|
|
func applyExecution(run *domain.RunRecord, execution domain.NodeExecutionRecord) {
|
|
index, _ := findNodeExecution(run, execution.NodeID)
|
|
if index >= 0 {
|
|
run.NodeExecutions[index] = execution
|
|
} else {
|
|
run.NodeExecutions = append(run.NodeExecutions, execution)
|
|
}
|
|
run.Logs = []domain.LogEntryRecord{}
|
|
run.ResourceSeries = []domain.ResourceSample{}
|
|
run.Artifacts = []domain.ExecutionArtifact{}
|
|
for _, item := range run.NodeExecutions {
|
|
for _, entry := range item.Logs {
|
|
appendLog(run, entry)
|
|
}
|
|
if len(item.ResourceSeries) > 0 {
|
|
run.ResourceSeries = append(run.ResourceSeries, item.ResourceSeries...)
|
|
}
|
|
if len(item.Artifacts) > 0 {
|
|
run.Artifacts = append(run.Artifacts, item.Artifacts...)
|
|
}
|
|
}
|
|
}
|
|
|
|
func globalConcurrencyLimit(settings *domain.SettingsRecord) int {
|
|
if settings != nil && settings.Worker.MaxConcurrentRuns > 0 {
|
|
return settings.Worker.MaxConcurrentRuns
|
|
}
|
|
return 1
|
|
}
|
|
|
|
func effectiveWorkflowConcurrencyLimit(workflow *domain.WorkflowRecord, settings *domain.SettingsRecord) int {
|
|
if workflow != nil && workflow.MaxConcurrentRuns > 0 {
|
|
return workflow.MaxConcurrentRuns
|
|
}
|
|
return globalConcurrencyLimit(settings)
|
|
}
|
|
|
|
func selectQueuedRunsForDispatch(runs []domain.RunRecord, workflows []domain.WorkflowRecord, settings *domain.SettingsRecord) []string {
|
|
globalLimit := globalConcurrencyLimit(settings)
|
|
workflowByID := make(map[string]domain.WorkflowRecord, len(workflows))
|
|
for _, workflow := range workflows {
|
|
workflowByID[workflow.ID] = workflow
|
|
}
|
|
now := time.Now().UTC()
|
|
|
|
activeByWorkflow := map[string]int{}
|
|
activeTotal := 0
|
|
queued := make([]domain.RunRecord, 0)
|
|
for _, run := range runs {
|
|
if _, ok := activeDispatchStatuses[run.Status]; ok {
|
|
activeTotal++
|
|
activeByWorkflow[run.WorkflowID]++
|
|
continue
|
|
}
|
|
if run.Status == domain.RunStatusQueued {
|
|
if !runDispatchReady(run, now) {
|
|
continue
|
|
}
|
|
queued = append(queued, run)
|
|
}
|
|
}
|
|
|
|
available := globalLimit - activeTotal
|
|
if available <= 0 {
|
|
return nil
|
|
}
|
|
|
|
sort.Slice(queued, func(left int, right int) bool {
|
|
leftAvailableAt := runAvailableAt(queued[left])
|
|
rightAvailableAt := runAvailableAt(queued[right])
|
|
if leftAvailableAt.Equal(rightAvailableAt) {
|
|
return queued[left].CreatedAt < queued[right].CreatedAt
|
|
}
|
|
return leftAvailableAt.Before(rightAvailableAt)
|
|
})
|
|
|
|
selected := make([]string, 0, available)
|
|
for _, run := range queued {
|
|
if available == 0 {
|
|
break
|
|
}
|
|
workflow := workflowByID[run.WorkflowID]
|
|
workflowLimit := effectiveWorkflowConcurrencyLimit(&workflow, settings)
|
|
if activeByWorkflow[run.WorkflowID] >= workflowLimit {
|
|
continue
|
|
}
|
|
selected = append(selected, run.ID)
|
|
activeByWorkflow[run.WorkflowID]++
|
|
available--
|
|
}
|
|
return selected
|
|
}
|
|
|
|
func (runtime *Runtime) enqueueStreamingWorkflows(ctx context.Context) error {
|
|
workflows, err := runtime.store.ListAllWorkflows(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
runs, err := runtime.store.ListRuns(ctx, "")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
settings, err := runtime.store.GetSettings(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
now := time.Now().UTC()
|
|
for _, workflow := range workflows {
|
|
if workflow.Type != domain.WorkflowTypeStreaming || workflow.Schedule == nil || workflow.Schedule.Interval <= 0 {
|
|
continue
|
|
}
|
|
inflightCount := 0
|
|
for _, run := range runs {
|
|
if run.WorkflowID == workflow.ID {
|
|
if _, ok := queuedOrActiveStatuses[run.Status]; ok {
|
|
inflightCount++
|
|
}
|
|
}
|
|
}
|
|
if inflightCount >= effectiveWorkflowConcurrencyLimit(&workflow, settings) {
|
|
continue
|
|
}
|
|
lastRunAt := time.Time{}
|
|
if workflow.LastRunAt != "" {
|
|
parsed, err := time.Parse(time.RFC3339Nano, workflow.LastRunAt)
|
|
if err == nil {
|
|
lastRunAt = parsed
|
|
}
|
|
}
|
|
if workflow.LastRunAt == "" || now.Sub(lastRunAt) >= time.Duration(workflow.Schedule.Interval)*time.Second {
|
|
if _, err := runtime.store.CreateRun(ctx, workflow.ID, domain.RunTriggerScheduled); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (runtime *Runtime) processClaimedRun(claimed *domain.RunRecord) {
|
|
if _, err := runtime.processRun(context.Background(), claimed); err != nil {
|
|
fmt.Printf("[worker] process run %s failed: %v\n", claimed.ID, err)
|
|
}
|
|
}
|
|
|
|
func (runtime *Runtime) ProcessAvailable(ctx context.Context) error {
|
|
if err := runtime.enqueueStreamingWorkflows(ctx); err != nil {
|
|
return err
|
|
}
|
|
settings, err := runtime.store.GetSettings(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
workflows, err := runtime.store.ListAllWorkflows(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
runs, err := runtime.store.ListRuns(ctx, "")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
for _, runID := range selectQueuedRunsForDispatch(runs, workflows, settings) {
|
|
claimed, err := runtime.store.ClaimQueuedRun(ctx, runID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if claimed == nil {
|
|
continue
|
|
}
|
|
go runtime.processClaimedRun(claimed)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func chooseImage(node domain.WorkflowNode, fallback string) string {
|
|
if node.Data.Image != "" {
|
|
return node.Data.Image
|
|
}
|
|
return fallback
|
|
}
|
|
|
|
func chooseValue(value string, fallback string) string {
|
|
if value == "" {
|
|
return fallback
|
|
}
|
|
return value
|
|
}
|
|
|
|
func workflowRunTimeout(workflow *domain.WorkflowRecord) time.Duration {
|
|
if workflow == nil || workflow.RunTimeoutSeconds <= 0 {
|
|
return 0
|
|
}
|
|
return time.Duration(workflow.RunTimeoutSeconds) * time.Second
|
|
}
|
|
|
|
func workflowNodeTimeout(workflow *domain.WorkflowRecord) time.Duration {
|
|
if workflow == nil || workflow.NodeTimeoutSeconds <= 0 {
|
|
return 0
|
|
}
|
|
return time.Duration(workflow.NodeTimeoutSeconds) * time.Second
|
|
}
|
|
|
|
func normalizeRunError(workflow *domain.WorkflowRecord, err error, executionCtx context.Context, nodeTimedOut bool) string {
|
|
if err == nil {
|
|
return ""
|
|
}
|
|
if workflow != nil && workflow.RunTimeoutSeconds > 0 && (errors.Is(err, context.DeadlineExceeded) || errors.Is(executionCtx.Err(), context.DeadlineExceeded)) {
|
|
return fmt.Sprintf("工作流执行超时(%d 秒)", workflow.RunTimeoutSeconds)
|
|
}
|
|
if workflow != nil && workflow.NodeTimeoutSeconds > 0 && nodeTimedOut {
|
|
return fmt.Sprintf("节点执行超时(%d 秒)", workflow.NodeTimeoutSeconds)
|
|
}
|
|
return err.Error()
|
|
}
|
|
|
|
func shouldAutoRetry(workflow *domain.WorkflowRecord, run *domain.RunRecord) bool {
|
|
if workflow == nil || run == nil || workflow.MaxRetryAttempts <= 0 {
|
|
return false
|
|
}
|
|
attempt := run.Attempt
|
|
if attempt <= 0 {
|
|
attempt = 1
|
|
}
|
|
return attempt <= workflow.MaxRetryAttempts
|
|
}
|
|
|
|
func nextRunAttempt(run *domain.RunRecord) int {
|
|
if run == nil || run.Attempt <= 0 {
|
|
return 1
|
|
}
|
|
return run.Attempt + 1
|
|
}
|
|
|
|
func nextRunAvailableAt(workflow *domain.WorkflowRecord) string {
|
|
if workflow == nil || workflow.RetryBackoffSeconds <= 0 {
|
|
return nowISO()
|
|
}
|
|
return time.Now().UTC().Add(time.Duration(workflow.RetryBackoffSeconds) * time.Second).Format(time.RFC3339Nano)
|
|
}
|
|
|
|
func (runtime *Runtime) enqueueRetryRun(ctx context.Context, workflow *domain.WorkflowRecord, failedRun *domain.RunRecord) (*domain.RunRecord, error) {
|
|
if workflow == nil || failedRun == nil {
|
|
return nil, nil
|
|
}
|
|
retryRun, err := runtime.store.CreateRun(ctx, workflow.ID, domain.RunTriggerRetry)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
retryRun.Attempt = nextRunAttempt(failedRun)
|
|
retryRun.ParentRunID = failedRun.ID
|
|
retryRun.AvailableAt = nextRunAvailableAt(workflow)
|
|
retryRun.UpdatedAt = nowISO()
|
|
if err := runtime.store.UpsertRun(ctx, *retryRun); err != nil {
|
|
return nil, err
|
|
}
|
|
return retryRun, nil
|
|
}
|
|
|
|
type nodeExecutionOutcome struct {
|
|
nodeID string
|
|
result executionResult
|
|
reference *k8s.JobReference
|
|
err error
|
|
nodeTimedOut bool
|
|
}
|
|
|
|
func runAvailableAt(run domain.RunRecord) time.Time {
|
|
if run.AvailableAt != "" {
|
|
if parsed, err := time.Parse(time.RFC3339Nano, run.AvailableAt); err == nil {
|
|
return parsed
|
|
}
|
|
}
|
|
if parsed, err := time.Parse(time.RFC3339Nano, run.CreatedAt); err == nil {
|
|
return parsed
|
|
}
|
|
return time.Time{}
|
|
}
|
|
|
|
func runDispatchReady(run domain.RunRecord, now time.Time) bool {
|
|
availableAt := runAvailableAt(run)
|
|
if availableAt.IsZero() {
|
|
return true
|
|
}
|
|
return !availableAt.After(now)
|
|
}
|
|
|
|
func syncCurrentNodeID(run *domain.RunRecord) {
|
|
run.CurrentNodeID = ""
|
|
for _, status := range []domain.NodeExecutionStatus{
|
|
domain.NodeExecutionStatusWaitingForAudit,
|
|
domain.NodeExecutionStatusRunning,
|
|
} {
|
|
for _, execution := range run.NodeExecutions {
|
|
if execution.Status == status {
|
|
run.CurrentNodeID = execution.NodeID
|
|
return
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func dependenciesComplete(workflow domain.WorkflowRecord, executionMap map[string]domain.NodeExecutionRecord, nodeID string) bool {
|
|
for _, candidate := range domain.GetIncomingEdges(workflow.Edges, nodeID) {
|
|
dependency, ok := executionMap[candidate.Source]
|
|
if !ok || dependency.Status != domain.NodeExecutionStatusCompleted {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
|
|
func queueNodeIfReady(workflow domain.WorkflowRecord, executionMap map[string]domain.NodeExecutionRecord, readyQueue *[]string, readySet map[string]struct{}, nodeID string) {
|
|
if _, queued := readySet[nodeID]; queued {
|
|
return
|
|
}
|
|
if execution, ok := executionMap[nodeID]; ok {
|
|
if execution.Status == domain.NodeExecutionStatusCompleted ||
|
|
execution.Status == domain.NodeExecutionStatusRunning ||
|
|
execution.Status == domain.NodeExecutionStatusWaitingForAudit {
|
|
return
|
|
}
|
|
}
|
|
if !dependenciesComplete(workflow, executionMap, nodeID) {
|
|
return
|
|
}
|
|
*readyQueue = append(*readyQueue, nodeID)
|
|
readySet[nodeID] = struct{}{}
|
|
}
|
|
|
|
func enqueueDependents(workflow domain.WorkflowRecord, executionMap map[string]domain.NodeExecutionRecord, readyQueue *[]string, readySet map[string]struct{}, nodeID string) {
|
|
for _, edge := range domain.GetOutgoingEdges(workflow.Edges, nodeID) {
|
|
queueNodeIfReady(workflow, executionMap, readyQueue, readySet, edge.Target)
|
|
}
|
|
}
|
|
|
|
func (runtime *Runtime) processRun(ctx context.Context, claimed *domain.RunRecord) (*domain.RunRecord, error) {
|
|
workflow, err := runtime.store.GetWorkflow(ctx, claimed.WorkflowID)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if workflow == nil {
|
|
claimed.Status = domain.RunStatusFailed
|
|
claimed.EndedAt = nowISO()
|
|
claimed.Error = fmt.Sprintf("workflow %s not found", claimed.WorkflowID)
|
|
claimed.UpdatedAt = nowISO()
|
|
return claimed, runtime.store.UpsertRun(ctx, *claimed)
|
|
}
|
|
|
|
currentRun, err := runtime.store.GetRun(ctx, claimed.ID)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if currentRun == nil {
|
|
currentRun = claimed
|
|
}
|
|
settings, err := runtime.store.GetSettings(ctx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
nodeConcurrencyLimit := effectiveWorkflowConcurrencyLimit(workflow, settings)
|
|
if nodeConcurrencyLimit <= 0 {
|
|
nodeConcurrencyLimit = 1
|
|
}
|
|
|
|
executionMap := map[string]domain.NodeExecutionRecord{}
|
|
for _, execution := range currentRun.NodeExecutions {
|
|
executionMap[execution.NodeID] = execution
|
|
}
|
|
readyQueue := make([]string, 0)
|
|
readySet := make(map[string]struct{})
|
|
auditQueue := make([]string, 0)
|
|
|
|
if currentRun.CurrentNodeID != "" {
|
|
tasks, err := runtime.store.ListAuditTasks(ctx, "all")
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
for _, task := range tasks {
|
|
if task.RunID == currentRun.ID && task.NodeID == currentRun.CurrentNodeID && task.Status != domain.AuditStatusPending {
|
|
node := domain.GetNodeByID(workflow.Nodes, currentRun.CurrentNodeID)
|
|
if node != nil {
|
|
execution := executionMap[node.ID]
|
|
execution.NodeID = node.ID
|
|
execution.NodeLabel = node.Data.Label
|
|
execution.Status = domain.NodeExecutionStatusCompleted
|
|
execution.EndedAt = nowISO()
|
|
execution.Logs = append(execution.Logs, logEntry("INFO", fmt.Sprintf("Audit decision: %s", task.Status), node.Data.Label))
|
|
executionMap[node.ID] = execution
|
|
applyExecution(currentRun, execution)
|
|
for _, target := range edgeTargets(domain.GetDecisionOutgoingEdges(workflow.Edges, node.ID, task.Status)) {
|
|
queueNodeIfReady(*workflow, executionMap, &readyQueue, readySet, target)
|
|
}
|
|
currentRun.Status = domain.RunStatusRunning
|
|
syncCurrentNodeID(currentRun)
|
|
currentRun.UpdatedAt = nowISO()
|
|
if err := runtime.store.UpsertRun(ctx, *currentRun); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
break
|
|
}
|
|
}
|
|
}
|
|
|
|
if len(readyQueue) == 0 && len(executionMap) == 0 {
|
|
for _, nodeID := range domain.GetStartNodeIDs(workflow.Nodes, workflow.Edges) {
|
|
queueNodeIfReady(*workflow, executionMap, &readyQueue, readySet, nodeID)
|
|
}
|
|
}
|
|
|
|
executionCtx := ctx
|
|
cancelExecution := func() {}
|
|
if timeout := workflowRunTimeout(workflow); timeout > 0 {
|
|
executionCtx, cancelExecution = context.WithTimeout(ctx, timeout)
|
|
} else {
|
|
executionCtx, cancelExecution = context.WithCancel(ctx)
|
|
}
|
|
defer cancelExecution()
|
|
results := make(chan nodeExecutionOutcome, nodeConcurrencyLimit)
|
|
runningNodes := make(map[string]struct{})
|
|
for {
|
|
launched := false
|
|
for len(runningNodes) < nodeConcurrencyLimit && len(readyQueue) > 0 {
|
|
nodeID := readyQueue[0]
|
|
readyQueue = readyQueue[1:]
|
|
delete(readySet, nodeID)
|
|
|
|
node := domain.GetNodeByID(workflow.Nodes, nodeID)
|
|
if node == nil {
|
|
continue
|
|
}
|
|
if existing, ok := executionMap[node.ID]; ok {
|
|
if existing.Status == domain.NodeExecutionStatusCompleted ||
|
|
existing.Status == domain.NodeExecutionStatusRunning ||
|
|
existing.Status == domain.NodeExecutionStatusWaitingForAudit {
|
|
continue
|
|
}
|
|
}
|
|
if !dependenciesComplete(*workflow, executionMap, node.ID) {
|
|
continue
|
|
}
|
|
if domain.NodeRequiresAudit(*node) {
|
|
auditQueue = append(auditQueue, node.ID)
|
|
continue
|
|
}
|
|
|
|
inputArtifacts := gatherInputArtifacts(*workflow, executionMap, node.ID)
|
|
execution := domain.NodeExecutionRecord{
|
|
NodeID: node.ID,
|
|
NodeLabel: node.Data.Label,
|
|
Status: domain.NodeExecutionStatusRunning,
|
|
StartedAt: nowISO(),
|
|
Logs: []domain.LogEntryRecord{logEntry("INFO", "Starting node execution", node.Data.Label)},
|
|
Artifacts: []domain.ExecutionArtifact{},
|
|
ResourceSeries: []domain.ResourceSample{},
|
|
}
|
|
executionMap[node.ID] = execution
|
|
applyExecution(currentRun, execution)
|
|
runningNodes[node.ID] = struct{}{}
|
|
launched = true
|
|
|
|
runSnapshot := *currentRun
|
|
nodeSnapshot := *node
|
|
inputSnapshot := cloneArtifacts(inputArtifacts)
|
|
nodeExecutionCtx := executionCtx
|
|
cancelNodeExecution := func() {}
|
|
if timeout := workflowNodeTimeout(workflow); timeout > 0 {
|
|
nodeExecutionCtx, cancelNodeExecution = context.WithTimeout(executionCtx, timeout)
|
|
}
|
|
go func() {
|
|
defer cancelNodeExecution()
|
|
result, reference, err := runtime.executeNode(nodeExecutionCtx, &runSnapshot, nodeSnapshot, inputSnapshot)
|
|
results <- nodeExecutionOutcome{
|
|
nodeID: nodeSnapshot.ID,
|
|
result: result,
|
|
reference: reference,
|
|
err: err,
|
|
nodeTimedOut: errors.Is(nodeExecutionCtx.Err(), context.DeadlineExceeded),
|
|
}
|
|
}()
|
|
}
|
|
|
|
if launched {
|
|
currentRun.Status = domain.RunStatusRunning
|
|
syncCurrentNodeID(currentRun)
|
|
currentRun.UpdatedAt = nowISO()
|
|
if err := runtime.store.UpsertRun(ctx, *currentRun); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
|
|
if len(runningNodes) == 0 {
|
|
if len(readyQueue) > 0 {
|
|
continue
|
|
}
|
|
if len(auditQueue) > 0 {
|
|
nodeID := auditQueue[0]
|
|
auditQueue = auditQueue[1:]
|
|
node := domain.GetNodeByID(workflow.Nodes, nodeID)
|
|
if node == nil {
|
|
continue
|
|
}
|
|
inputArtifacts := gatherInputArtifacts(*workflow, executionMap, node.ID)
|
|
execution := executionMap[node.ID]
|
|
if execution.NodeID == "" {
|
|
execution = domain.NodeExecutionRecord{
|
|
NodeID: node.ID,
|
|
NodeLabel: node.Data.Label,
|
|
Status: domain.NodeExecutionStatusRunning,
|
|
StartedAt: nowISO(),
|
|
Logs: []domain.LogEntryRecord{logEntry("INFO", "Starting node execution", node.Data.Label)},
|
|
Artifacts: []domain.ExecutionArtifact{},
|
|
ResourceSeries: []domain.ResourceSample{},
|
|
}
|
|
}
|
|
task := domain.AuditTaskRecord{
|
|
ID: domain.NewID("audit"),
|
|
RunID: currentRun.ID,
|
|
WorkflowID: workflow.ID,
|
|
ProjectID: workflow.ProjectID,
|
|
NodeID: node.ID,
|
|
WorkflowName: workflow.Name,
|
|
ProjectName: currentRun.ProjectName,
|
|
OperatorName: node.Data.Label,
|
|
CreatedAt: nowISO(),
|
|
UpdatedAt: nowISO(),
|
|
Status: domain.AuditStatusPending,
|
|
Data: createAuditPayload(*node, inputArtifacts),
|
|
}
|
|
if _, err := runtime.store.CreateAuditTask(ctx, task); err != nil {
|
|
return nil, err
|
|
}
|
|
execution.Status = domain.NodeExecutionStatusWaitingForAudit
|
|
execution.EndedAt = nowISO()
|
|
execution.Artifacts = cloneArtifacts(inputArtifacts)
|
|
execution.Logs = append(execution.Logs, logEntry("WARN", "Node is waiting for human review", node.Data.Label))
|
|
executionMap[node.ID] = execution
|
|
applyExecution(currentRun, execution)
|
|
currentRun.Status = domain.RunStatusWaitingForAudit
|
|
syncCurrentNodeID(currentRun)
|
|
currentRun.UpdatedAt = nowISO()
|
|
if err := runtime.store.UpsertRun(ctx, *currentRun); err != nil {
|
|
return nil, err
|
|
}
|
|
workflow.Status = domain.WorkflowStatusRunning
|
|
workflow.UpdatedAt = nowISO()
|
|
if err := runtime.store.UpsertWorkflow(ctx, *workflow); err != nil {
|
|
return nil, err
|
|
}
|
|
return currentRun, nil
|
|
}
|
|
break
|
|
}
|
|
|
|
outcome := <-results
|
|
delete(runningNodes, outcome.nodeID)
|
|
execution := executionMap[outcome.nodeID]
|
|
if outcome.err != nil {
|
|
cancelExecution()
|
|
execution.Status = domain.NodeExecutionStatusFailed
|
|
execution.EndedAt = nowISO()
|
|
execution.Error = normalizeRunError(workflow, outcome.err, executionCtx, outcome.nodeTimedOut)
|
|
execution.Logs = append(execution.Logs, logEntry("ERROR", execution.Error, execution.NodeLabel))
|
|
executionMap[outcome.nodeID] = execution
|
|
applyExecution(currentRun, execution)
|
|
currentRun.Status = domain.RunStatusFailed
|
|
currentRun.CurrentNodeID = outcome.nodeID
|
|
currentRun.EndedAt = nowISO()
|
|
currentRun.Error = execution.Error
|
|
currentRun.UpdatedAt = nowISO()
|
|
if err := runtime.store.UpsertRun(ctx, *currentRun); err != nil {
|
|
return nil, err
|
|
}
|
|
workflow.Status = domain.WorkflowStatusFailed
|
|
workflow.UpdatedAt = nowISO()
|
|
if err := runtime.store.UpsertWorkflow(ctx, *workflow); err != nil {
|
|
return nil, err
|
|
}
|
|
if shouldAutoRetry(workflow, currentRun) {
|
|
retryRun, err := runtime.enqueueRetryRun(ctx, workflow, currentRun)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if retryRun != nil {
|
|
appendLog(currentRun, logEntry("WARN", fmt.Sprintf("已自动创建重试运行,第 %d 次尝试已进入队列", retryRun.Attempt), execution.NodeLabel))
|
|
currentRun.UpdatedAt = nowISO()
|
|
if err := runtime.store.UpsertRun(ctx, *currentRun); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
}
|
|
return currentRun, nil
|
|
}
|
|
|
|
execution.Status = domain.NodeExecutionStatusCompleted
|
|
execution.EndedAt = nowISO()
|
|
execution.Logs = append(execution.Logs, outcome.result.Logs...)
|
|
execution.Artifacts = outcome.result.Artifacts
|
|
execution.ResourceSeries = outcome.result.ResourceSeries
|
|
executionMap[outcome.nodeID] = execution
|
|
applyExecution(currentRun, execution)
|
|
if outcome.reference != nil {
|
|
currentRun.K8sRefs = append(currentRun.K8sRefs, domain.K8sRef{NodeID: outcome.nodeID, JobName: outcome.reference.JobName, Namespace: outcome.reference.Namespace})
|
|
}
|
|
enqueueDependents(*workflow, executionMap, &readyQueue, readySet, outcome.nodeID)
|
|
currentRun.Status = domain.RunStatusRunning
|
|
syncCurrentNodeID(currentRun)
|
|
currentRun.UpdatedAt = nowISO()
|
|
if err := runtime.store.UpsertRun(ctx, *currentRun); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
|
|
currentRun.Status = domain.RunStatusCompleted
|
|
currentRun.EndedAt = nowISO()
|
|
syncCurrentNodeID(currentRun)
|
|
currentRun.UpdatedAt = nowISO()
|
|
if err := runtime.store.UpsertRun(ctx, *currentRun); err != nil {
|
|
return nil, err
|
|
}
|
|
workflow.Status = domain.WorkflowStatusCompleted
|
|
workflow.LastRunID = currentRun.ID
|
|
workflow.LastRunAt = nowISO()
|
|
workflow.UpdatedAt = nowISO()
|
|
if err := runtime.store.UpsertWorkflow(ctx, *workflow); err != nil {
|
|
return nil, err
|
|
}
|
|
return currentRun, nil
|
|
}
|
|
|
|
func edgeTargets(edges []domain.WorkflowEdge) []string {
|
|
items := make([]string, 0, len(edges))
|
|
for _, edge := range edges {
|
|
items = append(items, edge.Target)
|
|
}
|
|
return items
|
|
}
|
|
|
|
func (runtime *Runtime) ProcessOnce(ctx context.Context) (*domain.RunRecord, error) {
|
|
if err := runtime.enqueueStreamingWorkflows(ctx); err != nil {
|
|
return nil, err
|
|
}
|
|
claimed, err := runtime.store.ClaimNextQueuedRun(ctx)
|
|
if err != nil || claimed == nil {
|
|
return claimed, err
|
|
}
|
|
return runtime.processRun(ctx, claimed)
|
|
}
|