forked from longtao.wu/droboticflow
2935 lines
100 KiB
Go
2935 lines
100 KiB
Go
package worker
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"math"
|
|
"os"
|
|
"reflect"
|
|
"sort"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
"sync/atomic"
|
|
"time"
|
|
|
|
"droboticflow/internal/domain"
|
|
"droboticflow/internal/k8s"
|
|
"droboticflow/internal/lightweight"
|
|
"droboticflow/internal/store"
|
|
)
|
|
|
|
type jobLauncher interface {
|
|
Launch(context.Context, k8s.LaunchInput) (*k8s.LaunchResult, error)
|
|
Inspect(context.Context, string, string) (*k8s.JobInspection, error)
|
|
TerminateRun(context.Context, string, string) error
|
|
// SamplePodGPUForRun samples GPU usage inside the pod backing runID+nodeID.
|
|
// Used by the worker's progress reporter to stream live GPU metrics for
|
|
// custom-container operators during execution. Gracefully errors when no
|
|
// Running pod exists or pods/exec RBAC is unavailable.
|
|
SamplePodGPUForRun(context.Context, string, string) (k8s.GPUSample, error)
|
|
}
|
|
|
|
type gpuCapacityProvider interface {
|
|
GPUCapacity(context.Context) (*k8s.GPUCapacitySnapshot, error)
|
|
}
|
|
|
|
type orphanRuntimeReconciler interface {
|
|
ReconcileOrphanedRuntimes(context.Context, map[string]struct{}, time.Time, bool) ([]k8s.OrphanRuntime, error)
|
|
}
|
|
|
|
type nodePodLogTailer interface {
|
|
TailNodePodLogs(context.Context, string, string, int64) (string, error)
|
|
}
|
|
|
|
type BuildInfo struct {
|
|
BuildCommit string `json:"buildCommit,omitempty"`
|
|
BuildTime string `json:"buildTime,omitempty"`
|
|
Image string `json:"image,omitempty"`
|
|
}
|
|
|
|
type ReadinessSnapshot struct {
|
|
Ready bool `json:"ready"`
|
|
Status string `json:"status"`
|
|
Service string `json:"service"`
|
|
BuildCommit string `json:"buildCommit,omitempty"`
|
|
BuildTime string `json:"buildTime,omitempty"`
|
|
Image string `json:"image,omitempty"`
|
|
RunnerImage string `json:"runnerImage,omitempty"`
|
|
LastCycleAt string `json:"lastCycleAt,omitempty"`
|
|
LastSuccessfulCycleAt string `json:"lastSuccessfulCycleAt,omitempty"`
|
|
CycleDurationMS uint64 `json:"cycleDurationMs"`
|
|
ConfiguredPollIntervalMS int64 `json:"configuredPollIntervalMs"`
|
|
EffectivePollIntervalMS int64 `json:"effectivePollIntervalMs"`
|
|
StuckRunCount uint64 `json:"stuckRunCount"`
|
|
OrphanRuntimeCount uint64 `json:"orphanRuntimeCount"`
|
|
Reason string `json:"reason,omitempty"`
|
|
}
|
|
|
|
type Runtime struct {
|
|
store store.Store
|
|
operatorRunnerImage string
|
|
workspaceRoot string
|
|
globalBOSEndpoint string
|
|
launcher jobLauncher
|
|
metrics WorkerMetrics
|
|
processMu sync.Mutex
|
|
activeRuns map[string]struct{}
|
|
dispatchOwner string
|
|
runLeaseTTL time.Duration
|
|
buildMu sync.RWMutex
|
|
buildInfo BuildInfo
|
|
configuredPollMS atomic.Int64
|
|
effectivePollMS atomic.Int64
|
|
orphanCleanup bool
|
|
orphanMinimumAge time.Duration
|
|
orphanCheckInterval time.Duration
|
|
lastOrphanCheckNano atomic.Int64
|
|
pendingFailureAfter time.Duration
|
|
missingMaxRecoveries int
|
|
}
|
|
|
|
type ProcessOnceStatus string
|
|
|
|
const (
|
|
ProcessOnceStatusClaimed ProcessOnceStatus = "claimed"
|
|
ProcessOnceStatusSkippedLeaseActive ProcessOnceStatus = "skipped_lease_active"
|
|
ProcessOnceStatusNoQueuedRun ProcessOnceStatus = "no_queued_run"
|
|
)
|
|
|
|
var errRunLeaseLost = errors.New("run execution lease lost")
|
|
|
|
type ProcessOnceResult struct {
|
|
Status ProcessOnceStatus `json:"status"`
|
|
Run *domain.RunRecord `json:"-"`
|
|
RunRef *ProcessOnceRunSummary `json:"run,omitempty"`
|
|
}
|
|
|
|
type ProcessOnceRunSummary struct {
|
|
ID string `json:"id"`
|
|
Status domain.RunStatus `json:"status"`
|
|
WorkflowID string `json:"workflowId,omitempty"`
|
|
WorkflowName string `json:"workflowName,omitempty"`
|
|
ProjectID string `json:"projectId,omitempty"`
|
|
ProjectName string `json:"projectName,omitempty"`
|
|
CurrentNodeID string `json:"currentNodeId,omitempty"`
|
|
NodeExecutionCount int `json:"nodeExecutionCount"`
|
|
ArtifactCount int `json:"artifactCount"`
|
|
Attempt int `json:"attempt,omitempty"`
|
|
LeaseOwner string `json:"executionLeaseOwner,omitempty"`
|
|
LeaseExpiresAt string `json:"executionLeaseExpiresAt,omitempty"`
|
|
UpdatedAt string `json:"updatedAt,omitempty"`
|
|
}
|
|
|
|
func newProcessOnceResult(status ProcessOnceStatus, run *domain.RunRecord) *ProcessOnceResult {
|
|
result := &ProcessOnceResult{Status: status, Run: run}
|
|
if run == nil {
|
|
return result
|
|
}
|
|
artifactCount := len(run.Artifacts)
|
|
for _, execution := range run.NodeExecutions {
|
|
artifactCount += len(execution.Artifacts)
|
|
}
|
|
result.RunRef = &ProcessOnceRunSummary{
|
|
ID: run.ID,
|
|
Status: run.Status,
|
|
WorkflowID: run.WorkflowID,
|
|
WorkflowName: run.WorkflowName,
|
|
ProjectID: run.ProjectID,
|
|
ProjectName: run.ProjectName,
|
|
CurrentNodeID: run.CurrentNodeID,
|
|
NodeExecutionCount: len(run.NodeExecutions),
|
|
ArtifactCount: artifactCount,
|
|
Attempt: run.Attempt,
|
|
LeaseOwner: run.ExecutionLeaseOwner,
|
|
LeaseExpiresAt: run.ExecutionLeaseExpiresAt,
|
|
UpdatedAt: run.UpdatedAt,
|
|
}
|
|
return result
|
|
}
|
|
|
|
var globalActiveDispatchStatuses = map[domain.RunStatus]struct{}{
|
|
domain.RunStatusRunning: {},
|
|
domain.RunStatusStopping: {},
|
|
}
|
|
|
|
var workflowActiveDispatchStatuses = map[domain.RunStatus]struct{}{
|
|
domain.RunStatusRunning: {},
|
|
domain.RunStatusStopping: {},
|
|
domain.RunStatusWaitingForAudit: {},
|
|
}
|
|
|
|
const (
|
|
workerQueuedRunLimit = 100
|
|
workerReconcileRunLimit = 500
|
|
workerWorkflowQueryLimit = 500
|
|
)
|
|
|
|
func NewRuntime(dataStore store.Store, namespace string, imagePullSecretName string, operatorRunnerImage string, workspacePVCName string, workspaceMountPath string, gpuNodeSelector string, gpuTolerationsJSON string, gpuRuntimeClass string, jobTTLSeconds int, globalBOSEndpoint string) (*Runtime, error) {
|
|
launcher, err := k8s.NewLauncher(namespace, imagePullSecretName, workspacePVCName, workspaceMountPath, gpuNodeSelector, gpuTolerationsJSON, gpuRuntimeClass, jobTTLSeconds)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
trimmedWorkspaceRoot := strings.TrimSpace(workspaceMountPath)
|
|
if trimmedWorkspaceRoot == "" {
|
|
trimmedWorkspaceRoot = "/workspace"
|
|
}
|
|
return &Runtime{
|
|
store: dataStore,
|
|
operatorRunnerImage: operatorRunnerImage,
|
|
workspaceRoot: trimmedWorkspaceRoot,
|
|
globalBOSEndpoint: strings.TrimSpace(globalBOSEndpoint),
|
|
launcher: launcher,
|
|
dispatchOwner: defaultDispatchOwner(),
|
|
runLeaseTTL: defaultRunLeaseTTL(),
|
|
orphanCleanup: envBool("WORKER_ORPHAN_CLEANUP_ENABLED", false),
|
|
orphanMinimumAge: envDurationSeconds("WORKER_ORPHAN_MIN_AGE_SECONDS", 24*time.Hour),
|
|
orphanCheckInterval: envDurationSeconds("WORKER_ORPHAN_CHECK_INTERVAL_SECONDS", 5*time.Minute),
|
|
pendingFailureAfter: envDurationSeconds("WORKER_PENDING_FAILURE_SECONDS", 10*time.Minute),
|
|
missingMaxRecoveries: envPositiveInt("WORKER_MISSING_JOB_MAX_RECOVERIES", 3),
|
|
}, nil
|
|
}
|
|
|
|
func envBool(key string, fallback bool) bool {
|
|
raw := strings.TrimSpace(os.Getenv(key))
|
|
if raw == "" {
|
|
return fallback
|
|
}
|
|
parsed, err := strconv.ParseBool(raw)
|
|
if err != nil {
|
|
return fallback
|
|
}
|
|
return parsed
|
|
}
|
|
|
|
func envPositiveInt(key string, fallback int) int {
|
|
raw := strings.TrimSpace(os.Getenv(key))
|
|
if raw == "" {
|
|
return fallback
|
|
}
|
|
parsed, err := strconv.Atoi(raw)
|
|
if err != nil || parsed <= 0 {
|
|
return fallback
|
|
}
|
|
return parsed
|
|
}
|
|
|
|
func envDurationSeconds(key string, fallback time.Duration) time.Duration {
|
|
seconds := envPositiveInt(key, int(fallback/time.Second))
|
|
return time.Duration(seconds) * time.Second
|
|
}
|
|
|
|
func (runtime *Runtime) SetBuildInfo(info BuildInfo) {
|
|
if runtime == nil {
|
|
return
|
|
}
|
|
runtime.buildMu.Lock()
|
|
runtime.buildInfo = BuildInfo{
|
|
BuildCommit: strings.TrimSpace(info.BuildCommit),
|
|
BuildTime: strings.TrimSpace(info.BuildTime),
|
|
Image: strings.TrimSpace(info.Image),
|
|
}
|
|
runtime.buildMu.Unlock()
|
|
}
|
|
|
|
func (runtime *Runtime) BuildInfo() BuildInfo {
|
|
if runtime == nil {
|
|
return BuildInfo{}
|
|
}
|
|
runtime.buildMu.RLock()
|
|
defer runtime.buildMu.RUnlock()
|
|
return runtime.buildInfo
|
|
}
|
|
|
|
func runtimeOperatorGeneratedByAI(item domain.OperatorRecord) bool {
|
|
source := strings.ToLower(strings.TrimSpace(string(item.GenerationSource)))
|
|
if source == string(domain.OperatorGenerationSourceAI) {
|
|
return true
|
|
}
|
|
if item.RuntimeContract == nil {
|
|
return false
|
|
}
|
|
if sourceValue, ok := item.RuntimeContract["generationSource"].(string); ok && strings.EqualFold(strings.TrimSpace(sourceValue), string(domain.OperatorGenerationSourceAI)) {
|
|
return true
|
|
}
|
|
_, generated := item.RuntimeContract["generatedRepository"]
|
|
return generated
|
|
}
|
|
|
|
func runtimeOperatorPublishedStatus(item domain.OperatorRecord) domain.OperatorPublicationStatus {
|
|
status := domain.OperatorPublicationStatus(strings.ToLower(strings.TrimSpace(string(item.PublicationStatus))))
|
|
if status != "" {
|
|
return status
|
|
}
|
|
if runtimeOperatorGeneratedByAI(item) {
|
|
return domain.OperatorPublicationStatusDraft
|
|
}
|
|
return domain.OperatorPublicationStatusPublished
|
|
}
|
|
|
|
func runtimeNodeGeneratedByAI(node domain.WorkflowNode) bool {
|
|
return strings.EqualFold(strings.TrimSpace(string(node.Data.GenerationSource)), string(domain.OperatorGenerationSourceAI))
|
|
}
|
|
|
|
func runtimeNodePublishedStatus(node domain.WorkflowNode) domain.OperatorPublicationStatus {
|
|
status := domain.OperatorPublicationStatus(strings.ToLower(strings.TrimSpace(string(node.Data.PublicationStatus))))
|
|
if status != "" {
|
|
return status
|
|
}
|
|
if runtimeNodeGeneratedByAI(node) {
|
|
return domain.OperatorPublicationStatusDraft
|
|
}
|
|
return domain.OperatorPublicationStatusPublished
|
|
}
|
|
|
|
func (runtime *Runtime) workflowPublicationBlocker(ctx context.Context, workflow *domain.WorkflowRecord) string {
|
|
if workflow == nil || workflow.ExecutionScope == domain.WorkflowExecutionScopeSandbox {
|
|
return ""
|
|
}
|
|
for _, node := range workflow.Nodes {
|
|
sourceAI := runtimeNodeGeneratedByAI(node)
|
|
status := runtimeNodePublishedStatus(node)
|
|
if operatorID := strings.TrimSpace(node.Data.OperatorID); operatorID != "" {
|
|
operator, err := runtime.store.GetOperator(ctx, operatorID)
|
|
if err != nil {
|
|
return "工作流图非法:无法读取算子发布状态。"
|
|
}
|
|
if operator != nil {
|
|
sourceAI = runtimeOperatorGeneratedByAI(*operator)
|
|
status = runtimeOperatorPublishedStatus(*operator)
|
|
}
|
|
}
|
|
if !sourceAI || status == domain.OperatorPublicationStatusPublished {
|
|
continue
|
|
}
|
|
label := strings.TrimSpace(node.Data.Label)
|
|
if label == "" {
|
|
label = node.ID
|
|
}
|
|
return "生产工作流不能使用未发布的 AI 生成算子:" + label + "。请在沙盒工作流验证,或发布后再运行。"
|
|
}
|
|
return ""
|
|
}
|
|
|
|
func defaultDispatchOwner() string {
|
|
hostname, err := os.Hostname()
|
|
if err != nil || strings.TrimSpace(hostname) == "" {
|
|
hostname = "unknown-host"
|
|
}
|
|
return fmt.Sprintf("%s:%d", hostname, os.Getpid())
|
|
}
|
|
|
|
func defaultRunLeaseTTL() time.Duration {
|
|
raw := strings.TrimSpace(os.Getenv("WORKER_RUN_LEASE_TTL_SECONDS"))
|
|
if raw == "" {
|
|
return 15 * time.Minute
|
|
}
|
|
seconds, err := strconv.Atoi(raw)
|
|
if err != nil || seconds <= 0 {
|
|
return 15 * time.Minute
|
|
}
|
|
return time.Duration(seconds) * time.Second
|
|
}
|
|
|
|
func (runtime *Runtime) leaseOwner() string {
|
|
if runtime == nil {
|
|
return ""
|
|
}
|
|
if strings.TrimSpace(runtime.dispatchOwner) == "" {
|
|
runtime.dispatchOwner = defaultDispatchOwner()
|
|
}
|
|
return runtime.dispatchOwner
|
|
}
|
|
|
|
func (runtime *Runtime) leaseTTL() time.Duration {
|
|
if runtime == nil || runtime.runLeaseTTL <= 0 {
|
|
return defaultRunLeaseTTL()
|
|
}
|
|
return runtime.runLeaseTTL
|
|
}
|
|
|
|
func (runtime *Runtime) MetricsSnapshot() MetricsSnapshot {
|
|
return runtime.metrics.Snapshot()
|
|
}
|
|
|
|
const (
|
|
defaultPollIntervalMS int64 = 2000
|
|
minimumPollIntervalMS int64 = 250
|
|
maximumPollIntervalMS int64 = 60000
|
|
)
|
|
|
|
func normalizedPollIntervalMS(value int64) int64 {
|
|
if value <= 0 {
|
|
return defaultPollIntervalMS
|
|
}
|
|
if value < minimumPollIntervalMS {
|
|
return minimumPollIntervalMS
|
|
}
|
|
if value > maximumPollIntervalMS {
|
|
return maximumPollIntervalMS
|
|
}
|
|
return value
|
|
}
|
|
|
|
func (runtime *Runtime) SetConfiguredPollInterval(interval time.Duration) {
|
|
if runtime == nil {
|
|
return
|
|
}
|
|
configured := normalizedPollIntervalMS(interval.Milliseconds())
|
|
runtime.configuredPollMS.Store(configured)
|
|
if runtime.effectivePollMS.Load() == 0 {
|
|
runtime.effectivePollMS.Store(configured)
|
|
}
|
|
}
|
|
|
|
func (runtime *Runtime) updateEffectivePollInterval(settings *domain.SettingsRecord) {
|
|
configured := normalizedPollIntervalMS(runtime.configuredPollMS.Load())
|
|
effective := configured
|
|
if settings != nil && settings.Worker.PollIntervalMS > 0 {
|
|
effective = normalizedPollIntervalMS(int64(settings.Worker.PollIntervalMS))
|
|
}
|
|
runtime.configuredPollMS.Store(configured)
|
|
runtime.effectivePollMS.Store(effective)
|
|
}
|
|
|
|
func (runtime *Runtime) EffectivePollInterval() time.Duration {
|
|
if runtime == nil {
|
|
return time.Duration(defaultPollIntervalMS) * time.Millisecond
|
|
}
|
|
return time.Duration(normalizedPollIntervalMS(runtime.effectivePollMS.Load())) * time.Millisecond
|
|
}
|
|
|
|
func (runtime *Runtime) RunScheduler(ctx context.Context, configured time.Duration) {
|
|
if runtime == nil {
|
|
return
|
|
}
|
|
runtime.SetConfiguredPollInterval(configured)
|
|
timer := time.NewTimer(0)
|
|
defer timer.Stop()
|
|
for {
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case <-timer.C:
|
|
if err := runtime.ProcessAvailable(ctx); err != nil && !errors.Is(err, context.Canceled) {
|
|
fmt.Printf("[worker] processAvailable failed: %v\n", err)
|
|
}
|
|
timer.Reset(runtime.EffectivePollInterval())
|
|
}
|
|
}
|
|
}
|
|
|
|
func (runtime *Runtime) statusSnapshot() ReadinessSnapshot {
|
|
if runtime == nil {
|
|
return ReadinessSnapshot{Status: "not_ready", Service: "droboticflow-worker", Reason: "worker runtime is not initialized"}
|
|
}
|
|
metrics := runtime.MetricsSnapshot()
|
|
build := runtime.BuildInfo()
|
|
configured := normalizedPollIntervalMS(runtime.configuredPollMS.Load())
|
|
effective := normalizedPollIntervalMS(runtime.effectivePollMS.Load())
|
|
snapshot := ReadinessSnapshot{
|
|
Ready: false,
|
|
Status: "not_ready",
|
|
Service: "droboticflow-worker",
|
|
BuildCommit: build.BuildCommit,
|
|
BuildTime: build.BuildTime,
|
|
Image: build.Image,
|
|
RunnerImage: runtime.operatorRunnerImage,
|
|
LastCycleAt: metrics.LastCycleAt,
|
|
LastSuccessfulCycleAt: metrics.LastSuccessfulCycleAt,
|
|
CycleDurationMS: metrics.CycleDurationMS,
|
|
ConfiguredPollIntervalMS: configured,
|
|
EffectivePollIntervalMS: effective,
|
|
StuckRunCount: metrics.StuckRunCount,
|
|
OrphanRuntimeCount: metrics.OrphanRuntimeCount,
|
|
}
|
|
return snapshot
|
|
}
|
|
|
|
func (runtime *Runtime) Readiness(ctx context.Context) ReadinessSnapshot {
|
|
snapshot := runtime.statusSnapshot()
|
|
if runtime == nil {
|
|
return snapshot
|
|
}
|
|
if runtime.store == nil || runtime.launcher == nil {
|
|
snapshot.Reason = "worker runtime is not initialized"
|
|
return snapshot
|
|
}
|
|
if err := runtime.store.Ready(ctx); err != nil {
|
|
snapshot.Reason = "mongodb is not ready: " + err.Error()
|
|
return snapshot
|
|
}
|
|
lastSuccessful, ok := parseRFC3339Nano(snapshot.LastSuccessfulCycleAt)
|
|
if !ok {
|
|
snapshot.Reason = "worker has not completed a successful scheduling cycle"
|
|
return snapshot
|
|
}
|
|
maximumAge := 3 * time.Duration(snapshot.EffectivePollIntervalMS) * time.Millisecond
|
|
if maximumAge < 30*time.Second {
|
|
maximumAge = 30 * time.Second
|
|
}
|
|
if time.Since(lastSuccessful) > maximumAge {
|
|
snapshot.Reason = "last successful scheduling cycle is stale"
|
|
return snapshot
|
|
}
|
|
snapshot.Ready = true
|
|
snapshot.Status = "ready"
|
|
return snapshot
|
|
}
|
|
|
|
func (runtime *Runtime) GPUCapacity(ctx context.Context) (*k8s.GPUCapacitySnapshot, error) {
|
|
if runtime == nil || runtime.launcher == nil {
|
|
return nil, fmt.Errorf("worker launcher is not configured")
|
|
}
|
|
provider, ok := runtime.launcher.(gpuCapacityProvider)
|
|
if !ok {
|
|
return nil, fmt.Errorf("worker launcher does not expose gpu capacity")
|
|
}
|
|
return provider.GPUCapacity(ctx)
|
|
}
|
|
|
|
func nowISO() string {
|
|
return time.Now().UTC().Format(time.RFC3339Nano)
|
|
}
|
|
|
|
func logEntry(level string, message string, operator string) domain.LogEntryRecord {
|
|
return domain.LogEntryRecord{
|
|
Timestamp: nowISO(),
|
|
Level: level,
|
|
Message: message,
|
|
Operator: operator,
|
|
}
|
|
}
|
|
|
|
func findNodeExecution(run *domain.RunRecord, nodeID string) (int, *domain.NodeExecutionRecord) {
|
|
for index := range run.NodeExecutions {
|
|
if run.NodeExecutions[index].NodeID == nodeID {
|
|
return index, &run.NodeExecutions[index]
|
|
}
|
|
}
|
|
return -1, nil
|
|
}
|
|
|
|
func appendLog(run *domain.RunRecord, entry domain.LogEntryRecord) {
|
|
run.Logs = append(run.Logs, entry)
|
|
}
|
|
|
|
func applyExecution(run *domain.RunRecord, execution domain.NodeExecutionRecord) {
|
|
index, _ := findNodeExecution(run, execution.NodeID)
|
|
if index >= 0 {
|
|
run.NodeExecutions[index] = execution
|
|
} else {
|
|
run.NodeExecutions = append(run.NodeExecutions, execution)
|
|
}
|
|
run.Logs = []domain.LogEntryRecord{}
|
|
run.ResourceSeries = []domain.ResourceSample{}
|
|
run.Artifacts = []domain.ExecutionArtifact{}
|
|
for _, item := range run.NodeExecutions {
|
|
for _, entry := range item.Logs {
|
|
appendLog(run, entry)
|
|
}
|
|
if len(item.ResourceSeries) > 0 {
|
|
run.ResourceSeries = append(run.ResourceSeries, item.ResourceSeries...)
|
|
}
|
|
if len(item.Artifacts) > 0 {
|
|
run.Artifacts = append(run.Artifacts, item.Artifacts...)
|
|
}
|
|
}
|
|
}
|
|
|
|
func applyNodeExecutionSnapshot(execution *domain.NodeExecutionRecord, node domain.WorkflowNode, inputArtifacts []domain.ExecutionArtifact, inputLineage []domain.InputLineageRecord) {
|
|
if execution == nil {
|
|
return
|
|
}
|
|
resolvedImage := chooseImage(node, "")
|
|
execution.NodeID = node.ID
|
|
execution.NodeLabel = node.Data.Label
|
|
execution.OperatorID = strings.TrimSpace(node.Data.OperatorID)
|
|
execution.OperatorVersion = strings.TrimSpace(node.Data.OperatorVersion)
|
|
execution.OperatorImage = firstNonEmptyString(resolvedImage, node.Data.Image)
|
|
execution.OperatorImageID = firstNonEmptyString(node.Data.OperatorImageID, resolvedImage, node.Data.Image)
|
|
if execution.Attempt <= 0 {
|
|
execution.Attempt = 1
|
|
}
|
|
if inputArtifacts != nil {
|
|
execution.InputArtifacts = cloneArtifacts(inputArtifacts)
|
|
}
|
|
if inputLineage != nil {
|
|
execution.InputLineage = cloneInputLineage(inputLineage)
|
|
}
|
|
}
|
|
|
|
func globalConcurrencyLimit(settings *domain.SettingsRecord) int {
|
|
if settings != nil && settings.Worker.MaxConcurrentRuns > 0 {
|
|
return settings.Worker.MaxConcurrentRuns
|
|
}
|
|
return 1
|
|
}
|
|
|
|
func effectiveWorkflowConcurrencyLimit(workflow *domain.WorkflowRecord, settings *domain.SettingsRecord) int {
|
|
if workflow != nil && workflow.MaxConcurrentRuns > 0 {
|
|
return workflow.MaxConcurrentRuns
|
|
}
|
|
return globalConcurrencyLimit(settings)
|
|
}
|
|
|
|
func parseRFC3339Nano(value string) (time.Time, bool) {
|
|
trimmed := strings.TrimSpace(value)
|
|
if trimmed == "" {
|
|
return time.Time{}, false
|
|
}
|
|
parsed, err := time.Parse(time.RFC3339Nano, trimmed)
|
|
if err != nil {
|
|
return time.Time{}, false
|
|
}
|
|
return parsed, true
|
|
}
|
|
|
|
func isStaleQueuedNodeExecution(execution domain.NodeExecutionRecord, threshold time.Duration) bool {
|
|
startedAt, ok := parseRFC3339Nano(execution.StartedAt)
|
|
if !ok {
|
|
return false
|
|
}
|
|
return time.Since(startedAt) >= threshold
|
|
}
|
|
|
|
func firstNonEmptyString(values ...string) string {
|
|
for _, value := range values {
|
|
if strings.TrimSpace(value) != "" {
|
|
return value
|
|
}
|
|
}
|
|
return ""
|
|
}
|
|
|
|
func nextHourlyScheduleTime(schedule *domain.WorkflowSchedule, lastRunAt time.Time, now time.Time) (time.Time, bool) {
|
|
intervalHours := schedule.IntervalHours
|
|
if intervalHours <= 0 {
|
|
return time.Time{}, false
|
|
}
|
|
anchorAt, ok := parseRFC3339Nano(schedule.AnchorAt)
|
|
if !ok {
|
|
if !lastRunAt.IsZero() {
|
|
anchorAt = lastRunAt
|
|
} else {
|
|
anchorAt = now
|
|
}
|
|
}
|
|
interval := time.Duration(intervalHours) * time.Hour
|
|
if lastRunAt.IsZero() || lastRunAt.Before(anchorAt) {
|
|
return anchorAt, true
|
|
}
|
|
elapsed := lastRunAt.Sub(anchorAt)
|
|
slotsElapsed := int(elapsed/interval) + 1
|
|
return anchorAt.Add(time.Duration(slotsElapsed) * interval), true
|
|
}
|
|
|
|
func nextDailyScheduleTime(schedule *domain.WorkflowSchedule, lastRunAt time.Time, now time.Time) (time.Time, bool) {
|
|
if schedule.DayInterval <= 0 {
|
|
return time.Time{}, false
|
|
}
|
|
parts := strings.Split(strings.TrimSpace(schedule.TimeOfDay), ":")
|
|
if len(parts) != 2 {
|
|
return time.Time{}, false
|
|
}
|
|
hour, err := strconv.Atoi(parts[0])
|
|
if err != nil {
|
|
return time.Time{}, false
|
|
}
|
|
minute, err := strconv.Atoi(parts[1])
|
|
if err != nil {
|
|
return time.Time{}, false
|
|
}
|
|
location, err := time.LoadLocation(domain.WorkflowScheduleTimezone(schedule.Timezone))
|
|
if err != nil {
|
|
return time.Time{}, false
|
|
}
|
|
anchorAt, ok := parseRFC3339Nano(schedule.AnchorAt)
|
|
if !ok {
|
|
if !lastRunAt.IsZero() {
|
|
anchorAt = lastRunAt
|
|
} else {
|
|
anchorAt = now
|
|
}
|
|
}
|
|
anchorLocal := anchorAt.In(location)
|
|
firstDue := time.Date(anchorLocal.Year(), anchorLocal.Month(), anchorLocal.Day(), hour, minute, 0, 0, location)
|
|
if anchorLocal.After(firstDue) {
|
|
firstDue = firstDue.AddDate(0, 0, schedule.DayInterval)
|
|
}
|
|
if lastRunAt.IsZero() {
|
|
return firstDue, true
|
|
}
|
|
lastLocal := lastRunAt.In(location)
|
|
nextDue := firstDue
|
|
for !nextDue.After(lastLocal) {
|
|
nextDue = nextDue.AddDate(0, 0, schedule.DayInterval)
|
|
}
|
|
return nextDue, true
|
|
}
|
|
|
|
func shouldEnqueueScheduledRun(workflow domain.WorkflowRecord, now time.Time) bool {
|
|
if workflow.Type != domain.WorkflowTypeStreaming || !domain.WorkflowScheduleEnabled(workflow.Schedule) {
|
|
return false
|
|
}
|
|
lastRunAt, _ := parseRFC3339Nano(workflow.LastRunAt)
|
|
switch domain.EffectiveWorkflowScheduleMode(workflow.Schedule) {
|
|
case domain.WorkflowScheduleModeEveryHours:
|
|
nextDue, ok := nextHourlyScheduleTime(workflow.Schedule, lastRunAt, now)
|
|
return ok && !now.Before(nextDue)
|
|
case domain.WorkflowScheduleModeEveryDaysAtTime:
|
|
nextDue, ok := nextDailyScheduleTime(workflow.Schedule, lastRunAt, now)
|
|
return ok && !now.Before(nextDue)
|
|
case domain.WorkflowScheduleModeIntervalSeconds:
|
|
if workflow.Schedule.Interval <= 0 {
|
|
return false
|
|
}
|
|
if workflow.LastRunAt == "" {
|
|
return true
|
|
}
|
|
return !lastRunAt.IsZero() && now.Sub(lastRunAt) >= time.Duration(workflow.Schedule.Interval)*time.Second
|
|
default:
|
|
return false
|
|
}
|
|
}
|
|
|
|
func selectQueuedRunsForDispatch(runs []domain.RunRecord, workflows []domain.WorkflowRecord, settings *domain.SettingsRecord) []string {
|
|
selected, _, _ := selectQueuedRunsForDispatchWithStats(runs, workflows, settings)
|
|
return selected
|
|
}
|
|
|
|
func selectQueuedRunsForDispatchWithStats(runs []domain.RunRecord, workflows []domain.WorkflowRecord, settings *domain.SettingsRecord) ([]string, int, int) {
|
|
activeByWorkflow := map[string]int{}
|
|
activeTotal := 0
|
|
queued := make([]domain.RunRecord, 0)
|
|
for _, run := range runs {
|
|
if _, ok := workflowActiveDispatchStatuses[run.Status]; ok {
|
|
activeByWorkflow[run.WorkflowID]++
|
|
}
|
|
if _, ok := globalActiveDispatchStatuses[run.Status]; ok {
|
|
activeTotal++
|
|
continue
|
|
}
|
|
if run.Status == domain.RunStatusQueued {
|
|
queued = append(queued, run)
|
|
}
|
|
}
|
|
selected := selectQueuedRunsForDispatchSnapshot(queued, workflows, settings, activeByWorkflow, activeTotal)
|
|
return selected, len(queued), activeTotal
|
|
}
|
|
|
|
func selectQueuedRunsForDispatchSnapshot(
|
|
queued []domain.RunRecord,
|
|
workflows []domain.WorkflowRecord,
|
|
settings *domain.SettingsRecord,
|
|
activeByWorkflow map[string]int,
|
|
activeTotal int,
|
|
) []string {
|
|
globalLimit := globalConcurrencyLimit(settings)
|
|
workflowByID := make(map[string]domain.WorkflowRecord, len(workflows))
|
|
for _, workflow := range workflows {
|
|
workflowByID[workflow.ID] = workflow
|
|
}
|
|
now := time.Now().UTC()
|
|
ready := make([]domain.RunRecord, 0, len(queued))
|
|
for _, run := range queued {
|
|
if run.Status == domain.RunStatusQueued && runDispatchReady(run, now) {
|
|
ready = append(ready, run)
|
|
}
|
|
}
|
|
|
|
available := globalLimit - activeTotal
|
|
if available <= 0 {
|
|
return nil
|
|
}
|
|
|
|
sort.Slice(ready, func(left int, right int) bool {
|
|
leftAvailableAt := runAvailableAt(ready[left])
|
|
rightAvailableAt := runAvailableAt(ready[right])
|
|
if leftAvailableAt.Equal(rightAvailableAt) {
|
|
return ready[left].CreatedAt < ready[right].CreatedAt
|
|
}
|
|
return leftAvailableAt.Before(rightAvailableAt)
|
|
})
|
|
|
|
selected := make([]string, 0, available)
|
|
for _, run := range ready {
|
|
if available == 0 {
|
|
break
|
|
}
|
|
workflow := workflowByID[run.WorkflowID]
|
|
workflowLimit := effectiveWorkflowConcurrencyLimit(&workflow, settings)
|
|
if activeByWorkflow[run.WorkflowID] >= workflowLimit {
|
|
continue
|
|
}
|
|
selected = append(selected, run.ID)
|
|
activeByWorkflow[run.WorkflowID]++
|
|
available--
|
|
}
|
|
return selected
|
|
}
|
|
|
|
func (runtime *Runtime) enqueueStreamingWorkflows(ctx context.Context) error {
|
|
workflows, err := runtime.store.ListWorkerWorkflows(ctx, nil, workerWorkflowQueryLimit)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
counts, err := runtime.store.CountWorkerRuns(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
settings, err := runtime.store.GetSettings(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return runtime.enqueueStreamingWorkflowsWithCounts(ctx, workflows, &counts, settings)
|
|
}
|
|
|
|
func (runtime *Runtime) enqueueStreamingWorkflowsWithSnapshots(ctx context.Context, workflows []domain.WorkflowRecord, runs []domain.RunSummary, settings *domain.SettingsRecord) error {
|
|
counts := store.WorkerRunCounts{
|
|
ActiveByWorkflow: map[string]int{},
|
|
InflightByWorkflow: map[string]int{},
|
|
}
|
|
for _, run := range runs {
|
|
switch run.Status {
|
|
case domain.RunStatusQueued:
|
|
counts.QueuedTotal++
|
|
counts.InflightByWorkflow[run.WorkflowID]++
|
|
case domain.RunStatusRunning, domain.RunStatusStopping:
|
|
counts.ActiveTotal++
|
|
counts.ActiveByWorkflow[run.WorkflowID]++
|
|
counts.InflightByWorkflow[run.WorkflowID]++
|
|
case domain.RunStatusWaitingForAudit:
|
|
counts.ActiveByWorkflow[run.WorkflowID]++
|
|
counts.InflightByWorkflow[run.WorkflowID]++
|
|
}
|
|
}
|
|
return runtime.enqueueStreamingWorkflowsWithCounts(ctx, workflows, &counts, settings)
|
|
}
|
|
|
|
func (runtime *Runtime) enqueueStreamingWorkflowsWithCounts(ctx context.Context, workflows []domain.WorkflowRecord, counts *store.WorkerRunCounts, settings *domain.SettingsRecord) error {
|
|
now := time.Now().UTC()
|
|
for _, workflow := range workflows {
|
|
if !shouldEnqueueScheduledRun(workflow, now) {
|
|
continue
|
|
}
|
|
if strings.TrimSpace(workflow.ProjectID) != "" {
|
|
project, err := runtime.store.GetProject(ctx, workflow.ProjectID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if project == nil {
|
|
fmt.Printf("[worker] skipping streaming workflow %s because project %s is missing\n", workflow.ID, workflow.ProjectID)
|
|
continue
|
|
}
|
|
}
|
|
inflightCount := counts.InflightByWorkflow[workflow.ID]
|
|
if inflightCount >= effectiveWorkflowConcurrencyLimit(&workflow, settings) {
|
|
continue
|
|
}
|
|
manifests, err := lightweight.GenerateWorkflowCodeManifests(runtime.workspaceRoot, workflow)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
run, err := runtime.store.CreateRun(ctx, workflow.ID, domain.RunTriggerScheduled)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if len(manifests) > 0 {
|
|
lightweight.MergeRunMetadata(run, manifests)
|
|
run.UpdatedAt = nowISO()
|
|
if err := runtime.store.UpsertRun(ctx, *run); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
counts.QueuedTotal++
|
|
counts.InflightByWorkflow[workflow.ID]++
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func runRecordsFromSummaries(summaries []domain.RunSummary) []domain.RunRecord {
|
|
runs := make([]domain.RunRecord, 0, len(summaries))
|
|
for _, summary := range summaries {
|
|
runs = append(runs, domain.RunRecord{
|
|
ID: summary.ID,
|
|
OrganizationID: summary.OrganizationID,
|
|
WorkflowID: summary.WorkflowID,
|
|
WorkflowName: summary.WorkflowName,
|
|
ProjectID: summary.ProjectID,
|
|
ProjectName: summary.ProjectName,
|
|
Trigger: summary.Trigger,
|
|
Attempt: summary.Attempt,
|
|
ParentRunID: summary.ParentRunID,
|
|
RestartFromNodeID: summary.RestartFromNodeID,
|
|
AvailableAt: summary.AvailableAt,
|
|
ExecutionLeaseOwner: summary.ExecutionLeaseOwner,
|
|
ExecutionLeaseExpiresAt: summary.ExecutionLeaseExpiresAt,
|
|
Status: summary.Status,
|
|
StopMode: summary.StopMode,
|
|
StopRequestedAt: summary.StopRequestedAt,
|
|
StartedAt: summary.StartedAt,
|
|
EndedAt: summary.EndedAt,
|
|
DurationMS: summary.DurationMS,
|
|
CurrentNodeID: summary.CurrentNodeID,
|
|
Error: summary.Error,
|
|
CreatedAt: summary.CreatedAt,
|
|
UpdatedAt: summary.UpdatedAt,
|
|
})
|
|
}
|
|
return runs
|
|
}
|
|
|
|
func (runtime *Runtime) processClaimedRun(ctx context.Context, claimed *domain.RunRecord) {
|
|
if claimed == nil {
|
|
return
|
|
}
|
|
defer runtime.unmarkRunActive(claimed.ID)
|
|
defer func() {
|
|
if err := runtime.store.ReleaseRunLease(context.Background(), claimed.ID, runtime.leaseOwner()); err != nil {
|
|
fmt.Printf("[worker] release run lease %s failed: %v\n", claimed.ID, err)
|
|
}
|
|
}()
|
|
if _, err := runtime.processRun(ctx, claimed); err != nil {
|
|
fmt.Printf("[worker] process run %s failed: %v\n", claimed.ID, err)
|
|
runtime.bestEffortTerminalizeRun(claimed.ID, err)
|
|
}
|
|
}
|
|
|
|
func (runtime *Runtime) bestEffortTerminalizeRun(runID string, cause error) {
|
|
if strings.TrimSpace(runID) == "" || cause == nil || errors.Is(cause, errRunLeaseLost) || errors.Is(cause, context.Canceled) {
|
|
return
|
|
}
|
|
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
|
|
defer cancel()
|
|
run, err := runtime.store.GetRun(ctx, runID)
|
|
if err != nil || run == nil || isTerminalRunStatus(run.Status) {
|
|
return
|
|
}
|
|
message := "Worker failed to process run: " + boundedDiagnostic(cause.Error())
|
|
if err := runtime.failReconciledRun(ctx, run, message); err != nil {
|
|
fmt.Printf("[worker] terminalize run %s failed: %v\n", runID, err)
|
|
}
|
|
}
|
|
|
|
func (runtime *Runtime) renewRunLeaseUntil(ctx context.Context, runID string, cancelExecution context.CancelCauseFunc) {
|
|
ttl := runtime.leaseTTL()
|
|
interval := ttl / 3
|
|
if interval <= 0 || interval > time.Minute {
|
|
interval = time.Minute
|
|
}
|
|
ticker := time.NewTicker(interval)
|
|
defer ticker.Stop()
|
|
for {
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case <-ticker.C:
|
|
renewCtx, cancelRenew := context.WithTimeout(ctx, 10*time.Second)
|
|
ok, err := runtime.store.RenewRunLease(renewCtx, runID, runtime.leaseOwner(), ttl)
|
|
cancelRenew()
|
|
if err != nil {
|
|
if ctx.Err() != nil {
|
|
return
|
|
}
|
|
fmt.Printf("[worker] renew run lease %s failed: %v\n", runID, err)
|
|
continue
|
|
}
|
|
if !ok {
|
|
fmt.Printf("[worker] renew run lease %s skipped: owner changed or run missing\n", runID)
|
|
if cancelExecution != nil {
|
|
cancelExecution(errRunLeaseLost)
|
|
}
|
|
return
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func (runtime *Runtime) tryMarkRunActive(runID string) bool {
|
|
if strings.TrimSpace(runID) == "" {
|
|
return false
|
|
}
|
|
runtime.processMu.Lock()
|
|
defer runtime.processMu.Unlock()
|
|
if runtime.activeRuns == nil {
|
|
runtime.activeRuns = map[string]struct{}{}
|
|
}
|
|
if _, ok := runtime.activeRuns[runID]; ok {
|
|
return false
|
|
}
|
|
runtime.activeRuns[runID] = struct{}{}
|
|
return true
|
|
}
|
|
|
|
func (runtime *Runtime) unmarkRunActive(runID string) {
|
|
runtime.processMu.Lock()
|
|
defer runtime.processMu.Unlock()
|
|
delete(runtime.activeRuns, runID)
|
|
}
|
|
|
|
func (runtime *Runtime) orphanMinimumRuntimeAge() time.Duration {
|
|
if runtime == nil || runtime.orphanMinimumAge <= 0 {
|
|
return 24 * time.Hour
|
|
}
|
|
return runtime.orphanMinimumAge
|
|
}
|
|
|
|
func (runtime *Runtime) reconcileOrphanedRuntimes(ctx context.Context) {
|
|
now := time.Now().UTC()
|
|
interval := runtime.orphanCheckInterval
|
|
if interval <= 0 {
|
|
interval = 5 * time.Minute
|
|
}
|
|
lastCheck := runtime.lastOrphanCheckNano.Load()
|
|
if lastCheck > 0 && now.Sub(time.Unix(0, lastCheck)) < interval {
|
|
return
|
|
}
|
|
if !runtime.lastOrphanCheckNano.CompareAndSwap(lastCheck, now.UnixNano()) {
|
|
return
|
|
}
|
|
reconciler, ok := runtime.launcher.(orphanRuntimeReconciler)
|
|
if !ok {
|
|
runtime.metrics.RecordOrphanRuntimes(0)
|
|
return
|
|
}
|
|
summaries, err := runtime.store.ListRunSummaries(ctx, "")
|
|
if err != nil {
|
|
fmt.Printf("[worker] list run ids for orphan reconciliation failed: %v\n", err)
|
|
return
|
|
}
|
|
knownRunIDs := make(map[string]struct{}, len(summaries))
|
|
for _, summary := range summaries {
|
|
knownRunIDs[summary.ID] = struct{}{}
|
|
}
|
|
orphans, err := reconciler.ReconcileOrphanedRuntimes(ctx, knownRunIDs, now.Add(-runtime.orphanMinimumRuntimeAge()), runtime.orphanCleanup)
|
|
if err != nil {
|
|
fmt.Printf("[worker] orphan runtime reconciliation failed: %v\n", err)
|
|
return
|
|
}
|
|
runtime.metrics.RecordOrphanRuntimes(len(orphans))
|
|
}
|
|
|
|
func (runtime *Runtime) ProcessAvailable(ctx context.Context) error {
|
|
startedAt := time.Now()
|
|
var processErr error
|
|
defer func() {
|
|
runtime.metrics.RecordProcessAvailable(uint64(time.Since(startedAt).Milliseconds()), processErr)
|
|
}()
|
|
settings, err := runtime.store.GetSettings(ctx)
|
|
if err != nil {
|
|
processErr = err
|
|
return processErr
|
|
}
|
|
runtime.updateEffectivePollInterval(settings)
|
|
reconcileRuns, err := runtime.store.ListReconcileRunSummaries(ctx, workerReconcileRunLimit)
|
|
if err != nil {
|
|
processErr = err
|
|
return processErr
|
|
}
|
|
if err := runtime.reconcileActiveRunsWithSummaries(ctx, reconcileRuns); err != nil {
|
|
processErr = err
|
|
return processErr
|
|
}
|
|
queuedPage, err := runtime.store.ListRunSummariesPage(ctx, domain.RunSummaryQuery{
|
|
Status: domain.RunStatusQueued,
|
|
Limit: workerQueuedRunLimit,
|
|
})
|
|
if err != nil {
|
|
processErr = err
|
|
return processErr
|
|
}
|
|
counts, err := runtime.store.CountWorkerRuns(ctx)
|
|
if err != nil {
|
|
processErr = err
|
|
return processErr
|
|
}
|
|
workflowIDs := make([]string, 0, len(queuedPage.Items))
|
|
workflowSet := make(map[string]struct{}, len(queuedPage.Items))
|
|
for _, run := range queuedPage.Items {
|
|
if _, exists := workflowSet[run.WorkflowID]; exists {
|
|
continue
|
|
}
|
|
workflowSet[run.WorkflowID] = struct{}{}
|
|
workflowIDs = append(workflowIDs, run.WorkflowID)
|
|
}
|
|
workflows, err := runtime.store.ListWorkerWorkflows(ctx, workflowIDs, workerWorkflowQueryLimit)
|
|
if err != nil {
|
|
processErr = err
|
|
return processErr
|
|
}
|
|
if err := runtime.enqueueStreamingWorkflowsWithCounts(ctx, workflows, &counts, settings); err != nil {
|
|
processErr = err
|
|
return processErr
|
|
}
|
|
runtime.reconcileOrphanedRuntimes(ctx)
|
|
queuedRuns := runRecordsFromSummaries(queuedPage.Items)
|
|
selectedRunIDs := selectQueuedRunsForDispatchSnapshot(
|
|
queuedRuns,
|
|
workflows,
|
|
settings,
|
|
counts.ActiveByWorkflow,
|
|
counts.ActiveTotal,
|
|
)
|
|
runtime.metrics.RecordDispatch(counts.QueuedTotal, counts.ActiveTotal, len(selectedRunIDs))
|
|
for _, runID := range selectedRunIDs {
|
|
if !runtime.tryMarkRunActive(runID) {
|
|
continue
|
|
}
|
|
claimed, err := runtime.store.ClaimQueuedRun(ctx, runID, runtime.leaseOwner(), runtime.leaseTTL())
|
|
if err != nil {
|
|
runtime.unmarkRunActive(runID)
|
|
fmt.Printf("[worker] claim queued run %s failed: %v\n", runID, err)
|
|
continue
|
|
}
|
|
if claimed == nil {
|
|
runtime.unmarkRunActive(runID)
|
|
continue
|
|
}
|
|
go runtime.processClaimedRun(ctx, claimed)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func chooseImage(node domain.WorkflowNode, fallback string) string {
|
|
if digestPinnedImage(node.Data.OperatorImageID) {
|
|
return strings.TrimSpace(node.Data.OperatorImageID)
|
|
}
|
|
if node.Data.Image != "" {
|
|
if strings.Contains(node.Data.Image, "droboticflow-operator-runner") && fallback != "" {
|
|
return fallback
|
|
}
|
|
return node.Data.Image
|
|
}
|
|
return fallback
|
|
}
|
|
|
|
func digestPinnedImage(value string) bool {
|
|
trimmed := strings.TrimSpace(value)
|
|
if strings.ContainsAny(trimmed, " \t\r\n") {
|
|
return false
|
|
}
|
|
parts := strings.Split(trimmed, "@sha256:")
|
|
if len(parts) != 2 || parts[0] == "" || len(parts[1]) != 64 {
|
|
return false
|
|
}
|
|
for _, character := range parts[1] {
|
|
if (character < '0' || character > '9') && (character < 'a' || character > 'f') && (character < 'A' || character > 'F') {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
|
|
func chooseValue(value string, fallback string) string {
|
|
if value == "" {
|
|
return fallback
|
|
}
|
|
return value
|
|
}
|
|
|
|
func chooseDataSource(value string, fallback string) (string, error) {
|
|
normalized := strings.ToUpper(strings.TrimSpace(value))
|
|
if normalized == "" {
|
|
return fallback, nil
|
|
}
|
|
switch normalized {
|
|
case "INLINE", "LOCAL", "BOS", "UPSTREAM":
|
|
return normalized, nil
|
|
default:
|
|
return "", fmt.Errorf("DATA_SOURCE must be INLINE, LOCAL, BOS, or UPSTREAM")
|
|
}
|
|
}
|
|
|
|
const missingJobRecoveryLogPrefix = "Kubernetes job is missing; recovery attempt "
|
|
|
|
func (runtime *Runtime) missingJobRecoveryLimit() int {
|
|
if runtime == nil || runtime.missingMaxRecoveries <= 0 {
|
|
return 3
|
|
}
|
|
return runtime.missingMaxRecoveries
|
|
}
|
|
|
|
func (runtime *Runtime) pendingJobFailureThreshold() time.Duration {
|
|
if runtime == nil || runtime.pendingFailureAfter <= 0 {
|
|
return 10 * time.Minute
|
|
}
|
|
return runtime.pendingFailureAfter
|
|
}
|
|
|
|
func missingJobRecoveryCount(execution *domain.NodeExecutionRecord) int {
|
|
if execution == nil {
|
|
return 0
|
|
}
|
|
count := 0
|
|
for _, entry := range execution.Logs {
|
|
if strings.HasPrefix(strings.TrimSpace(entry.Message), missingJobRecoveryLogPrefix) {
|
|
count++
|
|
}
|
|
}
|
|
return count
|
|
}
|
|
|
|
func boundedDiagnostic(value string) string {
|
|
trimmed := strings.TrimSpace(value)
|
|
const maximum = 4096
|
|
if len(trimmed) <= maximum {
|
|
return trimmed
|
|
}
|
|
return trimmed[:maximum] + "..."
|
|
}
|
|
|
|
func (runtime *Runtime) failReconciledRun(ctx context.Context, run *domain.RunRecord, message string, structured ...domain.RunFailure) error {
|
|
if run == nil {
|
|
return nil
|
|
}
|
|
message = boundedDiagnostic(message)
|
|
if strings.TrimSpace(message) == "" {
|
|
message = "Kubernetes job failed"
|
|
}
|
|
if _, execution := findNodeExecution(run, run.CurrentNodeID); execution != nil {
|
|
execution.Status = domain.NodeExecutionStatusFailed
|
|
execution.EndedAt = nowISO()
|
|
execution.Error = message
|
|
execution.Logs = append(execution.Logs, logEntry("ERROR", message, execution.NodeLabel))
|
|
applyExecution(run, *execution)
|
|
}
|
|
run.Status = domain.RunStatusFailed
|
|
run.EndedAt = nowISO()
|
|
run.Error = message
|
|
failure := domain.RunFailure{Code: "worker_execution_failed", Stage: "execution", NodeID: run.CurrentNodeID, Message: message, Retryable: true, Suggestion: "查看节点日志后重试失败子链路"}
|
|
if len(structured) > 0 {
|
|
failure = structured[0]
|
|
failure.Message = message
|
|
if strings.TrimSpace(failure.NodeID) == "" {
|
|
failure.NodeID = run.CurrentNodeID
|
|
}
|
|
}
|
|
run.Failure = &failure
|
|
run.UpdatedAt = nowISO()
|
|
if err := runtime.store.UpsertRun(ctx, *run); err != nil {
|
|
return err
|
|
}
|
|
workflow, err := runtime.store.GetWorkflow(ctx, run.WorkflowID)
|
|
if err != nil || workflow == nil {
|
|
return err
|
|
}
|
|
return runtime.store.UpdateWorkflowRunState(ctx, workflow.ID, domain.WorkflowStatusFailed, run.ID, nowISO())
|
|
}
|
|
|
|
func (runtime *Runtime) recoverMissingJob(ctx context.Context, run *domain.RunRecord, execution *domain.NodeExecutionRecord) error {
|
|
if run != nil && run.Status == domain.RunStatusStopping {
|
|
workflow, err := runtime.store.GetWorkflow(ctx, run.WorkflowID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
executionMap := make(map[string]domain.NodeExecutionRecord, len(run.NodeExecutions))
|
|
for _, item := range run.NodeExecutions {
|
|
executionMap[item.NodeID] = item
|
|
}
|
|
_, err = runtime.finalizeCancelledRun(ctx, workflow, run, executionMap, "用户已停止工作流")
|
|
return err
|
|
}
|
|
if execution == nil {
|
|
run.Status = domain.RunStatusQueued
|
|
run.StartedAt = ""
|
|
run.CurrentNodeID = ""
|
|
run.AvailableAt = nowISO()
|
|
run.UpdatedAt = nowISO()
|
|
return runtime.store.UpsertRun(ctx, *run)
|
|
}
|
|
terminateCtx, cancelTerminate := context.WithTimeout(context.Background(), 30*time.Second)
|
|
err := runtime.launcher.TerminateRun(terminateCtx, "", run.ID)
|
|
cancelTerminate()
|
|
if err != nil {
|
|
return fmt.Errorf("clean up missing job runtime for run %s: %w", run.ID, err)
|
|
}
|
|
recoveryAttempt := missingJobRecoveryCount(execution) + 1
|
|
if recoveryAttempt > runtime.missingJobRecoveryLimit() {
|
|
return runtime.failReconciledRun(ctx, run, fmt.Sprintf("Kubernetes job remained missing after %d recovery attempts", runtime.missingJobRecoveryLimit()), domain.RunFailure{
|
|
Code: "kubernetes_job_missing", Stage: "execution", Retryable: true, Suggestion: "检查 Job 生命周期与集群事件后从失败节点继续",
|
|
})
|
|
}
|
|
message := fmt.Sprintf("%s%d/%d; node execution was re-queued", missingJobRecoveryLogPrefix, recoveryAttempt, runtime.missingJobRecoveryLimit())
|
|
execution.Status = domain.NodeExecutionStatusFailed
|
|
execution.EndedAt = nowISO()
|
|
execution.Error = message
|
|
execution.Logs = append(execution.Logs, logEntry("WARN", message, execution.NodeLabel))
|
|
applyExecution(run, *execution)
|
|
backoff := time.Duration(15*(1<<(recoveryAttempt-1))) * time.Second
|
|
run.Status = domain.RunStatusQueued
|
|
run.StartedAt = ""
|
|
run.CurrentNodeID = ""
|
|
run.AvailableAt = time.Now().UTC().Add(backoff).Format(time.RFC3339Nano)
|
|
run.UpdatedAt = nowISO()
|
|
return runtime.store.UpsertRun(ctx, *run)
|
|
}
|
|
|
|
func (runtime *Runtime) recoverSucceededJob(ctx context.Context, run *domain.RunRecord, inspection *k8s.JobInspection) error {
|
|
workflow, err := runtime.store.GetWorkflow(ctx, run.WorkflowID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if workflow == nil {
|
|
return fmt.Errorf("workflow %s not found while reconciling run %s", run.WorkflowID, run.ID)
|
|
}
|
|
node := domain.GetNodeByID(workflow.Nodes, run.CurrentNodeID)
|
|
if node == nil {
|
|
return fmt.Errorf("node %s not found while reconciling run %s", run.CurrentNodeID, run.ID)
|
|
}
|
|
executionMap := map[string]domain.NodeExecutionRecord{}
|
|
for _, item := range run.NodeExecutions {
|
|
executionMap[item.NodeID] = item
|
|
}
|
|
inputLineage := gatherInputLineage(*workflow, executionMap, node.ID)
|
|
inputArtifacts := inputArtifactsFromLineage(inputLineage)
|
|
_, existing := findNodeExecution(run, node.ID)
|
|
execution := domain.NodeExecutionRecord{
|
|
NodeID: node.ID,
|
|
NodeLabel: node.Data.Label,
|
|
Attempt: runExecutionAttempt(run),
|
|
Status: domain.NodeExecutionStatusRunning,
|
|
StartedAt: firstNonEmptyString(run.StartedAt, run.CurrentRunAt, run.CreatedAt),
|
|
Logs: []domain.LogEntryRecord{},
|
|
Artifacts: []domain.ExecutionArtifact{},
|
|
DataRefs: []domain.DataRefRecord{},
|
|
ResourceSeries: []domain.ResourceSample{},
|
|
}
|
|
if existing != nil {
|
|
execution = *existing
|
|
}
|
|
applyNodeExecutionSnapshot(&execution, *node, inputArtifacts, inputLineage)
|
|
parsed, err := parseExecutionOutput(inspection.Output, node.Data.Label)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
execution.Status = domain.NodeExecutionStatusCompleted
|
|
execution.EndedAt = nowISO()
|
|
execution.Logs = append(execution.Logs, parsed.Logs...)
|
|
execution.Artifacts = parsed.Artifacts
|
|
execution.DataRefs = parsed.DataRefs
|
|
execution.ResourceSeries = parsed.ResourceSeries
|
|
execution.Error = ""
|
|
executionMap[node.ID] = execution
|
|
applyExecution(run, execution)
|
|
if run.Status == domain.RunStatusStopping {
|
|
_, err := runtime.finalizeCancelledRun(ctx, workflow, run, executionMap, "用户已停止工作流")
|
|
return err
|
|
}
|
|
if allAllowedNodesCompleted(workflow, run, executionMap) {
|
|
_, err := runtime.finalizeCompletedRun(ctx, workflow, run)
|
|
return err
|
|
}
|
|
run.Status = domain.RunStatusQueued
|
|
run.StartedAt = ""
|
|
run.CurrentNodeID = ""
|
|
run.AvailableAt = nowISO()
|
|
run.UpdatedAt = nowISO()
|
|
return runtime.store.UpsertRun(ctx, *run)
|
|
}
|
|
|
|
func inspectionIsStuck(inspection *k8s.JobInspection, fallbackStartedAt string, threshold time.Duration) bool {
|
|
if inspection == nil || strings.TrimSpace(inspection.BlockingReason) == "" {
|
|
return false
|
|
}
|
|
startedAt, ok := parseRFC3339Nano(firstNonEmptyString(inspection.CreatedAt, fallbackStartedAt))
|
|
return ok && time.Since(startedAt) >= threshold
|
|
}
|
|
|
|
func nodeExecutionStartedAt(execution *domain.NodeExecutionRecord) string {
|
|
if execution == nil {
|
|
return ""
|
|
}
|
|
return execution.StartedAt
|
|
}
|
|
|
|
func (runtime *Runtime) reconcileActiveRun(ctx context.Context, candidate domain.RunSummary) (bool, error) {
|
|
run, err := runtime.store.GetRun(ctx, candidate.ID)
|
|
if err != nil || run == nil {
|
|
return false, err
|
|
}
|
|
var activeExecution *domain.NodeExecutionRecord
|
|
if strings.TrimSpace(run.CurrentNodeID) != "" {
|
|
_, activeExecution = findNodeExecution(run, run.CurrentNodeID)
|
|
}
|
|
if run.Status == domain.RunStatusQueued {
|
|
if activeExecution == nil || activeExecution.Status != domain.NodeExecutionStatusRunning || !isStaleQueuedNodeExecution(*activeExecution, 2*time.Minute) {
|
|
return false, nil
|
|
}
|
|
}
|
|
if run.Status == domain.RunStatusRunning && activeExecution != nil && activeExecution.Status == domain.NodeExecutionStatusRunning && !isStaleQueuedNodeExecution(*activeExecution, 2*time.Minute) {
|
|
return false, nil
|
|
}
|
|
if strings.TrimSpace(run.CurrentNodeID) == "" {
|
|
if updatedAt, ok := parseRFC3339Nano(run.UpdatedAt); ok && time.Since(updatedAt) < 2*time.Minute {
|
|
return false, nil
|
|
}
|
|
return false, runtime.recoverMissingJob(ctx, run, nil)
|
|
}
|
|
inspection, err := runtime.launcher.Inspect(ctx, run.ID, run.CurrentNodeID)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
switch inspection.State {
|
|
case k8s.JobStateMissing:
|
|
return true, runtime.recoverMissingJob(ctx, run, activeExecution)
|
|
case k8s.JobStateSucceeded:
|
|
return false, runtime.recoverSucceededJob(ctx, run, inspection)
|
|
case k8s.JobStateFailed:
|
|
message := "Kubernetes job failed"
|
|
if output := boundedDiagnostic(inspection.Output); output != "" {
|
|
message += ": " + output
|
|
}
|
|
return true, runtime.failReconciledRun(ctx, run, message, domain.RunFailure{
|
|
Code: "kubernetes_job_failed", Stage: "execution", Retryable: true, Suggestion: "查看 Pod 日志和事件后从失败节点继续",
|
|
})
|
|
case k8s.JobStatePending:
|
|
if inspectionIsStuck(inspection, firstNonEmptyString(nodeExecutionStartedAt(activeExecution), run.StartedAt), runtime.pendingJobFailureThreshold()) {
|
|
message := fmt.Sprintf("Kubernetes job remained blocked in %s for at least %s", inspection.BlockingReason, runtime.pendingJobFailureThreshold())
|
|
if details := boundedDiagnostic(inspection.BlockingMessage); details != "" {
|
|
message += ": " + details
|
|
}
|
|
failureCode := "kubernetes_runtime_blocked"
|
|
if inspection.BlockingReason == "ImagePullBackOff" || inspection.BlockingReason == "ErrImagePull" || inspection.BlockingReason == "InvalidImageName" {
|
|
failureCode = "kubernetes_image_pull_blocked"
|
|
} else if inspection.BlockingReason == "Unschedulable" {
|
|
failureCode = "kubernetes_unschedulable"
|
|
}
|
|
return true, runtime.failReconciledRun(ctx, run, message, domain.RunFailure{
|
|
Code: failureCode, Stage: "scheduling", Retryable: true, Suggestion: "修复镜像或调度条件后从失败节点继续",
|
|
})
|
|
}
|
|
return strings.TrimSpace(inspection.BlockingReason) != "", nil
|
|
default:
|
|
return false, nil
|
|
}
|
|
}
|
|
|
|
func (runtime *Runtime) reconcileActiveRunsWithSummaries(ctx context.Context, runs []domain.RunSummary) error {
|
|
stuckCount := 0
|
|
for _, candidate := range runs {
|
|
if candidate.Status != domain.RunStatusRunning && candidate.Status != domain.RunStatusStopping && (candidate.Status != domain.RunStatusQueued || strings.TrimSpace(candidate.CurrentNodeID) == "") {
|
|
continue
|
|
}
|
|
stuck, err := runtime.reconcileActiveRun(ctx, candidate)
|
|
if stuck {
|
|
stuckCount++
|
|
}
|
|
if err != nil {
|
|
// A single malformed or temporarily unavailable run must not prevent
|
|
// unrelated runs from reconciling and the queue from dispatching.
|
|
fmt.Printf("[worker] reconcile run %s failed: %v\n", candidate.ID, err)
|
|
}
|
|
}
|
|
runtime.metrics.RecordStuckRuns(stuckCount)
|
|
return nil
|
|
}
|
|
|
|
func (runtime *Runtime) reconcileActiveRuns(ctx context.Context) error {
|
|
runs, err := runtime.store.ListReconcileRunSummaries(ctx, workerReconcileRunLimit)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return runtime.reconcileActiveRunsWithSummaries(ctx, runs)
|
|
}
|
|
|
|
func upsertNodeHyperparam(items []domain.HyperParam, key string, value string) []domain.HyperParam {
|
|
for index := range items {
|
|
if items[index].Key == key {
|
|
items[index].Value = value
|
|
return items
|
|
}
|
|
}
|
|
return append(items, domain.HyperParam{Key: key, Value: value})
|
|
}
|
|
|
|
func bosSourcePath(bucket string, prefix string) string {
|
|
normalizedBucket := strings.TrimSpace(bucket)
|
|
if normalizedBucket == "" {
|
|
return ""
|
|
}
|
|
sourcePath := "bos://" + normalizedBucket
|
|
if normalizedPrefix := strings.Trim(strings.TrimSpace(prefix), "/"); normalizedPrefix != "" {
|
|
sourcePath += "/" + normalizedPrefix
|
|
}
|
|
return sourcePath
|
|
}
|
|
|
|
func (runtime *Runtime) defaultStorageEndpoint() string {
|
|
return strings.TrimSpace(runtime.globalBOSEndpoint)
|
|
}
|
|
|
|
func (runtime *Runtime) storageConnectionEndpoint(connection *domain.StorageConnectionRecord) string {
|
|
if connection == nil {
|
|
return runtime.defaultStorageEndpoint()
|
|
}
|
|
if endpoint := strings.TrimSpace(connection.Endpoint); endpoint != "" {
|
|
return endpoint
|
|
}
|
|
return runtime.defaultStorageEndpoint()
|
|
}
|
|
|
|
func nodeHyperparamsMap(node domain.WorkflowNode) map[string]string {
|
|
values := make(map[string]string, len(node.Data.Hyperparams))
|
|
for _, item := range node.Data.Hyperparams {
|
|
key := strings.TrimSpace(item.Key)
|
|
if key == "" {
|
|
continue
|
|
}
|
|
values[key] = item.Value
|
|
}
|
|
return values
|
|
}
|
|
|
|
func chooseScanMode(node domain.WorkflowNode, params map[string]string) (string, error) {
|
|
mode := strings.ToUpper(strings.TrimSpace(params["SCAN_MODE"]))
|
|
switch mode {
|
|
case "FULL", "INCREMENTAL":
|
|
return mode, nil
|
|
case "":
|
|
if node.Data.Label == "BOS 增量扫描" {
|
|
return "INCREMENTAL", nil
|
|
}
|
|
return "", nil
|
|
default:
|
|
return "", fmt.Errorf("SCAN_MODE must be FULL or INCREMENTAL")
|
|
}
|
|
}
|
|
|
|
func chooseIncrementalUnit(params map[string]string) (string, error) {
|
|
switch strings.ToUpper(strings.TrimSpace(params["INCREMENTAL_UNIT"])) {
|
|
case "":
|
|
return "OBJECT", nil
|
|
case "OBJECT":
|
|
return "OBJECT", nil
|
|
case "DIRECTORY":
|
|
return "DIRECTORY", nil
|
|
default:
|
|
return "", fmt.Errorf("INCREMENTAL_UNIT must be OBJECT or DIRECTORY")
|
|
}
|
|
}
|
|
|
|
func chooseDirectoryDepth(params map[string]string) (int, error) {
|
|
raw := strings.TrimSpace(params["INCREMENTAL_DIRECTORY_DEPTH"])
|
|
if raw == "" {
|
|
return 1, nil
|
|
}
|
|
parsed, err := strconv.Atoi(raw)
|
|
if err != nil || parsed <= 0 {
|
|
return 0, fmt.Errorf("INCREMENTAL_DIRECTORY_DEPTH must be a positive integer")
|
|
}
|
|
return parsed, nil
|
|
}
|
|
|
|
func scanWorkflowSourceNode(node domain.WorkflowNode) bool {
|
|
label := strings.TrimSpace(node.Data.Label)
|
|
return node.ID == "scan-source" || label == "BOS 增量扫描"
|
|
}
|
|
|
|
func sourceFingerprint(node domain.WorkflowNode, params map[string]string, dataSource string, scanMode string, incrementalUnit string, directoryDepth int) string {
|
|
return domain.ComputeSourceFingerprint(domain.SourceFingerprintInput{
|
|
ScanMode: scanMode,
|
|
IncrementalUnit: incrementalUnit,
|
|
DirectoryDepth: directoryDepth,
|
|
DataSource: dataSource,
|
|
SourcePath: params["SOURCE_PATH"],
|
|
Bucket: node.Data.Bucket,
|
|
Prefix: node.Data.Prefix,
|
|
Pattern: node.Data.Pattern,
|
|
Endpoint: params["BOS_ENDPOINT"],
|
|
})
|
|
}
|
|
|
|
func stringList(value any) []string {
|
|
if value == nil {
|
|
return nil
|
|
}
|
|
if items, ok := value.([]string); ok {
|
|
result := make([]string, 0, len(items))
|
|
for _, item := range items {
|
|
trimmed := strings.TrimSpace(item)
|
|
if trimmed != "" {
|
|
result = append(result, trimmed)
|
|
}
|
|
}
|
|
return result
|
|
}
|
|
items, ok := value.([]any)
|
|
if ok {
|
|
result := make([]string, 0, len(items))
|
|
for _, item := range items {
|
|
trimmed := strings.TrimSpace(fmt.Sprint(item))
|
|
if trimmed != "" {
|
|
result = append(result, trimmed)
|
|
}
|
|
}
|
|
return result
|
|
}
|
|
reflected := reflect.ValueOf(value)
|
|
if reflected.Kind() != reflect.Slice && reflected.Kind() != reflect.Array {
|
|
return nil
|
|
}
|
|
result := make([]string, 0, reflected.Len())
|
|
for index := 0; index < reflected.Len(); index++ {
|
|
trimmed := strings.TrimSpace(fmt.Sprint(reflected.Index(index).Interface()))
|
|
if trimmed != "" {
|
|
result = append(result, trimmed)
|
|
}
|
|
}
|
|
return result
|
|
}
|
|
|
|
func (runtime *Runtime) prepareExecutionNode(ctx context.Context, workflow *domain.WorkflowRecord, node domain.WorkflowNode) (domain.WorkflowNode, error) {
|
|
effectiveWorkflow, err := runtime.resolveScanWorkflow(ctx, workflow)
|
|
if err != nil {
|
|
return node, err
|
|
}
|
|
copy, err := scanExecutionNode(effectiveWorkflow, node)
|
|
if err != nil {
|
|
return node, err
|
|
}
|
|
copy, err = runtime.resolveNodeStorageConnection(ctx, copy)
|
|
if err != nil {
|
|
return node, err
|
|
}
|
|
copy, err = runtime.resolveNodeMongoConnection(ctx, copy)
|
|
if err != nil {
|
|
return node, err
|
|
}
|
|
params := nodeHyperparamsMap(copy)
|
|
dataSource, err := chooseDataSource(params["DATA_SOURCE"], "BOS")
|
|
if err != nil {
|
|
return node, err
|
|
}
|
|
scanMode, err := chooseScanMode(copy, params)
|
|
if err != nil {
|
|
return node, err
|
|
}
|
|
incrementalUnit, err := chooseIncrementalUnit(params)
|
|
if err != nil {
|
|
return node, err
|
|
}
|
|
directoryDepth, err := chooseDirectoryDepth(params)
|
|
if err != nil {
|
|
return node, err
|
|
}
|
|
if scanMode != "INCREMENTAL" || workflow == nil {
|
|
return copy, nil
|
|
}
|
|
fingerprint := sourceFingerprint(copy, params, dataSource, scanMode, incrementalUnit, directoryDepth)
|
|
state, err := runtime.store.GetSourceState(ctx, workflow.ID, copy.ID, fingerprint)
|
|
if err != nil {
|
|
return copy, err
|
|
}
|
|
payload := map[string]any{
|
|
"sourceFingerprint": fingerprint,
|
|
"scanMode": scanMode,
|
|
"incrementalUnit": incrementalUnit,
|
|
"directoryDepth": directoryDepth,
|
|
"dataSource": dataSource,
|
|
"sourcePath": strings.TrimSpace(params["SOURCE_PATH"]),
|
|
"bucket": strings.TrimSpace(copy.Data.Bucket),
|
|
"prefix": strings.TrimSpace(copy.Data.Prefix),
|
|
"pattern": strings.TrimSpace(copy.Data.Pattern),
|
|
"seenFingerprints": []string{},
|
|
}
|
|
if state != nil {
|
|
payload["seenFingerprints"] = append([]string(nil), state.SeenFingerprints...)
|
|
payload["lastRunId"] = state.LastRunID
|
|
payload["lastListedCount"] = state.LastListedCount
|
|
payload["lastNewCount"] = state.LastNewCount
|
|
payload["lastScanCompletedAt"] = state.LastScanCompletedAt
|
|
}
|
|
encoded, err := json.Marshal(payload)
|
|
if err != nil {
|
|
return copy, err
|
|
}
|
|
copy.Data.Hyperparams = append([]domain.HyperParam(nil), copy.Data.Hyperparams...)
|
|
copy.Data.Hyperparams = upsertNodeHyperparam(copy.Data.Hyperparams, "SCAN_MODE", scanMode)
|
|
copy.Data.Hyperparams = upsertNodeHyperparam(copy.Data.Hyperparams, "INCREMENTAL_UNIT", incrementalUnit)
|
|
copy.Data.Hyperparams = upsertNodeHyperparam(copy.Data.Hyperparams, "INCREMENTAL_DIRECTORY_DEPTH", strconv.Itoa(directoryDepth))
|
|
copy.Data.Hyperparams = upsertNodeHyperparam(copy.Data.Hyperparams, "SOURCE_FINGERPRINT", fingerprint)
|
|
copy.Data.Hyperparams = upsertNodeHyperparam(copy.Data.Hyperparams, "SOURCE_STATE_JSON", string(encoded))
|
|
return copy, nil
|
|
}
|
|
|
|
func (runtime *Runtime) resolveScanWorkflow(ctx context.Context, workflow *domain.WorkflowRecord) (*domain.WorkflowRecord, error) {
|
|
if workflow == nil || workflow.ScanConfig == nil || strings.TrimSpace(workflow.ScanConfig.ConnectionID) == "" {
|
|
return workflow, nil
|
|
}
|
|
connection, err := runtime.store.GetStorageConnection(ctx, workflow.ScanConfig.ConnectionID)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if connection == nil {
|
|
return nil, fmt.Errorf("storage connection %s not found", workflow.ScanConfig.ConnectionID)
|
|
}
|
|
copy := *workflow
|
|
scanConfig := *workflow.ScanConfig
|
|
scanConfig.Endpoint = runtime.storageConnectionEndpoint(connection)
|
|
if strings.TrimSpace(scanConfig.AccessKeyID) == "" {
|
|
scanConfig.AccessKeyID = connection.AccessKeyID
|
|
}
|
|
if strings.TrimSpace(scanConfig.SecretAccessKey) == "" {
|
|
scanConfig.SecretAccessKey = connection.SecretAccessKey
|
|
}
|
|
if strings.TrimSpace(scanConfig.SourcePath) == "" {
|
|
dataSource, err := chooseDataSource(scanConfig.DataSource, "BOS")
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if dataSource == "BOS" {
|
|
scanConfig.SourcePath = bosSourcePath(scanConfig.Bucket, scanConfig.Prefix)
|
|
}
|
|
}
|
|
copy.ScanConfig = &scanConfig
|
|
return ©, nil
|
|
}
|
|
|
|
func (runtime *Runtime) resolveNodeStorageConnection(ctx context.Context, node domain.WorkflowNode) (domain.WorkflowNode, error) {
|
|
connectionID := strings.TrimSpace(node.Data.ConnectionID)
|
|
if connectionID == "" {
|
|
return node, nil
|
|
}
|
|
connection, err := runtime.store.GetStorageConnection(ctx, connectionID)
|
|
if err != nil {
|
|
return node, err
|
|
}
|
|
if connection == nil {
|
|
return node, fmt.Errorf("storage connection %s not found", connectionID)
|
|
}
|
|
copy := node
|
|
copy.Data.Hyperparams = append([]domain.HyperParam(nil), node.Data.Hyperparams...)
|
|
sourcePath := strings.TrimSpace(nodeHyperparamsMap(copy)["SOURCE_PATH"])
|
|
if sourcePath == "" {
|
|
sourcePath = bosSourcePath(copy.Data.Bucket, copy.Data.Prefix)
|
|
}
|
|
copy.Data.Hyperparams = upsertNodeHyperparam(copy.Data.Hyperparams, "DATA_SOURCE", "BOS")
|
|
copy.Data.Hyperparams = upsertNodeHyperparam(copy.Data.Hyperparams, "SOURCE_PATH", sourcePath)
|
|
copy.Data.Hyperparams = upsertNodeHyperparam(copy.Data.Hyperparams, "BOS_ENDPOINT", runtime.storageConnectionEndpoint(connection))
|
|
copy.Data.Hyperparams = upsertNodeHyperparam(copy.Data.Hyperparams, "BOS_ACCESS_KEY_ID", connection.AccessKeyID)
|
|
copy.Data.Hyperparams = upsertNodeHyperparam(copy.Data.Hyperparams, "BOS_SECRET_ACCESS_KEY", connection.SecretAccessKey)
|
|
return copy, nil
|
|
}
|
|
|
|
func (runtime *Runtime) resolveNodeMongoConnection(ctx context.Context, node domain.WorkflowNode) (domain.WorkflowNode, error) {
|
|
connectionID := strings.TrimSpace(nodeHyperparamsMap(node)["MONGO_CONNECTION_ID"])
|
|
if connectionID == "" {
|
|
return node, nil
|
|
}
|
|
connection, err := runtime.store.GetMongoConnection(ctx, connectionID)
|
|
if err != nil {
|
|
return node, err
|
|
}
|
|
if connection == nil {
|
|
return node, fmt.Errorf("mongo connection %s not found", connectionID)
|
|
}
|
|
copy := node
|
|
copy.Data.Hyperparams = append([]domain.HyperParam(nil), node.Data.Hyperparams...)
|
|
copy.Data.Hyperparams = upsertNodeHyperparam(copy.Data.Hyperparams, "MONGO_URI", strings.TrimSpace(connection.URI))
|
|
copy.Data.Hyperparams = upsertNodeHyperparam(copy.Data.Hyperparams, "MONGO_DB", strings.TrimSpace(connection.Database))
|
|
copy.Data.Hyperparams = upsertNodeHyperparam(copy.Data.Hyperparams, "MONGO_TARGET", "business")
|
|
return copy, nil
|
|
}
|
|
|
|
func checkpointArtifact(execution domain.NodeExecutionRecord) *domain.ExecutionArtifact {
|
|
for index := range execution.Artifacts {
|
|
if execution.Artifacts[index].Name == "source-checkpoint.json" {
|
|
return &execution.Artifacts[index]
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func sourceCheckpointPayload(artifact *domain.ExecutionArtifact) (map[string]any, error) {
|
|
if artifact == nil {
|
|
return nil, nil
|
|
}
|
|
path := strings.TrimSpace(artifact.Path)
|
|
if path != "" {
|
|
data, err := os.ReadFile(path)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("read source checkpoint %s: %w", path, err)
|
|
}
|
|
var payload map[string]any
|
|
if err := json.Unmarshal(data, &payload); err != nil {
|
|
return nil, fmt.Errorf("parse source checkpoint %s: %w", path, err)
|
|
}
|
|
if len(payload) == 0 {
|
|
return nil, fmt.Errorf("source checkpoint %s must be a non-empty JSON object", path)
|
|
}
|
|
return payload, nil
|
|
}
|
|
if artifact.Data == nil {
|
|
return nil, nil
|
|
}
|
|
return artifact.Data, nil
|
|
}
|
|
|
|
func sourceStateFromCheckpoint(workflow *domain.WorkflowRecord, run *domain.RunRecord, execution domain.NodeExecutionRecord, artifact *domain.ExecutionArtifact) (*domain.SourceStateRecord, error) {
|
|
payload, err := sourceCheckpointPayload(artifact)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if workflow == nil || run == nil || payload == nil {
|
|
return nil, nil
|
|
}
|
|
fingerprint := strings.TrimSpace(fmt.Sprint(payload["sourceFingerprint"]))
|
|
if fingerprint == "" {
|
|
return nil, fmt.Errorf("source checkpoint for node %s is missing sourceFingerprint", execution.NodeID)
|
|
}
|
|
directoryDepth, err := checkpointIntValue(payload, "directoryDepth")
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if directoryDepth <= 0 {
|
|
return nil, fmt.Errorf("source checkpoint directoryDepth must be greater than 0")
|
|
}
|
|
listedCount, err := checkpointIntValue(payload, "listedCount")
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if listedCount < 0 {
|
|
return nil, fmt.Errorf("source checkpoint listedCount must be zero or greater")
|
|
}
|
|
newCount, err := checkpointIntValue(payload, "newCount")
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if newCount < 0 {
|
|
return nil, fmt.Errorf("source checkpoint newCount must be zero or greater")
|
|
}
|
|
return &domain.SourceStateRecord{
|
|
ID: workflow.ID + ":" + execution.NodeID + ":" + fingerprint,
|
|
WorkflowID: workflow.ID,
|
|
NodeID: execution.NodeID,
|
|
SourceFingerprint: fingerprint,
|
|
DataSource: strings.TrimSpace(fmt.Sprint(payload["dataSource"])),
|
|
SourcePath: strings.TrimSpace(fmt.Sprint(payload["sourcePath"])),
|
|
Bucket: strings.TrimSpace(fmt.Sprint(payload["bucket"])),
|
|
Prefix: strings.TrimSpace(fmt.Sprint(payload["prefix"])),
|
|
Pattern: strings.TrimSpace(fmt.Sprint(payload["pattern"])),
|
|
ScanMode: strings.TrimSpace(fmt.Sprint(payload["scanMode"])),
|
|
IncrementalUnit: strings.TrimSpace(fmt.Sprint(payload["incrementalUnit"])),
|
|
DirectoryDepth: directoryDepth,
|
|
SeenFingerprints: stringList(payload["seenFingerprints"]),
|
|
LastRunID: run.ID,
|
|
LastListedCount: listedCount,
|
|
LastNewCount: newCount,
|
|
LastScanStartedAt: strings.TrimSpace(fmt.Sprint(payload["scanStartedAt"])),
|
|
LastScanCompletedAt: strings.TrimSpace(fmt.Sprint(payload["scanCompletedAt"])),
|
|
UpdatedAt: nowISO(),
|
|
}, nil
|
|
}
|
|
|
|
func checkpointIntValue(payload map[string]any, key string) (int, error) {
|
|
value, ok := payload[key]
|
|
if !ok || value == nil {
|
|
return 0, fmt.Errorf("source checkpoint %s is required", key)
|
|
}
|
|
switch typed := value.(type) {
|
|
case int:
|
|
return typed, nil
|
|
case int32:
|
|
return int(typed), nil
|
|
case int64:
|
|
return int(typed), nil
|
|
case float32:
|
|
value := float64(typed)
|
|
if math.IsNaN(value) || math.IsInf(value, 0) || math.Trunc(value) != value {
|
|
return 0, fmt.Errorf("source checkpoint %s must be an integer", key)
|
|
}
|
|
return int(value), nil
|
|
case float64:
|
|
if math.IsNaN(typed) || math.IsInf(typed, 0) || math.Trunc(typed) != typed {
|
|
return 0, fmt.Errorf("source checkpoint %s must be an integer", key)
|
|
}
|
|
return int(typed), nil
|
|
case json.Number:
|
|
if parsed, err := typed.Int64(); err == nil {
|
|
return int(parsed), nil
|
|
}
|
|
case string:
|
|
trimmed := strings.TrimSpace(typed)
|
|
if parsed, err := strconv.Atoi(trimmed); err == nil && strconv.Itoa(parsed) == trimmed {
|
|
return parsed, nil
|
|
}
|
|
}
|
|
return 0, fmt.Errorf("source checkpoint %s must be an integer", key)
|
|
}
|
|
|
|
func (runtime *Runtime) commitSourceStates(ctx context.Context, workflow *domain.WorkflowRecord, run *domain.RunRecord) error {
|
|
if workflow == nil || run == nil {
|
|
return nil
|
|
}
|
|
sourceRun := run
|
|
persistedRun, err := runtime.store.GetRun(ctx, run.ID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if persistedRun != nil {
|
|
sourceRun = persistedRun
|
|
}
|
|
for _, execution := range sourceRun.NodeExecutions {
|
|
artifact := checkpointArtifact(execution)
|
|
if artifact == nil {
|
|
continue
|
|
}
|
|
state, err := sourceStateFromCheckpoint(workflow, sourceRun, execution, artifact)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if state == nil {
|
|
continue
|
|
}
|
|
if err := runtime.store.UpsertSourceState(ctx, *state); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func scanExecutionNode(workflow *domain.WorkflowRecord, node domain.WorkflowNode) (domain.WorkflowNode, error) {
|
|
if workflow == nil || workflow.ScanConfig == nil {
|
|
return node, nil
|
|
}
|
|
if !scanWorkflowSourceNode(node) {
|
|
return node, nil
|
|
}
|
|
dataSource, err := chooseDataSource(workflow.ScanConfig.DataSource, "BOS")
|
|
if err != nil {
|
|
return node, err
|
|
}
|
|
sourcePath := strings.TrimSpace(workflow.ScanConfig.SourcePath)
|
|
if sourcePath == "" && dataSource == "BOS" {
|
|
sourcePath = bosSourcePath(workflow.ScanConfig.Bucket, workflow.ScanConfig.Prefix)
|
|
}
|
|
copy := node
|
|
copy.Data.Bucket = workflow.ScanConfig.Bucket
|
|
copy.Data.Prefix = workflow.ScanConfig.Prefix
|
|
copy.Data.Pattern = workflow.ScanConfig.Pattern
|
|
copy.Data.Hyperparams = append([]domain.HyperParam(nil), node.Data.Hyperparams...)
|
|
copy.Data.Hyperparams = upsertNodeHyperparam(copy.Data.Hyperparams, "DATA_SOURCE", dataSource)
|
|
copy.Data.Hyperparams = upsertNodeHyperparam(copy.Data.Hyperparams, "SOURCE_PATH", sourcePath)
|
|
copy.Data.Hyperparams = upsertNodeHyperparam(copy.Data.Hyperparams, "BOS_ENDPOINT", workflow.ScanConfig.Endpoint)
|
|
copy.Data.Hyperparams = upsertNodeHyperparam(
|
|
copy.Data.Hyperparams,
|
|
"SCAN_MODE",
|
|
chooseValue(strings.ToUpper(strings.TrimSpace(workflow.ScanConfig.ScanMode)), "INCREMENTAL"),
|
|
)
|
|
bosAccessKeyID := ""
|
|
bosSecretAccessKey := ""
|
|
if dataSource == "BOS" {
|
|
bosAccessKeyID = workflow.ScanConfig.AccessKeyID
|
|
bosSecretAccessKey = workflow.ScanConfig.SecretAccessKey
|
|
}
|
|
copy.Data.Hyperparams = upsertNodeHyperparam(copy.Data.Hyperparams, "BOS_ACCESS_KEY_ID", bosAccessKeyID)
|
|
copy.Data.Hyperparams = upsertNodeHyperparam(copy.Data.Hyperparams, "BOS_SECRET_ACCESS_KEY", bosSecretAccessKey)
|
|
return copy, nil
|
|
}
|
|
|
|
func workflowRunTimeout(workflow *domain.WorkflowRecord) time.Duration {
|
|
if workflow == nil || workflow.RunTimeoutSeconds <= 0 {
|
|
return 0
|
|
}
|
|
return time.Duration(workflow.RunTimeoutSeconds) * time.Second
|
|
}
|
|
|
|
func workflowNodeTimeout(workflow *domain.WorkflowRecord) time.Duration {
|
|
if workflow == nil || workflow.NodeTimeoutSeconds <= 0 {
|
|
return 0
|
|
}
|
|
return time.Duration(workflow.NodeTimeoutSeconds) * time.Second
|
|
}
|
|
|
|
func normalizeRunError(workflow *domain.WorkflowRecord, err error, executionCtx context.Context, nodeTimedOut bool) string {
|
|
if err == nil {
|
|
return ""
|
|
}
|
|
if workflow != nil && workflow.RunTimeoutSeconds > 0 && (errors.Is(err, context.DeadlineExceeded) || errors.Is(executionCtx.Err(), context.DeadlineExceeded)) {
|
|
return fmt.Sprintf("工作流执行超时(%d 秒)", workflow.RunTimeoutSeconds)
|
|
}
|
|
if workflow != nil && workflow.NodeTimeoutSeconds > 0 && nodeTimedOut {
|
|
return fmt.Sprintf("节点执行超时(%d 秒)", workflow.NodeTimeoutSeconds)
|
|
}
|
|
return err.Error()
|
|
}
|
|
|
|
func shouldAutoRetry(workflow *domain.WorkflowRecord, run *domain.RunRecord) bool {
|
|
if workflow == nil || run == nil || workflow.MaxRetryAttempts <= 0 {
|
|
return false
|
|
}
|
|
attempt := run.Attempt
|
|
if attempt <= 0 {
|
|
attempt = 1
|
|
}
|
|
return attempt <= workflow.MaxRetryAttempts
|
|
}
|
|
|
|
func nextRunAttempt(run *domain.RunRecord) int {
|
|
if run == nil || run.Attempt <= 0 {
|
|
return 1
|
|
}
|
|
return run.Attempt + 1
|
|
}
|
|
|
|
func runExecutionAttempt(run *domain.RunRecord) int {
|
|
if run == nil || run.Attempt <= 0 {
|
|
return 1
|
|
}
|
|
return run.Attempt
|
|
}
|
|
|
|
func nextRunAvailableAt(workflow *domain.WorkflowRecord) string {
|
|
if workflow == nil || workflow.RetryBackoffSeconds <= 0 {
|
|
return nowISO()
|
|
}
|
|
return time.Now().UTC().Add(time.Duration(workflow.RetryBackoffSeconds) * time.Second).Format(time.RFC3339Nano)
|
|
}
|
|
|
|
func (runtime *Runtime) enqueueRetryRun(ctx context.Context, workflow *domain.WorkflowRecord, failedRun *domain.RunRecord) (*domain.RunRecord, error) {
|
|
if workflow == nil || failedRun == nil {
|
|
return nil, nil
|
|
}
|
|
manifests, err := lightweight.GenerateWorkflowCodeManifests(runtime.workspaceRoot, *workflow)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
retryRun, err := runtime.store.CreateRun(ctx, workflow.ID, domain.RunTriggerRetry)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
retryRun.Attempt = nextRunAttempt(failedRun)
|
|
retryRun.ParentRunID = failedRun.ID
|
|
retryRun.AvailableAt = nextRunAvailableAt(workflow)
|
|
lightweight.MergeRunMetadata(retryRun, manifests)
|
|
retryRun.UpdatedAt = nowISO()
|
|
if err := runtime.store.UpsertRun(ctx, *retryRun); err != nil {
|
|
return nil, err
|
|
}
|
|
return retryRun, nil
|
|
}
|
|
|
|
type nodeExecutionOutcome struct {
|
|
nodeID string
|
|
result executionResult
|
|
reference *k8s.JobReference
|
|
err error
|
|
nodeTimedOut bool
|
|
}
|
|
|
|
func runAvailableAt(run domain.RunRecord) time.Time {
|
|
if run.AvailableAt != "" {
|
|
if parsed, err := time.Parse(time.RFC3339Nano, run.AvailableAt); err == nil {
|
|
return parsed
|
|
}
|
|
}
|
|
if parsed, err := time.Parse(time.RFC3339Nano, run.CreatedAt); err == nil {
|
|
return parsed
|
|
}
|
|
return time.Time{}
|
|
}
|
|
|
|
func runDispatchReady(run domain.RunRecord, now time.Time) bool {
|
|
availableAt := runAvailableAt(run)
|
|
if availableAt.IsZero() {
|
|
return true
|
|
}
|
|
return !availableAt.After(now)
|
|
}
|
|
|
|
func syncCurrentNodeID(run *domain.RunRecord) {
|
|
run.CurrentNodeID = ""
|
|
for _, status := range []domain.NodeExecutionStatus{
|
|
domain.NodeExecutionStatusWaitingForAudit,
|
|
domain.NodeExecutionStatusRunning,
|
|
} {
|
|
for _, execution := range run.NodeExecutions {
|
|
if execution.Status == status {
|
|
run.CurrentNodeID = execution.NodeID
|
|
return
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func isTerminalRunStatus(status domain.RunStatus) bool {
|
|
return status == domain.RunStatusCompleted ||
|
|
status == domain.RunStatusFailed ||
|
|
status == domain.RunStatusCancelled
|
|
}
|
|
|
|
func (runtime *Runtime) loadRunControlState(ctx context.Context, runID string) (*domain.RunRecord, error) {
|
|
if strings.TrimSpace(runID) == "" {
|
|
return nil, nil
|
|
}
|
|
return runtime.store.GetRun(ctx, runID)
|
|
}
|
|
|
|
func syncStopMetadata(target *domain.RunRecord, source *domain.RunRecord) {
|
|
if target == nil || source == nil {
|
|
return
|
|
}
|
|
target.StopMode = source.StopMode
|
|
target.StopRequestedAt = source.StopRequestedAt
|
|
if source.Progress != nil {
|
|
progress := *source.Progress
|
|
target.Progress = &progress
|
|
}
|
|
}
|
|
|
|
func finalizeCancelledExecution(execution domain.NodeExecutionRecord, message string) (domain.NodeExecutionRecord, bool) {
|
|
if execution.Status == domain.NodeExecutionStatusCompleted ||
|
|
execution.Status == domain.NodeExecutionStatusFailed ||
|
|
execution.Status == domain.NodeExecutionStatusCancelled {
|
|
return execution, false
|
|
}
|
|
execution.Status = domain.NodeExecutionStatusCancelled
|
|
execution.EndedAt = nowISO()
|
|
execution.Error = message
|
|
execution.Logs = append(execution.Logs, logEntry("WARN", message, execution.NodeLabel))
|
|
return execution, true
|
|
}
|
|
|
|
func (runtime *Runtime) finalizeCancelledRun(ctx context.Context, workflow *domain.WorkflowRecord, currentRun *domain.RunRecord, executionMap map[string]domain.NodeExecutionRecord, message string) (*domain.RunRecord, error) {
|
|
for nodeID, execution := range executionMap {
|
|
updated, changed := finalizeCancelledExecution(execution, message)
|
|
if changed {
|
|
executionMap[nodeID] = updated
|
|
applyExecution(currentRun, updated)
|
|
}
|
|
}
|
|
currentRun.Status = domain.RunStatusCancelled
|
|
currentRun.EndedAt = nowISO()
|
|
if strings.TrimSpace(currentRun.Error) == "" {
|
|
currentRun.Error = message
|
|
}
|
|
syncCurrentNodeID(currentRun)
|
|
currentRun.UpdatedAt = nowISO()
|
|
if err := runtime.store.UpsertRun(ctx, *currentRun); err != nil {
|
|
return nil, err
|
|
}
|
|
if workflow != nil {
|
|
workflow.Status = domain.WorkflowStatusFailed
|
|
workflow.LastRunID = currentRun.ID
|
|
workflow.UpdatedAt = nowISO()
|
|
if err := runtime.store.UpdateWorkflowRunState(ctx, workflow.ID, workflow.Status, workflow.LastRunID, workflow.UpdatedAt); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
return currentRun, nil
|
|
}
|
|
|
|
func allAllowedNodesCompleted(workflow *domain.WorkflowRecord, run *domain.RunRecord, executionMap map[string]domain.NodeExecutionRecord) bool {
|
|
if workflow == nil || run == nil {
|
|
return false
|
|
}
|
|
allowedNodes := rerunAllowedNodeSet(*workflow, run)
|
|
for _, node := range workflow.Nodes {
|
|
if !nodeAllowedForExecution(allowedNodes, node.ID) {
|
|
continue
|
|
}
|
|
execution, ok := executionMap[node.ID]
|
|
if !ok || execution.Status != domain.NodeExecutionStatusCompleted {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
|
|
func (runtime *Runtime) finalizeCompletedRun(ctx context.Context, workflow *domain.WorkflowRecord, currentRun *domain.RunRecord) (*domain.RunRecord, error) {
|
|
if err := runtime.commitSourceStates(ctx, workflow, currentRun); err != nil {
|
|
return nil, err
|
|
}
|
|
currentRun.Status = domain.RunStatusCompleted
|
|
currentRun.EndedAt = nowISO()
|
|
syncCurrentNodeID(currentRun)
|
|
currentRun.UpdatedAt = nowISO()
|
|
if err := runtime.store.UpsertRun(ctx, *currentRun); err != nil {
|
|
return nil, err
|
|
}
|
|
workflow.Status = domain.WorkflowStatusCompleted
|
|
workflow.LastRunID = currentRun.ID
|
|
workflow.UpdatedAt = nowISO()
|
|
if err := runtime.store.UpdateWorkflowRunState(ctx, workflow.ID, workflow.Status, workflow.LastRunID, workflow.UpdatedAt); err != nil {
|
|
return nil, err
|
|
}
|
|
return currentRun, nil
|
|
}
|
|
|
|
func dependenciesComplete(workflow domain.WorkflowRecord, executionMap map[string]domain.NodeExecutionRecord, nodeID string) bool {
|
|
for _, candidate := range domain.GetIncomingEdges(workflow.Edges, nodeID) {
|
|
dependency, ok := executionMap[candidate.Source]
|
|
if !ok || dependency.Status != domain.NodeExecutionStatusCompleted {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
|
|
func rerunAllowedNodeSet(workflow domain.WorkflowRecord, run *domain.RunRecord) map[string]struct{} {
|
|
if run == nil || run.Trigger != domain.RunTriggerRerunSubtree || strings.TrimSpace(run.RestartFromNodeID) == "" {
|
|
return nil
|
|
}
|
|
visited := map[string]struct{}{}
|
|
queue := []string{strings.TrimSpace(run.RestartFromNodeID)}
|
|
for len(queue) > 0 {
|
|
nodeID := queue[0]
|
|
queue = queue[1:]
|
|
if _, ok := visited[nodeID]; ok {
|
|
continue
|
|
}
|
|
visited[nodeID] = struct{}{}
|
|
for _, edge := range domain.GetOutgoingEdges(workflow.Edges, nodeID) {
|
|
target := strings.TrimSpace(edge.Target)
|
|
if target != "" {
|
|
queue = append(queue, target)
|
|
}
|
|
}
|
|
}
|
|
return visited
|
|
}
|
|
|
|
func nodeAllowedForExecution(allowed map[string]struct{}, nodeID string) bool {
|
|
if allowed == nil {
|
|
return true
|
|
}
|
|
_, ok := allowed[nodeID]
|
|
return ok
|
|
}
|
|
|
|
func firstActiveExecution(executionMap map[string]domain.NodeExecutionRecord) (string, domain.NodeExecutionStatus) {
|
|
for _, status := range []domain.NodeExecutionStatus{
|
|
domain.NodeExecutionStatusWaitingForAudit,
|
|
domain.NodeExecutionStatusRunning,
|
|
domain.NodeExecutionStatusQueued,
|
|
} {
|
|
for nodeID, execution := range executionMap {
|
|
if execution.Status == status {
|
|
return nodeID, status
|
|
}
|
|
}
|
|
}
|
|
return "", ""
|
|
}
|
|
|
|
func queueNodeIfReady(workflow domain.WorkflowRecord, executionMap map[string]domain.NodeExecutionRecord, readyQueue *[]string, readySet map[string]struct{}, allowed map[string]struct{}, nodeID string) {
|
|
if !nodeAllowedForExecution(allowed, nodeID) {
|
|
return
|
|
}
|
|
if _, queued := readySet[nodeID]; queued {
|
|
return
|
|
}
|
|
if execution, ok := executionMap[nodeID]; ok {
|
|
if execution.Status == domain.NodeExecutionStatusCompleted ||
|
|
execution.Status == domain.NodeExecutionStatusRunning ||
|
|
execution.Status == domain.NodeExecutionStatusWaitingForAudit {
|
|
return
|
|
}
|
|
}
|
|
if !dependenciesComplete(workflow, executionMap, nodeID) {
|
|
return
|
|
}
|
|
*readyQueue = append(*readyQueue, nodeID)
|
|
readySet[nodeID] = struct{}{}
|
|
}
|
|
|
|
func enqueueDependents(workflow domain.WorkflowRecord, executionMap map[string]domain.NodeExecutionRecord, readyQueue *[]string, readySet map[string]struct{}, allowed map[string]struct{}, nodeID string) {
|
|
for _, edge := range domain.GetOutgoingEdges(workflow.Edges, nodeID) {
|
|
queueNodeIfReady(workflow, executionMap, readyQueue, readySet, allowed, edge.Target)
|
|
}
|
|
}
|
|
|
|
func enqueueReadyNodes(workflow domain.WorkflowRecord, executionMap map[string]domain.NodeExecutionRecord, readyQueue *[]string, readySet map[string]struct{}, allowed map[string]struct{}) {
|
|
for _, node := range workflow.Nodes {
|
|
queueNodeIfReady(workflow, executionMap, readyQueue, readySet, allowed, node.ID)
|
|
}
|
|
}
|
|
|
|
func (runtime *Runtime) processRun(ctx context.Context, claimed *domain.RunRecord) (*domain.RunRecord, error) {
|
|
if _, ok, err := runtime.store.AcquireRunLease(ctx, claimed.ID, runtime.leaseOwner(), runtime.leaseTTL()); err != nil {
|
|
return nil, err
|
|
} else if !ok {
|
|
return claimed, nil
|
|
}
|
|
leaseCtx, cancelLeaseExecution := context.WithCancelCause(ctx)
|
|
defer cancelLeaseExecution(context.Canceled)
|
|
go runtime.renewRunLeaseUntil(leaseCtx, claimed.ID, cancelLeaseExecution)
|
|
ctx = leaseCtx
|
|
workflow, err := runtime.store.GetWorkflow(ctx, claimed.WorkflowID)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if workflow == nil {
|
|
claimed.Status = domain.RunStatusFailed
|
|
claimed.EndedAt = nowISO()
|
|
claimed.Error = fmt.Sprintf("workflow %s not found", claimed.WorkflowID)
|
|
claimed.UpdatedAt = nowISO()
|
|
return claimed, runtime.store.UpsertRun(ctx, *claimed)
|
|
}
|
|
if conflicts := domain.FindInputPortConflicts(workflow.Nodes, workflow.Edges); len(conflicts) > 0 {
|
|
currentRun := claimed
|
|
message := "工作流图非法:同一输入端口存在多条上游连线。请拆分输入,或为每个上传结果添加单独的上传算子。"
|
|
currentRun.Status = domain.RunStatusFailed
|
|
currentRun.EndedAt = nowISO()
|
|
currentRun.Error = message
|
|
currentRun.UpdatedAt = nowISO()
|
|
appendLog(currentRun, logEntry("ERROR", message, "system"))
|
|
return currentRun, runtime.store.UpsertRun(ctx, *currentRun)
|
|
}
|
|
if graphIssues := domain.ValidateSingleConnectedWorkflowGraph(workflow.Nodes, workflow.Edges); len(graphIssues) > 0 {
|
|
currentRun := claimed
|
|
message := "工作流图非法:存在孤立算子或多条互不连接的独立链路。请将所有算子通过 artifact/lineage 连成一个单连通工作流。"
|
|
currentRun.Status = domain.RunStatusFailed
|
|
currentRun.EndedAt = nowISO()
|
|
currentRun.Error = message
|
|
currentRun.UpdatedAt = nowISO()
|
|
appendLog(currentRun, logEntry("ERROR", message, "system"))
|
|
return currentRun, runtime.store.UpsertRun(ctx, *currentRun)
|
|
}
|
|
if message := runtime.workflowPublicationBlocker(ctx, workflow); message != "" {
|
|
currentRun := claimed
|
|
currentRun.Status = domain.RunStatusFailed
|
|
currentRun.EndedAt = nowISO()
|
|
currentRun.Error = message
|
|
currentRun.UpdatedAt = nowISO()
|
|
appendLog(currentRun, logEntry("ERROR", message, "system"))
|
|
return currentRun, runtime.store.UpsertRun(ctx, *currentRun)
|
|
}
|
|
|
|
currentRun, err := runtime.store.GetRun(ctx, claimed.ID)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if currentRun == nil {
|
|
currentRun = claimed
|
|
}
|
|
if isTerminalRunStatus(currentRun.Status) {
|
|
return currentRun, nil
|
|
}
|
|
settings, err := runtime.store.GetSettings(ctx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
nodeConcurrencyLimit := effectiveWorkflowConcurrencyLimit(workflow, settings)
|
|
if nodeConcurrencyLimit <= 0 {
|
|
nodeConcurrencyLimit = 1
|
|
}
|
|
|
|
executionMap := map[string]domain.NodeExecutionRecord{}
|
|
for _, execution := range currentRun.NodeExecutions {
|
|
executionMap[execution.NodeID] = execution
|
|
}
|
|
allowedNodes := rerunAllowedNodeSet(*workflow, currentRun)
|
|
readyQueue := make([]string, 0)
|
|
readySet := make(map[string]struct{})
|
|
auditQueue := make([]string, 0)
|
|
|
|
if currentRun.CurrentNodeID != "" {
|
|
tasks, err := runtime.store.ListAuditTasks(ctx, "all")
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
for _, task := range tasks {
|
|
if task.RunID == currentRun.ID && task.NodeID == currentRun.CurrentNodeID && task.Status != domain.AuditStatusPending {
|
|
node := domain.GetNodeByID(workflow.Nodes, currentRun.CurrentNodeID)
|
|
if node != nil {
|
|
execution := executionMap[node.ID]
|
|
execution.NodeID = node.ID
|
|
execution.NodeLabel = node.Data.Label
|
|
execution.Status = domain.NodeExecutionStatusCompleted
|
|
execution.EndedAt = nowISO()
|
|
execution.Logs = append(execution.Logs, logEntry("INFO", fmt.Sprintf("Audit decision: %s", task.Status), node.Data.Label))
|
|
executionMap[node.ID] = execution
|
|
applyExecution(currentRun, execution)
|
|
for _, target := range edgeTargets(domain.GetDecisionOutgoingEdges(workflow.Edges, node.ID, task.Status)) {
|
|
queueNodeIfReady(*workflow, executionMap, &readyQueue, readySet, allowedNodes, target)
|
|
}
|
|
currentRun.Status = domain.RunStatusRunning
|
|
syncCurrentNodeID(currentRun)
|
|
currentRun.UpdatedAt = nowISO()
|
|
if err := runtime.store.UpsertRun(ctx, *currentRun); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
break
|
|
}
|
|
}
|
|
}
|
|
|
|
if activeNodeID, activeStatus := firstActiveExecution(executionMap); activeNodeID != "" {
|
|
if strings.TrimSpace(currentRun.CurrentNodeID) == "" {
|
|
currentRun.CurrentNodeID = activeNodeID
|
|
}
|
|
switch activeStatus {
|
|
case domain.NodeExecutionStatusWaitingForAudit:
|
|
currentRun.Status = domain.RunStatusWaitingForAudit
|
|
default:
|
|
currentRun.Status = domain.RunStatusRunning
|
|
}
|
|
currentRun.UpdatedAt = nowISO()
|
|
syncCurrentNodeID(currentRun)
|
|
if err := runtime.store.UpsertRun(ctx, *currentRun); err != nil {
|
|
return nil, err
|
|
}
|
|
workflow.Status = domain.WorkflowStatusRunning
|
|
workflow.LastRunID = currentRun.ID
|
|
workflow.UpdatedAt = nowISO()
|
|
if err := runtime.store.UpdateWorkflowRunState(ctx, workflow.ID, workflow.Status, workflow.LastRunID, workflow.UpdatedAt); err != nil {
|
|
return nil, err
|
|
}
|
|
return currentRun, nil
|
|
}
|
|
|
|
if len(readyQueue) == 0 && len(executionMap) == 0 {
|
|
if allowedNodes != nil && strings.TrimSpace(currentRun.RestartFromNodeID) != "" {
|
|
queueNodeIfReady(*workflow, executionMap, &readyQueue, readySet, allowedNodes, currentRun.RestartFromNodeID)
|
|
} else {
|
|
for _, nodeID := range domain.GetStartNodeIDs(workflow.Nodes, workflow.Edges) {
|
|
queueNodeIfReady(*workflow, executionMap, &readyQueue, readySet, allowedNodes, nodeID)
|
|
}
|
|
}
|
|
}
|
|
if len(readyQueue) == 0 && len(executionMap) > 0 && currentRun.CurrentNodeID == "" {
|
|
for _, execution := range currentRun.NodeExecutions {
|
|
if execution.Status != domain.NodeExecutionStatusCompleted {
|
|
continue
|
|
}
|
|
enqueueDependents(*workflow, executionMap, &readyQueue, readySet, allowedNodes, execution.NodeID)
|
|
}
|
|
for _, node := range workflow.Nodes {
|
|
queueNodeIfReady(*workflow, executionMap, &readyQueue, readySet, allowedNodes, node.ID)
|
|
}
|
|
}
|
|
|
|
executionCtx := ctx
|
|
cancelExecution := func() {}
|
|
if timeout := workflowRunTimeout(workflow); timeout > 0 {
|
|
executionCtx, cancelExecution = context.WithTimeout(ctx, timeout)
|
|
} else {
|
|
executionCtx, cancelExecution = context.WithCancel(ctx)
|
|
}
|
|
defer cancelExecution()
|
|
results := make(chan nodeExecutionOutcome, nodeConcurrencyLimit)
|
|
runningNodes := make(map[string]struct{})
|
|
nodeStartedAt := make(map[string]time.Time)
|
|
var persistedControl *domain.RunRecord
|
|
gracefulStopRequested := false
|
|
forceStopRequested := false
|
|
for {
|
|
if cause := context.Cause(ctx); cause != nil {
|
|
return currentRun, cause
|
|
}
|
|
persistedControl, err = runtime.loadRunControlState(ctx, currentRun.ID)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
gracefulStopRequested = false
|
|
forceStopRequested = false
|
|
if persistedControl != nil {
|
|
syncStopMetadata(currentRun, persistedControl)
|
|
if persistedControl.Status == domain.RunStatusStopping && persistedControl.StopMode == domain.RunStopModeGraceful {
|
|
gracefulStopRequested = true
|
|
currentRun.Status = domain.RunStatusStopping
|
|
}
|
|
if persistedControl.Status == domain.RunStatusCancelled {
|
|
forceStopRequested = persistedControl.StopMode == domain.RunStopModeForce
|
|
currentRun.Status = domain.RunStatusCancelled
|
|
if currentRun.EndedAt == "" {
|
|
currentRun.EndedAt = persistedControl.EndedAt
|
|
}
|
|
if strings.TrimSpace(currentRun.Error) == "" {
|
|
currentRun.Error = persistedControl.Error
|
|
}
|
|
}
|
|
}
|
|
if forceStopRequested {
|
|
cancelExecution()
|
|
}
|
|
|
|
launched := false
|
|
for !gracefulStopRequested && !forceStopRequested && len(runningNodes) < nodeConcurrencyLimit && len(readyQueue) > 0 {
|
|
nodeID := readyQueue[0]
|
|
readyQueue = readyQueue[1:]
|
|
delete(readySet, nodeID)
|
|
|
|
node := domain.GetNodeByID(workflow.Nodes, nodeID)
|
|
if node == nil {
|
|
continue
|
|
}
|
|
if !nodeAllowedForExecution(allowedNodes, node.ID) {
|
|
continue
|
|
}
|
|
if existing, ok := executionMap[node.ID]; ok {
|
|
if existing.Status == domain.NodeExecutionStatusCompleted ||
|
|
existing.Status == domain.NodeExecutionStatusRunning ||
|
|
existing.Status == domain.NodeExecutionStatusWaitingForAudit {
|
|
continue
|
|
}
|
|
}
|
|
if !dependenciesComplete(*workflow, executionMap, node.ID) {
|
|
continue
|
|
}
|
|
if domain.NodeRequiresAudit(*node) {
|
|
auditQueue = append(auditQueue, node.ID)
|
|
continue
|
|
}
|
|
|
|
inputLineage := gatherInputLineage(*workflow, executionMap, node.ID)
|
|
inputArtifacts := inputArtifactsFromLineage(inputLineage)
|
|
executionAttempt := runExecutionAttempt(currentRun)
|
|
executionLogs := []domain.LogEntryRecord{logEntry("INFO", "Starting node execution", node.Data.Label)}
|
|
if previous, ok := executionMap[node.ID]; ok && missingJobRecoveryCount(&previous) > 0 {
|
|
if previous.Attempt >= executionAttempt {
|
|
executionAttempt = previous.Attempt + 1
|
|
}
|
|
executionLogs = append(append([]domain.LogEntryRecord{}, previous.Logs...), executionLogs...)
|
|
}
|
|
execution := domain.NodeExecutionRecord{
|
|
NodeID: node.ID,
|
|
NodeLabel: node.Data.Label,
|
|
Attempt: executionAttempt,
|
|
Status: domain.NodeExecutionStatusRunning,
|
|
StartedAt: nowISO(),
|
|
Logs: executionLogs,
|
|
Artifacts: []domain.ExecutionArtifact{},
|
|
DataRefs: []domain.DataRefRecord{},
|
|
ResourceSeries: []domain.ResourceSample{},
|
|
StageManifestPath: "",
|
|
}
|
|
applyNodeExecutionSnapshot(&execution, *node, inputArtifacts, inputLineage)
|
|
executionMap[node.ID] = execution
|
|
applyExecution(currentRun, execution)
|
|
runningNodes[node.ID] = struct{}{}
|
|
nodeStartedAt[node.ID] = time.Now()
|
|
runtime.metrics.RecordNodeExecutionStarted()
|
|
launched = true
|
|
|
|
runSnapshot := *currentRun
|
|
nodeSnapshot, err := runtime.prepareExecutionNode(ctx, workflow, *node)
|
|
if err != nil {
|
|
results <- nodeExecutionOutcome{
|
|
nodeID: node.ID,
|
|
err: err,
|
|
}
|
|
continue
|
|
}
|
|
inputSnapshot := cloneArtifacts(inputArtifacts)
|
|
nodeExecutionCtx := executionCtx
|
|
cancelNodeExecution := func() {}
|
|
if timeout := workflowNodeTimeout(workflow); timeout > 0 {
|
|
nodeExecutionCtx, cancelNodeExecution = context.WithTimeout(executionCtx, timeout)
|
|
}
|
|
go func() {
|
|
defer cancelNodeExecution()
|
|
// Stream live GPU samples for custom-container nodes during execution so the
|
|
// resource monitor reflects usage before the node completes (builtin nodes
|
|
// self-report via operator-runner's ResourceSampler, so they are skipped).
|
|
progressCtx, progressCancel := context.WithCancel(nodeExecutionCtx)
|
|
var progressWG sync.WaitGroup
|
|
if shouldReportNodeProgress(runtime, nodeSnapshot) {
|
|
sampleGPU := runtime.operatorRunnerImage != "" && chooseImage(nodeSnapshot, runtime.operatorRunnerImage) != runtime.operatorRunnerImage
|
|
progressWG.Add(1)
|
|
go runtime.runNodeProgressReporter(progressCtx, runSnapshot.ID, nodeSnapshot.ID, sampleGPU, func() { progressWG.Done() })
|
|
}
|
|
result, reference, err := runtime.executeNode(nodeExecutionCtx, &runSnapshot, nodeSnapshot, inputSnapshot)
|
|
progressCancel()
|
|
progressWG.Wait()
|
|
results <- nodeExecutionOutcome{
|
|
nodeID: nodeSnapshot.ID,
|
|
result: result,
|
|
reference: reference,
|
|
err: err,
|
|
nodeTimedOut: errors.Is(nodeExecutionCtx.Err(), context.DeadlineExceeded),
|
|
}
|
|
}()
|
|
}
|
|
|
|
if launched {
|
|
currentRun.Status = domain.RunStatusRunning
|
|
syncCurrentNodeID(currentRun)
|
|
currentRun.UpdatedAt = nowISO()
|
|
if err := runtime.store.UpsertRun(ctx, *currentRun); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
|
|
if len(runningNodes) == 0 {
|
|
if forceStopRequested {
|
|
return runtime.finalizeCancelledRun(ctx, workflow, currentRun, executionMap, "用户已强制停止工作流")
|
|
}
|
|
if gracefulStopRequested || currentRun.Status == domain.RunStatusStopping {
|
|
return runtime.finalizeCancelledRun(ctx, workflow, currentRun, executionMap, "用户已停止工作流")
|
|
}
|
|
if len(readyQueue) == 0 && len(auditQueue) == 0 {
|
|
enqueueReadyNodes(*workflow, executionMap, &readyQueue, readySet, allowedNodes)
|
|
}
|
|
if len(readyQueue) > 0 {
|
|
continue
|
|
}
|
|
if len(auditQueue) > 0 {
|
|
nodeID := auditQueue[0]
|
|
auditQueue = auditQueue[1:]
|
|
node := domain.GetNodeByID(workflow.Nodes, nodeID)
|
|
if node == nil {
|
|
continue
|
|
}
|
|
inputLineage := gatherInputLineage(*workflow, executionMap, node.ID)
|
|
inputArtifacts := inputArtifactsFromLineage(inputLineage)
|
|
execution := executionMap[node.ID]
|
|
if execution.NodeID == "" {
|
|
execution = domain.NodeExecutionRecord{
|
|
NodeID: node.ID,
|
|
NodeLabel: node.Data.Label,
|
|
Attempt: runExecutionAttempt(currentRun),
|
|
Status: domain.NodeExecutionStatusRunning,
|
|
StartedAt: nowISO(),
|
|
Logs: []domain.LogEntryRecord{logEntry("INFO", "Starting node execution", node.Data.Label)},
|
|
Artifacts: []domain.ExecutionArtifact{},
|
|
DataRefs: []domain.DataRefRecord{},
|
|
ResourceSeries: []domain.ResourceSample{},
|
|
}
|
|
}
|
|
applyNodeExecutionSnapshot(&execution, *node, inputArtifacts, inputLineage)
|
|
task := domain.AuditTaskRecord{
|
|
ID: domain.NewID("audit"),
|
|
OrganizationID: currentRun.OrganizationID,
|
|
RunID: currentRun.ID,
|
|
WorkflowID: workflow.ID,
|
|
ProjectID: workflow.ProjectID,
|
|
NodeID: node.ID,
|
|
WorkflowName: workflow.Name,
|
|
ProjectName: currentRun.ProjectName,
|
|
OperatorName: node.Data.Label,
|
|
CreatedAt: nowISO(),
|
|
UpdatedAt: nowISO(),
|
|
Status: domain.AuditStatusPending,
|
|
Data: createAuditPayload(*node, inputArtifacts),
|
|
}
|
|
if _, err := runtime.store.CreateAuditTask(ctx, task); err != nil {
|
|
return nil, err
|
|
}
|
|
execution.Status = domain.NodeExecutionStatusWaitingForAudit
|
|
execution.EndedAt = nowISO()
|
|
execution.Artifacts = cloneArtifacts(inputArtifacts)
|
|
execution.InputArtifacts = cloneArtifacts(inputArtifacts)
|
|
execution.InputLineage = cloneInputLineage(inputLineage)
|
|
execution.Logs = append(execution.Logs, logEntry("WARN", "Node is waiting for human review", node.Data.Label))
|
|
executionMap[node.ID] = execution
|
|
applyExecution(currentRun, execution)
|
|
currentRun.Status = domain.RunStatusWaitingForAudit
|
|
syncCurrentNodeID(currentRun)
|
|
currentRun.UpdatedAt = nowISO()
|
|
if err := runtime.store.UpsertRun(ctx, *currentRun); err != nil {
|
|
return nil, err
|
|
}
|
|
workflow.Status = domain.WorkflowStatusRunning
|
|
workflow.UpdatedAt = nowISO()
|
|
if err := runtime.store.UpdateWorkflowRunState(ctx, workflow.ID, workflow.Status, workflow.LastRunID, workflow.UpdatedAt); err != nil {
|
|
return nil, err
|
|
}
|
|
return currentRun, nil
|
|
}
|
|
break
|
|
}
|
|
|
|
outcome := <-results
|
|
delete(runningNodes, outcome.nodeID)
|
|
startedAt := nodeStartedAt[outcome.nodeID]
|
|
delete(nodeStartedAt, outcome.nodeID)
|
|
nodeDurationMS := uint64(0)
|
|
if !startedAt.IsZero() {
|
|
nodeDurationMS = uint64(time.Since(startedAt).Milliseconds())
|
|
}
|
|
persistedControl, err := runtime.loadRunControlState(ctx, currentRun.ID)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
gracefulStopRequested := false
|
|
forceStopRequested := false
|
|
if persistedControl != nil {
|
|
syncStopMetadata(currentRun, persistedControl)
|
|
if persistedControl.Status == domain.RunStatusStopping && persistedControl.StopMode == domain.RunStopModeGraceful {
|
|
gracefulStopRequested = true
|
|
}
|
|
if persistedControl.Status == domain.RunStatusCancelled && persistedControl.StopMode == domain.RunStopModeForce {
|
|
forceStopRequested = true
|
|
}
|
|
}
|
|
execution := executionMap[outcome.nodeID]
|
|
if outcome.err != nil {
|
|
if cause := context.Cause(ctx); cause != nil {
|
|
return currentRun, cause
|
|
}
|
|
if forceStopRequested || (currentRun.StopMode == domain.RunStopModeForce && errors.Is(executionCtx.Err(), context.Canceled)) {
|
|
cancelledExecution, _ := finalizeCancelledExecution(execution, "用户已强制停止工作流")
|
|
executionMap[outcome.nodeID] = cancelledExecution
|
|
applyExecution(currentRun, cancelledExecution)
|
|
currentRun.Status = domain.RunStatusCancelled
|
|
currentRun.UpdatedAt = nowISO()
|
|
if err := runtime.store.UpsertRun(ctx, *currentRun); err != nil {
|
|
return nil, err
|
|
}
|
|
if len(runningNodes) == 0 {
|
|
return runtime.finalizeCancelledRun(ctx, workflow, currentRun, executionMap, "用户已强制停止工作流")
|
|
}
|
|
continue
|
|
}
|
|
runtime.metrics.RecordNodeExecutionFailed(nodeDurationMS)
|
|
cancelExecution()
|
|
execution.Status = domain.NodeExecutionStatusFailed
|
|
execution.EndedAt = nowISO()
|
|
execution.Error = normalizeRunError(workflow, outcome.err, executionCtx, outcome.nodeTimedOut)
|
|
execution.Logs = append(execution.Logs, logEntry("ERROR", execution.Error, execution.NodeLabel))
|
|
executionMap[outcome.nodeID] = execution
|
|
applyExecution(currentRun, execution)
|
|
currentRun.Status = domain.RunStatusFailed
|
|
currentRun.CurrentNodeID = outcome.nodeID
|
|
currentRun.EndedAt = nowISO()
|
|
currentRun.Error = execution.Error
|
|
currentRun.Failure = &domain.RunFailure{
|
|
Code: "node_execution_failed",
|
|
Stage: "execution",
|
|
NodeID: outcome.nodeID,
|
|
Message: execution.Error,
|
|
Retryable: true,
|
|
Suggestion: "查看节点日志后从失败节点继续",
|
|
}
|
|
currentRun.UpdatedAt = nowISO()
|
|
if err := runtime.store.UpsertRun(ctx, *currentRun); err != nil {
|
|
return nil, err
|
|
}
|
|
workflow.Status = domain.WorkflowStatusFailed
|
|
workflow.UpdatedAt = nowISO()
|
|
if err := runtime.store.UpdateWorkflowRunState(ctx, workflow.ID, workflow.Status, workflow.LastRunID, workflow.UpdatedAt); err != nil {
|
|
return nil, err
|
|
}
|
|
if shouldAutoRetry(workflow, currentRun) {
|
|
retryRun, err := runtime.enqueueRetryRun(ctx, workflow, currentRun)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if retryRun != nil {
|
|
appendLog(currentRun, logEntry("WARN", fmt.Sprintf("已自动创建重试运行,第 %d 次尝试已进入队列", retryRun.Attempt), execution.NodeLabel))
|
|
currentRun.UpdatedAt = nowISO()
|
|
if err := runtime.store.UpsertRun(ctx, *currentRun); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
}
|
|
return currentRun, nil
|
|
}
|
|
|
|
runtime.metrics.RecordNodeExecutionCompleted(nodeDurationMS)
|
|
execution.Status = domain.NodeExecutionStatusCompleted
|
|
execution.EndedAt = nowISO()
|
|
execution.Logs = append(execution.Logs, outcome.result.Logs...)
|
|
execution.Artifacts = outcome.result.Artifacts
|
|
execution.DataRefs = outcome.result.DataRefs
|
|
execution.StageManifestPath = outcome.result.StageManifestPath
|
|
execution.ResourceSeries = outcome.result.ResourceSeries
|
|
executionMap[outcome.nodeID] = execution
|
|
applyExecution(currentRun, execution)
|
|
if outcome.reference != nil {
|
|
currentRun.K8sRefs = append(currentRun.K8sRefs, domain.K8sRef{NodeID: outcome.nodeID, Attempt: runExecutionAttempt(currentRun), JobName: outcome.reference.JobName, Namespace: outcome.reference.Namespace})
|
|
}
|
|
if !gracefulStopRequested && !forceStopRequested {
|
|
enqueueDependents(*workflow, executionMap, &readyQueue, readySet, allowedNodes, outcome.nodeID)
|
|
}
|
|
if gracefulStopRequested {
|
|
currentRun.Status = domain.RunStatusStopping
|
|
} else if forceStopRequested {
|
|
currentRun.Status = domain.RunStatusCancelled
|
|
} else {
|
|
currentRun.Status = domain.RunStatusRunning
|
|
}
|
|
syncCurrentNodeID(currentRun)
|
|
currentRun.UpdatedAt = nowISO()
|
|
if err := runtime.store.UpsertRun(ctx, *currentRun); err != nil {
|
|
return nil, err
|
|
}
|
|
if forceStopRequested && len(runningNodes) == 0 {
|
|
return runtime.finalizeCancelledRun(ctx, workflow, currentRun, executionMap, "用户已强制停止工作流")
|
|
}
|
|
}
|
|
|
|
if currentRun.Status == domain.RunStatusStopping || currentRun.Status == domain.RunStatusCancelled {
|
|
message := "用户已停止工作流"
|
|
if currentRun.StopMode == domain.RunStopModeForce {
|
|
message = "用户已强制停止工作流"
|
|
}
|
|
return runtime.finalizeCancelledRun(ctx, workflow, currentRun, executionMap, message)
|
|
}
|
|
return runtime.finalizeCompletedRun(ctx, workflow, currentRun)
|
|
}
|
|
|
|
// nodeProgressSampleInterval bounds how often the worker reads structured progress markers
|
|
// from long-running pods and, for custom containers, pushes live GPU samples.
|
|
const nodeProgressSampleInterval = 15 * time.Second
|
|
|
|
// shouldReportNodeProgress reports whether a node needs live runtime polling. Custom
|
|
// containers need platform-side GPU samples, while long-running builtin and custom nodes
|
|
// can emit DROBOTICFLOW_PROGRESS markers that must be persisted before the job finishes.
|
|
func shouldReportNodeProgress(runtime *Runtime, node domain.WorkflowNode) bool {
|
|
return runtimeContractLongRunning(node.Data.RuntimeContract) ||
|
|
(runtime.operatorRunnerImage != "" && chooseImage(node, runtime.operatorRunnerImage) != runtime.operatorRunnerImage)
|
|
}
|
|
|
|
// mapGPUSample maps the launcher's k8s.GPUSample onto the resource monitor's domain
|
|
// representation — the same field mapping the executor uses when folding the launcher's
|
|
// final series into the node's resourceSeries.
|
|
func mapGPUSample(sample k8s.GPUSample) domain.ResourceSample {
|
|
return domain.ResourceSample{
|
|
Timestamp: sample.Timestamp,
|
|
GPU: sample.GPU,
|
|
GPUMemoryUsedMiB: sample.GPUMemoryUsedMiB,
|
|
GPUMemoryTotalMiB: sample.GPUMemoryTotalMiB,
|
|
GPUName: sample.GPUName,
|
|
GPUCount: sample.GPUCount,
|
|
}
|
|
}
|
|
|
|
// runNodeProgressReporter persists DROBOTICFLOW_PROGRESS markers while a long-running node is
|
|
// active and optionally samples GPU usage for custom containers. It stops before the final node
|
|
// result is persisted, preventing a stale live write from racing with completion.
|
|
const progressMarker = "DROBOTICFLOW_PROGRESS "
|
|
|
|
func latestStructuredProgress(logs string) (*domain.RunProgress, error) {
|
|
var latest *domain.RunProgress
|
|
var lastErr error
|
|
for _, line := range strings.Split(strings.ReplaceAll(logs, "\r\n", "\n"), "\n") {
|
|
markerIndex := strings.Index(line, progressMarker)
|
|
if markerIndex < 0 {
|
|
continue
|
|
}
|
|
payload := strings.TrimSpace(line[markerIndex+len(progressMarker):])
|
|
if payload == "" {
|
|
continue
|
|
}
|
|
var progress domain.RunProgress
|
|
if err := json.Unmarshal([]byte(payload), &progress); err != nil {
|
|
lastErr = fmt.Errorf("decode structured progress marker: %w", err)
|
|
continue
|
|
}
|
|
if strings.TrimSpace(progress.UpdatedAt) == "" {
|
|
progress.UpdatedAt = nowISO()
|
|
}
|
|
copy := progress
|
|
latest = ©
|
|
}
|
|
if latest != nil {
|
|
return latest, nil
|
|
}
|
|
return nil, lastErr
|
|
}
|
|
|
|
func progressFingerprint(progress *domain.RunProgress) string {
|
|
if progress == nil {
|
|
return ""
|
|
}
|
|
copy := *progress
|
|
copy.UpdatedAt = ""
|
|
payload, err := json.Marshal(copy)
|
|
if err != nil {
|
|
return ""
|
|
}
|
|
return string(payload)
|
|
}
|
|
|
|
func (runtime *Runtime) runNodeProgressReporter(ctx context.Context, runID string, nodeID string, sampleGPU bool, done func()) {
|
|
runtime.runNodeProgressReporterWithInterval(ctx, runID, nodeID, sampleGPU, nodeProgressSampleInterval, done)
|
|
}
|
|
|
|
func (runtime *Runtime) runNodeProgressReporterWithInterval(ctx context.Context, runID string, nodeID string, sampleGPU bool, interval time.Duration, done func()) {
|
|
defer done()
|
|
if interval <= 0 {
|
|
interval = nodeProgressSampleInterval
|
|
}
|
|
ticker := time.NewTicker(interval)
|
|
defer ticker.Stop()
|
|
lastProgressFingerprint := ""
|
|
tailer, canTailLogs := runtime.launcher.(nodePodLogTailer)
|
|
for {
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case <-ticker.C:
|
|
}
|
|
if canTailLogs {
|
|
logsCtx, cancelLogs := context.WithTimeout(ctx, 10*time.Second)
|
|
logs, err := tailer.TailNodePodLogs(logsCtx, runID, nodeID, 200)
|
|
cancelLogs()
|
|
if err == nil {
|
|
progress, parseErr := latestStructuredProgress(logs)
|
|
fingerprint := progressFingerprint(progress)
|
|
if parseErr == nil && progress != nil && fingerprint != "" && fingerprint != lastProgressFingerprint {
|
|
if err := runtime.store.UpdateRunProgress(ctx, runID, progress); err == nil {
|
|
lastProgressFingerprint = fingerprint
|
|
}
|
|
}
|
|
}
|
|
}
|
|
if !sampleGPU {
|
|
continue
|
|
}
|
|
sampleCtx, cancelSample := context.WithTimeout(ctx, 10*time.Second)
|
|
sample, err := runtime.launcher.SamplePodGPUForRun(sampleCtx, runID, nodeID)
|
|
cancelSample()
|
|
if err == nil && sample.Timestamp != "" {
|
|
_ = runtime.store.AppendNodeExecutionResourceSample(ctx, runID, nodeID, mapGPUSample(sample))
|
|
}
|
|
}
|
|
}
|
|
|
|
func edgeTargets(edges []domain.WorkflowEdge) []string {
|
|
items := make([]string, 0, len(edges))
|
|
for _, edge := range edges {
|
|
items = append(items, edge.Target)
|
|
}
|
|
return items
|
|
}
|
|
|
|
func (runtime *Runtime) ProcessOnce(ctx context.Context) (*ProcessOnceResult, error) {
|
|
if err := runtime.reconcileActiveRuns(ctx); err != nil {
|
|
return nil, err
|
|
}
|
|
if err := runtime.enqueueStreamingWorkflows(ctx); err != nil {
|
|
return nil, err
|
|
}
|
|
claimed, err := runtime.store.ClaimNextQueuedRun(ctx, runtime.leaseOwner(), runtime.leaseTTL())
|
|
if err != nil || claimed == nil {
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return newProcessOnceResult(ProcessOnceStatusNoQueuedRun, nil), nil
|
|
}
|
|
if !runtime.tryMarkRunActive(claimed.ID) {
|
|
// Another goroutine in this worker is already processing this run
|
|
// (tryMarkRunActive guards the in-memory activeRuns set). ClaimNextQueuedRun
|
|
// already set status=Running + claimed the lease for this worker, which is
|
|
// consistent with the active processor. Release the spurious lease so it
|
|
// does not block the next ProcessAvailable claim for the lease TTL (~15min),
|
|
// and leave status=Running rather than reverting to Queued — reverting kept
|
|
// currentNodeId set, producing an inconsistent "queued+currentNodeId" state
|
|
// that no other queued-writer emits. Reconcile / the active processor will
|
|
// advance the run.
|
|
if releaseErr := runtime.store.ReleaseRunLease(context.Background(), claimed.ID, runtime.leaseOwner()); releaseErr != nil {
|
|
fmt.Printf("[worker] release spurious run lease %s failed: %v\n", claimed.ID, releaseErr)
|
|
}
|
|
return newProcessOnceResult(ProcessOnceStatusSkippedLeaseActive, claimed), nil
|
|
}
|
|
defer runtime.unmarkRunActive(claimed.ID)
|
|
defer func() {
|
|
if err := runtime.store.ReleaseRunLease(context.Background(), claimed.ID, runtime.leaseOwner()); err != nil {
|
|
fmt.Printf("[worker] release run lease %s failed: %v\n", claimed.ID, err)
|
|
}
|
|
}()
|
|
run, err := runtime.processRun(ctx, claimed)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return newProcessOnceResult(ProcessOnceStatusClaimed, run), nil
|
|
}
|