package worker import ( "context" "fmt" "sync" "testing" "time" "droboticflow/internal/domain" "droboticflow/internal/store" ) // Keep claimed runs running without launching jobs, so the regression observes // the scheduler's real store query/selection/claim path. type dispatchProbeStore struct { store.Store mu sync.Mutex claimed []string maintenanceEntered chan struct{} maintenanceRelease chan struct{} once sync.Once } func (s *dispatchProbeStore) ClaimQueuedRun(ctx context.Context, id, owner string, ttl time.Duration) (*domain.RunRecord, error) { run, err := s.Store.ClaimQueuedRun(ctx, id, owner, ttl) if run != nil { s.mu.Lock() s.claimed = append(s.claimed, id) s.mu.Unlock() } return nil, err } func (s *dispatchProbeStore) ListWorkerWorkflows(ctx context.Context, ids []string, limit int) ([]domain.WorkflowRecord, error) { if len(ids) == 0 && s.maintenanceEntered != nil { s.once.Do(func() { close(s.maintenanceEntered) }) select { case <-s.maintenanceRelease: case <-ctx.Done(): return nil, ctx.Err() } } return s.Store.ListWorkerWorkflows(ctx, ids, limit) } func dispatchFixture(t *testing.T) *dispatchProbeStore { t.Helper() s := &dispatchProbeStore{Store: store.NewMemoryStore()} ctx := context.Background() settings, err := s.GetSettings(ctx) if err != nil { t.Fatal(err) } settings.Worker.MaxConcurrentRuns = 3 // Persisted settings intentionally override RunScheduler's fallback interval. settings.Worker.PollIntervalMS = 250 if err := s.UpsertSettings(ctx, *settings); err != nil { t.Fatal(err) } for _, id := range []string{"old-a", "old-b", "new-c"} { if err := s.UpsertWorkflow(ctx, domain.WorkflowRecord{ID: id, MaxConcurrentRuns: 2}); err != nil { t.Fatal(err) } } return s } func TestDispatchBacklogDoesNotHideOtherWorkflows(t *testing.T) { for _, crowded := range []string{"oldest", "newest"} { t.Run(crowded, func(t *testing.T) { s := dispatchFixture(t) ctx := context.Background() for _, id := range []string{"old-a", "old-b"} { if err := s.UpsertRun(ctx, domain.RunRecord{ID: id, WorkflowID: id, Status: domain.RunStatusQueued, CreatedAt: "2020-02-01T00:00:00Z"}); err != nil { t.Fatal(err) } } date := "2021-01-01T00:00:00Z" if crowded == "oldest" { date = "2019-01-01T00:00:00Z" } for i := 0; i < 125; i++ { if err := s.UpsertRun(ctx, domain.RunRecord{ID: fmt.Sprintf("crowded-%03d", i), WorkflowID: "new-c", Status: domain.RunStatusQueued, CreatedAt: date}); err != nil { t.Fatal(err) } } r := newTestRuntimeWithLauncher(t, s, testLauncher{}) if err := r.ProcessAvailable(ctx); err != nil { t.Fatal(err) } s.mu.Lock() defer s.mu.Unlock() if len(s.claimed) != 3 { t.Fatalf("dispatch left capacity idle behind one workflow: %v", s.claimed) } if crowded == "newest" && (s.claimed[0] != "old-a" || s.claimed[1] != "old-b") { t.Fatalf("older workflows hidden by newest page: %v", s.claimed) } }) } } func TestSchedulerDispatchContinuesDuringSlowMaintenance(t *testing.T) { s := dispatchFixture(t) s.maintenanceEntered, s.maintenanceRelease = make(chan struct{}), make(chan struct{}) ctx, cancel := context.WithCancel(context.Background()) done := make(chan struct{}) r := newTestRuntimeWithLauncher(t, s, testLauncher{}) go func() { defer close(done); r.RunScheduler(ctx, 250*time.Millisecond) }() defer func() { cancel(); close(s.maintenanceRelease); <-done }() select { case <-s.maintenanceEntered: case <-time.After(time.Second): t.Fatal("maintenance not started") } if err := s.UpsertRun(ctx, domain.RunRecord{ID: "arrived-during-scan", WorkflowID: "old-a", Status: domain.RunStatusQueued, CreatedAt: "2020-01-01T00:00:00Z"}); err != nil { t.Fatal(err) } deadline := time.Now().Add(time.Second) for time.Now().Before(deadline) { s.mu.Lock() n := len(s.claimed) s.mu.Unlock() if n == 1 { return } time.Sleep(10 * time.Millisecond) } t.Fatal("slow maintenance blocked queued work for every dispatch cycle") } func TestDispatchSkipsBackoffAndLeasesBeforeCandidateLimit(t *testing.T) { s := dispatchFixture(t) ctx := context.Background() for i := 0; i < 150; i++ { run := domain.RunRecord{ID: fmt.Sprintf("blocked-%03d", i), WorkflowID: "new-c", Status: domain.RunStatusQueued, CreatedAt: "2019-01-01T00:00:00Z"} if i%2 == 0 { run.AvailableAt = time.Now().Add(time.Hour).UTC().Format(time.RFC3339Nano) } else { run.ExecutionLeaseOwner = "live-worker" run.ExecutionLeaseExpiresAt = time.Now().Add(time.Hour).UTC().Format(time.RFC3339Nano) } if err := s.UpsertRun(ctx, run); err != nil { t.Fatal(err) } } if err := s.UpsertRun(ctx, domain.RunRecord{ID: "eligible", WorkflowID: "old-a", Status: domain.RunStatusQueued, CreatedAt: "2020-01-01T00:00:00Z"}); err != nil { t.Fatal(err) } r := newTestRuntimeWithLauncher(t, s, testLauncher{}) if err := r.ProcessAvailable(ctx); err != nil { t.Fatal(err) } if len(s.claimed) != 1 || s.claimed[0] != "eligible" { t.Fatalf("blocked candidates hid ready work: %v", s.claimed) } } func TestDispatchHumanReleaseBeyondSaturatedWorkflowPage(t *testing.T) { s := dispatchFixture(t) ctx := context.Background() for i := 0; i < 125; i++ { if err := s.UpsertRun(ctx, domain.RunRecord{ID: fmt.Sprintf("normal-%03d", i), WorkflowID: "old-a", Status: domain.RunStatusQueued, CreatedAt: "2019-01-01T00:00:00Z"}); err != nil { t.Fatal(err) } } if err := s.UpsertRun(ctx, domain.RunRecord{ID: "release", WorkflowID: "old-a", Trigger: domain.RunTriggerHumanRelease, Status: domain.RunStatusQueued, CreatedAt: "2020-01-01T00:00:00Z"}); err != nil { t.Fatal(err) } r := newTestRuntimeWithLauncher(t, s, testLauncher{}) if err := r.ProcessAvailable(ctx); err != nil { t.Fatal(err) } if len(s.claimed) != 3 || s.claimed[2] != "release" { t.Fatalf("human-release bypass lost: %v", s.claimed) } } func TestDispatchTwoWorkersAndRestartDoNotDuplicateClaims(t *testing.T) { s := dispatchFixture(t) ctx := context.Background() for i := 0; i < 25; i++ { if err := s.UpsertRun(ctx, domain.RunRecord{ID: fmt.Sprintf("run-%03d", i), WorkflowID: []string{"old-a", "old-b", "new-c"}[i%3], Status: domain.RunStatusQueued, CreatedAt: "2020-01-01T00:00:00Z"}); err != nil { t.Fatal(err) } } var wg sync.WaitGroup for i := 0; i < 2; i++ { r := newTestRuntimeWithLauncher(t, s, testLauncher{}) wg.Add(1) go func() { defer wg.Done() if err := r.ProcessAvailable(ctx); err != nil { t.Error(err) } }() } wg.Wait() if len(s.claimed) != 3 { t.Fatalf("unexpected concurrent claims: %v", s.claimed) } // A fresh scheduler has no in-memory cursor but must respect persisted slots. r := newTestRuntimeWithLauncher(t, s, testLauncher{}) if err := r.ProcessAvailable(ctx); err != nil { t.Fatal(err) } if len(s.claimed) != 3 { t.Fatalf("restart bypassed active limits: %v", s.claimed) } } func TestDispatchCapacityCanSpanCandidatePages(t *testing.T) { s := dispatchFixture(t) ctx := context.Background() settings, _ := s.GetSettings(ctx) settings.Worker.MaxConcurrentRuns = 125 if err := s.UpsertSettings(ctx, *settings); err != nil { t.Fatal(err) } if err := s.UpsertWorkflow(ctx, domain.WorkflowRecord{ID: "old-a", MaxConcurrentRuns: 125}); err != nil { t.Fatal(err) } for i := 0; i < 150; i++ { if err := s.UpsertRun(ctx, domain.RunRecord{ID: fmt.Sprintf("run-%03d", i), WorkflowID: "old-a", Status: domain.RunStatusQueued, CreatedAt: "2020-01-01T00:00:00Z"}); err != nil { t.Fatal(err) } } r := newTestRuntimeWithLauncher(t, s, testLauncher{}) if err := r.ProcessAvailable(ctx); err != nil { t.Fatal(err) } if len(s.claimed) != 125 { t.Fatalf("candidate page became a capacity limit: %d", len(s.claimed)) } } func TestDispatchReservesOneSlotForRecoveryRuns(t *testing.T) { s := dispatchFixture(t) ctx := context.Background() for _, run := range []domain.RunRecord{ {ID: "normal-old", WorkflowID: "old-a", Status: domain.RunStatusQueued, CreatedAt: "2019-01-01T00:00:00Z"}, {ID: "normal-middle", WorkflowID: "old-b", Status: domain.RunStatusQueued, CreatedAt: "2020-01-01T00:00:00Z"}, {ID: "normal-new", WorkflowID: "new-c", Status: domain.RunStatusQueued, CreatedAt: "2021-01-01T00:00:00Z"}, {ID: "recovery", WorkflowID: "new-c", Trigger: domain.RunTriggerRetry, Status: domain.RunStatusQueued, CreatedAt: "2022-01-01T00:00:00Z"}, } { if err := s.UpsertRun(ctx, run); err != nil { t.Fatal(err) } } r := newTestRuntimeWithLauncher(t, s, testLauncher{}) r.recoveryReservedSlots = 1 settings, err := s.GetSettings(ctx) if err != nil { t.Fatal(err) } counts, err := s.CountWorkerRuns(ctx) if err != nil { t.Fatal(err) } selected, err := r.dispatchCandidates(ctx, settings, counts) if err != nil { t.Fatal(err) } if len(selected) != 3 { t.Fatalf("dispatch selected %d runs, want 3: %v", len(selected), selected) } seen := make(map[string]bool, len(selected)) for _, id := range selected { seen[id] = true } if !seen["recovery"] { t.Fatalf("recovery run did not receive its reserved slot: %v", selected) } normalCount := 0 for _, id := range []string{"normal-old", "normal-middle", "normal-new"} { if seen[id] { normalCount++ } } if normalCount != 2 { t.Fatalf("recovery reservation displaced the wrong number of normal runs: %v", selected) } }