droboticflow/internal/worker/dispatch_regression_test.go
2026-10-01 03:08:27 -07:00

285 lines
9.2 KiB
Go

package worker
import (
"context"
"fmt"
"sync"
"testing"
"time"
"droboticflow/internal/domain"
"droboticflow/internal/store"
)
// Keep claimed runs running without launching jobs, so the regression observes
// the scheduler's real store query/selection/claim path.
type dispatchProbeStore struct {
store.Store
mu sync.Mutex
claimed []string
maintenanceEntered chan struct{}
maintenanceRelease chan struct{}
once sync.Once
}
func (s *dispatchProbeStore) ClaimQueuedRun(ctx context.Context, id, owner string, ttl time.Duration) (*domain.RunRecord, error) {
run, err := s.Store.ClaimQueuedRun(ctx, id, owner, ttl)
if run != nil {
s.mu.Lock()
s.claimed = append(s.claimed, id)
s.mu.Unlock()
}
return nil, err
}
func (s *dispatchProbeStore) ListWorkerWorkflows(ctx context.Context, ids []string, limit int) ([]domain.WorkflowRecord, error) {
if len(ids) == 0 && s.maintenanceEntered != nil {
s.once.Do(func() { close(s.maintenanceEntered) })
select {
case <-s.maintenanceRelease:
case <-ctx.Done():
return nil, ctx.Err()
}
}
return s.Store.ListWorkerWorkflows(ctx, ids, limit)
}
func dispatchFixture(t *testing.T) *dispatchProbeStore {
t.Helper()
s := &dispatchProbeStore{Store: store.NewMemoryStore()}
ctx := context.Background()
settings, err := s.GetSettings(ctx)
if err != nil {
t.Fatal(err)
}
settings.Worker.MaxConcurrentRuns = 3
// Persisted settings intentionally override RunScheduler's fallback interval.
settings.Worker.PollIntervalMS = 250
if err := s.UpsertSettings(ctx, *settings); err != nil {
t.Fatal(err)
}
for _, id := range []string{"old-a", "old-b", "new-c"} {
if err := s.UpsertWorkflow(ctx, domain.WorkflowRecord{ID: id, MaxConcurrentRuns: 2}); err != nil {
t.Fatal(err)
}
}
return s
}
func TestDispatchBacklogDoesNotHideOtherWorkflows(t *testing.T) {
for _, crowded := range []string{"oldest", "newest"} {
t.Run(crowded, func(t *testing.T) {
s := dispatchFixture(t)
ctx := context.Background()
for _, id := range []string{"old-a", "old-b"} {
if err := s.UpsertRun(ctx, domain.RunRecord{ID: id, WorkflowID: id, Status: domain.RunStatusQueued, CreatedAt: "2020-02-01T00:00:00Z"}); err != nil {
t.Fatal(err)
}
}
date := "2021-01-01T00:00:00Z"
if crowded == "oldest" {
date = "2019-01-01T00:00:00Z"
}
for i := 0; i < 125; i++ {
if err := s.UpsertRun(ctx, domain.RunRecord{ID: fmt.Sprintf("crowded-%03d", i), WorkflowID: "new-c", Status: domain.RunStatusQueued, CreatedAt: date}); err != nil {
t.Fatal(err)
}
}
r := newTestRuntimeWithLauncher(t, s, testLauncher{})
if err := r.ProcessAvailable(ctx); err != nil {
t.Fatal(err)
}
s.mu.Lock()
defer s.mu.Unlock()
if len(s.claimed) != 3 {
t.Fatalf("dispatch left capacity idle behind one workflow: %v", s.claimed)
}
if crowded == "newest" && (s.claimed[0] != "old-a" || s.claimed[1] != "old-b") {
t.Fatalf("older workflows hidden by newest page: %v", s.claimed)
}
})
}
}
func TestSchedulerDispatchContinuesDuringSlowMaintenance(t *testing.T) {
s := dispatchFixture(t)
s.maintenanceEntered, s.maintenanceRelease = make(chan struct{}), make(chan struct{})
ctx, cancel := context.WithCancel(context.Background())
done := make(chan struct{})
r := newTestRuntimeWithLauncher(t, s, testLauncher{})
go func() { defer close(done); r.RunScheduler(ctx, 250*time.Millisecond) }()
defer func() { cancel(); close(s.maintenanceRelease); <-done }()
select {
case <-s.maintenanceEntered:
case <-time.After(time.Second):
t.Fatal("maintenance not started")
}
if err := s.UpsertRun(ctx, domain.RunRecord{ID: "arrived-during-scan", WorkflowID: "old-a", Status: domain.RunStatusQueued, CreatedAt: "2020-01-01T00:00:00Z"}); err != nil {
t.Fatal(err)
}
deadline := time.Now().Add(time.Second)
for time.Now().Before(deadline) {
s.mu.Lock()
n := len(s.claimed)
s.mu.Unlock()
if n == 1 {
return
}
time.Sleep(10 * time.Millisecond)
}
t.Fatal("slow maintenance blocked queued work for every dispatch cycle")
}
func TestDispatchSkipsBackoffAndLeasesBeforeCandidateLimit(t *testing.T) {
s := dispatchFixture(t)
ctx := context.Background()
for i := 0; i < 150; i++ {
run := domain.RunRecord{ID: fmt.Sprintf("blocked-%03d", i), WorkflowID: "new-c", Status: domain.RunStatusQueued, CreatedAt: "2019-01-01T00:00:00Z"}
if i%2 == 0 {
run.AvailableAt = time.Now().Add(time.Hour).UTC().Format(time.RFC3339Nano)
} else {
run.ExecutionLeaseOwner = "live-worker"
run.ExecutionLeaseExpiresAt = time.Now().Add(time.Hour).UTC().Format(time.RFC3339Nano)
}
if err := s.UpsertRun(ctx, run); err != nil {
t.Fatal(err)
}
}
if err := s.UpsertRun(ctx, domain.RunRecord{ID: "eligible", WorkflowID: "old-a", Status: domain.RunStatusQueued, CreatedAt: "2020-01-01T00:00:00Z"}); err != nil {
t.Fatal(err)
}
r := newTestRuntimeWithLauncher(t, s, testLauncher{})
if err := r.ProcessAvailable(ctx); err != nil {
t.Fatal(err)
}
if len(s.claimed) != 1 || s.claimed[0] != "eligible" {
t.Fatalf("blocked candidates hid ready work: %v", s.claimed)
}
}
func TestDispatchHumanReleaseBeyondSaturatedWorkflowPage(t *testing.T) {
s := dispatchFixture(t)
ctx := context.Background()
for i := 0; i < 125; i++ {
if err := s.UpsertRun(ctx, domain.RunRecord{ID: fmt.Sprintf("normal-%03d", i), WorkflowID: "old-a", Status: domain.RunStatusQueued, CreatedAt: "2019-01-01T00:00:00Z"}); err != nil {
t.Fatal(err)
}
}
if err := s.UpsertRun(ctx, domain.RunRecord{ID: "release", WorkflowID: "old-a", Trigger: domain.RunTriggerHumanRelease, Status: domain.RunStatusQueued, CreatedAt: "2020-01-01T00:00:00Z"}); err != nil {
t.Fatal(err)
}
r := newTestRuntimeWithLauncher(t, s, testLauncher{})
if err := r.ProcessAvailable(ctx); err != nil {
t.Fatal(err)
}
if len(s.claimed) != 3 || s.claimed[2] != "release" {
t.Fatalf("human-release bypass lost: %v", s.claimed)
}
}
func TestDispatchTwoWorkersAndRestartDoNotDuplicateClaims(t *testing.T) {
s := dispatchFixture(t)
ctx := context.Background()
for i := 0; i < 25; i++ {
if err := s.UpsertRun(ctx, domain.RunRecord{ID: fmt.Sprintf("run-%03d", i), WorkflowID: []string{"old-a", "old-b", "new-c"}[i%3], Status: domain.RunStatusQueued, CreatedAt: "2020-01-01T00:00:00Z"}); err != nil {
t.Fatal(err)
}
}
var wg sync.WaitGroup
for i := 0; i < 2; i++ {
r := newTestRuntimeWithLauncher(t, s, testLauncher{})
wg.Add(1)
go func() {
defer wg.Done()
if err := r.ProcessAvailable(ctx); err != nil {
t.Error(err)
}
}()
}
wg.Wait()
if len(s.claimed) != 3 {
t.Fatalf("unexpected concurrent claims: %v", s.claimed)
}
// A fresh scheduler has no in-memory cursor but must respect persisted slots.
r := newTestRuntimeWithLauncher(t, s, testLauncher{})
if err := r.ProcessAvailable(ctx); err != nil {
t.Fatal(err)
}
if len(s.claimed) != 3 {
t.Fatalf("restart bypassed active limits: %v", s.claimed)
}
}
func TestDispatchCapacityCanSpanCandidatePages(t *testing.T) {
s := dispatchFixture(t)
ctx := context.Background()
settings, _ := s.GetSettings(ctx)
settings.Worker.MaxConcurrentRuns = 125
if err := s.UpsertSettings(ctx, *settings); err != nil {
t.Fatal(err)
}
if err := s.UpsertWorkflow(ctx, domain.WorkflowRecord{ID: "old-a", MaxConcurrentRuns: 125}); err != nil {
t.Fatal(err)
}
for i := 0; i < 150; i++ {
if err := s.UpsertRun(ctx, domain.RunRecord{ID: fmt.Sprintf("run-%03d", i), WorkflowID: "old-a", Status: domain.RunStatusQueued, CreatedAt: "2020-01-01T00:00:00Z"}); err != nil {
t.Fatal(err)
}
}
r := newTestRuntimeWithLauncher(t, s, testLauncher{})
if err := r.ProcessAvailable(ctx); err != nil {
t.Fatal(err)
}
if len(s.claimed) != 125 {
t.Fatalf("candidate page became a capacity limit: %d", len(s.claimed))
}
}
func TestDispatchReservesOneSlotForRecoveryRuns(t *testing.T) {
s := dispatchFixture(t)
ctx := context.Background()
for _, run := range []domain.RunRecord{
{ID: "normal-old", WorkflowID: "old-a", Status: domain.RunStatusQueued, CreatedAt: "2019-01-01T00:00:00Z"},
{ID: "normal-middle", WorkflowID: "old-b", Status: domain.RunStatusQueued, CreatedAt: "2020-01-01T00:00:00Z"},
{ID: "normal-new", WorkflowID: "new-c", Status: domain.RunStatusQueued, CreatedAt: "2021-01-01T00:00:00Z"},
{ID: "recovery", WorkflowID: "new-c", Trigger: domain.RunTriggerRetry, Status: domain.RunStatusQueued, CreatedAt: "2022-01-01T00:00:00Z"},
} {
if err := s.UpsertRun(ctx, run); err != nil {
t.Fatal(err)
}
}
r := newTestRuntimeWithLauncher(t, s, testLauncher{})
r.recoveryReservedSlots = 1
settings, err := s.GetSettings(ctx)
if err != nil {
t.Fatal(err)
}
counts, err := s.CountWorkerRuns(ctx)
if err != nil {
t.Fatal(err)
}
selected, err := r.dispatchCandidates(ctx, settings, counts)
if err != nil {
t.Fatal(err)
}
if len(selected) != 3 {
t.Fatalf("dispatch selected %d runs, want 3: %v", len(selected), selected)
}
seen := make(map[string]bool, len(selected))
for _, id := range selected {
seen[id] = true
}
if !seen["recovery"] {
t.Fatalf("recovery run did not receive its reserved slot: %v", selected)
}
normalCount := 0
for _, id := range []string{"normal-old", "normal-middle", "normal-new"} {
if seen[id] {
normalCount++
}
}
if normalCount != 2 {
t.Fatalf("recovery reservation displaced the wrong number of normal runs: %v", selected)
}
}