285 lines
9.2 KiB
Go
285 lines
9.2 KiB
Go
package worker
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"droboticflow/internal/domain"
|
|
"droboticflow/internal/store"
|
|
)
|
|
|
|
// Keep claimed runs running without launching jobs, so the regression observes
|
|
// the scheduler's real store query/selection/claim path.
|
|
type dispatchProbeStore struct {
|
|
store.Store
|
|
mu sync.Mutex
|
|
claimed []string
|
|
maintenanceEntered chan struct{}
|
|
maintenanceRelease chan struct{}
|
|
once sync.Once
|
|
}
|
|
|
|
func (s *dispatchProbeStore) ClaimQueuedRun(ctx context.Context, id, owner string, ttl time.Duration) (*domain.RunRecord, error) {
|
|
run, err := s.Store.ClaimQueuedRun(ctx, id, owner, ttl)
|
|
if run != nil {
|
|
s.mu.Lock()
|
|
s.claimed = append(s.claimed, id)
|
|
s.mu.Unlock()
|
|
}
|
|
return nil, err
|
|
}
|
|
|
|
func (s *dispatchProbeStore) ListWorkerWorkflows(ctx context.Context, ids []string, limit int) ([]domain.WorkflowRecord, error) {
|
|
if len(ids) == 0 && s.maintenanceEntered != nil {
|
|
s.once.Do(func() { close(s.maintenanceEntered) })
|
|
select {
|
|
case <-s.maintenanceRelease:
|
|
case <-ctx.Done():
|
|
return nil, ctx.Err()
|
|
}
|
|
}
|
|
return s.Store.ListWorkerWorkflows(ctx, ids, limit)
|
|
}
|
|
|
|
func dispatchFixture(t *testing.T) *dispatchProbeStore {
|
|
t.Helper()
|
|
s := &dispatchProbeStore{Store: store.NewMemoryStore()}
|
|
ctx := context.Background()
|
|
settings, err := s.GetSettings(ctx)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
settings.Worker.MaxConcurrentRuns = 3
|
|
// Persisted settings intentionally override RunScheduler's fallback interval.
|
|
settings.Worker.PollIntervalMS = 250
|
|
if err := s.UpsertSettings(ctx, *settings); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for _, id := range []string{"old-a", "old-b", "new-c"} {
|
|
if err := s.UpsertWorkflow(ctx, domain.WorkflowRecord{ID: id, MaxConcurrentRuns: 2}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
return s
|
|
}
|
|
|
|
func TestDispatchBacklogDoesNotHideOtherWorkflows(t *testing.T) {
|
|
for _, crowded := range []string{"oldest", "newest"} {
|
|
t.Run(crowded, func(t *testing.T) {
|
|
s := dispatchFixture(t)
|
|
ctx := context.Background()
|
|
for _, id := range []string{"old-a", "old-b"} {
|
|
if err := s.UpsertRun(ctx, domain.RunRecord{ID: id, WorkflowID: id, Status: domain.RunStatusQueued, CreatedAt: "2020-02-01T00:00:00Z"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
date := "2021-01-01T00:00:00Z"
|
|
if crowded == "oldest" {
|
|
date = "2019-01-01T00:00:00Z"
|
|
}
|
|
for i := 0; i < 125; i++ {
|
|
if err := s.UpsertRun(ctx, domain.RunRecord{ID: fmt.Sprintf("crowded-%03d", i), WorkflowID: "new-c", Status: domain.RunStatusQueued, CreatedAt: date}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
r := newTestRuntimeWithLauncher(t, s, testLauncher{})
|
|
if err := r.ProcessAvailable(ctx); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
if len(s.claimed) != 3 {
|
|
t.Fatalf("dispatch left capacity idle behind one workflow: %v", s.claimed)
|
|
}
|
|
if crowded == "newest" && (s.claimed[0] != "old-a" || s.claimed[1] != "old-b") {
|
|
t.Fatalf("older workflows hidden by newest page: %v", s.claimed)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestSchedulerDispatchContinuesDuringSlowMaintenance(t *testing.T) {
|
|
s := dispatchFixture(t)
|
|
s.maintenanceEntered, s.maintenanceRelease = make(chan struct{}), make(chan struct{})
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
done := make(chan struct{})
|
|
r := newTestRuntimeWithLauncher(t, s, testLauncher{})
|
|
go func() { defer close(done); r.RunScheduler(ctx, 250*time.Millisecond) }()
|
|
defer func() { cancel(); close(s.maintenanceRelease); <-done }()
|
|
select {
|
|
case <-s.maintenanceEntered:
|
|
case <-time.After(time.Second):
|
|
t.Fatal("maintenance not started")
|
|
}
|
|
if err := s.UpsertRun(ctx, domain.RunRecord{ID: "arrived-during-scan", WorkflowID: "old-a", Status: domain.RunStatusQueued, CreatedAt: "2020-01-01T00:00:00Z"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
deadline := time.Now().Add(time.Second)
|
|
for time.Now().Before(deadline) {
|
|
s.mu.Lock()
|
|
n := len(s.claimed)
|
|
s.mu.Unlock()
|
|
if n == 1 {
|
|
return
|
|
}
|
|
time.Sleep(10 * time.Millisecond)
|
|
}
|
|
t.Fatal("slow maintenance blocked queued work for every dispatch cycle")
|
|
}
|
|
|
|
func TestDispatchSkipsBackoffAndLeasesBeforeCandidateLimit(t *testing.T) {
|
|
s := dispatchFixture(t)
|
|
ctx := context.Background()
|
|
for i := 0; i < 150; i++ {
|
|
run := domain.RunRecord{ID: fmt.Sprintf("blocked-%03d", i), WorkflowID: "new-c", Status: domain.RunStatusQueued, CreatedAt: "2019-01-01T00:00:00Z"}
|
|
if i%2 == 0 {
|
|
run.AvailableAt = time.Now().Add(time.Hour).UTC().Format(time.RFC3339Nano)
|
|
} else {
|
|
run.ExecutionLeaseOwner = "live-worker"
|
|
run.ExecutionLeaseExpiresAt = time.Now().Add(time.Hour).UTC().Format(time.RFC3339Nano)
|
|
}
|
|
if err := s.UpsertRun(ctx, run); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
if err := s.UpsertRun(ctx, domain.RunRecord{ID: "eligible", WorkflowID: "old-a", Status: domain.RunStatusQueued, CreatedAt: "2020-01-01T00:00:00Z"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
r := newTestRuntimeWithLauncher(t, s, testLauncher{})
|
|
if err := r.ProcessAvailable(ctx); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(s.claimed) != 1 || s.claimed[0] != "eligible" {
|
|
t.Fatalf("blocked candidates hid ready work: %v", s.claimed)
|
|
}
|
|
}
|
|
|
|
func TestDispatchHumanReleaseBeyondSaturatedWorkflowPage(t *testing.T) {
|
|
s := dispatchFixture(t)
|
|
ctx := context.Background()
|
|
for i := 0; i < 125; i++ {
|
|
if err := s.UpsertRun(ctx, domain.RunRecord{ID: fmt.Sprintf("normal-%03d", i), WorkflowID: "old-a", Status: domain.RunStatusQueued, CreatedAt: "2019-01-01T00:00:00Z"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
if err := s.UpsertRun(ctx, domain.RunRecord{ID: "release", WorkflowID: "old-a", Trigger: domain.RunTriggerHumanRelease, Status: domain.RunStatusQueued, CreatedAt: "2020-01-01T00:00:00Z"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
r := newTestRuntimeWithLauncher(t, s, testLauncher{})
|
|
if err := r.ProcessAvailable(ctx); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(s.claimed) != 3 || s.claimed[2] != "release" {
|
|
t.Fatalf("human-release bypass lost: %v", s.claimed)
|
|
}
|
|
}
|
|
|
|
func TestDispatchTwoWorkersAndRestartDoNotDuplicateClaims(t *testing.T) {
|
|
s := dispatchFixture(t)
|
|
ctx := context.Background()
|
|
for i := 0; i < 25; i++ {
|
|
if err := s.UpsertRun(ctx, domain.RunRecord{ID: fmt.Sprintf("run-%03d", i), WorkflowID: []string{"old-a", "old-b", "new-c"}[i%3], Status: domain.RunStatusQueued, CreatedAt: "2020-01-01T00:00:00Z"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
var wg sync.WaitGroup
|
|
for i := 0; i < 2; i++ {
|
|
r := newTestRuntimeWithLauncher(t, s, testLauncher{})
|
|
wg.Add(1)
|
|
go func() {
|
|
defer wg.Done()
|
|
if err := r.ProcessAvailable(ctx); err != nil {
|
|
t.Error(err)
|
|
}
|
|
}()
|
|
}
|
|
wg.Wait()
|
|
if len(s.claimed) != 3 {
|
|
t.Fatalf("unexpected concurrent claims: %v", s.claimed)
|
|
}
|
|
// A fresh scheduler has no in-memory cursor but must respect persisted slots.
|
|
r := newTestRuntimeWithLauncher(t, s, testLauncher{})
|
|
if err := r.ProcessAvailable(ctx); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(s.claimed) != 3 {
|
|
t.Fatalf("restart bypassed active limits: %v", s.claimed)
|
|
}
|
|
}
|
|
|
|
func TestDispatchCapacityCanSpanCandidatePages(t *testing.T) {
|
|
s := dispatchFixture(t)
|
|
ctx := context.Background()
|
|
settings, _ := s.GetSettings(ctx)
|
|
settings.Worker.MaxConcurrentRuns = 125
|
|
if err := s.UpsertSettings(ctx, *settings); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := s.UpsertWorkflow(ctx, domain.WorkflowRecord{ID: "old-a", MaxConcurrentRuns: 125}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for i := 0; i < 150; i++ {
|
|
if err := s.UpsertRun(ctx, domain.RunRecord{ID: fmt.Sprintf("run-%03d", i), WorkflowID: "old-a", Status: domain.RunStatusQueued, CreatedAt: "2020-01-01T00:00:00Z"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
r := newTestRuntimeWithLauncher(t, s, testLauncher{})
|
|
if err := r.ProcessAvailable(ctx); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(s.claimed) != 125 {
|
|
t.Fatalf("candidate page became a capacity limit: %d", len(s.claimed))
|
|
}
|
|
}
|
|
|
|
func TestDispatchReservesOneSlotForRecoveryRuns(t *testing.T) {
|
|
s := dispatchFixture(t)
|
|
ctx := context.Background()
|
|
for _, run := range []domain.RunRecord{
|
|
{ID: "normal-old", WorkflowID: "old-a", Status: domain.RunStatusQueued, CreatedAt: "2019-01-01T00:00:00Z"},
|
|
{ID: "normal-middle", WorkflowID: "old-b", Status: domain.RunStatusQueued, CreatedAt: "2020-01-01T00:00:00Z"},
|
|
{ID: "normal-new", WorkflowID: "new-c", Status: domain.RunStatusQueued, CreatedAt: "2021-01-01T00:00:00Z"},
|
|
{ID: "recovery", WorkflowID: "new-c", Trigger: domain.RunTriggerRetry, Status: domain.RunStatusQueued, CreatedAt: "2022-01-01T00:00:00Z"},
|
|
} {
|
|
if err := s.UpsertRun(ctx, run); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
r := newTestRuntimeWithLauncher(t, s, testLauncher{})
|
|
r.recoveryReservedSlots = 1
|
|
settings, err := s.GetSettings(ctx)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
counts, err := s.CountWorkerRuns(ctx)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
selected, err := r.dispatchCandidates(ctx, settings, counts)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(selected) != 3 {
|
|
t.Fatalf("dispatch selected %d runs, want 3: %v", len(selected), selected)
|
|
}
|
|
seen := make(map[string]bool, len(selected))
|
|
for _, id := range selected {
|
|
seen[id] = true
|
|
}
|
|
if !seen["recovery"] {
|
|
t.Fatalf("recovery run did not receive its reserved slot: %v", selected)
|
|
}
|
|
normalCount := 0
|
|
for _, id := range []string{"normal-old", "normal-middle", "normal-new"} {
|
|
if seen[id] {
|
|
normalCount++
|
|
}
|
|
}
|
|
if normalCount != 2 {
|
|
t.Fatalf("recovery reservation displaced the wrong number of normal runs: %v", selected)
|
|
}
|
|
}
|