droboticflow/internal/api/annotation_index_lease_test.go
d-robotics 3405d2da4f 🛡️ Keep index jobs running through transient lease renewal errors
The index heartbeat cancelled the job on any TouchAnnotationIndex error, so
a Mongo failover or timeout during a long source validation aborted it and
burned one of its five attempts; five blips marked a healthy source as
failed. Cancel only when the lease is provably lost, or when renewal errors
persist until the claimed lease is within a safety margin of lapsing, so
another processor never reclaims a job that is still running. Renewal
starts from the lease the claim actually stored.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-18 22:28:02 +08:00

82 lines
2.5 KiB
Go

package api
import (
"context"
"errors"
"sync"
"testing"
"time"
)
type leaseTouchScript struct {
mu sync.Mutex
calls int
reply func(call int) (bool, error)
}
func (s *leaseTouchScript) touch(context.Context, time.Time) (bool, error) {
s.mu.Lock()
s.calls++
call := s.calls
s.mu.Unlock()
return s.reply(call)
}
func (s *leaseTouchScript) count() int {
s.mu.Lock()
defer s.mu.Unlock()
return s.calls
}
func runLeaseScript(t *testing.T, reply func(call int) (bool, error), observe time.Duration) (cancelledAfter time.Duration, cancelled bool, calls int) {
t.Helper()
const every, lease, safety = 10 * time.Millisecond, 120 * time.Millisecond, 40 * time.Millisecond
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
script := &leaseTouchScript{reply: reply}
started := time.Now()
stop := renewAnnotationIndexLease(ctx, script.touch, cancel, started.Add(lease), every, lease, safety)
defer stop()
select {
case <-ctx.Done():
return time.Since(started), true, script.count()
case <-time.After(observe):
return 0, false, script.count()
}
}
func TestAnnotationIndexLeaseSurvivesTransientTouchErrors(t *testing.T) {
// Every third renewal fails; successful renewals keep extending the lease.
_, cancelled, calls := runLeaseScript(t, func(call int) (bool, error) {
if call%3 == 0 {
return false, errors.New("mongo failover")
}
return true, nil
}, 300*time.Millisecond)
if cancelled {
t.Fatal("transient renewal error cancelled a job whose lease was still valid")
}
if calls < 5 {
t.Fatalf("lease renewed %d times, want continued renewal", calls)
}
}
func TestAnnotationIndexLeaseCancelsWhenLeaseIsLost(t *testing.T) {
after, cancelled, _ := runLeaseScript(t, func(int) (bool, error) { return false, nil }, 300*time.Millisecond)
if !cancelled || after > 100*time.Millisecond {
t.Fatalf("lost lease cancelled=%v after %v, want prompt cancellation", cancelled, after)
}
}
func TestAnnotationIndexLeaseCancelsBeforePersistentErrorsLetItLapse(t *testing.T) {
// No renewal ever succeeds: keep working while the claimed lease is
// comfortably valid, then stop before it can lapse (120ms lease, 40ms safety).
after, cancelled, _ := runLeaseScript(t, func(int) (bool, error) { return false, errors.New("mongo unavailable") }, 500*time.Millisecond)
if !cancelled {
t.Fatal("persistent renewal errors never cancelled the job")
}
if after < 60*time.Millisecond || after >= 120*time.Millisecond {
t.Fatalf("cancelled after %v, want after transient tolerance and before the 120ms lease lapses", after)
}
}