forked from longtao.wu/droboticflow
The index heartbeat cancelled the job on any TouchAnnotationIndex error, so a Mongo failover or timeout during a long source validation aborted it and burned one of its five attempts; five blips marked a healthy source as failed. Cancel only when the lease is provably lost, or when renewal errors persist until the claimed lease is within a safety margin of lapsing, so another processor never reclaims a job that is still running. Renewal starts from the lease the claim actually stored. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
82 lines
2.5 KiB
Go
82 lines
2.5 KiB
Go
package api
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
type leaseTouchScript struct {
|
|
mu sync.Mutex
|
|
calls int
|
|
reply func(call int) (bool, error)
|
|
}
|
|
|
|
func (s *leaseTouchScript) touch(context.Context, time.Time) (bool, error) {
|
|
s.mu.Lock()
|
|
s.calls++
|
|
call := s.calls
|
|
s.mu.Unlock()
|
|
return s.reply(call)
|
|
}
|
|
|
|
func (s *leaseTouchScript) count() int {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
return s.calls
|
|
}
|
|
|
|
func runLeaseScript(t *testing.T, reply func(call int) (bool, error), observe time.Duration) (cancelledAfter time.Duration, cancelled bool, calls int) {
|
|
t.Helper()
|
|
const every, lease, safety = 10 * time.Millisecond, 120 * time.Millisecond, 40 * time.Millisecond
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
script := &leaseTouchScript{reply: reply}
|
|
started := time.Now()
|
|
stop := renewAnnotationIndexLease(ctx, script.touch, cancel, started.Add(lease), every, lease, safety)
|
|
defer stop()
|
|
select {
|
|
case <-ctx.Done():
|
|
return time.Since(started), true, script.count()
|
|
case <-time.After(observe):
|
|
return 0, false, script.count()
|
|
}
|
|
}
|
|
|
|
func TestAnnotationIndexLeaseSurvivesTransientTouchErrors(t *testing.T) {
|
|
// Every third renewal fails; successful renewals keep extending the lease.
|
|
_, cancelled, calls := runLeaseScript(t, func(call int) (bool, error) {
|
|
if call%3 == 0 {
|
|
return false, errors.New("mongo failover")
|
|
}
|
|
return true, nil
|
|
}, 300*time.Millisecond)
|
|
if cancelled {
|
|
t.Fatal("transient renewal error cancelled a job whose lease was still valid")
|
|
}
|
|
if calls < 5 {
|
|
t.Fatalf("lease renewed %d times, want continued renewal", calls)
|
|
}
|
|
}
|
|
|
|
func TestAnnotationIndexLeaseCancelsWhenLeaseIsLost(t *testing.T) {
|
|
after, cancelled, _ := runLeaseScript(t, func(int) (bool, error) { return false, nil }, 300*time.Millisecond)
|
|
if !cancelled || after > 100*time.Millisecond {
|
|
t.Fatalf("lost lease cancelled=%v after %v, want prompt cancellation", cancelled, after)
|
|
}
|
|
}
|
|
|
|
func TestAnnotationIndexLeaseCancelsBeforePersistentErrorsLetItLapse(t *testing.T) {
|
|
// No renewal ever succeeds: keep working while the claimed lease is
|
|
// comfortably valid, then stop before it can lapse (120ms lease, 40ms safety).
|
|
after, cancelled, _ := runLeaseScript(t, func(int) (bool, error) { return false, errors.New("mongo unavailable") }, 500*time.Millisecond)
|
|
if !cancelled {
|
|
t.Fatal("persistent renewal errors never cancelled the job")
|
|
}
|
|
if after < 60*time.Millisecond || after >= 120*time.Millisecond {
|
|
t.Fatalf("cancelled after %v, want after transient tolerance and before the 120ms lease lapses", after)
|
|
}
|
|
}
|