droboticflow/internal/api/annotation_index_lease.go
d-robotics 3405d2da4f 🛡️ Keep index jobs running through transient lease renewal errors
The index heartbeat cancelled the job on any TouchAnnotationIndex error, so
a Mongo failover or timeout during a long source validation aborted it and
burned one of its five attempts; five blips marked a healthy source as
failed. Cancel only when the lease is provably lost, or when renewal errors
persist until the claimed lease is within a safety margin of lapsing, so
another processor never reclaims a job that is still running. Renewal
starts from the lease the claim actually stored.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-18 22:28:02 +08:00

68 lines
2.2 KiB
Go

package api
import (
"context"
"droboticflow/internal/domain"
"droboticflow/internal/store"
"time"
)
const (
annotationIndexLeaseDuration = 90 * time.Second
annotationIndexLeaseRenewal = 15 * time.Second
// Stop before the claimed lease can lapse, so another processor never
// reclaims a job this one is still working on.
annotationIndexLeaseSafety = 15 * time.Second
)
func keepAnnotationIndexLease(ctx context.Context, db store.AnnotationProcessingStore, j domain.AnnotationIndexJob, cancel context.CancelFunc) func() {
touch := func(ctx context.Context, until time.Time) (bool, error) {
return db.TouchAnnotationIndex(ctx, j.ID, j.Owner, until.UTC().Format(time.RFC3339Nano))
}
// The claim already set the first lease; fall back to a full lease from now
// only if the stored value is unreadable.
leaseUntil, e := time.Parse(time.RFC3339Nano, j.LeaseUntil)
if e != nil {
leaseUntil = time.Now().Add(annotationIndexLeaseDuration)
}
return renewAnnotationIndexLease(ctx, touch, cancel, leaseUntil, annotationIndexLeaseRenewal, annotationIndexLeaseDuration, annotationIndexLeaseSafety)
}
// renewAnnotationIndexLease cancels the job only when the lease is provably
// lost (another owner or a finished job) or when transient renewal errors have
// lasted long enough that the lease may lapse. A single failed renewal, such
// as a Mongo failover, must not abort a long source validation and burn one
// of its limited attempts.
func renewAnnotationIndexLease(ctx context.Context, touch func(context.Context, time.Time) (bool, error), cancel context.CancelFunc, leaseUntil time.Time, every, lease, safety time.Duration) func() {
done, stopped := make(chan struct{}), make(chan struct{})
go func() {
defer close(stopped)
tick := time.NewTicker(every)
defer tick.Stop()
for {
select {
case <-done:
return
case <-ctx.Done():
return
case <-tick.C:
until := time.Now().Add(lease)
pulse, stop := context.WithTimeout(ctx, min(10*time.Second, every))
ok, e := touch(pulse, until)
stop()
if e == nil && !ok {
cancel()
return
}
if e == nil {
leaseUntil = until
} else if time.Until(leaseUntil) <= safety {
cancel()
return
}
}
}
}()
return func() { close(done); <-stopped }
}