forked from longtao.wu/droboticflow
The index heartbeat cancelled the job on any TouchAnnotationIndex error, so a Mongo failover or timeout during a long source validation aborted it and burned one of its five attempts; five blips marked a healthy source as failed. Cancel only when the lease is provably lost, or when renewal errors persist until the claimed lease is within a safety margin of lapsing, so another processor never reclaims a job that is still running. Renewal starts from the lease the claim actually stored. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
68 lines
2.2 KiB
Go
68 lines
2.2 KiB
Go
package api
|
|
|
|
import (
|
|
"context"
|
|
"droboticflow/internal/domain"
|
|
"droboticflow/internal/store"
|
|
"time"
|
|
)
|
|
|
|
const (
|
|
annotationIndexLeaseDuration = 90 * time.Second
|
|
annotationIndexLeaseRenewal = 15 * time.Second
|
|
// Stop before the claimed lease can lapse, so another processor never
|
|
// reclaims a job this one is still working on.
|
|
annotationIndexLeaseSafety = 15 * time.Second
|
|
)
|
|
|
|
func keepAnnotationIndexLease(ctx context.Context, db store.AnnotationProcessingStore, j domain.AnnotationIndexJob, cancel context.CancelFunc) func() {
|
|
touch := func(ctx context.Context, until time.Time) (bool, error) {
|
|
return db.TouchAnnotationIndex(ctx, j.ID, j.Owner, until.UTC().Format(time.RFC3339Nano))
|
|
}
|
|
// The claim already set the first lease; fall back to a full lease from now
|
|
// only if the stored value is unreadable.
|
|
leaseUntil, e := time.Parse(time.RFC3339Nano, j.LeaseUntil)
|
|
if e != nil {
|
|
leaseUntil = time.Now().Add(annotationIndexLeaseDuration)
|
|
}
|
|
return renewAnnotationIndexLease(ctx, touch, cancel, leaseUntil, annotationIndexLeaseRenewal, annotationIndexLeaseDuration, annotationIndexLeaseSafety)
|
|
}
|
|
|
|
// renewAnnotationIndexLease cancels the job only when the lease is provably
|
|
// lost (another owner or a finished job) or when transient renewal errors have
|
|
// lasted long enough that the lease may lapse. A single failed renewal, such
|
|
// as a Mongo failover, must not abort a long source validation and burn one
|
|
// of its limited attempts.
|
|
func renewAnnotationIndexLease(ctx context.Context, touch func(context.Context, time.Time) (bool, error), cancel context.CancelFunc, leaseUntil time.Time, every, lease, safety time.Duration) func() {
|
|
done, stopped := make(chan struct{}), make(chan struct{})
|
|
go func() {
|
|
defer close(stopped)
|
|
tick := time.NewTicker(every)
|
|
defer tick.Stop()
|
|
for {
|
|
select {
|
|
case <-done:
|
|
return
|
|
case <-ctx.Done():
|
|
return
|
|
case <-tick.C:
|
|
until := time.Now().Add(lease)
|
|
pulse, stop := context.WithTimeout(ctx, min(10*time.Second, every))
|
|
ok, e := touch(pulse, until)
|
|
stop()
|
|
if e == nil && !ok {
|
|
cancel()
|
|
return
|
|
}
|
|
if e == nil {
|
|
leaseUntil = until
|
|
} else if time.Until(leaseUntil) <= safety {
|
|
cancel()
|
|
return
|
|
}
|
|
}
|
|
}
|
|
}()
|
|
return func() { close(done); <-stopped }
|
|
}
|