forked from longtao.wu/droboticflow
Archived heads from protocol v1 carry only a SHA-256 receipt, so every sync that still holds those files ends partial and they can never be cleaned. Add an administrator-approved digest backfill job run by the coordinator. It reads each legacy object with conditional (If-Match) ranged GETs, hashes MD5 and SHA-256 in one pass, cross-checks every surviving receipt (retired head sha256, drosync object metadata, BOS stored digests), and records the MD5 on the head with a compare-and-set write, a bound attestation and an audit event. It never writes to BOS and never accepts a client digest. The cleanup proof now accepts an object without drosync-md5sum metadata only through an attestation bound to its live ETag, version and size. The next ordinary sync plans those files unchanged, completes, and they become cleanable; closed partial sessions are left untouched. Includes the design doc comparing remediation options, store/coordinator/ API unit tests and an end-to-end recovery test on MemoryStore and fake object storage. Nothing is deployed. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
409 lines
13 KiB
Go
409 lines
13 KiB
Go
package objectsync
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"sync"
|
|
"sync/atomic"
|
|
"time"
|
|
|
|
"droboticflow/internal/domain"
|
|
)
|
|
|
|
const (
|
|
phasePlanning = iota
|
|
phaseCommit
|
|
phaseCleanup
|
|
phaseReconcile
|
|
phaseRelease
|
|
phaseAudit
|
|
phaseDigestBackfill
|
|
phaseCount
|
|
commitConcurrency = 4
|
|
cleanupConcurrency = 2
|
|
planConcurrency = 16
|
|
)
|
|
|
|
var phaseNames = [phaseCount]string{"planning", "commit", "cleanup", "reconcile", "release", "audit", "digest_backfill"}
|
|
var errClaimLost = errors.New("coordinator claim or object lease lost")
|
|
|
|
type phaseCounters struct{ cycles, errors, completed, itemErrors, inFlight, durationMS, totalDurationMS atomic.Int64 }
|
|
type PhaseMetrics struct{ Cycles, Errors, Completed, ItemErrors, InFlight, DurationMS, TotalDurationMS int64 }
|
|
|
|
func (coordinator *Coordinator) executeStage(ctx context.Context, phase int) (ProcessResult, error) {
|
|
return coordinator.measureStage(phase, func() (ProcessResult, error) { return coordinator.processStage(ctx, phase) })
|
|
}
|
|
|
|
func (coordinator *Coordinator) executeSingleItem(ctx context.Context, phase int) (ProcessResult, error) {
|
|
return coordinator.measureStage(phase, func() (ProcessResult, error) {
|
|
result := ProcessResult{}
|
|
count, err := coordinator.processItems(ctx, phase == phaseCleanup, 1, 1)
|
|
if phase == phaseCommit {
|
|
result.Committed = count
|
|
} else {
|
|
result.Cleaned = count
|
|
}
|
|
return result, err
|
|
})
|
|
}
|
|
|
|
func (coordinator *Coordinator) measureStage(phase int, work func() (ProcessResult, error)) (ProcessResult, error) {
|
|
metrics := &coordinator.phases[phase]
|
|
metrics.inFlight.Add(1)
|
|
started := time.Now()
|
|
result, err := work()
|
|
elapsed := time.Since(started).Milliseconds()
|
|
metrics.inFlight.Add(-1)
|
|
metrics.cycles.Add(1)
|
|
metrics.durationMS.Store(elapsed)
|
|
metrics.totalDurationMS.Add(elapsed)
|
|
completed := result.Planned + result.Committed + result.Cleaned + result.Reconciled + result.Released + result.Sealed + result.Backfilled
|
|
metrics.completed.Add(int64(completed))
|
|
coordinator.planned.Add(int64(result.Planned))
|
|
coordinator.committed.Add(int64(result.Committed))
|
|
coordinator.cleaned.Add(int64(result.Cleaned))
|
|
coordinator.reconciled.Add(int64(result.Reconciled))
|
|
coordinator.released.Add(int64(result.Released))
|
|
coordinator.sealed.Add(int64(result.Sealed))
|
|
coordinator.backfilled.Add(int64(result.Backfilled))
|
|
if err != nil && !errors.Is(err, context.Canceled) {
|
|
metrics.errors.Add(1)
|
|
coordinator.errors.Add(1)
|
|
fmt.Printf("[drosync-coordinator] %s error: %v\n", phaseNames[phase], err)
|
|
}
|
|
if phase == phaseCommit {
|
|
coordinator.cycles.Add(1)
|
|
coordinator.lastCycle.Store(time.Now().Unix())
|
|
coordinator.durationMS.Store(elapsed)
|
|
if err == nil {
|
|
coordinator.lastOK.Store(time.Now().Unix())
|
|
}
|
|
}
|
|
return result, err
|
|
}
|
|
|
|
func (coordinator *Coordinator) processStage(ctx context.Context, phase int) (ProcessResult, error) {
|
|
result := ProcessResult{}
|
|
switch phase {
|
|
case phasePlanning:
|
|
session, err := coordinator.syncStore.ClaimObjectSyncPlanningSession(ctx, coordinator.owner, claimTTL)
|
|
if err != nil || session == nil {
|
|
return result, err
|
|
}
|
|
token := session.CoordinatorOwner
|
|
err = coordinator.withRenewingClaim(ctx, func(callCtx context.Context) (bool, error) {
|
|
return coordinator.syncStore.RenewObjectSyncSessionClaim(callCtx, session.ID, token, claimTTL)
|
|
}, func(callCtx context.Context) error { return coordinator.planSession(callCtx, session) })
|
|
if err != nil {
|
|
if !errors.Is(err, errClaimLost) && !errors.Is(err, context.Canceled) {
|
|
session.Status = domain.ObjectSyncSessionFailed
|
|
session.ErrorSummary = err.Error()
|
|
session.UpdatedAt = nowISO()
|
|
session.CoordinatorOwner = ""
|
|
session.CoordinatorLeaseUntil = time.Time{}
|
|
saved, saveErr := coordinator.syncStore.SaveClaimedObjectSyncSession(ctx, *session, token)
|
|
if !saved && saveErr == nil {
|
|
saveErr = errClaimLost
|
|
}
|
|
err = errors.Join(err, saveErr)
|
|
}
|
|
return result, err
|
|
}
|
|
result.Planned = 1
|
|
case phaseCommit, phaseCleanup:
|
|
count, err := coordinator.processItemBatch(ctx, phase == phaseCleanup)
|
|
if phase == phaseCommit {
|
|
result.Committed = count
|
|
} else {
|
|
result.Cleaned = count
|
|
}
|
|
return result, err
|
|
case phaseReconcile:
|
|
item, err := coordinator.syncStore.ClaimObjectSyncReconcile(ctx, coordinator.owner, claimTTL)
|
|
if err != nil || item == nil {
|
|
return result, err
|
|
}
|
|
token, id := item.CoordinatorOwner, item.ID
|
|
err = coordinator.withRenewingClaim(ctx, func(callCtx context.Context) (bool, error) {
|
|
return coordinator.syncStore.RenewObjectSyncReconcileClaim(callCtx, id, token, claimTTL)
|
|
}, func(callCtx context.Context) error { return coordinator.reconcileSpace(callCtx, item) })
|
|
if err != nil {
|
|
if !errors.Is(err, errClaimLost) && !errors.Is(err, context.Canceled) {
|
|
item.Status = "failed"
|
|
item.Error = err.Error()
|
|
item.UpdatedAt = nowISO()
|
|
item.CompletedAt = item.UpdatedAt
|
|
item.CoordinatorOwner = ""
|
|
item.CoordinatorLeaseUntil = time.Time{}
|
|
saved, saveErr := coordinator.syncStore.SaveClaimedObjectSyncReconcile(ctx, *item, token)
|
|
if !saved && saveErr == nil {
|
|
saveErr = errClaimLost
|
|
}
|
|
err = errors.Join(err, saveErr)
|
|
}
|
|
return result, err
|
|
}
|
|
result.Reconciled = 1
|
|
case phaseRelease:
|
|
item, err := coordinator.syncStore.ClaimObjectSyncRelease(ctx, coordinator.owner, claimTTL)
|
|
if err != nil || item == nil {
|
|
return result, err
|
|
}
|
|
token, id := item.CoordinatorOwner, item.ID
|
|
err = coordinator.withRenewingClaim(ctx, func(callCtx context.Context) (bool, error) {
|
|
return coordinator.syncStore.RenewObjectSyncReleaseClaim(callCtx, id, token, claimTTL)
|
|
}, func(callCtx context.Context) error { return coordinator.publishRelease(callCtx, item) })
|
|
if err != nil {
|
|
if !errors.Is(err, errClaimLost) && !errors.Is(err, context.Canceled) {
|
|
item.Status = "failed"
|
|
item.Error = err.Error()
|
|
item.UpdatedAt = nowISO()
|
|
item.CoordinatorOwner = ""
|
|
item.CoordinatorLeaseUntil = time.Time{}
|
|
saved, saveErr := coordinator.syncStore.SaveClaimedObjectSyncRelease(ctx, *item, token)
|
|
if !saved && saveErr == nil {
|
|
saveErr = errClaimLost
|
|
}
|
|
err = errors.Join(err, saveErr)
|
|
}
|
|
return result, err
|
|
}
|
|
result.Released = 1
|
|
case phaseAudit:
|
|
if coordinator.AuditSigningConfigured() {
|
|
count, err := coordinator.sealPreviousAuditDay(ctx, time.Now().UTC())
|
|
result.Sealed = count
|
|
return result, err
|
|
}
|
|
case phaseDigestBackfill:
|
|
job, err := coordinator.syncStore.ClaimObjectSyncDigestBackfill(ctx, coordinator.owner, claimTTL)
|
|
if err != nil || job == nil {
|
|
return result, err
|
|
}
|
|
token, id := job.CoordinatorOwner, job.ID
|
|
err = coordinator.withRenewingClaim(ctx, func(callCtx context.Context) (bool, error) {
|
|
return coordinator.syncStore.RenewObjectSyncDigestBackfillClaim(callCtx, id, token, claimTTL)
|
|
}, func(callCtx context.Context) error { return coordinator.runDigestBackfill(callCtx, job) })
|
|
if err != nil {
|
|
if !errors.Is(err, errClaimLost) && !errors.Is(err, context.Canceled) {
|
|
job.Status = "failed"
|
|
job.Error = err.Error()
|
|
job.UpdatedAt = nowISO()
|
|
job.CompletedAt = job.UpdatedAt
|
|
job.CoordinatorOwner = ""
|
|
job.CoordinatorLeaseUntil = time.Time{}
|
|
saved, saveErr := coordinator.syncStore.SaveClaimedObjectSyncDigestBackfill(ctx, *job, token)
|
|
if !saved && saveErr == nil {
|
|
saveErr = errClaimLost
|
|
}
|
|
err = errors.Join(err, saveErr)
|
|
}
|
|
return result, err
|
|
}
|
|
result.Backfilled = 1
|
|
}
|
|
return result, nil
|
|
}
|
|
|
|
func (coordinator *Coordinator) withRenewingClaim(ctx context.Context, renew func(context.Context) (bool, error), work func(context.Context) error) error {
|
|
owned, err := renew(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if !owned {
|
|
return errClaimLost
|
|
}
|
|
workCtx, cancel := context.WithCancelCause(ctx)
|
|
defer cancel(nil)
|
|
interval := coordinator.renewInterval
|
|
if interval <= 0 {
|
|
interval = 30 * time.Second
|
|
}
|
|
stop := make(chan struct{})
|
|
done := make(chan struct{})
|
|
go func() {
|
|
defer close(done)
|
|
ticker := time.NewTicker(interval)
|
|
defer ticker.Stop()
|
|
for {
|
|
select {
|
|
case <-stop:
|
|
return
|
|
case <-workCtx.Done():
|
|
return
|
|
case <-ticker.C:
|
|
renewCtx, renewCancel := context.WithTimeout(workCtx, 20*time.Second)
|
|
owned, err := renew(renewCtx)
|
|
renewCancel()
|
|
if err != nil || !owned {
|
|
cancel(errors.Join(errClaimLost, err))
|
|
return
|
|
}
|
|
}
|
|
}
|
|
}()
|
|
err = work(workCtx)
|
|
close(stop)
|
|
<-done
|
|
if err == nil {
|
|
return nil
|
|
}
|
|
if cause := context.Cause(workCtx); cause != nil {
|
|
return cause
|
|
}
|
|
return err
|
|
}
|
|
|
|
func (coordinator *Coordinator) processItemBatch(ctx context.Context, cleanup bool) (int, error) {
|
|
workers, limit := commitConcurrency, 16
|
|
if cleanup {
|
|
workers, limit = cleanupConcurrency, 8
|
|
}
|
|
return coordinator.processItems(ctx, cleanup, workers, limit)
|
|
}
|
|
|
|
func (coordinator *Coordinator) processItems(ctx context.Context, cleanup bool, workers, limit int) (int, error) {
|
|
phase := phaseCommit
|
|
if cleanup {
|
|
phase = phaseCleanup
|
|
}
|
|
var attempts, completed atomic.Int64
|
|
var mu sync.Mutex
|
|
var failures []error
|
|
var group sync.WaitGroup
|
|
recordFailure := func(err error) { mu.Lock(); failures = append(failures, err); mu.Unlock() }
|
|
for range workers {
|
|
group.Add(1)
|
|
go func() {
|
|
defer group.Done()
|
|
for {
|
|
if ctx.Err() != nil {
|
|
return
|
|
}
|
|
if attempts.Add(1) > int64(limit) {
|
|
return
|
|
}
|
|
var item *domain.ObjectSyncItemRecord
|
|
var err error
|
|
if cleanup {
|
|
item, err = coordinator.syncStore.ClaimObjectSyncCleanupItem(ctx, coordinator.owner, time.Now().UTC().Add(-stagingRetention), claimTTL)
|
|
} else {
|
|
item, err = coordinator.syncStore.ClaimObjectSyncItem(ctx, coordinator.owner, claimTTL)
|
|
}
|
|
if err != nil {
|
|
recordFailure(err)
|
|
return
|
|
}
|
|
if item == nil {
|
|
return
|
|
}
|
|
token, id, sessionID, leaseID, fence := item.CoordinatorOwner, item.ID, item.SessionID, item.LeaseID, item.FencingToken
|
|
err = coordinator.withRenewingClaim(ctx, func(callCtx context.Context) (bool, error) {
|
|
owned, renewErr := coordinator.syncStore.RenewObjectSyncItemClaim(callCtx, id, token, claimTTL)
|
|
if renewErr != nil || !owned || cleanup {
|
|
return owned, renewErr
|
|
}
|
|
return coordinator.syncStore.RenewObjectSyncLease(callCtx, leaseID, sessionID, id, fence, claimTTL)
|
|
}, func(callCtx context.Context) error {
|
|
if cleanup {
|
|
return coordinator.cleanupStaging(callCtx, item)
|
|
}
|
|
return coordinator.commitItem(callCtx, item)
|
|
})
|
|
if err != nil {
|
|
coordinator.phases[phase].itemErrors.Add(1)
|
|
if !errors.Is(err, errClaimLost) && !errors.Is(err, context.Canceled) {
|
|
if !cleanup {
|
|
item.Status = domain.ObjectSyncItemFailed
|
|
}
|
|
item.LastError = err.Error()
|
|
item.AttemptCount++
|
|
item.CoordinatorOwner = ""
|
|
item.CoordinatorLeaseUntil = time.Time{}
|
|
item.UpdatedAt = nowISO()
|
|
saved, saveErr := coordinator.syncStore.SaveClaimedObjectSyncItem(ctx, *item, token)
|
|
if !saved && saveErr == nil {
|
|
saveErr = errClaimLost
|
|
}
|
|
err = errors.Join(err, saveErr)
|
|
}
|
|
recordFailure(fmt.Errorf("item %s: %w", id, err))
|
|
continue
|
|
}
|
|
completed.Add(1)
|
|
}
|
|
}()
|
|
}
|
|
group.Wait()
|
|
return int(completed.Load()), errors.Join(failures...)
|
|
}
|
|
|
|
func (coordinator *Coordinator) saveClaimedItem(ctx context.Context, item domain.ObjectSyncItemRecord, token string) error {
|
|
if err := ctx.Err(); err != nil {
|
|
return err
|
|
}
|
|
saved, err := coordinator.syncStore.SaveClaimedObjectSyncItem(ctx, item, token)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if !saved {
|
|
return errClaimLost
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (coordinator *Coordinator) recordAuxiliaryFailure(phase int, operation string, err error) {
|
|
coordinator.errors.Add(1)
|
|
coordinator.phases[phase].itemErrors.Add(1)
|
|
fmt.Printf("[drosync-coordinator] %s %s: %v\n", phaseNames[phase], operation, err)
|
|
}
|
|
|
|
// Queue observation is bounded and decoupled from the metrics scrape path.
|
|
// A failed observation leaves its prior timestamp intact, making staleness visible.
|
|
type queueSnapshot struct {
|
|
stats domain.ObjectSyncQueueStats
|
|
sampleAt int64
|
|
}
|
|
|
|
func (coordinator *Coordinator) refreshQueueStats(ctx context.Context) error {
|
|
queryCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
|
defer cancel()
|
|
stats, err := coordinator.syncStore.GetObjectSyncQueueStats(queryCtx)
|
|
if err != nil {
|
|
if !errors.Is(err, context.Canceled) {
|
|
coordinator.errors.Add(1)
|
|
fmt.Printf("[drosync-coordinator] queue sample failed: %v\n", err)
|
|
}
|
|
return err
|
|
}
|
|
coordinator.queueSnapshot.Store(&queueSnapshot{stats: stats, sampleAt: time.Now().Unix()})
|
|
return nil
|
|
}
|
|
func (coordinator *Coordinator) runQueueSampler(ctx context.Context) {
|
|
for ctx.Err() == nil {
|
|
_ = coordinator.refreshQueueStats(ctx)
|
|
timer := time.NewTimer(30 * time.Second)
|
|
select {
|
|
case <-ctx.Done():
|
|
timer.Stop()
|
|
return
|
|
case <-timer.C:
|
|
}
|
|
}
|
|
}
|
|
|
|
func requireOwnedClaim(ctx context.Context, renew func(context.Context) (bool, error)) error {
|
|
if err := ctx.Err(); err != nil {
|
|
return err
|
|
}
|
|
owned, err := renew(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if !owned {
|
|
return errClaimLost
|
|
}
|
|
return nil
|
|
}
|