droboticflow/internal/objectsync/coordinator_workers.go
eust-w a56882bedc ✨ Verify legacy SHA256-only archives and backfill their MD5 server-side
Archived heads from protocol v1 carry only a SHA-256 receipt, so every sync
that still holds those files ends partial and they can never be cleaned.

Add an administrator-approved digest backfill job run by the coordinator.
It reads each legacy object with conditional (If-Match) ranged GETs, hashes
MD5 and SHA-256 in one pass, cross-checks every surviving receipt (retired
head sha256, drosync object metadata, BOS stored digests), and records the
MD5 on the head with a compare-and-set write, a bound attestation and an
audit event. It never writes to BOS and never accepts a client digest.

The cleanup proof now accepts an object without drosync-md5sum metadata
only through an attestation bound to its live ETag, version and size. The
next ordinary sync plans those files unchanged, completes, and they become
cleanable; closed partial sessions are left untouched.

Includes the design doc comparing remediation options, store/coordinator/
API unit tests and an end-to-end recovery test on MemoryStore and fake
object storage. Nothing is deployed.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-30 15:44:14 +08:00

409 lines
13 KiB
Go

package objectsync
import (
"context"
"errors"
"fmt"
"sync"
"sync/atomic"
"time"
"droboticflow/internal/domain"
)
const (
phasePlanning = iota
phaseCommit
phaseCleanup
phaseReconcile
phaseRelease
phaseAudit
phaseDigestBackfill
phaseCount
commitConcurrency = 4
cleanupConcurrency = 2
planConcurrency = 16
)
var phaseNames = [phaseCount]string{"planning", "commit", "cleanup", "reconcile", "release", "audit", "digest_backfill"}
var errClaimLost = errors.New("coordinator claim or object lease lost")
type phaseCounters struct{ cycles, errors, completed, itemErrors, inFlight, durationMS, totalDurationMS atomic.Int64 }
type PhaseMetrics struct{ Cycles, Errors, Completed, ItemErrors, InFlight, DurationMS, TotalDurationMS int64 }
func (coordinator *Coordinator) executeStage(ctx context.Context, phase int) (ProcessResult, error) {
return coordinator.measureStage(phase, func() (ProcessResult, error) { return coordinator.processStage(ctx, phase) })
}
func (coordinator *Coordinator) executeSingleItem(ctx context.Context, phase int) (ProcessResult, error) {
return coordinator.measureStage(phase, func() (ProcessResult, error) {
result := ProcessResult{}
count, err := coordinator.processItems(ctx, phase == phaseCleanup, 1, 1)
if phase == phaseCommit {
result.Committed = count
} else {
result.Cleaned = count
}
return result, err
})
}
func (coordinator *Coordinator) measureStage(phase int, work func() (ProcessResult, error)) (ProcessResult, error) {
metrics := &coordinator.phases[phase]
metrics.inFlight.Add(1)
started := time.Now()
result, err := work()
elapsed := time.Since(started).Milliseconds()
metrics.inFlight.Add(-1)
metrics.cycles.Add(1)
metrics.durationMS.Store(elapsed)
metrics.totalDurationMS.Add(elapsed)
completed := result.Planned + result.Committed + result.Cleaned + result.Reconciled + result.Released + result.Sealed + result.Backfilled
metrics.completed.Add(int64(completed))
coordinator.planned.Add(int64(result.Planned))
coordinator.committed.Add(int64(result.Committed))
coordinator.cleaned.Add(int64(result.Cleaned))
coordinator.reconciled.Add(int64(result.Reconciled))
coordinator.released.Add(int64(result.Released))
coordinator.sealed.Add(int64(result.Sealed))
coordinator.backfilled.Add(int64(result.Backfilled))
if err != nil && !errors.Is(err, context.Canceled) {
metrics.errors.Add(1)
coordinator.errors.Add(1)
fmt.Printf("[drosync-coordinator] %s error: %v\n", phaseNames[phase], err)
}
if phase == phaseCommit {
coordinator.cycles.Add(1)
coordinator.lastCycle.Store(time.Now().Unix())
coordinator.durationMS.Store(elapsed)
if err == nil {
coordinator.lastOK.Store(time.Now().Unix())
}
}
return result, err
}
func (coordinator *Coordinator) processStage(ctx context.Context, phase int) (ProcessResult, error) {
result := ProcessResult{}
switch phase {
case phasePlanning:
session, err := coordinator.syncStore.ClaimObjectSyncPlanningSession(ctx, coordinator.owner, claimTTL)
if err != nil || session == nil {
return result, err
}
token := session.CoordinatorOwner
err = coordinator.withRenewingClaim(ctx, func(callCtx context.Context) (bool, error) {
return coordinator.syncStore.RenewObjectSyncSessionClaim(callCtx, session.ID, token, claimTTL)
}, func(callCtx context.Context) error { return coordinator.planSession(callCtx, session) })
if err != nil {
if !errors.Is(err, errClaimLost) && !errors.Is(err, context.Canceled) {
session.Status = domain.ObjectSyncSessionFailed
session.ErrorSummary = err.Error()
session.UpdatedAt = nowISO()
session.CoordinatorOwner = ""
session.CoordinatorLeaseUntil = time.Time{}
saved, saveErr := coordinator.syncStore.SaveClaimedObjectSyncSession(ctx, *session, token)
if !saved && saveErr == nil {
saveErr = errClaimLost
}
err = errors.Join(err, saveErr)
}
return result, err
}
result.Planned = 1
case phaseCommit, phaseCleanup:
count, err := coordinator.processItemBatch(ctx, phase == phaseCleanup)
if phase == phaseCommit {
result.Committed = count
} else {
result.Cleaned = count
}
return result, err
case phaseReconcile:
item, err := coordinator.syncStore.ClaimObjectSyncReconcile(ctx, coordinator.owner, claimTTL)
if err != nil || item == nil {
return result, err
}
token, id := item.CoordinatorOwner, item.ID
err = coordinator.withRenewingClaim(ctx, func(callCtx context.Context) (bool, error) {
return coordinator.syncStore.RenewObjectSyncReconcileClaim(callCtx, id, token, claimTTL)
}, func(callCtx context.Context) error { return coordinator.reconcileSpace(callCtx, item) })
if err != nil {
if !errors.Is(err, errClaimLost) && !errors.Is(err, context.Canceled) {
item.Status = "failed"
item.Error = err.Error()
item.UpdatedAt = nowISO()
item.CompletedAt = item.UpdatedAt
item.CoordinatorOwner = ""
item.CoordinatorLeaseUntil = time.Time{}
saved, saveErr := coordinator.syncStore.SaveClaimedObjectSyncReconcile(ctx, *item, token)
if !saved && saveErr == nil {
saveErr = errClaimLost
}
err = errors.Join(err, saveErr)
}
return result, err
}
result.Reconciled = 1
case phaseRelease:
item, err := coordinator.syncStore.ClaimObjectSyncRelease(ctx, coordinator.owner, claimTTL)
if err != nil || item == nil {
return result, err
}
token, id := item.CoordinatorOwner, item.ID
err = coordinator.withRenewingClaim(ctx, func(callCtx context.Context) (bool, error) {
return coordinator.syncStore.RenewObjectSyncReleaseClaim(callCtx, id, token, claimTTL)
}, func(callCtx context.Context) error { return coordinator.publishRelease(callCtx, item) })
if err != nil {
if !errors.Is(err, errClaimLost) && !errors.Is(err, context.Canceled) {
item.Status = "failed"
item.Error = err.Error()
item.UpdatedAt = nowISO()
item.CoordinatorOwner = ""
item.CoordinatorLeaseUntil = time.Time{}
saved, saveErr := coordinator.syncStore.SaveClaimedObjectSyncRelease(ctx, *item, token)
if !saved && saveErr == nil {
saveErr = errClaimLost
}
err = errors.Join(err, saveErr)
}
return result, err
}
result.Released = 1
case phaseAudit:
if coordinator.AuditSigningConfigured() {
count, err := coordinator.sealPreviousAuditDay(ctx, time.Now().UTC())
result.Sealed = count
return result, err
}
case phaseDigestBackfill:
job, err := coordinator.syncStore.ClaimObjectSyncDigestBackfill(ctx, coordinator.owner, claimTTL)
if err != nil || job == nil {
return result, err
}
token, id := job.CoordinatorOwner, job.ID
err = coordinator.withRenewingClaim(ctx, func(callCtx context.Context) (bool, error) {
return coordinator.syncStore.RenewObjectSyncDigestBackfillClaim(callCtx, id, token, claimTTL)
}, func(callCtx context.Context) error { return coordinator.runDigestBackfill(callCtx, job) })
if err != nil {
if !errors.Is(err, errClaimLost) && !errors.Is(err, context.Canceled) {
job.Status = "failed"
job.Error = err.Error()
job.UpdatedAt = nowISO()
job.CompletedAt = job.UpdatedAt
job.CoordinatorOwner = ""
job.CoordinatorLeaseUntil = time.Time{}
saved, saveErr := coordinator.syncStore.SaveClaimedObjectSyncDigestBackfill(ctx, *job, token)
if !saved && saveErr == nil {
saveErr = errClaimLost
}
err = errors.Join(err, saveErr)
}
return result, err
}
result.Backfilled = 1
}
return result, nil
}
func (coordinator *Coordinator) withRenewingClaim(ctx context.Context, renew func(context.Context) (bool, error), work func(context.Context) error) error {
owned, err := renew(ctx)
if err != nil {
return err
}
if !owned {
return errClaimLost
}
workCtx, cancel := context.WithCancelCause(ctx)
defer cancel(nil)
interval := coordinator.renewInterval
if interval <= 0 {
interval = 30 * time.Second
}
stop := make(chan struct{})
done := make(chan struct{})
go func() {
defer close(done)
ticker := time.NewTicker(interval)
defer ticker.Stop()
for {
select {
case <-stop:
return
case <-workCtx.Done():
return
case <-ticker.C:
renewCtx, renewCancel := context.WithTimeout(workCtx, 20*time.Second)
owned, err := renew(renewCtx)
renewCancel()
if err != nil || !owned {
cancel(errors.Join(errClaimLost, err))
return
}
}
}
}()
err = work(workCtx)
close(stop)
<-done
if err == nil {
return nil
}
if cause := context.Cause(workCtx); cause != nil {
return cause
}
return err
}
func (coordinator *Coordinator) processItemBatch(ctx context.Context, cleanup bool) (int, error) {
workers, limit := commitConcurrency, 16
if cleanup {
workers, limit = cleanupConcurrency, 8
}
return coordinator.processItems(ctx, cleanup, workers, limit)
}
func (coordinator *Coordinator) processItems(ctx context.Context, cleanup bool, workers, limit int) (int, error) {
phase := phaseCommit
if cleanup {
phase = phaseCleanup
}
var attempts, completed atomic.Int64
var mu sync.Mutex
var failures []error
var group sync.WaitGroup
recordFailure := func(err error) { mu.Lock(); failures = append(failures, err); mu.Unlock() }
for range workers {
group.Add(1)
go func() {
defer group.Done()
for {
if ctx.Err() != nil {
return
}
if attempts.Add(1) > int64(limit) {
return
}
var item *domain.ObjectSyncItemRecord
var err error
if cleanup {
item, err = coordinator.syncStore.ClaimObjectSyncCleanupItem(ctx, coordinator.owner, time.Now().UTC().Add(-stagingRetention), claimTTL)
} else {
item, err = coordinator.syncStore.ClaimObjectSyncItem(ctx, coordinator.owner, claimTTL)
}
if err != nil {
recordFailure(err)
return
}
if item == nil {
return
}
token, id, sessionID, leaseID, fence := item.CoordinatorOwner, item.ID, item.SessionID, item.LeaseID, item.FencingToken
err = coordinator.withRenewingClaim(ctx, func(callCtx context.Context) (bool, error) {
owned, renewErr := coordinator.syncStore.RenewObjectSyncItemClaim(callCtx, id, token, claimTTL)
if renewErr != nil || !owned || cleanup {
return owned, renewErr
}
return coordinator.syncStore.RenewObjectSyncLease(callCtx, leaseID, sessionID, id, fence, claimTTL)
}, func(callCtx context.Context) error {
if cleanup {
return coordinator.cleanupStaging(callCtx, item)
}
return coordinator.commitItem(callCtx, item)
})
if err != nil {
coordinator.phases[phase].itemErrors.Add(1)
if !errors.Is(err, errClaimLost) && !errors.Is(err, context.Canceled) {
if !cleanup {
item.Status = domain.ObjectSyncItemFailed
}
item.LastError = err.Error()
item.AttemptCount++
item.CoordinatorOwner = ""
item.CoordinatorLeaseUntil = time.Time{}
item.UpdatedAt = nowISO()
saved, saveErr := coordinator.syncStore.SaveClaimedObjectSyncItem(ctx, *item, token)
if !saved && saveErr == nil {
saveErr = errClaimLost
}
err = errors.Join(err, saveErr)
}
recordFailure(fmt.Errorf("item %s: %w", id, err))
continue
}
completed.Add(1)
}
}()
}
group.Wait()
return int(completed.Load()), errors.Join(failures...)
}
func (coordinator *Coordinator) saveClaimedItem(ctx context.Context, item domain.ObjectSyncItemRecord, token string) error {
if err := ctx.Err(); err != nil {
return err
}
saved, err := coordinator.syncStore.SaveClaimedObjectSyncItem(ctx, item, token)
if err != nil {
return err
}
if !saved {
return errClaimLost
}
return nil
}
func (coordinator *Coordinator) recordAuxiliaryFailure(phase int, operation string, err error) {
coordinator.errors.Add(1)
coordinator.phases[phase].itemErrors.Add(1)
fmt.Printf("[drosync-coordinator] %s %s: %v\n", phaseNames[phase], operation, err)
}
// Queue observation is bounded and decoupled from the metrics scrape path.
// A failed observation leaves its prior timestamp intact, making staleness visible.
type queueSnapshot struct {
stats domain.ObjectSyncQueueStats
sampleAt int64
}
func (coordinator *Coordinator) refreshQueueStats(ctx context.Context) error {
queryCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
defer cancel()
stats, err := coordinator.syncStore.GetObjectSyncQueueStats(queryCtx)
if err != nil {
if !errors.Is(err, context.Canceled) {
coordinator.errors.Add(1)
fmt.Printf("[drosync-coordinator] queue sample failed: %v\n", err)
}
return err
}
coordinator.queueSnapshot.Store(&queueSnapshot{stats: stats, sampleAt: time.Now().Unix()})
return nil
}
func (coordinator *Coordinator) runQueueSampler(ctx context.Context) {
for ctx.Err() == nil {
_ = coordinator.refreshQueueStats(ctx)
timer := time.NewTimer(30 * time.Second)
select {
case <-ctx.Done():
timer.Stop()
return
case <-timer.C:
}
}
}
func requireOwnedClaim(ctx context.Context, renew func(context.Context) (bool, error)) error {
if err := ctx.Err(); err != nil {
return err
}
owned, err := renew(ctx)
if err != nil {
return err
}
if !owned {
return errClaimLost
}
return nil
}