eust-w 0a943abecc 🐛 Make staging cleanup idempotent without weakening deletion safety
Retire confirmed missing staging resources while rejecting ambiguous errors, noncanonical targets and malformed or already-retired Mongo keys. Validate Mongo 8.0.17 and real BOS default 64 MiB ranges, including replacement between ranges, and preserve regression and exact-resource cleanup evidence without deploying services or changing the client release.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
2026-11-01 01:35:59 +08:00

1145 lines
38 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

package objectsync
import (
"bufio"
"bytes"
"context"
"crypto/ed25519"
"crypto/sha256"
"encoding/base64"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"os"
"path"
"strings"
"sync"
"sync/atomic"
"time"
"droboticflow/internal/domain"
"droboticflow/internal/store"
"golang.org/x/sync/errgroup"
)
const (
claimTTL = 10 * time.Minute
versionRetention = 365 * 24 * time.Hour
stagingRetention = 24 * time.Hour
)
type Coordinator struct {
store store.Store
syncStore store.ObjectSyncStore
publisher Publisher
owner string
signingKey ed25519.PrivateKey
signingID string
cycles atomic.Int64
errors atomic.Int64
planned atomic.Int64
committed atomic.Int64
cleaned atomic.Int64
reconciled atomic.Int64
released atomic.Int64
sealed atomic.Int64
backfilled atomic.Int64
lastCycle atomic.Int64
lastOK atomic.Int64
durationMS atomic.Int64
phases [phaseCount]phaseCounters
renewInterval time.Duration
queueSnapshot atomic.Pointer[queueSnapshot]
}
func NewCoordinator(base store.Store, publisher Publisher, owner string) (*Coordinator, error) {
syncStore, ok := store.AsObjectSyncStore(base)
if !ok {
return nil, fmt.Errorf("store does not implement object sync persistence")
}
if publisher == nil {
return nil, fmt.Errorf("object publisher is required")
}
if strings.TrimSpace(owner) == "" {
owner = fmt.Sprintf("coordinator-%d", time.Now().UnixNano())
}
return &Coordinator{store: base, syncStore: syncStore, publisher: publisher, owner: owner}, nil
}
func (coordinator *Coordinator) ConfigureAuditSigning(encodedPrivateKey, keyID string) error {
encodedPrivateKey = strings.TrimSpace(encodedPrivateKey)
if encodedPrivateKey == "" {
coordinator.signingKey = nil
coordinator.signingID = ""
return nil
}
raw, err := base64.StdEncoding.DecodeString(encodedPrivateKey)
if err != nil {
raw, err = base64.RawStdEncoding.DecodeString(encodedPrivateKey)
}
if err != nil {
return fmt.Errorf("decode audit signing key: %w", err)
}
switch len(raw) {
case ed25519.SeedSize:
coordinator.signingKey = ed25519.NewKeyFromSeed(raw)
case ed25519.PrivateKeySize:
coordinator.signingKey = ed25519.PrivateKey(append([]byte(nil), raw...))
default:
return fmt.Errorf("audit signing key must be a %d-byte Ed25519 seed or %d-byte private key", ed25519.SeedSize, ed25519.PrivateKeySize)
}
coordinator.signingID = strings.TrimSpace(keyID)
if coordinator.signingID == "" {
return fmt.Errorf("audit signing key id is required")
}
return nil
}
func (coordinator *Coordinator) AuditSigningConfigured() bool {
return len(coordinator.signingKey) == ed25519.PrivateKeySize && coordinator.signingID != ""
}
// Each phase owns its own loop: slow planning, cleanup or audit cannot stop
// subsequent commit batches, and each phase has bounded internal parallelism.
func (coordinator *Coordinator) Run(ctx context.Context, interval time.Duration) {
if interval <= 0 {
interval = 2 * time.Second
}
var group sync.WaitGroup
group.Add(1)
go func() { defer group.Done(); coordinator.runQueueSampler(ctx) }()
for phase := range phaseCount {
workers := 1
if phase == phaseCommit {
workers = commitConcurrency
}
if phase == phaseCleanup {
workers = cleanupConcurrency
}
for range workers {
group.Add(1)
go func() {
defer group.Done()
for ctx.Err() == nil {
if phase == phaseCommit || phase == phaseCleanup {
result, err := coordinator.executeSingleItem(ctx, phase)
if err == nil && result.Committed+result.Cleaned > 0 {
continue
}
} else {
_, _ = coordinator.executeStage(ctx, phase)
}
timer := time.NewTimer(interval)
select {
case <-ctx.Done():
timer.Stop()
return
case <-timer.C:
}
}
}()
}
}
group.Wait()
}
type Metrics struct {
QueueStats domain.ObjectSyncQueueStats
QueueSampleAt int64
Phases map[string]PhaseMetrics
Cycles int64
Errors int64
Planned int64
Committed int64
Cleaned int64
Reconciled int64
Released int64
Sealed int64
Backfilled int64
LastCycle int64
LastOK int64
DurationMS int64
}
func (coordinator *Coordinator) Metrics() Metrics {
phases := make(map[string]PhaseMetrics, phaseCount)
for index, name := range phaseNames {
item := &coordinator.phases[index]
phases[name] = PhaseMetrics{Cycles: item.cycles.Load(), Errors: item.errors.Load(), Completed: item.completed.Load(), ItemErrors: item.itemErrors.Load(), InFlight: item.inFlight.Load(), DurationMS: item.durationMS.Load(), TotalDurationMS: item.totalDurationMS.Load()}
}
queue := coordinator.queueSnapshot.Load()
if queue == nil {
queue = &queueSnapshot{}
}
return Metrics{
QueueStats: queue.stats,
QueueSampleAt: queue.sampleAt,
Phases: phases,
Cycles: coordinator.cycles.Load(),
Errors: coordinator.errors.Load(),
Planned: coordinator.planned.Load(),
Committed: coordinator.committed.Load(),
Cleaned: coordinator.cleaned.Load(),
Reconciled: coordinator.reconciled.Load(),
Released: coordinator.released.Load(),
Sealed: coordinator.sealed.Load(),
Backfilled: coordinator.backfilled.Load(),
LastCycle: coordinator.lastCycle.Load(),
LastOK: coordinator.lastOK.Load(),
DurationMS: coordinator.durationMS.Load(),
}
}
func (coordinator *Coordinator) recordSystemAudit(ctx context.Context, event domain.ObjectSyncAuditEvent) {
if err := AppendAudit(ctx, coordinator.syncStore, event); err != nil {
coordinator.errors.Add(1)
fmt.Printf("[drosync-coordinator] audit persistence error for %s: %v\n", event.Action, err)
}
}
type ProcessResult struct {
Planned int `json:"planned"`
Committed int `json:"committed"`
Cleaned int `json:"cleaned"`
Reconciled int `json:"reconciled"`
Released int `json:"released"`
Sealed int `json:"sealed"`
Backfilled int `json:"backfilled"`
}
func (coordinator *Coordinator) ProcessOnce(ctx context.Context) (ProcessResult, error) {
type outcome struct {
result ProcessResult
err error
}
results := make(chan outcome, phaseCount)
for phase := range phaseCount {
go func() { value, err := coordinator.executeStage(ctx, phase); results <- outcome{value, err} }()
}
result := ProcessResult{}
var failures []error
for range phaseCount {
value := <-results
result.Planned += value.result.Planned
result.Committed += value.result.Committed
result.Cleaned += value.result.Cleaned
result.Reconciled += value.result.Reconciled
result.Released += value.result.Released
result.Sealed += value.result.Sealed
result.Backfilled += value.result.Backfilled
if value.err != nil {
failures = append(failures, value.err)
}
}
return result, errors.Join(failures...)
}
type auditSealSigningPayload struct {
OrganizationID string `json:"organizationId"`
DayUTC string `json:"dayUtc"`
EventCount int64 `json:"eventCount"`
FirstEventHash string `json:"firstEventHash,omitempty"`
LastEventHash string `json:"lastEventHash,omitempty"`
AggregateHash string `json:"aggregateHash"`
SigningKeyID string `json:"signingKeyId"`
}
type auditSealDocument struct {
auditSealSigningPayload
Signature string `json:"signature"`
}
func validateAuditEventHash(event domain.ObjectSyncAuditEvent) error {
unsigned := event
unsigned.EventHash = ""
payload, err := json.Marshal(unsigned)
if err != nil {
return err
}
sum := sha256.Sum256(append([]byte(event.PreviousHash), payload...))
if !strings.EqualFold(event.EventHash, hex.EncodeToString(sum[:])) {
return fmt.Errorf("audit event %s hash mismatch", event.ID)
}
return nil
}
func (coordinator *Coordinator) sealPreviousAuditDay(ctx context.Context, now time.Time) (int, error) {
dayStart := now.UTC().Truncate(24 * time.Hour).Add(-24 * time.Hour)
dayEnd := dayStart.Add(24 * time.Hour)
dayUTC := dayStart.Format("2006-01-02")
organizations, err := coordinator.store.ListOrganizations(ctx)
if err != nil {
return 0, err
}
sealed := 0
for _, organization := range organizations {
existing, err := coordinator.syncStore.GetObjectSyncAuditSeal(ctx, organization.ID, dayUTC)
if err != nil {
return sealed, err
}
if existing != nil {
continue
}
ok, err := coordinator.sealAuditDay(ctx, organization.ID, dayUTC, dayStart, dayEnd)
if err != nil {
return sealed, err
}
if ok {
sealed++
}
}
return sealed, nil
}
func (coordinator *Coordinator) sealAuditDay(ctx context.Context, organizationID, dayUTC string, dayStart, dayEnd time.Time) (bool, error) {
resourceSum := sha256.Sum256([]byte("audit-seal\x00" + organizationID + "\x00" + dayUTC))
resourceHash := hex.EncodeToString(resourceSum[:])
leaseOwner := coordinator.owner + ":" + dayUTC
lease, acquired, err := coordinator.syncStore.AcquireObjectSyncLease(ctx, domain.ObjectSyncLeaseRecord{
ID: "osl_seal_" + resourceHash,
OrganizationID: organizationID,
ResourceHash: resourceHash,
ResourceType: "audit_seal",
KeyOrPrefix: dayUTC,
OwnerSessionID: leaseOwner,
OwnerItemID: resourceHash,
}, claimTTL)
if err != nil {
return false, err
}
if !acquired {
return false, nil
}
defer func() {
if err := coordinator.syncStore.ReleaseObjectSyncLease(context.Background(), resourceHash, leaseOwner, resourceHash, lease.FencingToken); err != nil {
coordinator.recordAuxiliaryFailure(phaseAudit, "release audit lease", err)
}
}()
existing, err := coordinator.syncStore.GetObjectSyncAuditSeal(ctx, organizationID, dayUTC)
if err != nil || existing != nil {
return false, err
}
hasher := sha256.New()
var count int64
var firstHash string
var lastHash string
cursor := ""
for {
page, err := coordinator.syncStore.ListObjectSyncAuditRange(
ctx,
organizationID,
dayStart.Format(time.RFC3339Nano),
dayEnd.Format(time.RFC3339Nano),
domain.ObjectSyncListQuery{Cursor: cursor, Limit: 1000},
)
if err != nil {
return false, err
}
for _, event := range page.Items {
if err := validateAuditEventHash(event); err != nil {
return false, err
}
if count > 0 && event.PreviousHash != lastHash {
return false, fmt.Errorf("audit chain discontinuity before event %s", event.ID)
}
if count == 0 {
firstHash = event.EventHash
}
lastHash = event.EventHash
_, _ = io.WriteString(hasher, event.EventHash+"\n")
count++
}
if page.NextCursor == "" {
break
}
cursor = page.NextCursor
}
aggregateHash := hex.EncodeToString(hasher.Sum(nil))
signingPayload := auditSealSigningPayload{
OrganizationID: organizationID,
DayUTC: dayUTC,
EventCount: count,
FirstEventHash: firstHash,
LastEventHash: lastHash,
AggregateHash: aggregateHash,
SigningKeyID: coordinator.signingID,
}
signingBytes, err := json.Marshal(signingPayload)
if err != nil {
return false, err
}
signature := base64.StdEncoding.EncodeToString(ed25519.Sign(coordinator.signingKey, signingBytes))
documentBytes, err := json.MarshalIndent(auditSealDocument{
auditSealSigningPayload: signingPayload,
Signature: signature,
}, "", " ")
if err != nil {
return false, err
}
documentBytes = append(documentBytes, '\n')
spaces, err := coordinator.syncStore.ListObjectSyncSpaces(ctx, organizationID)
if err != nil {
return false, err
}
if len(spaces) == 0 {
if count == 0 {
return false, nil
}
return false, fmt.Errorf("organization %s has audit events but no sync space for audit sealing", organizationID)
}
space := spaces[0]
connection, err := coordinator.store.GetStorageConnection(ctx, space.ConnectionID)
if err != nil || connection == nil {
return false, fmt.Errorf("audit seal storage connection is unavailable")
}
objectKey := connection.InternalStorageKey(path.Join(".drosync-internal/audit", organizationID, dayUTC+".json"))
object, err := coordinator.publisher.PutInternal(
ctx,
*connection,
space.Bucket,
objectKey,
bytes.NewReader(documentBytes),
int64(len(documentBytes)),
"application/json",
map[string]string{
"drosync-audit-day": dayUTC,
"drosync-audit-hash": aggregateHash,
"drosync-signing-key": coordinator.signingID,
},
)
if IsPreconditionFailure(err) {
object, err = coordinator.publisher.Head(ctx, *connection, space.Bucket, objectKey)
if err == nil && (!object.Exists || object.SizeBytes != int64(len(documentBytes))) {
err = fmt.Errorf("existing audit seal object does not match the expected size")
}
}
if err != nil {
return false, err
}
recordIDHash := sha256.Sum256([]byte(organizationID + "\x00" + dayUTC))
record := domain.ObjectSyncAuditSealRecord{
ID: "osas_" + hex.EncodeToString(recordIDHash[:12]),
OrganizationID: organizationID,
DayUTC: dayUTC,
EventCount: count,
FirstEventHash: firstHash,
LastEventHash: lastHash,
AggregateHash: aggregateHash,
Signature: signature,
SigningKeyID: coordinator.signingID,
ObjectKey: objectKey,
ObjectETag: object.ETag,
ObjectVersionID: object.VersionID,
CreatedAt: nowISO(),
PurgeAt: time.Now().UTC().Add(versionRetention),
}
if err := coordinator.syncStore.InsertObjectSyncAuditSeal(ctx, record); err != nil {
return false, err
}
return true, nil
}
func (coordinator *Coordinator) reconcileSpace(ctx context.Context, reconcile *domain.ObjectSyncReconcileRecord) error {
token, id := reconcile.CoordinatorOwner, reconcile.ID
if token == "" {
return errClaimLost
}
requireClaim := func() error {
return requireOwnedClaim(ctx, func(callCtx context.Context) (bool, error) {
return coordinator.syncStore.RenewObjectSyncReconcileClaim(callCtx, id, token, claimTTL)
})
}
space, err := coordinator.syncStore.GetObjectSyncSpace(ctx, reconcile.SyncSpaceID)
if err != nil || space == nil || space.OrganizationID != reconcile.OrganizationID {
return fmt.Errorf("sync space is unavailable")
}
connection, err := coordinator.store.GetStorageConnection(ctx, space.ConnectionID)
if err != nil || connection == nil {
return fmt.Errorf("storage connection is unavailable")
}
var scanned int64
var drifted int64
cursor := ""
for {
page, err := coordinator.syncStore.ListObjectSyncHeads(ctx, reconcile.OrganizationID, reconcile.SyncSpaceID, cursor, 1000)
if err != nil {
return err
}
for _, head := range page.Items {
remote, err := coordinator.publisher.Head(ctx, *connection, space.Bucket, head.Key)
if err != nil {
return fmt.Errorf("head %s: %w", head.Key, err)
}
if err := requireClaim(); err != nil {
return err
}
headDrifted := !remote.Exists || strings.Trim(remote.ETag, `"`) != strings.Trim(head.RemoteETag, `"`)
if headDrifted {
drifted++
}
scanned++
// The page was read before the slow HEADs above. A commit or digest
// attestation that replaced this receipt since then is not what the
// verdict was computed against, so the store skips it.
if _, err := coordinator.syncStore.SetObjectSyncHeadDrifted(ctx, head, headDrifted, nowISO()); err != nil {
return err
}
}
reconcile.Scanned = scanned
reconcile.Drifted = drifted
reconcile.UpdatedAt = nowISO()
if err := requireClaim(); err != nil {
return err
}
reconcile.CoordinatorLeaseUntil = time.Now().UTC().Add(claimTTL)
saved, err := coordinator.syncStore.SaveClaimedObjectSyncReconcile(ctx, *reconcile, token)
if err != nil {
return err
}
if !saved {
return errClaimLost
}
if page.NextCursor == "" {
break
}
cursor = page.NextCursor
}
if err := requireClaim(); err != nil {
return err
}
space.Enabled = false
space.Mode = "blocked"
if drifted > 0 {
space.Capability.State = domain.ObjectSyncCapabilityDrifted
} else {
switch {
case space.Capability.ConditionalCreate &&
space.Capability.ConditionalUpdate &&
space.Capability.ConditionalMultipart &&
space.Capability.ExclusivePrefixConfirmed:
space.Capability.State = domain.ObjectSyncCapabilityUpdateEnabled
case space.Capability.ConditionalCreate:
space.Capability.State = domain.ObjectSyncCapabilityCreateOnly
default:
space.Capability.State = domain.ObjectSyncCapabilityUnsupported
}
}
space.Policy.AllowConditionalUpdate = space.Capability.State == domain.ObjectSyncCapabilityUpdateEnabled
if space.Capability.State != domain.ObjectSyncCapabilityUpdateEnabled {
space.Policy.AllowForceOverwrite = false
}
space.UpdatedAt = nowISO()
if err := coordinator.syncStore.UpsertObjectSyncSpace(ctx, *space); err != nil {
return err
}
reconcile.Status = "completed"
reconcile.CompletedAt = nowISO()
reconcile.UpdatedAt = reconcile.CompletedAt
reconcile.CoordinatorOwner = ""
reconcile.CoordinatorLeaseUntil = time.Time{}
saved, err := coordinator.syncStore.SaveClaimedObjectSyncReconcile(ctx, *reconcile, token)
if err != nil {
return err
}
if !saved {
return errClaimLost
}
coordinator.recordSystemAudit(ctx, domain.ObjectSyncAuditEvent{
OrganizationID: reconcile.OrganizationID,
ActorType: "system",
ActorID: coordinator.owner,
Action: "space.reconcile.completed",
Target: map[string]any{
"reconcileId": reconcile.ID,
"spaceId": reconcile.SyncSpaceID,
},
After: map[string]any{"scanned": scanned, "drifted": drifted},
Result: "success",
})
return nil
}
func (coordinator *Coordinator) cleanupStaging(ctx context.Context, item *domain.ObjectSyncItemRecord) error {
claimOwner := item.CoordinatorOwner
if claimOwner == "" {
return errClaimLost
}
if err := validateCleanupObjectKey(item.StagingKey); err != nil {
return err
}
if err := validateCleanupObjectKey(item.TargetKey); err != nil {
return err
}
session, err := coordinator.syncStore.GetObjectSyncSession(ctx, item.SessionID)
if err != nil || session == nil {
return fmt.Errorf("session is unavailable")
}
space, err := coordinator.syncStore.GetObjectSyncSpace(ctx, session.SyncSpaceID)
if err != nil || space == nil {
return fmt.Errorf("sync space is unavailable")
}
connection, err := coordinator.store.GetStorageConnection(ctx, space.ConnectionID)
if err != nil || connection == nil {
return fmt.Errorf("storage connection is unavailable")
}
if item.UploadID != "" {
if err := coordinator.publisher.AbortMultipart(ctx, *connection, space.Bucket, item.StagingKey, item.UploadID); err != nil {
return fmt.Errorf("abort staging multipart: %w", err)
}
}
if item.StagingKey != item.TargetKey {
if err := coordinator.publisher.Delete(ctx, *connection, space.Bucket, item.StagingKey); err != nil {
return err
}
}
if item.Status == domain.ObjectSyncItemUploading || item.Status == domain.ObjectSyncItemUploaded {
item.Status = domain.ObjectSyncItemExpired
}
item.StagingKey = ""
item.UploadID = ""
item.UploadedParts = nil
item.CoordinatorOwner = ""
item.CoordinatorLeaseUntil = time.Time{}
item.UpdatedAt = nowISO()
if err := coordinator.saveClaimedItem(ctx, *item, claimOwner); err != nil {
return err
}
if item.LeaseID != "" {
if err := coordinator.syncStore.ReleaseObjectSyncLease(ctx, item.LeaseID, item.SessionID, item.ID, item.FencingToken); err != nil {
coordinator.recordAuxiliaryFailure(phaseCleanup, "release cleaned lease", err)
}
}
return nil
}
func nowISO() string {
return time.Now().UTC().Format(time.RFC3339Nano)
}
func firstNonEmpty(values ...string) string {
for _, value := range values {
if strings.TrimSpace(value) != "" {
return value
}
}
return ""
}
func (coordinator *Coordinator) planSession(ctx context.Context, session *domain.ObjectSyncSessionRecord) error {
token := session.CoordinatorOwner
if token == "" {
return errClaimLost
}
space, err := coordinator.syncStore.GetObjectSyncSpace(ctx, session.SyncSpaceID)
if err != nil || space == nil {
return fmt.Errorf("sync space is unavailable")
}
connection, err := coordinator.store.GetStorageConnection(ctx, space.ConnectionID)
if err != nil || connection == nil {
return fmt.Errorf("storage connection is unavailable")
}
manifestCount := session.Counts.Manifest
cursor := ""
for {
page, err := coordinator.syncStore.ListObjectSyncItems(ctx, domain.ObjectSyncListQuery{SessionID: session.ID, Cursor: cursor, Limit: 1000})
if err != nil {
return err
}
type observed struct {
remote ObjectMetadata
head *domain.ObjectSyncObjectHeadRecord
}
facts := make([]observed, len(page.Items))
group, queryCtx := errgroup.WithContext(ctx)
group.SetLimit(planConcurrency)
for index, item := range page.Items {
if item.Status != domain.ObjectSyncItemDiscovered {
continue
}
group.Go(func() error {
head, err := coordinator.syncStore.GetObjectSyncHead(queryCtx, session.OrganizationID, space.ID, item.TargetKey)
if err != nil {
return err
}
var remote ObjectMetadata
// Formal publication receipts are authoritative for upload-once planning.
// Only new paths need a collision check against object storage.
if head == nil {
remote, err = coordinator.publisher.Head(queryCtx, *connection, space.Bucket, item.TargetKey)
if err != nil {
return fmt.Errorf("head %s: %w", item.TargetKey, err)
}
}
facts[index] = observed{remote, head}
return nil
})
}
if err := group.Wait(); err != nil {
return err
}
updates := make([]domain.ObjectSyncItemRecord, 0, len(page.Items))
for index, item := range page.Items {
if item.Status != domain.ObjectSyncItemDiscovered {
continue
}
remote, head := facts[index].remote, facts[index].head
item.ExpectedBase = domain.ObjectSyncRemoteBase{Exists: remote.Exists, ETag: remote.ETag, SizeBytes: remote.SizeBytes, LastModified: remote.LastModified, VersionID: remote.VersionID}
switch {
case head != nil && len(head.MD5Sum) != 32:
item.Action = "deny_archived_hash_change"
item.Status = domain.ObjectSyncItemDenied
item.LastError = "已归档对象缺少 MD5 回执,暂停上传;旧 SHA256 回执退出内容比较和清理支持,保留云端原文件"
case head != nil && head.MD5Sum == item.MD5Sum && head.SizeBytes == item.SizeBytes:
item.Action = "unchanged"
item.Status = domain.ObjectSyncItemUnchanged
item.ExpectedBase = domain.ObjectSyncRemoteBase{Exists: true, ETag: head.RemoteETag, SizeBytes: head.SizeBytes, VersionID: head.VersionID, PlatformVersion: head.PlatformVersion}
case head != nil:
item.Action = "deny_archived_hash_change"
item.Status = domain.ObjectSyncItemDenied
item.LastError = fmt.Sprintf("同名文件内容冲突,暂停上传:%s;云端已提交 MD5Sum=%s,本地 MD5Sum=%s;保留云端原文件", item.TargetKey, head.MD5Sum, item.MD5Sum)
case remote.Exists:
item.Action = "deny_existing_untracked"
item.Status = domain.ObjectSyncItemDenied
item.LastError = "目标路径已有对象但缺少可信提交记录,暂停上传;保留云端原文件"
default:
item.Action = "create"
item.Status = domain.ObjectSyncItemPlanned
}
item.UpdatedAt = nowISO()
updates = append(updates, item)
}
owned, err := coordinator.syncStore.RenewObjectSyncSessionClaim(ctx, session.ID, token, claimTTL)
if err != nil {
return err
}
if !owned {
return errClaimLost
}
if err := ctx.Err(); err != nil {
return err
}
if err := coordinator.syncStore.UpsertObjectSyncItems(ctx, updates); err != nil {
return err
}
if page.NextCursor == "" {
break
}
cursor = page.NextCursor
}
summaries, err := coordinator.syncStore.SummarizeObjectSyncItems(ctx, []string{session.ID})
if err != nil {
return err
}
session.Counts = summaries[session.ID].Counts
session.Counts.Manifest = manifestCount
session.Status = domain.ObjectSyncSessionPlanned
session.UpdatedAt = nowISO()
session.CoordinatorOwner = ""
session.CoordinatorLeaseUntil = time.Time{}
saved, err := coordinator.syncStore.SaveClaimedObjectSyncSession(ctx, *session, token)
if err != nil {
return err
}
if !saved {
return errClaimLost
}
return nil
}
func conflictKey(original, itemID string, now time.Time) string {
shortID := strings.TrimPrefix(itemID, "osi_")
if len(shortID) > 10 {
shortID = shortID[:10]
}
return fmt.Sprintf("%s.__conflict__%s_%s", original, now.UTC().Format("20060102T150405Z"), shortID)
}
func (coordinator *Coordinator) commitItem(ctx context.Context, item *domain.ObjectSyncItemRecord) error {
return coordinator.commitItemWithObservation(ctx, item, nil)
}
func (coordinator *Coordinator) commitItemWithObservation(ctx context.Context, item *domain.ObjectSyncItemRecord, observed *ObjectMetadata) error {
claimOwner := item.CoordinatorOwner
if claimOwner == "" {
return errClaimLost
}
session, err := coordinator.syncStore.GetObjectSyncSession(ctx, item.SessionID)
if err != nil || session == nil {
return fmt.Errorf("session is unavailable")
}
space, err := coordinator.syncStore.GetObjectSyncSpace(ctx, session.SyncSpaceID)
if err != nil || space == nil {
return fmt.Errorf("sync space is unavailable")
}
connection, err := coordinator.store.GetStorageConnection(ctx, space.ConnectionID)
if err != nil || connection == nil {
return fmt.Errorf("storage connection is unavailable")
}
renewed, err := coordinator.syncStore.RenewObjectSyncLease(ctx, item.LeaseID, session.ID, item.ID, item.FencingToken, claimTTL)
if err != nil || !renewed {
return fmt.Errorf("lease is stale; publish rejected by fencing token")
}
sourceKey := item.StagingKey
preserveSource := false
if item.Action == "force_overwrite" {
sourceKey = item.ForceSourceKey
preserveSource = true
}
metadata := map[string]string{
"drosync-session": session.ID,
"drosync-item": item.ID,
"drosync-md5sum": item.MD5Sum,
"drosync-fence": fmt.Sprintf("%d", item.FencingToken),
}
if preserveSource {
metadata["drosync-force-source"] = sourceKey
metadata["drosync-force-actor"] = item.ForceActorID
}
targetKey := item.TargetKey
logicalTarget := item.LogicalTargetKey
keptBoth := !preserveSource && logicalTarget != "" && logicalTarget != targetKey
if keptBoth {
metadata["drosync-logical-target"] = logicalTarget
}
// A previous worker may have published and then lost its response or claim.
// This bounded HEAD avoids copying those bytes again; existence alone is not
// a receipt. Every operation marker, content hash and size must agree.
var current ObjectMetadata
if observed != nil {
if err := connection.CheckStoragePath(space.Bucket, targetKey, false); err != nil {
return err
}
if preserveSource || sourceKey != targetKey || item.Action != "create" {
return fmt.Errorf("observation cannot authorize a copy or overwrite")
}
current = *observed
} else {
current, err = coordinator.publisher.Head(ctx, *connection, space.Bucket, targetKey)
}
if err != nil {
return fmt.Errorf("verify publication target: %w", err)
}
ifMatch, ifNoneMatch := "", ""
switch item.Action {
case "create":
ifNoneMatch = "*"
case "update":
if space.Capability.State != domain.ObjectSyncCapabilityUpdateEnabled || !space.Policy.AllowConditionalUpdate {
return fmt.Errorf("conditional update gate is no longer enabled")
}
ifMatch = item.ExpectedBase.ETag
case "force_overwrite":
if space.Capability.State != domain.ObjectSyncCapabilityUpdateEnabled ||
!space.Capability.ExclusivePrefixConfirmed ||
!space.Policy.AllowConditionalUpdate ||
!space.Policy.AllowForceOverwrite {
return fmt.Errorf("force overwrite gate is no longer enabled")
}
if current.Exists {
ifMatch = current.ETag
} else {
ifNoneMatch = "*"
}
default:
return fmt.Errorf("unsupported commit action %q", item.Action)
}
if keptBoth {
ifMatch, ifNoneMatch = "", "*"
}
directUpload := !preserveSource && sourceKey == targetKey
if directUpload && !publicationMatches(current, item.SizeBytes, metadata) {
return fmt.Errorf("direct upload target is missing or its bound identity/hash/size does not match")
}
published := current
if !publicationMatches(current, item.SizeBytes, metadata) {
staging, headErr := coordinator.publisher.Head(ctx, *connection, space.Bucket, sourceKey)
if headErr != nil {
return headErr
}
if !staging.Exists || staging.SizeBytes != item.SizeBytes {
return fmt.Errorf("staging object is missing or size changed")
}
var confirmedConflict bool
published, err, confirmedConflict = coordinator.publishAndConfirm(ctx, *connection, space.Bucket, sourceKey, targetKey, item.SizeBytes, staging.ETag, ifMatch, ifNoneMatch, metadata)
if confirmedConflict && !preserveSource && !keptBoth {
logicalTarget = targetKey
targetKey = conflictKey(targetKey, item.ID, time.Now())
metadata["drosync-logical-target"] = logicalTarget
// Persist the sole conflict destination before any remote write. A
// restart must inspect the same object, never invent another suffix.
item.TargetKey, item.LogicalTargetKey = targetKey, logicalTarget
item.UpdatedAt = nowISO()
if saveErr := coordinator.saveClaimedItem(ctx, *item, claimOwner); saveErr != nil {
return saveErr
}
published, err, _ = coordinator.publishAndConfirm(ctx, *connection, space.Bucket, sourceKey, targetKey, item.SizeBytes, staging.ETag, "", "*", metadata)
keptBoth = err == nil
}
if err != nil {
return err
}
}
if err := ctx.Err(); err != nil {
return err
}
owned, claimErr := coordinator.syncStore.RenewObjectSyncItemClaim(ctx, item.ID, claimOwner, claimTTL)
if claimErr != nil {
return claimErr
}
if !owned {
return errClaimLost
}
previousHead, err := coordinator.syncStore.GetObjectSyncHead(ctx, session.OrganizationID, space.ID, targetKey)
if err != nil {
return err
}
platformVersion := int64(1)
if previousHead != nil {
platformVersion = previousHead.PlatformVersion + 1
// Head/version writes may have succeeded before the item receipt was
// saved. Reuse that exact publication instead of creating version N+1.
if previousHead.ItemID == item.ID && previousHead.SessionID == session.ID &&
previousHead.RemoteETag == published.ETag && previousHead.VersionID == published.VersionID &&
previousHead.MD5Sum == item.MD5Sum && previousHead.SizeBytes == published.SizeBytes && previousHead.PlatformVersion > 0 {
platformVersion = previousHead.PlatformVersion
}
}
now := time.Now().UTC()
head := domain.ObjectSyncObjectHeadRecord{
OrganizationID: session.OrganizationID,
SyncSpaceID: space.ID,
Key: targetKey,
PlatformVersion: platformVersion,
RemoteETag: published.ETag,
VersionID: published.VersionID,
SizeBytes: published.SizeBytes,
MD5Sum: item.MD5Sum,
SessionID: session.ID,
ItemID: item.ID,
UpdatedAt: now.Format(time.RFC3339Nano),
}
if err := coordinator.syncStore.UpsertObjectSyncHead(ctx, head); err != nil {
return err
}
if err := coordinator.syncStore.InsertObjectSyncVersion(ctx, domain.ObjectSyncVersionRecord{
ID: fmt.Sprintf("osv_%s_%d", item.ID, platformVersion),
OrganizationID: session.OrganizationID,
SyncSpaceID: space.ID,
Key: targetKey,
LogicalTargetKey: logicalTarget,
SourceKey: item.ForceSourceKey,
PlatformVersion: platformVersion,
RemoteETag: published.ETag,
VersionID: published.VersionID,
SizeBytes: published.SizeBytes,
MD5Sum: item.MD5Sum,
SessionID: session.ID,
ItemID: item.ID,
ActorID: firstNonEmpty(item.ForceActorID, session.DeviceID),
CommittedAt: now.Format(time.RFC3339Nano),
PurgeAt: now.Add(versionRetention),
}); err != nil {
return err
}
item.TargetKey = targetKey
item.LogicalTargetKey = logicalTarget
item.LastError = ""
item.CommittedVersion = domain.ObjectSyncRemoteBase{
Exists: true,
ETag: published.ETag,
SizeBytes: published.SizeBytes,
VersionID: published.VersionID,
PlatformVersion: platformVersion,
}
if keptBoth {
item.Status = domain.ObjectSyncItemKeptBoth
} else {
item.Status = domain.ObjectSyncItemCommitted
}
item.CoordinatorOwner = ""
item.CoordinatorLeaseUntil = time.Time{}
item.UpdatedAt = now.Format(time.RFC3339Nano)
if err := coordinator.saveClaimedItem(ctx, *item, claimOwner); err != nil {
return err
}
if !preserveSource && !directUpload {
// Publication already committed. Keep staging metadata for the cleanup
// worker on failure, and expose the cleanup failure without undoing it.
if err := coordinator.publisher.Delete(ctx, *connection, space.Bucket, sourceKey); err != nil {
coordinator.recordAuxiliaryFailure(phaseCleanup, "delete published staging", err)
}
}
if err := coordinator.syncStore.ReleaseObjectSyncLease(ctx, item.LeaseID, session.ID, item.ID, item.FencingToken); err != nil {
coordinator.recordAuxiliaryFailure(phaseCommit, "release committed lease", err)
}
coordinator.recordSystemAudit(ctx, domain.ObjectSyncAuditEvent{
OrganizationID: session.OrganizationID,
ActorType: "system",
ActorID: coordinator.owner,
DeviceID: session.DeviceID,
SessionID: session.ID,
ItemID: item.ID,
Action: "item." + item.Action + ".committed",
Target: map[string]any{
"key": item.TargetKey,
"sourceKey": item.ForceSourceKey,
},
After: map[string]any{
"etag": item.CommittedVersion.ETag,
"versionId": item.CommittedVersion.VersionID,
"platformVersion": item.CommittedVersion.PlatformVersion,
"sizeBytes": item.CommittedVersion.SizeBytes,
},
Result: "success",
})
return nil
}
func (coordinator *Coordinator) publishRelease(ctx context.Context, release *domain.ObjectSyncReleaseRecord) error {
token, id := release.CoordinatorOwner, release.ID
if token == "" {
return errClaimLost
}
requireClaim := func() error {
return requireOwnedClaim(ctx, func(callCtx context.Context) (bool, error) {
return coordinator.syncStore.RenewObjectSyncReleaseClaim(callCtx, id, token, claimTTL)
})
}
temp, err := os.CreateTemp("", "drosync-release-*.ndjson")
if err != nil {
return err
}
tempName := temp.Name()
defer os.Remove(tempName)
hasher := sha256.New()
writer := bufio.NewWriter(io.MultiWriter(temp, hasher))
var objectCount int64
var totalBytes int64
var firstSession *domain.ObjectSyncSessionRecord
for _, sessionID := range release.SessionIDs {
session, err := coordinator.syncStore.GetObjectSyncSession(ctx, sessionID)
if err != nil || session == nil || session.OrganizationID != release.OrganizationID {
_ = temp.Close()
return fmt.Errorf("release session %s is unavailable", sessionID)
}
if release.SyncSpaceID == "" {
release.SyncSpaceID = session.SyncSpaceID
} else if session.SyncSpaceID != release.SyncSpaceID {
_ = temp.Close()
return fmt.Errorf("release session %s belongs to a different sync space", sessionID)
}
if firstSession == nil {
firstSession = session
}
cursor := ""
for {
page, err := coordinator.syncStore.ListObjectSyncItems(ctx, domain.ObjectSyncListQuery{SessionID: session.ID, Cursor: cursor, Limit: 1000})
if err != nil {
_ = temp.Close()
return err
}
for _, item := range page.Items {
if item.Status != domain.ObjectSyncItemCommitted && item.Status != domain.ObjectSyncItemKeptBoth && item.Status != domain.ObjectSyncItemUnchanged {
continue
}
line := map[string]any{
"sessionId": item.SessionID,
"itemId": item.ID,
"logicalTargetKey": item.LogicalTargetKey,
"key": item.TargetKey,
"sizeBytes": item.SizeBytes,
"md5sum": item.MD5Sum,
"remote": item.CommittedVersion,
"status": item.Status,
}
payload, _ := json.Marshal(line)
if _, err := writer.Write(append(payload, '\n')); err != nil {
_ = temp.Close()
return err
}
objectCount++
totalBytes += item.SizeBytes
}
if page.NextCursor == "" {
break
}
cursor = page.NextCursor
}
}
if err := writer.Flush(); err != nil {
_ = temp.Close()
return err
}
if err := temp.Close(); err != nil {
return err
}
if firstSession == nil {
return fmt.Errorf("release has no sessions")
}
space, err := coordinator.syncStore.GetObjectSyncSpace(ctx, release.SyncSpaceID)
if err != nil || space == nil {
return fmt.Errorf("release sync space is unavailable")
}
connection, err := coordinator.store.GetStorageConnection(ctx, space.ConnectionID)
if err != nil || connection == nil {
return fmt.Errorf("release storage connection is unavailable")
}
info, err := os.Stat(tempName)
if err != nil {
return err
}
reader, err := os.Open(tempName)
if err != nil {
return err
}
defer reader.Close()
manifestKey := connection.InternalStorageKey(path.Join(".drosync-internal/releases", release.OrganizationID, release.ID+".ndjson"))
digest := hex.EncodeToString(hasher.Sum(nil))
if err := requireClaim(); err != nil {
return err
}
if _, err := coordinator.publisher.PutInternal(ctx, *connection, space.Bucket, manifestKey, reader, info.Size(), "application/x-ndjson", map[string]string{
"drosync-release": release.ID,
"drosync-sha256": digest,
}); err != nil {
return err
}
if err := requireClaim(); err != nil {
return err
}
release.ManifestKey = manifestKey
release.ManifestSHA256 = digest
release.ObjectCount = objectCount
release.TotalBytes = totalBytes
release.Status = "published"
release.PublishedAt = nowISO()
release.CoordinatorOwner = ""
release.CoordinatorLeaseUntil = time.Time{}
saved, err := coordinator.syncStore.SaveClaimedObjectSyncRelease(ctx, *release, token)
if err != nil {
return err
}
if !saved {
return errClaimLost
}
coordinator.recordSystemAudit(ctx, domain.ObjectSyncAuditEvent{
OrganizationID: release.OrganizationID,
ActorType: "system",
ActorID: coordinator.owner,
Action: "release.published",
Target: map[string]any{"releaseId": release.ID, "manifestKey": release.ManifestKey},
After: map[string]any{
"manifestSha256": release.ManifestSHA256,
"objectCount": release.ObjectCount,
"totalBytes": release.TotalBytes,
},
Result: "success",
})
return nil
}