package objectsync import ( "bufio" "bytes" "context" "crypto/ed25519" "crypto/sha256" "encoding/base64" "encoding/hex" "encoding/json" "errors" "fmt" "io" "os" "path" "strings" "sync/atomic" "time" "droboticflow/internal/domain" "droboticflow/internal/store" ) const ( claimTTL = 10 * time.Minute versionRetention = 365 * 24 * time.Hour stagingRetention = 24 * time.Hour ) type Coordinator struct { store store.Store syncStore store.ObjectSyncStore publisher Publisher owner string signingKey ed25519.PrivateKey signingID string cycles atomic.Int64 errors atomic.Int64 planned atomic.Int64 committed atomic.Int64 cleaned atomic.Int64 reconciled atomic.Int64 released atomic.Int64 sealed atomic.Int64 lastCycle atomic.Int64 lastOK atomic.Int64 durationMS atomic.Int64 } func NewCoordinator(base store.Store, publisher Publisher, owner string) (*Coordinator, error) { syncStore, ok := store.AsObjectSyncStore(base) if !ok { return nil, fmt.Errorf("store does not implement object sync persistence") } if publisher == nil { return nil, fmt.Errorf("object publisher is required") } if strings.TrimSpace(owner) == "" { owner = fmt.Sprintf("coordinator-%d", time.Now().UnixNano()) } return &Coordinator{store: base, syncStore: syncStore, publisher: publisher, owner: owner}, nil } func (coordinator *Coordinator) ConfigureAuditSigning(encodedPrivateKey, keyID string) error { encodedPrivateKey = strings.TrimSpace(encodedPrivateKey) if encodedPrivateKey == "" { coordinator.signingKey = nil coordinator.signingID = "" return nil } raw, err := base64.StdEncoding.DecodeString(encodedPrivateKey) if err != nil { raw, err = base64.RawStdEncoding.DecodeString(encodedPrivateKey) } if err != nil { return fmt.Errorf("decode audit signing key: %w", err) } switch len(raw) { case ed25519.SeedSize: coordinator.signingKey = ed25519.NewKeyFromSeed(raw) case ed25519.PrivateKeySize: coordinator.signingKey = ed25519.PrivateKey(append([]byte(nil), raw...)) default: return fmt.Errorf("audit signing key must be a %d-byte Ed25519 seed or %d-byte private key", ed25519.SeedSize, ed25519.PrivateKeySize) } coordinator.signingID = strings.TrimSpace(keyID) if coordinator.signingID == "" { return fmt.Errorf("audit signing key id is required") } return nil } func (coordinator *Coordinator) AuditSigningConfigured() bool { return len(coordinator.signingKey) == ed25519.PrivateKeySize && coordinator.signingID != "" } func (coordinator *Coordinator) Run(ctx context.Context, interval time.Duration) { if interval <= 0 { interval = 2 * time.Second } ticker := time.NewTicker(interval) defer ticker.Stop() for { started := time.Now() result, err := coordinator.ProcessOnce(ctx) finished := time.Now() coordinator.cycles.Add(1) coordinator.lastCycle.Store(finished.Unix()) coordinator.durationMS.Store(finished.Sub(started).Milliseconds()) coordinator.planned.Add(int64(result.Planned)) coordinator.committed.Add(int64(result.Committed)) coordinator.cleaned.Add(int64(result.Cleaned)) coordinator.reconciled.Add(int64(result.Reconciled)) coordinator.released.Add(int64(result.Released)) coordinator.sealed.Add(int64(result.Sealed)) if err != nil && !errors.Is(err, context.Canceled) { coordinator.errors.Add(1) fmt.Printf("[drosync-coordinator] process error: %v\n", err) } else if err == nil { coordinator.lastOK.Store(finished.Unix()) } select { case <-ctx.Done(): return case <-ticker.C: } } } type Metrics struct { Cycles int64 Errors int64 Planned int64 Committed int64 Cleaned int64 Reconciled int64 Released int64 Sealed int64 LastCycle int64 LastOK int64 DurationMS int64 } func (coordinator *Coordinator) Metrics() Metrics { return Metrics{ Cycles: coordinator.cycles.Load(), Errors: coordinator.errors.Load(), Planned: coordinator.planned.Load(), Committed: coordinator.committed.Load(), Cleaned: coordinator.cleaned.Load(), Reconciled: coordinator.reconciled.Load(), Released: coordinator.released.Load(), Sealed: coordinator.sealed.Load(), LastCycle: coordinator.lastCycle.Load(), LastOK: coordinator.lastOK.Load(), DurationMS: coordinator.durationMS.Load(), } } func (coordinator *Coordinator) recordSystemAudit(ctx context.Context, event domain.ObjectSyncAuditEvent) { if err := AppendAudit(ctx, coordinator.syncStore, event); err != nil { coordinator.errors.Add(1) fmt.Printf("[drosync-coordinator] audit persistence error for %s: %v\n", event.Action, err) } } type ProcessResult struct { Planned int `json:"planned"` Committed int `json:"committed"` Cleaned int `json:"cleaned"` Reconciled int `json:"reconciled"` Released int `json:"released"` Sealed int `json:"sealed"` } func (coordinator *Coordinator) ProcessOnce(ctx context.Context) (ProcessResult, error) { result := ProcessResult{} session, err := coordinator.syncStore.ClaimObjectSyncSession(ctx, domain.ObjectSyncSessionPlanning, domain.ObjectSyncSessionPlanned, coordinator.owner) if err != nil { return result, err } if session != nil { if err := coordinator.planSession(ctx, session); err != nil { session.Status = domain.ObjectSyncSessionFailed session.ErrorSummary = err.Error() session.UpdatedAt = nowISO() _ = coordinator.syncStore.UpsertObjectSyncSession(ctx, *session) return result, err } result.Planned++ } for index := 0; index < 16; index++ { item, claimErr := coordinator.syncStore.ClaimObjectSyncItem(ctx, coordinator.owner, claimTTL) if claimErr != nil { return result, claimErr } if item == nil { break } if err := coordinator.commitItem(ctx, item); err != nil { item.Status = domain.ObjectSyncItemFailed item.LastError = err.Error() item.AttemptCount++ item.CoordinatorOwner = "" item.CoordinatorLeaseUntil = time.Time{} item.UpdatedAt = nowISO() _ = coordinator.syncStore.UpsertObjectSyncItems(ctx, []domain.ObjectSyncItemRecord{*item}) if session, sessionErr := coordinator.syncStore.GetObjectSyncSession(ctx, item.SessionID); sessionErr == nil && session != nil { _ = coordinator.recountSession(ctx, session) } continue } result.Committed++ } for index := 0; index < 8; index++ { item, claimErr := coordinator.syncStore.ClaimObjectSyncCleanupItem(ctx, coordinator.owner, time.Now().UTC().Add(-stagingRetention), claimTTL) if claimErr != nil { return result, claimErr } if item == nil { break } if err := coordinator.cleanupStaging(ctx, item); err != nil { item.LastError = "staging cleanup: " + err.Error() item.CoordinatorOwner = "" item.CoordinatorLeaseUntil = time.Time{} item.UpdatedAt = nowISO() _ = coordinator.syncStore.UpsertObjectSyncItems(ctx, []domain.ObjectSyncItemRecord{*item}) continue } result.Cleaned++ } reconcile, err := coordinator.syncStore.ClaimObjectSyncReconcile(ctx, coordinator.owner, claimTTL) if err != nil { return result, err } if reconcile != nil { if err := coordinator.reconcileSpace(ctx, reconcile); err != nil { reconcile.Status = "failed" reconcile.Error = err.Error() reconcile.UpdatedAt = nowISO() reconcile.CompletedAt = reconcile.UpdatedAt reconcile.CoordinatorOwner = "" reconcile.CoordinatorLeaseUntil = time.Time{} _ = coordinator.syncStore.UpsertObjectSyncReconcile(ctx, *reconcile) } else { result.Reconciled++ } } release, err := coordinator.syncStore.ClaimObjectSyncRelease(ctx, coordinator.owner, claimTTL) if err != nil { return result, err } if release != nil { if err := coordinator.publishRelease(ctx, release); err != nil { release.Status = "failed" release.CoordinatorOwner = "" release.CoordinatorLeaseUntil = time.Time{} _ = coordinator.syncStore.UpsertObjectSyncRelease(ctx, *release) return result, err } result.Released++ } if coordinator.AuditSigningConfigured() { sealed, sealErr := coordinator.sealPreviousAuditDay(ctx, time.Now().UTC()) if sealErr != nil { return result, sealErr } result.Sealed += sealed } return result, nil } type auditSealSigningPayload struct { OrganizationID string `json:"organizationId"` DayUTC string `json:"dayUtc"` EventCount int64 `json:"eventCount"` FirstEventHash string `json:"firstEventHash,omitempty"` LastEventHash string `json:"lastEventHash,omitempty"` AggregateHash string `json:"aggregateHash"` SigningKeyID string `json:"signingKeyId"` } type auditSealDocument struct { auditSealSigningPayload Signature string `json:"signature"` } func validateAuditEventHash(event domain.ObjectSyncAuditEvent) error { unsigned := event unsigned.EventHash = "" payload, err := json.Marshal(unsigned) if err != nil { return err } sum := sha256.Sum256(append([]byte(event.PreviousHash), payload...)) if !strings.EqualFold(event.EventHash, hex.EncodeToString(sum[:])) { return fmt.Errorf("audit event %s hash mismatch", event.ID) } return nil } func (coordinator *Coordinator) sealPreviousAuditDay(ctx context.Context, now time.Time) (int, error) { dayStart := now.UTC().Truncate(24 * time.Hour).Add(-24 * time.Hour) dayEnd := dayStart.Add(24 * time.Hour) dayUTC := dayStart.Format("2006-01-02") organizations, err := coordinator.store.ListOrganizations(ctx) if err != nil { return 0, err } sealed := 0 for _, organization := range organizations { existing, err := coordinator.syncStore.GetObjectSyncAuditSeal(ctx, organization.ID, dayUTC) if err != nil { return sealed, err } if existing != nil { continue } ok, err := coordinator.sealAuditDay(ctx, organization.ID, dayUTC, dayStart, dayEnd) if err != nil { return sealed, err } if ok { sealed++ } } return sealed, nil } func (coordinator *Coordinator) sealAuditDay(ctx context.Context, organizationID, dayUTC string, dayStart, dayEnd time.Time) (bool, error) { resourceSum := sha256.Sum256([]byte("audit-seal\x00" + organizationID + "\x00" + dayUTC)) resourceHash := hex.EncodeToString(resourceSum[:]) leaseOwner := coordinator.owner + ":" + dayUTC lease, acquired, err := coordinator.syncStore.AcquireObjectSyncLease(ctx, domain.ObjectSyncLeaseRecord{ ID: "osl_seal_" + resourceHash, OrganizationID: organizationID, ResourceHash: resourceHash, ResourceType: "audit_seal", KeyOrPrefix: dayUTC, OwnerSessionID: leaseOwner, OwnerItemID: resourceHash, }, claimTTL) if err != nil { return false, err } if !acquired { return false, nil } defer func() { _ = coordinator.syncStore.ReleaseObjectSyncLease(context.Background(), resourceHash, leaseOwner, resourceHash, lease.FencingToken) }() existing, err := coordinator.syncStore.GetObjectSyncAuditSeal(ctx, organizationID, dayUTC) if err != nil || existing != nil { return false, err } hasher := sha256.New() var count int64 var firstHash string var lastHash string cursor := "" for { page, err := coordinator.syncStore.ListObjectSyncAuditRange( ctx, organizationID, dayStart.Format(time.RFC3339Nano), dayEnd.Format(time.RFC3339Nano), cursor, 1000, ) if err != nil { return false, err } for _, event := range page.Items { if err := validateAuditEventHash(event); err != nil { return false, err } if count > 0 && event.PreviousHash != lastHash { return false, fmt.Errorf("audit chain discontinuity before event %s", event.ID) } if count == 0 { firstHash = event.EventHash } lastHash = event.EventHash _, _ = io.WriteString(hasher, event.EventHash+"\n") count++ } if page.NextCursor == "" { break } cursor = page.NextCursor } aggregateHash := hex.EncodeToString(hasher.Sum(nil)) signingPayload := auditSealSigningPayload{ OrganizationID: organizationID, DayUTC: dayUTC, EventCount: count, FirstEventHash: firstHash, LastEventHash: lastHash, AggregateHash: aggregateHash, SigningKeyID: coordinator.signingID, } signingBytes, err := json.Marshal(signingPayload) if err != nil { return false, err } signature := base64.StdEncoding.EncodeToString(ed25519.Sign(coordinator.signingKey, signingBytes)) documentBytes, err := json.MarshalIndent(auditSealDocument{ auditSealSigningPayload: signingPayload, Signature: signature, }, "", " ") if err != nil { return false, err } documentBytes = append(documentBytes, '\n') spaces, err := coordinator.syncStore.ListObjectSyncSpaces(ctx, organizationID) if err != nil { return false, err } if len(spaces) == 0 { if count == 0 { return false, nil } return false, fmt.Errorf("organization %s has audit events but no sync space for audit sealing", organizationID) } space := spaces[0] connection, err := coordinator.store.GetStorageConnection(ctx, space.ConnectionID) if err != nil || connection == nil { return false, fmt.Errorf("audit seal storage connection is unavailable") } objectKey := path.Join(".drosync-internal/audit", organizationID, dayUTC+".json") object, err := coordinator.publisher.PutInternal( ctx, *connection, space.Bucket, objectKey, bytes.NewReader(documentBytes), int64(len(documentBytes)), "application/json", map[string]string{ "drosync-audit-day": dayUTC, "drosync-audit-hash": aggregateHash, "drosync-signing-key": coordinator.signingID, }, ) if IsPreconditionFailure(err) { object, err = coordinator.publisher.Head(ctx, *connection, space.Bucket, objectKey) if err == nil && (!object.Exists || object.SizeBytes != int64(len(documentBytes))) { err = fmt.Errorf("existing audit seal object does not match the expected size") } } if err != nil { return false, err } recordIDHash := sha256.Sum256([]byte(organizationID + "\x00" + dayUTC)) record := domain.ObjectSyncAuditSealRecord{ ID: "osas_" + hex.EncodeToString(recordIDHash[:12]), OrganizationID: organizationID, DayUTC: dayUTC, EventCount: count, FirstEventHash: firstHash, LastEventHash: lastHash, AggregateHash: aggregateHash, Signature: signature, SigningKeyID: coordinator.signingID, ObjectKey: objectKey, ObjectETag: object.ETag, ObjectVersionID: object.VersionID, CreatedAt: nowISO(), PurgeAt: time.Now().UTC().Add(versionRetention), } if err := coordinator.syncStore.InsertObjectSyncAuditSeal(ctx, record); err != nil { return false, err } return true, nil } func (coordinator *Coordinator) reconcileSpace(ctx context.Context, reconcile *domain.ObjectSyncReconcileRecord) error { space, err := coordinator.syncStore.GetObjectSyncSpace(ctx, reconcile.SyncSpaceID) if err != nil || space == nil || space.OrganizationID != reconcile.OrganizationID { return fmt.Errorf("sync space is unavailable") } connection, err := coordinator.store.GetStorageConnection(ctx, space.ConnectionID) if err != nil || connection == nil { return fmt.Errorf("storage connection is unavailable") } var scanned int64 var drifted int64 cursor := "" for { page, err := coordinator.syncStore.ListObjectSyncHeads(ctx, reconcile.OrganizationID, reconcile.SyncSpaceID, cursor, 1000) if err != nil { return err } for _, head := range page.Items { remote, err := coordinator.publisher.Head(ctx, *connection, space.Bucket, head.Key) if err != nil { return fmt.Errorf("head %s: %w", head.Key, err) } head.Drifted = !remote.Exists || strings.Trim(remote.ETag, `"`) != strings.Trim(head.RemoteETag, `"`) head.UpdatedAt = nowISO() if head.Drifted { drifted++ } scanned++ if err := coordinator.syncStore.UpsertObjectSyncHead(ctx, head); err != nil { return err } } reconcile.Scanned = scanned reconcile.Drifted = drifted reconcile.UpdatedAt = nowISO() if err := coordinator.syncStore.UpsertObjectSyncReconcile(ctx, *reconcile); err != nil { return err } if page.NextCursor == "" { break } cursor = page.NextCursor } space.Enabled = false space.Mode = "blocked" if drifted > 0 { space.Capability.State = domain.ObjectSyncCapabilityDrifted } else { switch { case space.Capability.ConditionalCreate && space.Capability.ConditionalUpdate && space.Capability.ConditionalMultipart && space.Capability.ExclusivePrefixConfirmed: space.Capability.State = domain.ObjectSyncCapabilityUpdateEnabled case space.Capability.ConditionalCreate: space.Capability.State = domain.ObjectSyncCapabilityCreateOnly default: space.Capability.State = domain.ObjectSyncCapabilityUnsupported } } space.Policy.AllowConditionalUpdate = space.Capability.State == domain.ObjectSyncCapabilityUpdateEnabled if space.Capability.State != domain.ObjectSyncCapabilityUpdateEnabled { space.Policy.AllowForceOverwrite = false } space.UpdatedAt = nowISO() if err := coordinator.syncStore.UpsertObjectSyncSpace(ctx, *space); err != nil { return err } reconcile.Status = "completed" reconcile.CompletedAt = nowISO() reconcile.UpdatedAt = reconcile.CompletedAt reconcile.CoordinatorOwner = "" reconcile.CoordinatorLeaseUntil = time.Time{} if err := coordinator.syncStore.UpsertObjectSyncReconcile(ctx, *reconcile); err != nil { return err } coordinator.recordSystemAudit(ctx, domain.ObjectSyncAuditEvent{ OrganizationID: reconcile.OrganizationID, ActorType: "system", ActorID: coordinator.owner, Action: "space.reconcile.completed", Target: map[string]any{ "reconcileId": reconcile.ID, "spaceId": reconcile.SyncSpaceID, }, After: map[string]any{"scanned": scanned, "drifted": drifted}, Result: "success", }) return nil } func (coordinator *Coordinator) cleanupStaging(ctx context.Context, item *domain.ObjectSyncItemRecord) error { session, err := coordinator.syncStore.GetObjectSyncSession(ctx, item.SessionID) if err != nil || session == nil { return fmt.Errorf("session is unavailable") } space, err := coordinator.syncStore.GetObjectSyncSpace(ctx, session.SyncSpaceID) if err != nil || space == nil { return fmt.Errorf("sync space is unavailable") } connection, err := coordinator.store.GetStorageConnection(ctx, space.ConnectionID) if err != nil || connection == nil { return fmt.Errorf("storage connection is unavailable") } if item.UploadID != "" { _ = coordinator.publisher.AbortMultipart(ctx, *connection, space.Bucket, item.StagingKey, item.UploadID) } if err := coordinator.publisher.Delete(ctx, *connection, space.Bucket, item.StagingKey); err != nil { return err } if item.Status == domain.ObjectSyncItemUploading || item.Status == domain.ObjectSyncItemUploaded { item.Status = domain.ObjectSyncItemExpired } item.StagingKey = "" item.UploadID = "" item.UploadedParts = nil item.CoordinatorOwner = "" item.CoordinatorLeaseUntil = time.Time{} item.UpdatedAt = nowISO() if err := coordinator.syncStore.UpsertObjectSyncItems(ctx, []domain.ObjectSyncItemRecord{*item}); err != nil { return err } if item.LeaseID != "" { _ = coordinator.syncStore.ReleaseObjectSyncLease(ctx, item.LeaseID, item.SessionID, item.ID, item.FencingToken) } return coordinator.recountSession(ctx, session) } func nowISO() string { return time.Now().UTC().Format(time.RFC3339Nano) } func firstNonEmpty(values ...string) string { for _, value := range values { if strings.TrimSpace(value) != "" { return value } } return "" } func (coordinator *Coordinator) planSession(ctx context.Context, session *domain.ObjectSyncSessionRecord) error { space, err := coordinator.syncStore.GetObjectSyncSpace(ctx, session.SyncSpaceID) if err != nil || space == nil { return fmt.Errorf("sync space is unavailable") } connection, err := coordinator.store.GetStorageConnection(ctx, space.ConnectionID) if err != nil || connection == nil { return fmt.Errorf("storage connection is unavailable") } counts := domain.ObjectSyncSessionCounts{Manifest: session.Counts.Manifest} cursor := "" for { page, err := coordinator.syncStore.ListObjectSyncItems(ctx, domain.ObjectSyncListQuery{SessionID: session.ID, Cursor: cursor, Limit: 1000}) if err != nil { return err } updates := make([]domain.ObjectSyncItemRecord, 0, len(page.Items)) for _, item := range page.Items { if item.Status != domain.ObjectSyncItemDiscovered { updates = append(updates, item) continue } remote, err := coordinator.publisher.Head(ctx, *connection, space.Bucket, item.TargetKey) if err != nil { return fmt.Errorf("head %s: %w", item.TargetKey, err) } head, err := coordinator.syncStore.GetObjectSyncHead(ctx, session.OrganizationID, space.ID, item.TargetKey) if err != nil { return err } item.ExpectedBase = domain.ObjectSyncRemoteBase{ Exists: remote.Exists, ETag: remote.ETag, SizeBytes: remote.SizeBytes, LastModified: remote.LastModified, VersionID: remote.VersionID, } switch { case !remote.Exists: item.Action = "create" item.Status = domain.ObjectSyncItemPlanned counts.Create++ case head == nil: // Untracked objects are never silently overwritten. Conditional create // deliberately falls through to keep-both during publish. item.Action = "create" item.ExpectedBase = domain.ObjectSyncRemoteBase{Exists: false} item.Status = domain.ObjectSyncItemPlanned counts.Create++ case head.RemoteETag != remote.ETag: head.Drifted = true head.UpdatedAt = nowISO() _ = coordinator.syncStore.UpsertObjectSyncHead(ctx, *head) space.Capability.State = domain.ObjectSyncCapabilityDrifted space.Enabled = false space.Mode = "blocked" space.UpdatedAt = nowISO() _ = coordinator.syncStore.UpsertObjectSyncSpace(ctx, *space) item.Action = "deny_drift" item.Status = domain.ObjectSyncItemDenied item.LastError = "remote object drifted outside drosync" counts.Denied++ case head.SHA256 == item.SHA256 && head.SizeBytes == item.SizeBytes: item.Action = "unchanged" item.Status = domain.ObjectSyncItemUnchanged counts.Unchanged++ case space.Capability.State == domain.ObjectSyncCapabilityUpdateEnabled && space.Policy.AllowConditionalUpdate: item.Action = "update" item.ExpectedBase.PlatformVersion = head.PlatformVersion item.Status = domain.ObjectSyncItemPlanned counts.Update++ default: item.Action = "deny_update" item.Status = domain.ObjectSyncItemDenied item.LastError = "sync space is create-only" counts.Denied++ } item.UpdatedAt = nowISO() updates = append(updates, item) } if err := coordinator.syncStore.UpsertObjectSyncItems(ctx, updates); err != nil { return err } if page.NextCursor == "" { break } cursor = page.NextCursor } session.Counts = counts session.Status = domain.ObjectSyncSessionPlanned session.UpdatedAt = nowISO() return coordinator.syncStore.UpsertObjectSyncSession(ctx, *session) } func conflictKey(original, itemID string, now time.Time) string { shortID := strings.TrimPrefix(itemID, "osi_") if len(shortID) > 10 { shortID = shortID[:10] } return fmt.Sprintf("%s.__conflict__%s_%s", original, now.UTC().Format("20060102T150405Z"), shortID) } func (coordinator *Coordinator) commitItem(ctx context.Context, item *domain.ObjectSyncItemRecord) error { session, err := coordinator.syncStore.GetObjectSyncSession(ctx, item.SessionID) if err != nil || session == nil { return fmt.Errorf("session is unavailable") } space, err := coordinator.syncStore.GetObjectSyncSpace(ctx, session.SyncSpaceID) if err != nil || space == nil { return fmt.Errorf("sync space is unavailable") } connection, err := coordinator.store.GetStorageConnection(ctx, space.ConnectionID) if err != nil || connection == nil { return fmt.Errorf("storage connection is unavailable") } renewed, err := coordinator.syncStore.RenewObjectSyncLease(ctx, item.LeaseID, session.ID, item.ID, item.FencingToken, claimTTL) if err != nil || !renewed { return fmt.Errorf("lease is stale; publish rejected by fencing token") } sourceKey := item.StagingKey preserveSource := false if item.Action == "force_overwrite" { sourceKey = item.ForceSourceKey preserveSource = true } staging, err := coordinator.publisher.Head(ctx, *connection, space.Bucket, sourceKey) if err != nil { return err } if !staging.Exists || staging.SizeBytes != item.SizeBytes { return fmt.Errorf("staging object is missing or size changed") } ifMatch := "" ifNoneMatch := "" switch item.Action { case "create": ifNoneMatch = "*" case "update": if space.Capability.State != domain.ObjectSyncCapabilityUpdateEnabled || !space.Policy.AllowConditionalUpdate { return fmt.Errorf("conditional update gate is no longer enabled") } ifMatch = item.ExpectedBase.ETag case "force_overwrite": if space.Capability.State != domain.ObjectSyncCapabilityUpdateEnabled || !space.Capability.ExclusivePrefixConfirmed || !space.Policy.AllowConditionalUpdate || !space.Policy.AllowForceOverwrite { return fmt.Errorf("force overwrite gate is no longer enabled") } current, headErr := coordinator.publisher.Head(ctx, *connection, space.Bucket, item.TargetKey) if headErr != nil { return headErr } if current.Exists { ifMatch = current.ETag } else { ifNoneMatch = "*" } default: return fmt.Errorf("unsupported commit action %q", item.Action) } metadata := map[string]string{ "drosync-session": session.ID, "drosync-item": item.ID, "drosync-sha256": item.SHA256, "drosync-fence": fmt.Sprintf("%d", item.FencingToken), } if preserveSource { metadata["drosync-force-source"] = sourceKey metadata["drosync-force-actor"] = item.ForceActorID } targetKey := item.TargetKey logicalTarget := item.LogicalTargetKey published, err := coordinator.publisher.PublishConditional(ctx, *connection, space.Bucket, sourceKey, targetKey, item.SizeBytes, staging.ETag, ifMatch, ifNoneMatch, metadata) keptBoth := false if IsPreconditionFailure(err) && !preserveSource { logicalTarget = item.TargetKey targetKey = conflictKey(item.TargetKey, item.ID, time.Now()) metadata["drosync-logical-target"] = logicalTarget published, err = coordinator.publisher.PublishConditional(ctx, *connection, space.Bucket, sourceKey, targetKey, item.SizeBytes, staging.ETag, "", "*", metadata) keptBoth = err == nil } if err != nil { return err } previousHead, err := coordinator.syncStore.GetObjectSyncHead(ctx, session.OrganizationID, space.ID, targetKey) if err != nil { return err } platformVersion := int64(1) if previousHead != nil { platformVersion = previousHead.PlatformVersion + 1 } now := time.Now().UTC() head := domain.ObjectSyncObjectHeadRecord{ OrganizationID: session.OrganizationID, SyncSpaceID: space.ID, Key: targetKey, PlatformVersion: platformVersion, RemoteETag: published.ETag, VersionID: published.VersionID, SizeBytes: published.SizeBytes, SHA256: item.SHA256, SessionID: session.ID, ItemID: item.ID, UpdatedAt: now.Format(time.RFC3339Nano), } if err := coordinator.syncStore.UpsertObjectSyncHead(ctx, head); err != nil { return err } if err := coordinator.syncStore.InsertObjectSyncVersion(ctx, domain.ObjectSyncVersionRecord{ ID: fmt.Sprintf("osv_%s_%d", item.ID, platformVersion), OrganizationID: session.OrganizationID, SyncSpaceID: space.ID, Key: targetKey, LogicalTargetKey: logicalTarget, SourceKey: item.ForceSourceKey, PlatformVersion: platformVersion, RemoteETag: published.ETag, VersionID: published.VersionID, SizeBytes: published.SizeBytes, SHA256: item.SHA256, SessionID: session.ID, ItemID: item.ID, ActorID: firstNonEmpty(item.ForceActorID, session.DeviceID), CommittedAt: now.Format(time.RFC3339Nano), PurgeAt: now.Add(versionRetention), }); err != nil { return err } item.TargetKey = targetKey item.LogicalTargetKey = logicalTarget item.CommittedVersion = domain.ObjectSyncRemoteBase{ Exists: true, ETag: published.ETag, SizeBytes: published.SizeBytes, VersionID: published.VersionID, PlatformVersion: platformVersion, } if keptBoth { item.Status = domain.ObjectSyncItemKeptBoth } else { item.Status = domain.ObjectSyncItemCommitted } item.CoordinatorOwner = "" item.CoordinatorLeaseUntil = time.Time{} item.UpdatedAt = now.Format(time.RFC3339Nano) if err := coordinator.syncStore.UpsertObjectSyncItems(ctx, []domain.ObjectSyncItemRecord{*item}); err != nil { return err } if !preserveSource { _ = coordinator.publisher.Delete(ctx, *connection, space.Bucket, sourceKey) } _ = coordinator.syncStore.ReleaseObjectSyncLease(ctx, item.LeaseID, session.ID, item.ID, item.FencingToken) if err := coordinator.recountSession(ctx, session); err != nil { return err } coordinator.recordSystemAudit(ctx, domain.ObjectSyncAuditEvent{ OrganizationID: session.OrganizationID, ActorType: "system", ActorID: coordinator.owner, DeviceID: session.DeviceID, SessionID: session.ID, ItemID: item.ID, Action: "item." + item.Action + ".committed", Target: map[string]any{ "key": item.TargetKey, "sourceKey": item.ForceSourceKey, }, After: map[string]any{ "etag": item.CommittedVersion.ETag, "versionId": item.CommittedVersion.VersionID, "platformVersion": item.CommittedVersion.PlatformVersion, "sizeBytes": item.CommittedVersion.SizeBytes, }, Result: "success", }) return nil } func (coordinator *Coordinator) recountSession(ctx context.Context, session *domain.ObjectSyncSessionRecord) error { counts := domain.ObjectSyncSessionCounts{Manifest: session.Counts.Manifest} cursor := "" for { page, err := coordinator.syncStore.ListObjectSyncItems(ctx, domain.ObjectSyncListQuery{SessionID: session.ID, Cursor: cursor, Limit: 1000}) if err != nil { return err } for _, item := range page.Items { switch item.Status { case domain.ObjectSyncItemCommitted: counts.Committed++ case domain.ObjectSyncItemKeptBoth: counts.KeptBoth++ counts.Committed++ case domain.ObjectSyncItemUnchanged: counts.Unchanged++ case domain.ObjectSyncItemSkipped: counts.Skipped++ case domain.ObjectSyncItemDenied: counts.Denied++ case domain.ObjectSyncItemFailed: counts.Failed++ } switch item.Action { case "create": counts.Create++ case "update": counts.Update++ } } if page.NextCursor == "" { break } cursor = page.NextCursor } fresh, err := coordinator.syncStore.GetObjectSyncSession(ctx, session.ID) if err != nil || fresh == nil { return err } fresh.Counts = counts fresh.UpdatedAt = nowISO() return coordinator.syncStore.UpsertObjectSyncSession(ctx, *fresh) } func (coordinator *Coordinator) publishRelease(ctx context.Context, release *domain.ObjectSyncReleaseRecord) error { temp, err := os.CreateTemp("", "drosync-release-*.ndjson") if err != nil { return err } tempName := temp.Name() defer os.Remove(tempName) hasher := sha256.New() writer := bufio.NewWriter(io.MultiWriter(temp, hasher)) var objectCount int64 var totalBytes int64 var firstSession *domain.ObjectSyncSessionRecord for _, sessionID := range release.SessionIDs { session, err := coordinator.syncStore.GetObjectSyncSession(ctx, sessionID) if err != nil || session == nil || session.OrganizationID != release.OrganizationID { _ = temp.Close() return fmt.Errorf("release session %s is unavailable", sessionID) } if release.SyncSpaceID == "" { release.SyncSpaceID = session.SyncSpaceID } else if session.SyncSpaceID != release.SyncSpaceID { _ = temp.Close() return fmt.Errorf("release session %s belongs to a different sync space", sessionID) } if firstSession == nil { firstSession = session } cursor := "" for { page, err := coordinator.syncStore.ListObjectSyncItems(ctx, domain.ObjectSyncListQuery{SessionID: session.ID, Cursor: cursor, Limit: 1000}) if err != nil { _ = temp.Close() return err } for _, item := range page.Items { if item.Status != domain.ObjectSyncItemCommitted && item.Status != domain.ObjectSyncItemKeptBoth && item.Status != domain.ObjectSyncItemUnchanged { continue } line := map[string]any{ "sessionId": item.SessionID, "itemId": item.ID, "logicalTargetKey": item.LogicalTargetKey, "key": item.TargetKey, "sizeBytes": item.SizeBytes, "sha256": item.SHA256, "remote": item.CommittedVersion, "status": item.Status, } payload, _ := json.Marshal(line) if _, err := writer.Write(append(payload, '\n')); err != nil { _ = temp.Close() return err } objectCount++ totalBytes += item.SizeBytes } if page.NextCursor == "" { break } cursor = page.NextCursor } } if err := writer.Flush(); err != nil { _ = temp.Close() return err } if err := temp.Close(); err != nil { return err } if firstSession == nil { return fmt.Errorf("release has no sessions") } space, err := coordinator.syncStore.GetObjectSyncSpace(ctx, release.SyncSpaceID) if err != nil || space == nil { return fmt.Errorf("release sync space is unavailable") } connection, err := coordinator.store.GetStorageConnection(ctx, space.ConnectionID) if err != nil || connection == nil { return fmt.Errorf("release storage connection is unavailable") } info, err := os.Stat(tempName) if err != nil { return err } reader, err := os.Open(tempName) if err != nil { return err } defer reader.Close() manifestKey := path.Join(".drosync-internal/releases", release.OrganizationID, release.ID+".ndjson") digest := hex.EncodeToString(hasher.Sum(nil)) if _, err := coordinator.publisher.PutInternal(ctx, *connection, space.Bucket, manifestKey, reader, info.Size(), "application/x-ndjson", map[string]string{ "drosync-release": release.ID, "drosync-sha256": digest, }); err != nil { return err } release.ManifestKey = manifestKey release.ManifestSHA256 = digest release.ObjectCount = objectCount release.TotalBytes = totalBytes release.Status = "published" release.PublishedAt = nowISO() release.CoordinatorOwner = "" release.CoordinatorLeaseUntil = time.Time{} if err := coordinator.syncStore.UpsertObjectSyncRelease(ctx, *release); err != nil { return err } coordinator.recordSystemAudit(ctx, domain.ObjectSyncAuditEvent{ OrganizationID: release.OrganizationID, ActorType: "system", ActorID: coordinator.owner, Action: "release.published", Target: map[string]any{"releaseId": release.ID, "manifestKey": release.ManifestKey}, After: map[string]any{ "manifestSha256": release.ManifestSHA256, "objectCount": release.ObjectCount, "totalBytes": release.TotalBytes, }, Result: "success", }) return nil }