2026-07-28 20:43:46 +08:00

1070 lines
34 KiB
Go

package objectsync
import (
"bufio"
"bytes"
"context"
"crypto/ed25519"
"crypto/sha256"
"encoding/base64"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"os"
"path"
"strings"
"sync/atomic"
"time"
"droboticflow/internal/domain"
"droboticflow/internal/store"
)
const (
claimTTL = 10 * time.Minute
versionRetention = 365 * 24 * time.Hour
stagingRetention = 24 * time.Hour
)
type Coordinator struct {
store store.Store
syncStore store.ObjectSyncStore
publisher Publisher
owner string
signingKey ed25519.PrivateKey
signingID string
cycles atomic.Int64
errors atomic.Int64
planned atomic.Int64
committed atomic.Int64
cleaned atomic.Int64
reconciled atomic.Int64
released atomic.Int64
sealed atomic.Int64
lastCycle atomic.Int64
lastOK atomic.Int64
durationMS atomic.Int64
}
func NewCoordinator(base store.Store, publisher Publisher, owner string) (*Coordinator, error) {
syncStore, ok := store.AsObjectSyncStore(base)
if !ok {
return nil, fmt.Errorf("store does not implement object sync persistence")
}
if publisher == nil {
return nil, fmt.Errorf("object publisher is required")
}
if strings.TrimSpace(owner) == "" {
owner = fmt.Sprintf("coordinator-%d", time.Now().UnixNano())
}
return &Coordinator{store: base, syncStore: syncStore, publisher: publisher, owner: owner}, nil
}
func (coordinator *Coordinator) ConfigureAuditSigning(encodedPrivateKey, keyID string) error {
encodedPrivateKey = strings.TrimSpace(encodedPrivateKey)
if encodedPrivateKey == "" {
coordinator.signingKey = nil
coordinator.signingID = ""
return nil
}
raw, err := base64.StdEncoding.DecodeString(encodedPrivateKey)
if err != nil {
raw, err = base64.RawStdEncoding.DecodeString(encodedPrivateKey)
}
if err != nil {
return fmt.Errorf("decode audit signing key: %w", err)
}
switch len(raw) {
case ed25519.SeedSize:
coordinator.signingKey = ed25519.NewKeyFromSeed(raw)
case ed25519.PrivateKeySize:
coordinator.signingKey = ed25519.PrivateKey(append([]byte(nil), raw...))
default:
return fmt.Errorf("audit signing key must be a %d-byte Ed25519 seed or %d-byte private key", ed25519.SeedSize, ed25519.PrivateKeySize)
}
coordinator.signingID = strings.TrimSpace(keyID)
if coordinator.signingID == "" {
return fmt.Errorf("audit signing key id is required")
}
return nil
}
func (coordinator *Coordinator) AuditSigningConfigured() bool {
return len(coordinator.signingKey) == ed25519.PrivateKeySize && coordinator.signingID != ""
}
func (coordinator *Coordinator) Run(ctx context.Context, interval time.Duration) {
if interval <= 0 {
interval = 2 * time.Second
}
ticker := time.NewTicker(interval)
defer ticker.Stop()
for {
started := time.Now()
result, err := coordinator.ProcessOnce(ctx)
finished := time.Now()
coordinator.cycles.Add(1)
coordinator.lastCycle.Store(finished.Unix())
coordinator.durationMS.Store(finished.Sub(started).Milliseconds())
coordinator.planned.Add(int64(result.Planned))
coordinator.committed.Add(int64(result.Committed))
coordinator.cleaned.Add(int64(result.Cleaned))
coordinator.reconciled.Add(int64(result.Reconciled))
coordinator.released.Add(int64(result.Released))
coordinator.sealed.Add(int64(result.Sealed))
if err != nil && !errors.Is(err, context.Canceled) {
coordinator.errors.Add(1)
fmt.Printf("[drosync-coordinator] process error: %v\n", err)
} else if err == nil {
coordinator.lastOK.Store(finished.Unix())
}
select {
case <-ctx.Done():
return
case <-ticker.C:
}
}
}
type Metrics struct {
Cycles int64
Errors int64
Planned int64
Committed int64
Cleaned int64
Reconciled int64
Released int64
Sealed int64
LastCycle int64
LastOK int64
DurationMS int64
}
func (coordinator *Coordinator) Metrics() Metrics {
return Metrics{
Cycles: coordinator.cycles.Load(),
Errors: coordinator.errors.Load(),
Planned: coordinator.planned.Load(),
Committed: coordinator.committed.Load(),
Cleaned: coordinator.cleaned.Load(),
Reconciled: coordinator.reconciled.Load(),
Released: coordinator.released.Load(),
Sealed: coordinator.sealed.Load(),
LastCycle: coordinator.lastCycle.Load(),
LastOK: coordinator.lastOK.Load(),
DurationMS: coordinator.durationMS.Load(),
}
}
func (coordinator *Coordinator) recordSystemAudit(ctx context.Context, event domain.ObjectSyncAuditEvent) {
if err := AppendAudit(ctx, coordinator.syncStore, event); err != nil {
coordinator.errors.Add(1)
fmt.Printf("[drosync-coordinator] audit persistence error for %s: %v\n", event.Action, err)
}
}
type ProcessResult struct {
Planned int `json:"planned"`
Committed int `json:"committed"`
Cleaned int `json:"cleaned"`
Reconciled int `json:"reconciled"`
Released int `json:"released"`
Sealed int `json:"sealed"`
}
func (coordinator *Coordinator) ProcessOnce(ctx context.Context) (ProcessResult, error) {
result := ProcessResult{}
session, err := coordinator.syncStore.ClaimObjectSyncSession(ctx, domain.ObjectSyncSessionPlanning, domain.ObjectSyncSessionPlanned, coordinator.owner)
if err != nil {
return result, err
}
if session != nil {
if err := coordinator.planSession(ctx, session); err != nil {
session.Status = domain.ObjectSyncSessionFailed
session.ErrorSummary = err.Error()
session.UpdatedAt = nowISO()
_ = coordinator.syncStore.UpsertObjectSyncSession(ctx, *session)
return result, err
}
result.Planned++
}
for index := 0; index < 16; index++ {
item, claimErr := coordinator.syncStore.ClaimObjectSyncItem(ctx, coordinator.owner, claimTTL)
if claimErr != nil {
return result, claimErr
}
if item == nil {
break
}
if err := coordinator.commitItem(ctx, item); err != nil {
item.Status = domain.ObjectSyncItemFailed
item.LastError = err.Error()
item.AttemptCount++
item.CoordinatorOwner = ""
item.CoordinatorLeaseUntil = time.Time{}
item.UpdatedAt = nowISO()
_ = coordinator.syncStore.UpsertObjectSyncItems(ctx, []domain.ObjectSyncItemRecord{*item})
if session, sessionErr := coordinator.syncStore.GetObjectSyncSession(ctx, item.SessionID); sessionErr == nil && session != nil {
_ = coordinator.recountSession(ctx, session)
}
continue
}
result.Committed++
}
for index := 0; index < 8; index++ {
item, claimErr := coordinator.syncStore.ClaimObjectSyncCleanupItem(ctx, coordinator.owner, time.Now().UTC().Add(-stagingRetention), claimTTL)
if claimErr != nil {
return result, claimErr
}
if item == nil {
break
}
if err := coordinator.cleanupStaging(ctx, item); err != nil {
item.LastError = "staging cleanup: " + err.Error()
item.CoordinatorOwner = ""
item.CoordinatorLeaseUntil = time.Time{}
item.UpdatedAt = nowISO()
_ = coordinator.syncStore.UpsertObjectSyncItems(ctx, []domain.ObjectSyncItemRecord{*item})
continue
}
result.Cleaned++
}
reconcile, err := coordinator.syncStore.ClaimObjectSyncReconcile(ctx, coordinator.owner, claimTTL)
if err != nil {
return result, err
}
if reconcile != nil {
if err := coordinator.reconcileSpace(ctx, reconcile); err != nil {
reconcile.Status = "failed"
reconcile.Error = err.Error()
reconcile.UpdatedAt = nowISO()
reconcile.CompletedAt = reconcile.UpdatedAt
reconcile.CoordinatorOwner = ""
reconcile.CoordinatorLeaseUntil = time.Time{}
_ = coordinator.syncStore.UpsertObjectSyncReconcile(ctx, *reconcile)
} else {
result.Reconciled++
}
}
release, err := coordinator.syncStore.ClaimObjectSyncRelease(ctx, coordinator.owner, claimTTL)
if err != nil {
return result, err
}
if release != nil {
if err := coordinator.publishRelease(ctx, release); err != nil {
release.Status = "failed"
release.CoordinatorOwner = ""
release.CoordinatorLeaseUntil = time.Time{}
_ = coordinator.syncStore.UpsertObjectSyncRelease(ctx, *release)
return result, err
}
result.Released++
}
if coordinator.AuditSigningConfigured() {
sealed, sealErr := coordinator.sealPreviousAuditDay(ctx, time.Now().UTC())
if sealErr != nil {
return result, sealErr
}
result.Sealed += sealed
}
return result, nil
}
type auditSealSigningPayload struct {
OrganizationID string `json:"organizationId"`
DayUTC string `json:"dayUtc"`
EventCount int64 `json:"eventCount"`
FirstEventHash string `json:"firstEventHash,omitempty"`
LastEventHash string `json:"lastEventHash,omitempty"`
AggregateHash string `json:"aggregateHash"`
SigningKeyID string `json:"signingKeyId"`
}
type auditSealDocument struct {
auditSealSigningPayload
Signature string `json:"signature"`
}
func validateAuditEventHash(event domain.ObjectSyncAuditEvent) error {
unsigned := event
unsigned.EventHash = ""
payload, err := json.Marshal(unsigned)
if err != nil {
return err
}
sum := sha256.Sum256(append([]byte(event.PreviousHash), payload...))
if !strings.EqualFold(event.EventHash, hex.EncodeToString(sum[:])) {
return fmt.Errorf("audit event %s hash mismatch", event.ID)
}
return nil
}
func (coordinator *Coordinator) sealPreviousAuditDay(ctx context.Context, now time.Time) (int, error) {
dayStart := now.UTC().Truncate(24 * time.Hour).Add(-24 * time.Hour)
dayEnd := dayStart.Add(24 * time.Hour)
dayUTC := dayStart.Format("2006-01-02")
organizations, err := coordinator.store.ListOrganizations(ctx)
if err != nil {
return 0, err
}
sealed := 0
for _, organization := range organizations {
existing, err := coordinator.syncStore.GetObjectSyncAuditSeal(ctx, organization.ID, dayUTC)
if err != nil {
return sealed, err
}
if existing != nil {
continue
}
ok, err := coordinator.sealAuditDay(ctx, organization.ID, dayUTC, dayStart, dayEnd)
if err != nil {
return sealed, err
}
if ok {
sealed++
}
}
return sealed, nil
}
func (coordinator *Coordinator) sealAuditDay(ctx context.Context, organizationID, dayUTC string, dayStart, dayEnd time.Time) (bool, error) {
resourceSum := sha256.Sum256([]byte("audit-seal\x00" + organizationID + "\x00" + dayUTC))
resourceHash := hex.EncodeToString(resourceSum[:])
leaseOwner := coordinator.owner + ":" + dayUTC
lease, acquired, err := coordinator.syncStore.AcquireObjectSyncLease(ctx, domain.ObjectSyncLeaseRecord{
ID: "osl_seal_" + resourceHash,
OrganizationID: organizationID,
ResourceHash: resourceHash,
ResourceType: "audit_seal",
KeyOrPrefix: dayUTC,
OwnerSessionID: leaseOwner,
OwnerItemID: resourceHash,
}, claimTTL)
if err != nil {
return false, err
}
if !acquired {
return false, nil
}
defer func() {
_ = coordinator.syncStore.ReleaseObjectSyncLease(context.Background(), resourceHash, leaseOwner, resourceHash, lease.FencingToken)
}()
existing, err := coordinator.syncStore.GetObjectSyncAuditSeal(ctx, organizationID, dayUTC)
if err != nil || existing != nil {
return false, err
}
hasher := sha256.New()
var count int64
var firstHash string
var lastHash string
cursor := ""
for {
page, err := coordinator.syncStore.ListObjectSyncAuditRange(
ctx,
organizationID,
dayStart.Format(time.RFC3339Nano),
dayEnd.Format(time.RFC3339Nano),
cursor,
1000,
)
if err != nil {
return false, err
}
for _, event := range page.Items {
if err := validateAuditEventHash(event); err != nil {
return false, err
}
if count > 0 && event.PreviousHash != lastHash {
return false, fmt.Errorf("audit chain discontinuity before event %s", event.ID)
}
if count == 0 {
firstHash = event.EventHash
}
lastHash = event.EventHash
_, _ = io.WriteString(hasher, event.EventHash+"\n")
count++
}
if page.NextCursor == "" {
break
}
cursor = page.NextCursor
}
aggregateHash := hex.EncodeToString(hasher.Sum(nil))
signingPayload := auditSealSigningPayload{
OrganizationID: organizationID,
DayUTC: dayUTC,
EventCount: count,
FirstEventHash: firstHash,
LastEventHash: lastHash,
AggregateHash: aggregateHash,
SigningKeyID: coordinator.signingID,
}
signingBytes, err := json.Marshal(signingPayload)
if err != nil {
return false, err
}
signature := base64.StdEncoding.EncodeToString(ed25519.Sign(coordinator.signingKey, signingBytes))
documentBytes, err := json.MarshalIndent(auditSealDocument{
auditSealSigningPayload: signingPayload,
Signature: signature,
}, "", " ")
if err != nil {
return false, err
}
documentBytes = append(documentBytes, '\n')
spaces, err := coordinator.syncStore.ListObjectSyncSpaces(ctx, organizationID)
if err != nil {
return false, err
}
if len(spaces) == 0 {
if count == 0 {
return false, nil
}
return false, fmt.Errorf("organization %s has audit events but no sync space for audit sealing", organizationID)
}
space := spaces[0]
connection, err := coordinator.store.GetStorageConnection(ctx, space.ConnectionID)
if err != nil || connection == nil {
return false, fmt.Errorf("audit seal storage connection is unavailable")
}
objectKey := path.Join(".drosync-internal/audit", organizationID, dayUTC+".json")
object, err := coordinator.publisher.PutInternal(
ctx,
*connection,
space.Bucket,
objectKey,
bytes.NewReader(documentBytes),
int64(len(documentBytes)),
"application/json",
map[string]string{
"drosync-audit-day": dayUTC,
"drosync-audit-hash": aggregateHash,
"drosync-signing-key": coordinator.signingID,
},
)
if IsPreconditionFailure(err) {
object, err = coordinator.publisher.Head(ctx, *connection, space.Bucket, objectKey)
if err == nil && (!object.Exists || object.SizeBytes != int64(len(documentBytes))) {
err = fmt.Errorf("existing audit seal object does not match the expected size")
}
}
if err != nil {
return false, err
}
recordIDHash := sha256.Sum256([]byte(organizationID + "\x00" + dayUTC))
record := domain.ObjectSyncAuditSealRecord{
ID: "osas_" + hex.EncodeToString(recordIDHash[:12]),
OrganizationID: organizationID,
DayUTC: dayUTC,
EventCount: count,
FirstEventHash: firstHash,
LastEventHash: lastHash,
AggregateHash: aggregateHash,
Signature: signature,
SigningKeyID: coordinator.signingID,
ObjectKey: objectKey,
ObjectETag: object.ETag,
ObjectVersionID: object.VersionID,
CreatedAt: nowISO(),
PurgeAt: time.Now().UTC().Add(versionRetention),
}
if err := coordinator.syncStore.InsertObjectSyncAuditSeal(ctx, record); err != nil {
return false, err
}
return true, nil
}
func (coordinator *Coordinator) reconcileSpace(ctx context.Context, reconcile *domain.ObjectSyncReconcileRecord) error {
space, err := coordinator.syncStore.GetObjectSyncSpace(ctx, reconcile.SyncSpaceID)
if err != nil || space == nil || space.OrganizationID != reconcile.OrganizationID {
return fmt.Errorf("sync space is unavailable")
}
connection, err := coordinator.store.GetStorageConnection(ctx, space.ConnectionID)
if err != nil || connection == nil {
return fmt.Errorf("storage connection is unavailable")
}
var scanned int64
var drifted int64
cursor := ""
for {
page, err := coordinator.syncStore.ListObjectSyncHeads(ctx, reconcile.OrganizationID, reconcile.SyncSpaceID, cursor, 1000)
if err != nil {
return err
}
for _, head := range page.Items {
remote, err := coordinator.publisher.Head(ctx, *connection, space.Bucket, head.Key)
if err != nil {
return fmt.Errorf("head %s: %w", head.Key, err)
}
head.Drifted = !remote.Exists || strings.Trim(remote.ETag, `"`) != strings.Trim(head.RemoteETag, `"`)
head.UpdatedAt = nowISO()
if head.Drifted {
drifted++
}
scanned++
if err := coordinator.syncStore.UpsertObjectSyncHead(ctx, head); err != nil {
return err
}
}
reconcile.Scanned = scanned
reconcile.Drifted = drifted
reconcile.UpdatedAt = nowISO()
if err := coordinator.syncStore.UpsertObjectSyncReconcile(ctx, *reconcile); err != nil {
return err
}
if page.NextCursor == "" {
break
}
cursor = page.NextCursor
}
space.Enabled = false
space.Mode = "blocked"
if drifted > 0 {
space.Capability.State = domain.ObjectSyncCapabilityDrifted
} else {
switch {
case space.Capability.ConditionalCreate &&
space.Capability.ConditionalUpdate &&
space.Capability.ConditionalMultipart &&
space.Capability.ExclusivePrefixConfirmed:
space.Capability.State = domain.ObjectSyncCapabilityUpdateEnabled
case space.Capability.ConditionalCreate:
space.Capability.State = domain.ObjectSyncCapabilityCreateOnly
default:
space.Capability.State = domain.ObjectSyncCapabilityUnsupported
}
}
space.Policy.AllowConditionalUpdate = space.Capability.State == domain.ObjectSyncCapabilityUpdateEnabled
if space.Capability.State != domain.ObjectSyncCapabilityUpdateEnabled {
space.Policy.AllowForceOverwrite = false
}
space.UpdatedAt = nowISO()
if err := coordinator.syncStore.UpsertObjectSyncSpace(ctx, *space); err != nil {
return err
}
reconcile.Status = "completed"
reconcile.CompletedAt = nowISO()
reconcile.UpdatedAt = reconcile.CompletedAt
reconcile.CoordinatorOwner = ""
reconcile.CoordinatorLeaseUntil = time.Time{}
if err := coordinator.syncStore.UpsertObjectSyncReconcile(ctx, *reconcile); err != nil {
return err
}
coordinator.recordSystemAudit(ctx, domain.ObjectSyncAuditEvent{
OrganizationID: reconcile.OrganizationID,
ActorType: "system",
ActorID: coordinator.owner,
Action: "space.reconcile.completed",
Target: map[string]any{
"reconcileId": reconcile.ID,
"spaceId": reconcile.SyncSpaceID,
},
After: map[string]any{"scanned": scanned, "drifted": drifted},
Result: "success",
})
return nil
}
func (coordinator *Coordinator) cleanupStaging(ctx context.Context, item *domain.ObjectSyncItemRecord) error {
session, err := coordinator.syncStore.GetObjectSyncSession(ctx, item.SessionID)
if err != nil || session == nil {
return fmt.Errorf("session is unavailable")
}
space, err := coordinator.syncStore.GetObjectSyncSpace(ctx, session.SyncSpaceID)
if err != nil || space == nil {
return fmt.Errorf("sync space is unavailable")
}
connection, err := coordinator.store.GetStorageConnection(ctx, space.ConnectionID)
if err != nil || connection == nil {
return fmt.Errorf("storage connection is unavailable")
}
if item.UploadID != "" {
_ = coordinator.publisher.AbortMultipart(ctx, *connection, space.Bucket, item.StagingKey, item.UploadID)
}
if err := coordinator.publisher.Delete(ctx, *connection, space.Bucket, item.StagingKey); err != nil {
return err
}
if item.Status == domain.ObjectSyncItemUploading || item.Status == domain.ObjectSyncItemUploaded {
item.Status = domain.ObjectSyncItemExpired
}
item.StagingKey = ""
item.UploadID = ""
item.UploadedParts = nil
item.CoordinatorOwner = ""
item.CoordinatorLeaseUntil = time.Time{}
item.UpdatedAt = nowISO()
if err := coordinator.syncStore.UpsertObjectSyncItems(ctx, []domain.ObjectSyncItemRecord{*item}); err != nil {
return err
}
if item.LeaseID != "" {
_ = coordinator.syncStore.ReleaseObjectSyncLease(ctx, item.LeaseID, item.SessionID, item.ID, item.FencingToken)
}
return coordinator.recountSession(ctx, session)
}
func nowISO() string {
return time.Now().UTC().Format(time.RFC3339Nano)
}
func firstNonEmpty(values ...string) string {
for _, value := range values {
if strings.TrimSpace(value) != "" {
return value
}
}
return ""
}
func (coordinator *Coordinator) planSession(ctx context.Context, session *domain.ObjectSyncSessionRecord) error {
space, err := coordinator.syncStore.GetObjectSyncSpace(ctx, session.SyncSpaceID)
if err != nil || space == nil {
return fmt.Errorf("sync space is unavailable")
}
connection, err := coordinator.store.GetStorageConnection(ctx, space.ConnectionID)
if err != nil || connection == nil {
return fmt.Errorf("storage connection is unavailable")
}
counts := domain.ObjectSyncSessionCounts{Manifest: session.Counts.Manifest}
cursor := ""
for {
page, err := coordinator.syncStore.ListObjectSyncItems(ctx, domain.ObjectSyncListQuery{SessionID: session.ID, Cursor: cursor, Limit: 1000})
if err != nil {
return err
}
updates := make([]domain.ObjectSyncItemRecord, 0, len(page.Items))
for _, item := range page.Items {
if item.Status != domain.ObjectSyncItemDiscovered {
updates = append(updates, item)
continue
}
remote, err := coordinator.publisher.Head(ctx, *connection, space.Bucket, item.TargetKey)
if err != nil {
return fmt.Errorf("head %s: %w", item.TargetKey, err)
}
head, err := coordinator.syncStore.GetObjectSyncHead(ctx, session.OrganizationID, space.ID, item.TargetKey)
if err != nil {
return err
}
item.ExpectedBase = domain.ObjectSyncRemoteBase{
Exists: remote.Exists,
ETag: remote.ETag,
SizeBytes: remote.SizeBytes,
LastModified: remote.LastModified,
VersionID: remote.VersionID,
}
switch {
case !remote.Exists:
item.Action = "create"
item.Status = domain.ObjectSyncItemPlanned
counts.Create++
case head == nil:
// Untracked objects are never silently overwritten. Conditional create
// deliberately falls through to keep-both during publish.
item.Action = "create"
item.ExpectedBase = domain.ObjectSyncRemoteBase{Exists: false}
item.Status = domain.ObjectSyncItemPlanned
counts.Create++
case head.RemoteETag != remote.ETag:
head.Drifted = true
head.UpdatedAt = nowISO()
_ = coordinator.syncStore.UpsertObjectSyncHead(ctx, *head)
space.Capability.State = domain.ObjectSyncCapabilityDrifted
space.Enabled = false
space.Mode = "blocked"
space.UpdatedAt = nowISO()
_ = coordinator.syncStore.UpsertObjectSyncSpace(ctx, *space)
item.Action = "deny_drift"
item.Status = domain.ObjectSyncItemDenied
item.LastError = "remote object drifted outside drosync"
counts.Denied++
case head.SHA256 == item.SHA256 && head.SizeBytes == item.SizeBytes:
item.Action = "unchanged"
item.Status = domain.ObjectSyncItemUnchanged
counts.Unchanged++
case space.Capability.State == domain.ObjectSyncCapabilityUpdateEnabled && space.Policy.AllowConditionalUpdate:
item.Action = "update"
item.ExpectedBase.PlatformVersion = head.PlatformVersion
item.Status = domain.ObjectSyncItemPlanned
counts.Update++
default:
item.Action = "deny_update"
item.Status = domain.ObjectSyncItemDenied
item.LastError = "sync space is create-only"
counts.Denied++
}
item.UpdatedAt = nowISO()
updates = append(updates, item)
}
if err := coordinator.syncStore.UpsertObjectSyncItems(ctx, updates); err != nil {
return err
}
if page.NextCursor == "" {
break
}
cursor = page.NextCursor
}
session.Counts = counts
session.Status = domain.ObjectSyncSessionPlanned
session.UpdatedAt = nowISO()
return coordinator.syncStore.UpsertObjectSyncSession(ctx, *session)
}
func conflictKey(original, itemID string, now time.Time) string {
shortID := strings.TrimPrefix(itemID, "osi_")
if len(shortID) > 10 {
shortID = shortID[:10]
}
return fmt.Sprintf("%s.__conflict__%s_%s", original, now.UTC().Format("20060102T150405Z"), shortID)
}
func (coordinator *Coordinator) commitItem(ctx context.Context, item *domain.ObjectSyncItemRecord) error {
session, err := coordinator.syncStore.GetObjectSyncSession(ctx, item.SessionID)
if err != nil || session == nil {
return fmt.Errorf("session is unavailable")
}
space, err := coordinator.syncStore.GetObjectSyncSpace(ctx, session.SyncSpaceID)
if err != nil || space == nil {
return fmt.Errorf("sync space is unavailable")
}
connection, err := coordinator.store.GetStorageConnection(ctx, space.ConnectionID)
if err != nil || connection == nil {
return fmt.Errorf("storage connection is unavailable")
}
renewed, err := coordinator.syncStore.RenewObjectSyncLease(ctx, item.LeaseID, session.ID, item.ID, item.FencingToken, claimTTL)
if err != nil || !renewed {
return fmt.Errorf("lease is stale; publish rejected by fencing token")
}
sourceKey := item.StagingKey
preserveSource := false
if item.Action == "force_overwrite" {
sourceKey = item.ForceSourceKey
preserveSource = true
}
staging, err := coordinator.publisher.Head(ctx, *connection, space.Bucket, sourceKey)
if err != nil {
return err
}
if !staging.Exists || staging.SizeBytes != item.SizeBytes {
return fmt.Errorf("staging object is missing or size changed")
}
ifMatch := ""
ifNoneMatch := ""
switch item.Action {
case "create":
ifNoneMatch = "*"
case "update":
if space.Capability.State != domain.ObjectSyncCapabilityUpdateEnabled || !space.Policy.AllowConditionalUpdate {
return fmt.Errorf("conditional update gate is no longer enabled")
}
ifMatch = item.ExpectedBase.ETag
case "force_overwrite":
if space.Capability.State != domain.ObjectSyncCapabilityUpdateEnabled ||
!space.Capability.ExclusivePrefixConfirmed ||
!space.Policy.AllowConditionalUpdate ||
!space.Policy.AllowForceOverwrite {
return fmt.Errorf("force overwrite gate is no longer enabled")
}
current, headErr := coordinator.publisher.Head(ctx, *connection, space.Bucket, item.TargetKey)
if headErr != nil {
return headErr
}
if current.Exists {
ifMatch = current.ETag
} else {
ifNoneMatch = "*"
}
default:
return fmt.Errorf("unsupported commit action %q", item.Action)
}
metadata := map[string]string{
"drosync-session": session.ID,
"drosync-item": item.ID,
"drosync-sha256": item.SHA256,
"drosync-fence": fmt.Sprintf("%d", item.FencingToken),
}
if preserveSource {
metadata["drosync-force-source"] = sourceKey
metadata["drosync-force-actor"] = item.ForceActorID
}
targetKey := item.TargetKey
logicalTarget := item.LogicalTargetKey
published, err := coordinator.publisher.PublishConditional(ctx, *connection, space.Bucket, sourceKey, targetKey, item.SizeBytes, staging.ETag, ifMatch, ifNoneMatch, metadata)
keptBoth := false
if IsPreconditionFailure(err) && !preserveSource {
logicalTarget = item.TargetKey
targetKey = conflictKey(item.TargetKey, item.ID, time.Now())
metadata["drosync-logical-target"] = logicalTarget
published, err = coordinator.publisher.PublishConditional(ctx, *connection, space.Bucket, sourceKey, targetKey, item.SizeBytes, staging.ETag, "", "*", metadata)
keptBoth = err == nil
}
if err != nil {
return err
}
previousHead, err := coordinator.syncStore.GetObjectSyncHead(ctx, session.OrganizationID, space.ID, targetKey)
if err != nil {
return err
}
platformVersion := int64(1)
if previousHead != nil {
platformVersion = previousHead.PlatformVersion + 1
}
now := time.Now().UTC()
head := domain.ObjectSyncObjectHeadRecord{
OrganizationID: session.OrganizationID,
SyncSpaceID: space.ID,
Key: targetKey,
PlatformVersion: platformVersion,
RemoteETag: published.ETag,
VersionID: published.VersionID,
SizeBytes: published.SizeBytes,
SHA256: item.SHA256,
SessionID: session.ID,
ItemID: item.ID,
UpdatedAt: now.Format(time.RFC3339Nano),
}
if err := coordinator.syncStore.UpsertObjectSyncHead(ctx, head); err != nil {
return err
}
if err := coordinator.syncStore.InsertObjectSyncVersion(ctx, domain.ObjectSyncVersionRecord{
ID: fmt.Sprintf("osv_%s_%d", item.ID, platformVersion),
OrganizationID: session.OrganizationID,
SyncSpaceID: space.ID,
Key: targetKey,
LogicalTargetKey: logicalTarget,
SourceKey: item.ForceSourceKey,
PlatformVersion: platformVersion,
RemoteETag: published.ETag,
VersionID: published.VersionID,
SizeBytes: published.SizeBytes,
SHA256: item.SHA256,
SessionID: session.ID,
ItemID: item.ID,
ActorID: firstNonEmpty(item.ForceActorID, session.DeviceID),
CommittedAt: now.Format(time.RFC3339Nano),
PurgeAt: now.Add(versionRetention),
}); err != nil {
return err
}
item.TargetKey = targetKey
item.LogicalTargetKey = logicalTarget
item.CommittedVersion = domain.ObjectSyncRemoteBase{
Exists: true,
ETag: published.ETag,
SizeBytes: published.SizeBytes,
VersionID: published.VersionID,
PlatformVersion: platformVersion,
}
if keptBoth {
item.Status = domain.ObjectSyncItemKeptBoth
} else {
item.Status = domain.ObjectSyncItemCommitted
}
item.CoordinatorOwner = ""
item.CoordinatorLeaseUntil = time.Time{}
item.UpdatedAt = now.Format(time.RFC3339Nano)
if err := coordinator.syncStore.UpsertObjectSyncItems(ctx, []domain.ObjectSyncItemRecord{*item}); err != nil {
return err
}
if !preserveSource {
_ = coordinator.publisher.Delete(ctx, *connection, space.Bucket, sourceKey)
}
_ = coordinator.syncStore.ReleaseObjectSyncLease(ctx, item.LeaseID, session.ID, item.ID, item.FencingToken)
if err := coordinator.recountSession(ctx, session); err != nil {
return err
}
coordinator.recordSystemAudit(ctx, domain.ObjectSyncAuditEvent{
OrganizationID: session.OrganizationID,
ActorType: "system",
ActorID: coordinator.owner,
DeviceID: session.DeviceID,
SessionID: session.ID,
ItemID: item.ID,
Action: "item." + item.Action + ".committed",
Target: map[string]any{
"key": item.TargetKey,
"sourceKey": item.ForceSourceKey,
},
After: map[string]any{
"etag": item.CommittedVersion.ETag,
"versionId": item.CommittedVersion.VersionID,
"platformVersion": item.CommittedVersion.PlatformVersion,
"sizeBytes": item.CommittedVersion.SizeBytes,
},
Result: "success",
})
return nil
}
func (coordinator *Coordinator) recountSession(ctx context.Context, session *domain.ObjectSyncSessionRecord) error {
counts := domain.ObjectSyncSessionCounts{Manifest: session.Counts.Manifest}
cursor := ""
for {
page, err := coordinator.syncStore.ListObjectSyncItems(ctx, domain.ObjectSyncListQuery{SessionID: session.ID, Cursor: cursor, Limit: 1000})
if err != nil {
return err
}
for _, item := range page.Items {
switch item.Status {
case domain.ObjectSyncItemCommitted:
counts.Committed++
case domain.ObjectSyncItemKeptBoth:
counts.KeptBoth++
counts.Committed++
case domain.ObjectSyncItemUnchanged:
counts.Unchanged++
case domain.ObjectSyncItemSkipped:
counts.Skipped++
case domain.ObjectSyncItemDenied:
counts.Denied++
case domain.ObjectSyncItemFailed:
counts.Failed++
}
switch item.Action {
case "create":
counts.Create++
case "update":
counts.Update++
}
}
if page.NextCursor == "" {
break
}
cursor = page.NextCursor
}
fresh, err := coordinator.syncStore.GetObjectSyncSession(ctx, session.ID)
if err != nil || fresh == nil {
return err
}
fresh.Counts = counts
fresh.UpdatedAt = nowISO()
return coordinator.syncStore.UpsertObjectSyncSession(ctx, *fresh)
}
func (coordinator *Coordinator) publishRelease(ctx context.Context, release *domain.ObjectSyncReleaseRecord) error {
temp, err := os.CreateTemp("", "drosync-release-*.ndjson")
if err != nil {
return err
}
tempName := temp.Name()
defer os.Remove(tempName)
hasher := sha256.New()
writer := bufio.NewWriter(io.MultiWriter(temp, hasher))
var objectCount int64
var totalBytes int64
var firstSession *domain.ObjectSyncSessionRecord
for _, sessionID := range release.SessionIDs {
session, err := coordinator.syncStore.GetObjectSyncSession(ctx, sessionID)
if err != nil || session == nil || session.OrganizationID != release.OrganizationID {
_ = temp.Close()
return fmt.Errorf("release session %s is unavailable", sessionID)
}
if release.SyncSpaceID == "" {
release.SyncSpaceID = session.SyncSpaceID
} else if session.SyncSpaceID != release.SyncSpaceID {
_ = temp.Close()
return fmt.Errorf("release session %s belongs to a different sync space", sessionID)
}
if firstSession == nil {
firstSession = session
}
cursor := ""
for {
page, err := coordinator.syncStore.ListObjectSyncItems(ctx, domain.ObjectSyncListQuery{SessionID: session.ID, Cursor: cursor, Limit: 1000})
if err != nil {
_ = temp.Close()
return err
}
for _, item := range page.Items {
if item.Status != domain.ObjectSyncItemCommitted && item.Status != domain.ObjectSyncItemKeptBoth && item.Status != domain.ObjectSyncItemUnchanged {
continue
}
line := map[string]any{
"sessionId": item.SessionID,
"itemId": item.ID,
"logicalTargetKey": item.LogicalTargetKey,
"key": item.TargetKey,
"sizeBytes": item.SizeBytes,
"sha256": item.SHA256,
"remote": item.CommittedVersion,
"status": item.Status,
}
payload, _ := json.Marshal(line)
if _, err := writer.Write(append(payload, '\n')); err != nil {
_ = temp.Close()
return err
}
objectCount++
totalBytes += item.SizeBytes
}
if page.NextCursor == "" {
break
}
cursor = page.NextCursor
}
}
if err := writer.Flush(); err != nil {
_ = temp.Close()
return err
}
if err := temp.Close(); err != nil {
return err
}
if firstSession == nil {
return fmt.Errorf("release has no sessions")
}
space, err := coordinator.syncStore.GetObjectSyncSpace(ctx, release.SyncSpaceID)
if err != nil || space == nil {
return fmt.Errorf("release sync space is unavailable")
}
connection, err := coordinator.store.GetStorageConnection(ctx, space.ConnectionID)
if err != nil || connection == nil {
return fmt.Errorf("release storage connection is unavailable")
}
info, err := os.Stat(tempName)
if err != nil {
return err
}
reader, err := os.Open(tempName)
if err != nil {
return err
}
defer reader.Close()
manifestKey := path.Join(".drosync-internal/releases", release.OrganizationID, release.ID+".ndjson")
digest := hex.EncodeToString(hasher.Sum(nil))
if _, err := coordinator.publisher.PutInternal(ctx, *connection, space.Bucket, manifestKey, reader, info.Size(), "application/x-ndjson", map[string]string{
"drosync-release": release.ID,
"drosync-sha256": digest,
}); err != nil {
return err
}
release.ManifestKey = manifestKey
release.ManifestSHA256 = digest
release.ObjectCount = objectCount
release.TotalBytes = totalBytes
release.Status = "published"
release.PublishedAt = nowISO()
release.CoordinatorOwner = ""
release.CoordinatorLeaseUntil = time.Time{}
if err := coordinator.syncStore.UpsertObjectSyncRelease(ctx, *release); err != nil {
return err
}
coordinator.recordSystemAudit(ctx, domain.ObjectSyncAuditEvent{
OrganizationID: release.OrganizationID,
ActorType: "system",
ActorID: coordinator.owner,
Action: "release.published",
Target: map[string]any{"releaseId": release.ID, "manifestKey": release.ManifestKey},
After: map[string]any{
"manifestSha256": release.ManifestSHA256,
"objectCount": release.ObjectCount,
"totalBytes": release.TotalBytes,
},
Result: "success",
})
return nil
}