forked from longtao.wu/droboticflow
1070 lines
34 KiB
Go
1070 lines
34 KiB
Go
package objectsync
|
|
|
|
import (
|
|
"bufio"
|
|
"bytes"
|
|
"context"
|
|
"crypto/ed25519"
|
|
"crypto/sha256"
|
|
"encoding/base64"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"os"
|
|
"path"
|
|
"strings"
|
|
"sync/atomic"
|
|
"time"
|
|
|
|
"droboticflow/internal/domain"
|
|
"droboticflow/internal/store"
|
|
)
|
|
|
|
const (
|
|
claimTTL = 10 * time.Minute
|
|
versionRetention = 365 * 24 * time.Hour
|
|
stagingRetention = 24 * time.Hour
|
|
)
|
|
|
|
type Coordinator struct {
|
|
store store.Store
|
|
syncStore store.ObjectSyncStore
|
|
publisher Publisher
|
|
owner string
|
|
signingKey ed25519.PrivateKey
|
|
signingID string
|
|
cycles atomic.Int64
|
|
errors atomic.Int64
|
|
planned atomic.Int64
|
|
committed atomic.Int64
|
|
cleaned atomic.Int64
|
|
reconciled atomic.Int64
|
|
released atomic.Int64
|
|
sealed atomic.Int64
|
|
lastCycle atomic.Int64
|
|
lastOK atomic.Int64
|
|
durationMS atomic.Int64
|
|
}
|
|
|
|
func NewCoordinator(base store.Store, publisher Publisher, owner string) (*Coordinator, error) {
|
|
syncStore, ok := store.AsObjectSyncStore(base)
|
|
if !ok {
|
|
return nil, fmt.Errorf("store does not implement object sync persistence")
|
|
}
|
|
if publisher == nil {
|
|
return nil, fmt.Errorf("object publisher is required")
|
|
}
|
|
if strings.TrimSpace(owner) == "" {
|
|
owner = fmt.Sprintf("coordinator-%d", time.Now().UnixNano())
|
|
}
|
|
return &Coordinator{store: base, syncStore: syncStore, publisher: publisher, owner: owner}, nil
|
|
}
|
|
|
|
func (coordinator *Coordinator) ConfigureAuditSigning(encodedPrivateKey, keyID string) error {
|
|
encodedPrivateKey = strings.TrimSpace(encodedPrivateKey)
|
|
if encodedPrivateKey == "" {
|
|
coordinator.signingKey = nil
|
|
coordinator.signingID = ""
|
|
return nil
|
|
}
|
|
raw, err := base64.StdEncoding.DecodeString(encodedPrivateKey)
|
|
if err != nil {
|
|
raw, err = base64.RawStdEncoding.DecodeString(encodedPrivateKey)
|
|
}
|
|
if err != nil {
|
|
return fmt.Errorf("decode audit signing key: %w", err)
|
|
}
|
|
switch len(raw) {
|
|
case ed25519.SeedSize:
|
|
coordinator.signingKey = ed25519.NewKeyFromSeed(raw)
|
|
case ed25519.PrivateKeySize:
|
|
coordinator.signingKey = ed25519.PrivateKey(append([]byte(nil), raw...))
|
|
default:
|
|
return fmt.Errorf("audit signing key must be a %d-byte Ed25519 seed or %d-byte private key", ed25519.SeedSize, ed25519.PrivateKeySize)
|
|
}
|
|
coordinator.signingID = strings.TrimSpace(keyID)
|
|
if coordinator.signingID == "" {
|
|
return fmt.Errorf("audit signing key id is required")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (coordinator *Coordinator) AuditSigningConfigured() bool {
|
|
return len(coordinator.signingKey) == ed25519.PrivateKeySize && coordinator.signingID != ""
|
|
}
|
|
|
|
func (coordinator *Coordinator) Run(ctx context.Context, interval time.Duration) {
|
|
if interval <= 0 {
|
|
interval = 2 * time.Second
|
|
}
|
|
ticker := time.NewTicker(interval)
|
|
defer ticker.Stop()
|
|
for {
|
|
started := time.Now()
|
|
result, err := coordinator.ProcessOnce(ctx)
|
|
finished := time.Now()
|
|
coordinator.cycles.Add(1)
|
|
coordinator.lastCycle.Store(finished.Unix())
|
|
coordinator.durationMS.Store(finished.Sub(started).Milliseconds())
|
|
coordinator.planned.Add(int64(result.Planned))
|
|
coordinator.committed.Add(int64(result.Committed))
|
|
coordinator.cleaned.Add(int64(result.Cleaned))
|
|
coordinator.reconciled.Add(int64(result.Reconciled))
|
|
coordinator.released.Add(int64(result.Released))
|
|
coordinator.sealed.Add(int64(result.Sealed))
|
|
if err != nil && !errors.Is(err, context.Canceled) {
|
|
coordinator.errors.Add(1)
|
|
fmt.Printf("[drosync-coordinator] process error: %v\n", err)
|
|
} else if err == nil {
|
|
coordinator.lastOK.Store(finished.Unix())
|
|
}
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case <-ticker.C:
|
|
}
|
|
}
|
|
}
|
|
|
|
type Metrics struct {
|
|
Cycles int64
|
|
Errors int64
|
|
Planned int64
|
|
Committed int64
|
|
Cleaned int64
|
|
Reconciled int64
|
|
Released int64
|
|
Sealed int64
|
|
LastCycle int64
|
|
LastOK int64
|
|
DurationMS int64
|
|
}
|
|
|
|
func (coordinator *Coordinator) Metrics() Metrics {
|
|
return Metrics{
|
|
Cycles: coordinator.cycles.Load(),
|
|
Errors: coordinator.errors.Load(),
|
|
Planned: coordinator.planned.Load(),
|
|
Committed: coordinator.committed.Load(),
|
|
Cleaned: coordinator.cleaned.Load(),
|
|
Reconciled: coordinator.reconciled.Load(),
|
|
Released: coordinator.released.Load(),
|
|
Sealed: coordinator.sealed.Load(),
|
|
LastCycle: coordinator.lastCycle.Load(),
|
|
LastOK: coordinator.lastOK.Load(),
|
|
DurationMS: coordinator.durationMS.Load(),
|
|
}
|
|
}
|
|
|
|
func (coordinator *Coordinator) recordSystemAudit(ctx context.Context, event domain.ObjectSyncAuditEvent) {
|
|
if err := AppendAudit(ctx, coordinator.syncStore, event); err != nil {
|
|
coordinator.errors.Add(1)
|
|
fmt.Printf("[drosync-coordinator] audit persistence error for %s: %v\n", event.Action, err)
|
|
}
|
|
}
|
|
|
|
type ProcessResult struct {
|
|
Planned int `json:"planned"`
|
|
Committed int `json:"committed"`
|
|
Cleaned int `json:"cleaned"`
|
|
Reconciled int `json:"reconciled"`
|
|
Released int `json:"released"`
|
|
Sealed int `json:"sealed"`
|
|
}
|
|
|
|
func (coordinator *Coordinator) ProcessOnce(ctx context.Context) (ProcessResult, error) {
|
|
result := ProcessResult{}
|
|
session, err := coordinator.syncStore.ClaimObjectSyncSession(ctx, domain.ObjectSyncSessionPlanning, domain.ObjectSyncSessionPlanned, coordinator.owner)
|
|
if err != nil {
|
|
return result, err
|
|
}
|
|
if session != nil {
|
|
if err := coordinator.planSession(ctx, session); err != nil {
|
|
session.Status = domain.ObjectSyncSessionFailed
|
|
session.ErrorSummary = err.Error()
|
|
session.UpdatedAt = nowISO()
|
|
_ = coordinator.syncStore.UpsertObjectSyncSession(ctx, *session)
|
|
return result, err
|
|
}
|
|
result.Planned++
|
|
}
|
|
for index := 0; index < 16; index++ {
|
|
item, claimErr := coordinator.syncStore.ClaimObjectSyncItem(ctx, coordinator.owner, claimTTL)
|
|
if claimErr != nil {
|
|
return result, claimErr
|
|
}
|
|
if item == nil {
|
|
break
|
|
}
|
|
if err := coordinator.commitItem(ctx, item); err != nil {
|
|
item.Status = domain.ObjectSyncItemFailed
|
|
item.LastError = err.Error()
|
|
item.AttemptCount++
|
|
item.CoordinatorOwner = ""
|
|
item.CoordinatorLeaseUntil = time.Time{}
|
|
item.UpdatedAt = nowISO()
|
|
_ = coordinator.syncStore.UpsertObjectSyncItems(ctx, []domain.ObjectSyncItemRecord{*item})
|
|
if session, sessionErr := coordinator.syncStore.GetObjectSyncSession(ctx, item.SessionID); sessionErr == nil && session != nil {
|
|
_ = coordinator.recountSession(ctx, session)
|
|
}
|
|
continue
|
|
}
|
|
result.Committed++
|
|
}
|
|
for index := 0; index < 8; index++ {
|
|
item, claimErr := coordinator.syncStore.ClaimObjectSyncCleanupItem(ctx, coordinator.owner, time.Now().UTC().Add(-stagingRetention), claimTTL)
|
|
if claimErr != nil {
|
|
return result, claimErr
|
|
}
|
|
if item == nil {
|
|
break
|
|
}
|
|
if err := coordinator.cleanupStaging(ctx, item); err != nil {
|
|
item.LastError = "staging cleanup: " + err.Error()
|
|
item.CoordinatorOwner = ""
|
|
item.CoordinatorLeaseUntil = time.Time{}
|
|
item.UpdatedAt = nowISO()
|
|
_ = coordinator.syncStore.UpsertObjectSyncItems(ctx, []domain.ObjectSyncItemRecord{*item})
|
|
continue
|
|
}
|
|
result.Cleaned++
|
|
}
|
|
reconcile, err := coordinator.syncStore.ClaimObjectSyncReconcile(ctx, coordinator.owner, claimTTL)
|
|
if err != nil {
|
|
return result, err
|
|
}
|
|
if reconcile != nil {
|
|
if err := coordinator.reconcileSpace(ctx, reconcile); err != nil {
|
|
reconcile.Status = "failed"
|
|
reconcile.Error = err.Error()
|
|
reconcile.UpdatedAt = nowISO()
|
|
reconcile.CompletedAt = reconcile.UpdatedAt
|
|
reconcile.CoordinatorOwner = ""
|
|
reconcile.CoordinatorLeaseUntil = time.Time{}
|
|
_ = coordinator.syncStore.UpsertObjectSyncReconcile(ctx, *reconcile)
|
|
} else {
|
|
result.Reconciled++
|
|
}
|
|
}
|
|
release, err := coordinator.syncStore.ClaimObjectSyncRelease(ctx, coordinator.owner, claimTTL)
|
|
if err != nil {
|
|
return result, err
|
|
}
|
|
if release != nil {
|
|
if err := coordinator.publishRelease(ctx, release); err != nil {
|
|
release.Status = "failed"
|
|
release.CoordinatorOwner = ""
|
|
release.CoordinatorLeaseUntil = time.Time{}
|
|
_ = coordinator.syncStore.UpsertObjectSyncRelease(ctx, *release)
|
|
return result, err
|
|
}
|
|
result.Released++
|
|
}
|
|
if coordinator.AuditSigningConfigured() {
|
|
sealed, sealErr := coordinator.sealPreviousAuditDay(ctx, time.Now().UTC())
|
|
if sealErr != nil {
|
|
return result, sealErr
|
|
}
|
|
result.Sealed += sealed
|
|
}
|
|
return result, nil
|
|
}
|
|
|
|
type auditSealSigningPayload struct {
|
|
OrganizationID string `json:"organizationId"`
|
|
DayUTC string `json:"dayUtc"`
|
|
EventCount int64 `json:"eventCount"`
|
|
FirstEventHash string `json:"firstEventHash,omitempty"`
|
|
LastEventHash string `json:"lastEventHash,omitempty"`
|
|
AggregateHash string `json:"aggregateHash"`
|
|
SigningKeyID string `json:"signingKeyId"`
|
|
}
|
|
|
|
type auditSealDocument struct {
|
|
auditSealSigningPayload
|
|
Signature string `json:"signature"`
|
|
}
|
|
|
|
func validateAuditEventHash(event domain.ObjectSyncAuditEvent) error {
|
|
unsigned := event
|
|
unsigned.EventHash = ""
|
|
payload, err := json.Marshal(unsigned)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
sum := sha256.Sum256(append([]byte(event.PreviousHash), payload...))
|
|
if !strings.EqualFold(event.EventHash, hex.EncodeToString(sum[:])) {
|
|
return fmt.Errorf("audit event %s hash mismatch", event.ID)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (coordinator *Coordinator) sealPreviousAuditDay(ctx context.Context, now time.Time) (int, error) {
|
|
dayStart := now.UTC().Truncate(24 * time.Hour).Add(-24 * time.Hour)
|
|
dayEnd := dayStart.Add(24 * time.Hour)
|
|
dayUTC := dayStart.Format("2006-01-02")
|
|
organizations, err := coordinator.store.ListOrganizations(ctx)
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
sealed := 0
|
|
for _, organization := range organizations {
|
|
existing, err := coordinator.syncStore.GetObjectSyncAuditSeal(ctx, organization.ID, dayUTC)
|
|
if err != nil {
|
|
return sealed, err
|
|
}
|
|
if existing != nil {
|
|
continue
|
|
}
|
|
ok, err := coordinator.sealAuditDay(ctx, organization.ID, dayUTC, dayStart, dayEnd)
|
|
if err != nil {
|
|
return sealed, err
|
|
}
|
|
if ok {
|
|
sealed++
|
|
}
|
|
}
|
|
return sealed, nil
|
|
}
|
|
|
|
func (coordinator *Coordinator) sealAuditDay(ctx context.Context, organizationID, dayUTC string, dayStart, dayEnd time.Time) (bool, error) {
|
|
resourceSum := sha256.Sum256([]byte("audit-seal\x00" + organizationID + "\x00" + dayUTC))
|
|
resourceHash := hex.EncodeToString(resourceSum[:])
|
|
leaseOwner := coordinator.owner + ":" + dayUTC
|
|
lease, acquired, err := coordinator.syncStore.AcquireObjectSyncLease(ctx, domain.ObjectSyncLeaseRecord{
|
|
ID: "osl_seal_" + resourceHash,
|
|
OrganizationID: organizationID,
|
|
ResourceHash: resourceHash,
|
|
ResourceType: "audit_seal",
|
|
KeyOrPrefix: dayUTC,
|
|
OwnerSessionID: leaseOwner,
|
|
OwnerItemID: resourceHash,
|
|
}, claimTTL)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
if !acquired {
|
|
return false, nil
|
|
}
|
|
defer func() {
|
|
_ = coordinator.syncStore.ReleaseObjectSyncLease(context.Background(), resourceHash, leaseOwner, resourceHash, lease.FencingToken)
|
|
}()
|
|
existing, err := coordinator.syncStore.GetObjectSyncAuditSeal(ctx, organizationID, dayUTC)
|
|
if err != nil || existing != nil {
|
|
return false, err
|
|
}
|
|
|
|
hasher := sha256.New()
|
|
var count int64
|
|
var firstHash string
|
|
var lastHash string
|
|
cursor := ""
|
|
for {
|
|
page, err := coordinator.syncStore.ListObjectSyncAuditRange(
|
|
ctx,
|
|
organizationID,
|
|
dayStart.Format(time.RFC3339Nano),
|
|
dayEnd.Format(time.RFC3339Nano),
|
|
cursor,
|
|
1000,
|
|
)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
for _, event := range page.Items {
|
|
if err := validateAuditEventHash(event); err != nil {
|
|
return false, err
|
|
}
|
|
if count > 0 && event.PreviousHash != lastHash {
|
|
return false, fmt.Errorf("audit chain discontinuity before event %s", event.ID)
|
|
}
|
|
if count == 0 {
|
|
firstHash = event.EventHash
|
|
}
|
|
lastHash = event.EventHash
|
|
_, _ = io.WriteString(hasher, event.EventHash+"\n")
|
|
count++
|
|
}
|
|
if page.NextCursor == "" {
|
|
break
|
|
}
|
|
cursor = page.NextCursor
|
|
}
|
|
aggregateHash := hex.EncodeToString(hasher.Sum(nil))
|
|
signingPayload := auditSealSigningPayload{
|
|
OrganizationID: organizationID,
|
|
DayUTC: dayUTC,
|
|
EventCount: count,
|
|
FirstEventHash: firstHash,
|
|
LastEventHash: lastHash,
|
|
AggregateHash: aggregateHash,
|
|
SigningKeyID: coordinator.signingID,
|
|
}
|
|
signingBytes, err := json.Marshal(signingPayload)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
signature := base64.StdEncoding.EncodeToString(ed25519.Sign(coordinator.signingKey, signingBytes))
|
|
documentBytes, err := json.MarshalIndent(auditSealDocument{
|
|
auditSealSigningPayload: signingPayload,
|
|
Signature: signature,
|
|
}, "", " ")
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
documentBytes = append(documentBytes, '\n')
|
|
|
|
spaces, err := coordinator.syncStore.ListObjectSyncSpaces(ctx, organizationID)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
if len(spaces) == 0 {
|
|
if count == 0 {
|
|
return false, nil
|
|
}
|
|
return false, fmt.Errorf("organization %s has audit events but no sync space for audit sealing", organizationID)
|
|
}
|
|
space := spaces[0]
|
|
connection, err := coordinator.store.GetStorageConnection(ctx, space.ConnectionID)
|
|
if err != nil || connection == nil {
|
|
return false, fmt.Errorf("audit seal storage connection is unavailable")
|
|
}
|
|
objectKey := path.Join(".drosync-internal/audit", organizationID, dayUTC+".json")
|
|
object, err := coordinator.publisher.PutInternal(
|
|
ctx,
|
|
*connection,
|
|
space.Bucket,
|
|
objectKey,
|
|
bytes.NewReader(documentBytes),
|
|
int64(len(documentBytes)),
|
|
"application/json",
|
|
map[string]string{
|
|
"drosync-audit-day": dayUTC,
|
|
"drosync-audit-hash": aggregateHash,
|
|
"drosync-signing-key": coordinator.signingID,
|
|
},
|
|
)
|
|
if IsPreconditionFailure(err) {
|
|
object, err = coordinator.publisher.Head(ctx, *connection, space.Bucket, objectKey)
|
|
if err == nil && (!object.Exists || object.SizeBytes != int64(len(documentBytes))) {
|
|
err = fmt.Errorf("existing audit seal object does not match the expected size")
|
|
}
|
|
}
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
recordIDHash := sha256.Sum256([]byte(organizationID + "\x00" + dayUTC))
|
|
record := domain.ObjectSyncAuditSealRecord{
|
|
ID: "osas_" + hex.EncodeToString(recordIDHash[:12]),
|
|
OrganizationID: organizationID,
|
|
DayUTC: dayUTC,
|
|
EventCount: count,
|
|
FirstEventHash: firstHash,
|
|
LastEventHash: lastHash,
|
|
AggregateHash: aggregateHash,
|
|
Signature: signature,
|
|
SigningKeyID: coordinator.signingID,
|
|
ObjectKey: objectKey,
|
|
ObjectETag: object.ETag,
|
|
ObjectVersionID: object.VersionID,
|
|
CreatedAt: nowISO(),
|
|
PurgeAt: time.Now().UTC().Add(versionRetention),
|
|
}
|
|
if err := coordinator.syncStore.InsertObjectSyncAuditSeal(ctx, record); err != nil {
|
|
return false, err
|
|
}
|
|
return true, nil
|
|
}
|
|
|
|
func (coordinator *Coordinator) reconcileSpace(ctx context.Context, reconcile *domain.ObjectSyncReconcileRecord) error {
|
|
space, err := coordinator.syncStore.GetObjectSyncSpace(ctx, reconcile.SyncSpaceID)
|
|
if err != nil || space == nil || space.OrganizationID != reconcile.OrganizationID {
|
|
return fmt.Errorf("sync space is unavailable")
|
|
}
|
|
connection, err := coordinator.store.GetStorageConnection(ctx, space.ConnectionID)
|
|
if err != nil || connection == nil {
|
|
return fmt.Errorf("storage connection is unavailable")
|
|
}
|
|
var scanned int64
|
|
var drifted int64
|
|
cursor := ""
|
|
for {
|
|
page, err := coordinator.syncStore.ListObjectSyncHeads(ctx, reconcile.OrganizationID, reconcile.SyncSpaceID, cursor, 1000)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
for _, head := range page.Items {
|
|
remote, err := coordinator.publisher.Head(ctx, *connection, space.Bucket, head.Key)
|
|
if err != nil {
|
|
return fmt.Errorf("head %s: %w", head.Key, err)
|
|
}
|
|
head.Drifted = !remote.Exists || strings.Trim(remote.ETag, `"`) != strings.Trim(head.RemoteETag, `"`)
|
|
head.UpdatedAt = nowISO()
|
|
if head.Drifted {
|
|
drifted++
|
|
}
|
|
scanned++
|
|
if err := coordinator.syncStore.UpsertObjectSyncHead(ctx, head); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
reconcile.Scanned = scanned
|
|
reconcile.Drifted = drifted
|
|
reconcile.UpdatedAt = nowISO()
|
|
if err := coordinator.syncStore.UpsertObjectSyncReconcile(ctx, *reconcile); err != nil {
|
|
return err
|
|
}
|
|
if page.NextCursor == "" {
|
|
break
|
|
}
|
|
cursor = page.NextCursor
|
|
}
|
|
space.Enabled = false
|
|
space.Mode = "blocked"
|
|
if drifted > 0 {
|
|
space.Capability.State = domain.ObjectSyncCapabilityDrifted
|
|
} else {
|
|
switch {
|
|
case space.Capability.ConditionalCreate &&
|
|
space.Capability.ConditionalUpdate &&
|
|
space.Capability.ConditionalMultipart &&
|
|
space.Capability.ExclusivePrefixConfirmed:
|
|
space.Capability.State = domain.ObjectSyncCapabilityUpdateEnabled
|
|
case space.Capability.ConditionalCreate:
|
|
space.Capability.State = domain.ObjectSyncCapabilityCreateOnly
|
|
default:
|
|
space.Capability.State = domain.ObjectSyncCapabilityUnsupported
|
|
}
|
|
}
|
|
space.Policy.AllowConditionalUpdate = space.Capability.State == domain.ObjectSyncCapabilityUpdateEnabled
|
|
if space.Capability.State != domain.ObjectSyncCapabilityUpdateEnabled {
|
|
space.Policy.AllowForceOverwrite = false
|
|
}
|
|
space.UpdatedAt = nowISO()
|
|
if err := coordinator.syncStore.UpsertObjectSyncSpace(ctx, *space); err != nil {
|
|
return err
|
|
}
|
|
reconcile.Status = "completed"
|
|
reconcile.CompletedAt = nowISO()
|
|
reconcile.UpdatedAt = reconcile.CompletedAt
|
|
reconcile.CoordinatorOwner = ""
|
|
reconcile.CoordinatorLeaseUntil = time.Time{}
|
|
if err := coordinator.syncStore.UpsertObjectSyncReconcile(ctx, *reconcile); err != nil {
|
|
return err
|
|
}
|
|
coordinator.recordSystemAudit(ctx, domain.ObjectSyncAuditEvent{
|
|
OrganizationID: reconcile.OrganizationID,
|
|
ActorType: "system",
|
|
ActorID: coordinator.owner,
|
|
Action: "space.reconcile.completed",
|
|
Target: map[string]any{
|
|
"reconcileId": reconcile.ID,
|
|
"spaceId": reconcile.SyncSpaceID,
|
|
},
|
|
After: map[string]any{"scanned": scanned, "drifted": drifted},
|
|
Result: "success",
|
|
})
|
|
return nil
|
|
}
|
|
|
|
func (coordinator *Coordinator) cleanupStaging(ctx context.Context, item *domain.ObjectSyncItemRecord) error {
|
|
session, err := coordinator.syncStore.GetObjectSyncSession(ctx, item.SessionID)
|
|
if err != nil || session == nil {
|
|
return fmt.Errorf("session is unavailable")
|
|
}
|
|
space, err := coordinator.syncStore.GetObjectSyncSpace(ctx, session.SyncSpaceID)
|
|
if err != nil || space == nil {
|
|
return fmt.Errorf("sync space is unavailable")
|
|
}
|
|
connection, err := coordinator.store.GetStorageConnection(ctx, space.ConnectionID)
|
|
if err != nil || connection == nil {
|
|
return fmt.Errorf("storage connection is unavailable")
|
|
}
|
|
if item.UploadID != "" {
|
|
_ = coordinator.publisher.AbortMultipart(ctx, *connection, space.Bucket, item.StagingKey, item.UploadID)
|
|
}
|
|
if err := coordinator.publisher.Delete(ctx, *connection, space.Bucket, item.StagingKey); err != nil {
|
|
return err
|
|
}
|
|
if item.Status == domain.ObjectSyncItemUploading || item.Status == domain.ObjectSyncItemUploaded {
|
|
item.Status = domain.ObjectSyncItemExpired
|
|
}
|
|
item.StagingKey = ""
|
|
item.UploadID = ""
|
|
item.UploadedParts = nil
|
|
item.CoordinatorOwner = ""
|
|
item.CoordinatorLeaseUntil = time.Time{}
|
|
item.UpdatedAt = nowISO()
|
|
if err := coordinator.syncStore.UpsertObjectSyncItems(ctx, []domain.ObjectSyncItemRecord{*item}); err != nil {
|
|
return err
|
|
}
|
|
if item.LeaseID != "" {
|
|
_ = coordinator.syncStore.ReleaseObjectSyncLease(ctx, item.LeaseID, item.SessionID, item.ID, item.FencingToken)
|
|
}
|
|
return coordinator.recountSession(ctx, session)
|
|
}
|
|
|
|
func nowISO() string {
|
|
return time.Now().UTC().Format(time.RFC3339Nano)
|
|
}
|
|
|
|
func firstNonEmpty(values ...string) string {
|
|
for _, value := range values {
|
|
if strings.TrimSpace(value) != "" {
|
|
return value
|
|
}
|
|
}
|
|
return ""
|
|
}
|
|
|
|
func (coordinator *Coordinator) planSession(ctx context.Context, session *domain.ObjectSyncSessionRecord) error {
|
|
space, err := coordinator.syncStore.GetObjectSyncSpace(ctx, session.SyncSpaceID)
|
|
if err != nil || space == nil {
|
|
return fmt.Errorf("sync space is unavailable")
|
|
}
|
|
connection, err := coordinator.store.GetStorageConnection(ctx, space.ConnectionID)
|
|
if err != nil || connection == nil {
|
|
return fmt.Errorf("storage connection is unavailable")
|
|
}
|
|
counts := domain.ObjectSyncSessionCounts{Manifest: session.Counts.Manifest}
|
|
cursor := ""
|
|
for {
|
|
page, err := coordinator.syncStore.ListObjectSyncItems(ctx, domain.ObjectSyncListQuery{SessionID: session.ID, Cursor: cursor, Limit: 1000})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
updates := make([]domain.ObjectSyncItemRecord, 0, len(page.Items))
|
|
for _, item := range page.Items {
|
|
if item.Status != domain.ObjectSyncItemDiscovered {
|
|
updates = append(updates, item)
|
|
continue
|
|
}
|
|
remote, err := coordinator.publisher.Head(ctx, *connection, space.Bucket, item.TargetKey)
|
|
if err != nil {
|
|
return fmt.Errorf("head %s: %w", item.TargetKey, err)
|
|
}
|
|
head, err := coordinator.syncStore.GetObjectSyncHead(ctx, session.OrganizationID, space.ID, item.TargetKey)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
item.ExpectedBase = domain.ObjectSyncRemoteBase{
|
|
Exists: remote.Exists,
|
|
ETag: remote.ETag,
|
|
SizeBytes: remote.SizeBytes,
|
|
LastModified: remote.LastModified,
|
|
VersionID: remote.VersionID,
|
|
}
|
|
switch {
|
|
case !remote.Exists:
|
|
item.Action = "create"
|
|
item.Status = domain.ObjectSyncItemPlanned
|
|
counts.Create++
|
|
case head == nil:
|
|
// Untracked objects are never silently overwritten. Conditional create
|
|
// deliberately falls through to keep-both during publish.
|
|
item.Action = "create"
|
|
item.ExpectedBase = domain.ObjectSyncRemoteBase{Exists: false}
|
|
item.Status = domain.ObjectSyncItemPlanned
|
|
counts.Create++
|
|
case head.RemoteETag != remote.ETag:
|
|
head.Drifted = true
|
|
head.UpdatedAt = nowISO()
|
|
_ = coordinator.syncStore.UpsertObjectSyncHead(ctx, *head)
|
|
space.Capability.State = domain.ObjectSyncCapabilityDrifted
|
|
space.Enabled = false
|
|
space.Mode = "blocked"
|
|
space.UpdatedAt = nowISO()
|
|
_ = coordinator.syncStore.UpsertObjectSyncSpace(ctx, *space)
|
|
item.Action = "deny_drift"
|
|
item.Status = domain.ObjectSyncItemDenied
|
|
item.LastError = "remote object drifted outside drosync"
|
|
counts.Denied++
|
|
case head.SHA256 == item.SHA256 && head.SizeBytes == item.SizeBytes:
|
|
item.Action = "unchanged"
|
|
item.Status = domain.ObjectSyncItemUnchanged
|
|
counts.Unchanged++
|
|
case space.Capability.State == domain.ObjectSyncCapabilityUpdateEnabled && space.Policy.AllowConditionalUpdate:
|
|
item.Action = "update"
|
|
item.ExpectedBase.PlatformVersion = head.PlatformVersion
|
|
item.Status = domain.ObjectSyncItemPlanned
|
|
counts.Update++
|
|
default:
|
|
item.Action = "deny_update"
|
|
item.Status = domain.ObjectSyncItemDenied
|
|
item.LastError = "sync space is create-only"
|
|
counts.Denied++
|
|
}
|
|
item.UpdatedAt = nowISO()
|
|
updates = append(updates, item)
|
|
}
|
|
if err := coordinator.syncStore.UpsertObjectSyncItems(ctx, updates); err != nil {
|
|
return err
|
|
}
|
|
if page.NextCursor == "" {
|
|
break
|
|
}
|
|
cursor = page.NextCursor
|
|
}
|
|
session.Counts = counts
|
|
session.Status = domain.ObjectSyncSessionPlanned
|
|
session.UpdatedAt = nowISO()
|
|
return coordinator.syncStore.UpsertObjectSyncSession(ctx, *session)
|
|
}
|
|
|
|
func conflictKey(original, itemID string, now time.Time) string {
|
|
shortID := strings.TrimPrefix(itemID, "osi_")
|
|
if len(shortID) > 10 {
|
|
shortID = shortID[:10]
|
|
}
|
|
return fmt.Sprintf("%s.__conflict__%s_%s", original, now.UTC().Format("20060102T150405Z"), shortID)
|
|
}
|
|
|
|
func (coordinator *Coordinator) commitItem(ctx context.Context, item *domain.ObjectSyncItemRecord) error {
|
|
session, err := coordinator.syncStore.GetObjectSyncSession(ctx, item.SessionID)
|
|
if err != nil || session == nil {
|
|
return fmt.Errorf("session is unavailable")
|
|
}
|
|
space, err := coordinator.syncStore.GetObjectSyncSpace(ctx, session.SyncSpaceID)
|
|
if err != nil || space == nil {
|
|
return fmt.Errorf("sync space is unavailable")
|
|
}
|
|
connection, err := coordinator.store.GetStorageConnection(ctx, space.ConnectionID)
|
|
if err != nil || connection == nil {
|
|
return fmt.Errorf("storage connection is unavailable")
|
|
}
|
|
renewed, err := coordinator.syncStore.RenewObjectSyncLease(ctx, item.LeaseID, session.ID, item.ID, item.FencingToken, claimTTL)
|
|
if err != nil || !renewed {
|
|
return fmt.Errorf("lease is stale; publish rejected by fencing token")
|
|
}
|
|
sourceKey := item.StagingKey
|
|
preserveSource := false
|
|
if item.Action == "force_overwrite" {
|
|
sourceKey = item.ForceSourceKey
|
|
preserveSource = true
|
|
}
|
|
staging, err := coordinator.publisher.Head(ctx, *connection, space.Bucket, sourceKey)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if !staging.Exists || staging.SizeBytes != item.SizeBytes {
|
|
return fmt.Errorf("staging object is missing or size changed")
|
|
}
|
|
ifMatch := ""
|
|
ifNoneMatch := ""
|
|
switch item.Action {
|
|
case "create":
|
|
ifNoneMatch = "*"
|
|
case "update":
|
|
if space.Capability.State != domain.ObjectSyncCapabilityUpdateEnabled || !space.Policy.AllowConditionalUpdate {
|
|
return fmt.Errorf("conditional update gate is no longer enabled")
|
|
}
|
|
ifMatch = item.ExpectedBase.ETag
|
|
case "force_overwrite":
|
|
if space.Capability.State != domain.ObjectSyncCapabilityUpdateEnabled ||
|
|
!space.Capability.ExclusivePrefixConfirmed ||
|
|
!space.Policy.AllowConditionalUpdate ||
|
|
!space.Policy.AllowForceOverwrite {
|
|
return fmt.Errorf("force overwrite gate is no longer enabled")
|
|
}
|
|
current, headErr := coordinator.publisher.Head(ctx, *connection, space.Bucket, item.TargetKey)
|
|
if headErr != nil {
|
|
return headErr
|
|
}
|
|
if current.Exists {
|
|
ifMatch = current.ETag
|
|
} else {
|
|
ifNoneMatch = "*"
|
|
}
|
|
default:
|
|
return fmt.Errorf("unsupported commit action %q", item.Action)
|
|
}
|
|
metadata := map[string]string{
|
|
"drosync-session": session.ID,
|
|
"drosync-item": item.ID,
|
|
"drosync-sha256": item.SHA256,
|
|
"drosync-fence": fmt.Sprintf("%d", item.FencingToken),
|
|
}
|
|
if preserveSource {
|
|
metadata["drosync-force-source"] = sourceKey
|
|
metadata["drosync-force-actor"] = item.ForceActorID
|
|
}
|
|
targetKey := item.TargetKey
|
|
logicalTarget := item.LogicalTargetKey
|
|
published, err := coordinator.publisher.PublishConditional(ctx, *connection, space.Bucket, sourceKey, targetKey, item.SizeBytes, staging.ETag, ifMatch, ifNoneMatch, metadata)
|
|
keptBoth := false
|
|
if IsPreconditionFailure(err) && !preserveSource {
|
|
logicalTarget = item.TargetKey
|
|
targetKey = conflictKey(item.TargetKey, item.ID, time.Now())
|
|
metadata["drosync-logical-target"] = logicalTarget
|
|
published, err = coordinator.publisher.PublishConditional(ctx, *connection, space.Bucket, sourceKey, targetKey, item.SizeBytes, staging.ETag, "", "*", metadata)
|
|
keptBoth = err == nil
|
|
}
|
|
if err != nil {
|
|
return err
|
|
}
|
|
previousHead, err := coordinator.syncStore.GetObjectSyncHead(ctx, session.OrganizationID, space.ID, targetKey)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
platformVersion := int64(1)
|
|
if previousHead != nil {
|
|
platformVersion = previousHead.PlatformVersion + 1
|
|
}
|
|
now := time.Now().UTC()
|
|
head := domain.ObjectSyncObjectHeadRecord{
|
|
OrganizationID: session.OrganizationID,
|
|
SyncSpaceID: space.ID,
|
|
Key: targetKey,
|
|
PlatformVersion: platformVersion,
|
|
RemoteETag: published.ETag,
|
|
VersionID: published.VersionID,
|
|
SizeBytes: published.SizeBytes,
|
|
SHA256: item.SHA256,
|
|
SessionID: session.ID,
|
|
ItemID: item.ID,
|
|
UpdatedAt: now.Format(time.RFC3339Nano),
|
|
}
|
|
if err := coordinator.syncStore.UpsertObjectSyncHead(ctx, head); err != nil {
|
|
return err
|
|
}
|
|
if err := coordinator.syncStore.InsertObjectSyncVersion(ctx, domain.ObjectSyncVersionRecord{
|
|
ID: fmt.Sprintf("osv_%s_%d", item.ID, platformVersion),
|
|
OrganizationID: session.OrganizationID,
|
|
SyncSpaceID: space.ID,
|
|
Key: targetKey,
|
|
LogicalTargetKey: logicalTarget,
|
|
SourceKey: item.ForceSourceKey,
|
|
PlatformVersion: platformVersion,
|
|
RemoteETag: published.ETag,
|
|
VersionID: published.VersionID,
|
|
SizeBytes: published.SizeBytes,
|
|
SHA256: item.SHA256,
|
|
SessionID: session.ID,
|
|
ItemID: item.ID,
|
|
ActorID: firstNonEmpty(item.ForceActorID, session.DeviceID),
|
|
CommittedAt: now.Format(time.RFC3339Nano),
|
|
PurgeAt: now.Add(versionRetention),
|
|
}); err != nil {
|
|
return err
|
|
}
|
|
item.TargetKey = targetKey
|
|
item.LogicalTargetKey = logicalTarget
|
|
item.CommittedVersion = domain.ObjectSyncRemoteBase{
|
|
Exists: true,
|
|
ETag: published.ETag,
|
|
SizeBytes: published.SizeBytes,
|
|
VersionID: published.VersionID,
|
|
PlatformVersion: platformVersion,
|
|
}
|
|
if keptBoth {
|
|
item.Status = domain.ObjectSyncItemKeptBoth
|
|
} else {
|
|
item.Status = domain.ObjectSyncItemCommitted
|
|
}
|
|
item.CoordinatorOwner = ""
|
|
item.CoordinatorLeaseUntil = time.Time{}
|
|
item.UpdatedAt = now.Format(time.RFC3339Nano)
|
|
if err := coordinator.syncStore.UpsertObjectSyncItems(ctx, []domain.ObjectSyncItemRecord{*item}); err != nil {
|
|
return err
|
|
}
|
|
if !preserveSource {
|
|
_ = coordinator.publisher.Delete(ctx, *connection, space.Bucket, sourceKey)
|
|
}
|
|
_ = coordinator.syncStore.ReleaseObjectSyncLease(ctx, item.LeaseID, session.ID, item.ID, item.FencingToken)
|
|
if err := coordinator.recountSession(ctx, session); err != nil {
|
|
return err
|
|
}
|
|
coordinator.recordSystemAudit(ctx, domain.ObjectSyncAuditEvent{
|
|
OrganizationID: session.OrganizationID,
|
|
ActorType: "system",
|
|
ActorID: coordinator.owner,
|
|
DeviceID: session.DeviceID,
|
|
SessionID: session.ID,
|
|
ItemID: item.ID,
|
|
Action: "item." + item.Action + ".committed",
|
|
Target: map[string]any{
|
|
"key": item.TargetKey,
|
|
"sourceKey": item.ForceSourceKey,
|
|
},
|
|
After: map[string]any{
|
|
"etag": item.CommittedVersion.ETag,
|
|
"versionId": item.CommittedVersion.VersionID,
|
|
"platformVersion": item.CommittedVersion.PlatformVersion,
|
|
"sizeBytes": item.CommittedVersion.SizeBytes,
|
|
},
|
|
Result: "success",
|
|
})
|
|
return nil
|
|
}
|
|
|
|
func (coordinator *Coordinator) recountSession(ctx context.Context, session *domain.ObjectSyncSessionRecord) error {
|
|
counts := domain.ObjectSyncSessionCounts{Manifest: session.Counts.Manifest}
|
|
cursor := ""
|
|
for {
|
|
page, err := coordinator.syncStore.ListObjectSyncItems(ctx, domain.ObjectSyncListQuery{SessionID: session.ID, Cursor: cursor, Limit: 1000})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
for _, item := range page.Items {
|
|
switch item.Status {
|
|
case domain.ObjectSyncItemCommitted:
|
|
counts.Committed++
|
|
case domain.ObjectSyncItemKeptBoth:
|
|
counts.KeptBoth++
|
|
counts.Committed++
|
|
case domain.ObjectSyncItemUnchanged:
|
|
counts.Unchanged++
|
|
case domain.ObjectSyncItemSkipped:
|
|
counts.Skipped++
|
|
case domain.ObjectSyncItemDenied:
|
|
counts.Denied++
|
|
case domain.ObjectSyncItemFailed:
|
|
counts.Failed++
|
|
}
|
|
switch item.Action {
|
|
case "create":
|
|
counts.Create++
|
|
case "update":
|
|
counts.Update++
|
|
}
|
|
}
|
|
if page.NextCursor == "" {
|
|
break
|
|
}
|
|
cursor = page.NextCursor
|
|
}
|
|
fresh, err := coordinator.syncStore.GetObjectSyncSession(ctx, session.ID)
|
|
if err != nil || fresh == nil {
|
|
return err
|
|
}
|
|
fresh.Counts = counts
|
|
fresh.UpdatedAt = nowISO()
|
|
return coordinator.syncStore.UpsertObjectSyncSession(ctx, *fresh)
|
|
}
|
|
|
|
func (coordinator *Coordinator) publishRelease(ctx context.Context, release *domain.ObjectSyncReleaseRecord) error {
|
|
temp, err := os.CreateTemp("", "drosync-release-*.ndjson")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
tempName := temp.Name()
|
|
defer os.Remove(tempName)
|
|
hasher := sha256.New()
|
|
writer := bufio.NewWriter(io.MultiWriter(temp, hasher))
|
|
var objectCount int64
|
|
var totalBytes int64
|
|
var firstSession *domain.ObjectSyncSessionRecord
|
|
for _, sessionID := range release.SessionIDs {
|
|
session, err := coordinator.syncStore.GetObjectSyncSession(ctx, sessionID)
|
|
if err != nil || session == nil || session.OrganizationID != release.OrganizationID {
|
|
_ = temp.Close()
|
|
return fmt.Errorf("release session %s is unavailable", sessionID)
|
|
}
|
|
if release.SyncSpaceID == "" {
|
|
release.SyncSpaceID = session.SyncSpaceID
|
|
} else if session.SyncSpaceID != release.SyncSpaceID {
|
|
_ = temp.Close()
|
|
return fmt.Errorf("release session %s belongs to a different sync space", sessionID)
|
|
}
|
|
if firstSession == nil {
|
|
firstSession = session
|
|
}
|
|
cursor := ""
|
|
for {
|
|
page, err := coordinator.syncStore.ListObjectSyncItems(ctx, domain.ObjectSyncListQuery{SessionID: session.ID, Cursor: cursor, Limit: 1000})
|
|
if err != nil {
|
|
_ = temp.Close()
|
|
return err
|
|
}
|
|
for _, item := range page.Items {
|
|
if item.Status != domain.ObjectSyncItemCommitted && item.Status != domain.ObjectSyncItemKeptBoth && item.Status != domain.ObjectSyncItemUnchanged {
|
|
continue
|
|
}
|
|
line := map[string]any{
|
|
"sessionId": item.SessionID,
|
|
"itemId": item.ID,
|
|
"logicalTargetKey": item.LogicalTargetKey,
|
|
"key": item.TargetKey,
|
|
"sizeBytes": item.SizeBytes,
|
|
"sha256": item.SHA256,
|
|
"remote": item.CommittedVersion,
|
|
"status": item.Status,
|
|
}
|
|
payload, _ := json.Marshal(line)
|
|
if _, err := writer.Write(append(payload, '\n')); err != nil {
|
|
_ = temp.Close()
|
|
return err
|
|
}
|
|
objectCount++
|
|
totalBytes += item.SizeBytes
|
|
}
|
|
if page.NextCursor == "" {
|
|
break
|
|
}
|
|
cursor = page.NextCursor
|
|
}
|
|
}
|
|
if err := writer.Flush(); err != nil {
|
|
_ = temp.Close()
|
|
return err
|
|
}
|
|
if err := temp.Close(); err != nil {
|
|
return err
|
|
}
|
|
if firstSession == nil {
|
|
return fmt.Errorf("release has no sessions")
|
|
}
|
|
space, err := coordinator.syncStore.GetObjectSyncSpace(ctx, release.SyncSpaceID)
|
|
if err != nil || space == nil {
|
|
return fmt.Errorf("release sync space is unavailable")
|
|
}
|
|
connection, err := coordinator.store.GetStorageConnection(ctx, space.ConnectionID)
|
|
if err != nil || connection == nil {
|
|
return fmt.Errorf("release storage connection is unavailable")
|
|
}
|
|
info, err := os.Stat(tempName)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
reader, err := os.Open(tempName)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer reader.Close()
|
|
manifestKey := path.Join(".drosync-internal/releases", release.OrganizationID, release.ID+".ndjson")
|
|
digest := hex.EncodeToString(hasher.Sum(nil))
|
|
if _, err := coordinator.publisher.PutInternal(ctx, *connection, space.Bucket, manifestKey, reader, info.Size(), "application/x-ndjson", map[string]string{
|
|
"drosync-release": release.ID,
|
|
"drosync-sha256": digest,
|
|
}); err != nil {
|
|
return err
|
|
}
|
|
release.ManifestKey = manifestKey
|
|
release.ManifestSHA256 = digest
|
|
release.ObjectCount = objectCount
|
|
release.TotalBytes = totalBytes
|
|
release.Status = "published"
|
|
release.PublishedAt = nowISO()
|
|
release.CoordinatorOwner = ""
|
|
release.CoordinatorLeaseUntil = time.Time{}
|
|
if err := coordinator.syncStore.UpsertObjectSyncRelease(ctx, *release); err != nil {
|
|
return err
|
|
}
|
|
coordinator.recordSystemAudit(ctx, domain.ObjectSyncAuditEvent{
|
|
OrganizationID: release.OrganizationID,
|
|
ActorType: "system",
|
|
ActorID: coordinator.owner,
|
|
Action: "release.published",
|
|
Target: map[string]any{"releaseId": release.ID, "manifestKey": release.ManifestKey},
|
|
After: map[string]any{
|
|
"manifestSha256": release.ManifestSHA256,
|
|
"objectCount": release.ObjectCount,
|
|
"totalBytes": release.TotalBytes,
|
|
},
|
|
Result: "success",
|
|
})
|
|
return nil
|
|
}
|