forked from longtao.wu/droboticflow
reconcileSpace lists a page of up to 1000 heads, HEADs each object, then wrote the stale copy back with UpsertObjectSyncHead. A commit (new ETag, MD5 and platformVersion) or a digest attestation (md5sum and md5Attestation) that landed in that window was reverted, leaving the head drifted or legacy again. Reconcile now records its verdict with SetObjectSyncHeadDrifted. It sets only drifted and updatedAt, and only while the head still has the remoteETag, versionId, platformVersion and md5sum that reconcile read; a changed head is skipped. The Mongo filter matches missing legacy fields with $in [null, ""] like the attestation CAS, and never upserts. The scanned/drifted counters and the end-of-run space capability are unchanged. Tests run a real commit and a real digest backfill on a second coordinator replica while reconcile's HEAD is in flight and check that both receipts survive unchanged; genuinely drifted heads are still flagged. Store tests cover each compared field and the Mongo filter. The design doc no longer lists this as a follow-up. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
1139 lines
37 KiB
Go
1139 lines
37 KiB
Go
package objectsync
|
||
|
||
import (
|
||
"bufio"
|
||
"bytes"
|
||
"context"
|
||
"crypto/ed25519"
|
||
"crypto/sha256"
|
||
"encoding/base64"
|
||
"encoding/hex"
|
||
"encoding/json"
|
||
"errors"
|
||
"fmt"
|
||
"io"
|
||
"os"
|
||
"path"
|
||
"strings"
|
||
"sync"
|
||
"sync/atomic"
|
||
"time"
|
||
|
||
"droboticflow/internal/domain"
|
||
"droboticflow/internal/store"
|
||
|
||
"golang.org/x/sync/errgroup"
|
||
)
|
||
|
||
const (
|
||
claimTTL = 10 * time.Minute
|
||
versionRetention = 365 * 24 * time.Hour
|
||
stagingRetention = 24 * time.Hour
|
||
)
|
||
|
||
type Coordinator struct {
|
||
store store.Store
|
||
syncStore store.ObjectSyncStore
|
||
publisher Publisher
|
||
owner string
|
||
signingKey ed25519.PrivateKey
|
||
signingID string
|
||
cycles atomic.Int64
|
||
errors atomic.Int64
|
||
planned atomic.Int64
|
||
committed atomic.Int64
|
||
cleaned atomic.Int64
|
||
reconciled atomic.Int64
|
||
released atomic.Int64
|
||
sealed atomic.Int64
|
||
backfilled atomic.Int64
|
||
lastCycle atomic.Int64
|
||
lastOK atomic.Int64
|
||
durationMS atomic.Int64
|
||
phases [phaseCount]phaseCounters
|
||
renewInterval time.Duration
|
||
queueSnapshot atomic.Pointer[queueSnapshot]
|
||
}
|
||
|
||
func NewCoordinator(base store.Store, publisher Publisher, owner string) (*Coordinator, error) {
|
||
syncStore, ok := store.AsObjectSyncStore(base)
|
||
if !ok {
|
||
return nil, fmt.Errorf("store does not implement object sync persistence")
|
||
}
|
||
if publisher == nil {
|
||
return nil, fmt.Errorf("object publisher is required")
|
||
}
|
||
if strings.TrimSpace(owner) == "" {
|
||
owner = fmt.Sprintf("coordinator-%d", time.Now().UnixNano())
|
||
}
|
||
return &Coordinator{store: base, syncStore: syncStore, publisher: publisher, owner: owner}, nil
|
||
}
|
||
|
||
func (coordinator *Coordinator) ConfigureAuditSigning(encodedPrivateKey, keyID string) error {
|
||
encodedPrivateKey = strings.TrimSpace(encodedPrivateKey)
|
||
if encodedPrivateKey == "" {
|
||
coordinator.signingKey = nil
|
||
coordinator.signingID = ""
|
||
return nil
|
||
}
|
||
raw, err := base64.StdEncoding.DecodeString(encodedPrivateKey)
|
||
if err != nil {
|
||
raw, err = base64.RawStdEncoding.DecodeString(encodedPrivateKey)
|
||
}
|
||
if err != nil {
|
||
return fmt.Errorf("decode audit signing key: %w", err)
|
||
}
|
||
switch len(raw) {
|
||
case ed25519.SeedSize:
|
||
coordinator.signingKey = ed25519.NewKeyFromSeed(raw)
|
||
case ed25519.PrivateKeySize:
|
||
coordinator.signingKey = ed25519.PrivateKey(append([]byte(nil), raw...))
|
||
default:
|
||
return fmt.Errorf("audit signing key must be a %d-byte Ed25519 seed or %d-byte private key", ed25519.SeedSize, ed25519.PrivateKeySize)
|
||
}
|
||
coordinator.signingID = strings.TrimSpace(keyID)
|
||
if coordinator.signingID == "" {
|
||
return fmt.Errorf("audit signing key id is required")
|
||
}
|
||
return nil
|
||
}
|
||
|
||
func (coordinator *Coordinator) AuditSigningConfigured() bool {
|
||
return len(coordinator.signingKey) == ed25519.PrivateKeySize && coordinator.signingID != ""
|
||
}
|
||
|
||
// Each phase owns its own loop: slow planning, cleanup or audit cannot stop
|
||
// subsequent commit batches, and each phase has bounded internal parallelism.
|
||
func (coordinator *Coordinator) Run(ctx context.Context, interval time.Duration) {
|
||
if interval <= 0 {
|
||
interval = 2 * time.Second
|
||
}
|
||
var group sync.WaitGroup
|
||
group.Add(1)
|
||
go func() { defer group.Done(); coordinator.runQueueSampler(ctx) }()
|
||
for phase := range phaseCount {
|
||
workers := 1
|
||
if phase == phaseCommit {
|
||
workers = commitConcurrency
|
||
}
|
||
if phase == phaseCleanup {
|
||
workers = cleanupConcurrency
|
||
}
|
||
for range workers {
|
||
group.Add(1)
|
||
go func() {
|
||
defer group.Done()
|
||
for ctx.Err() == nil {
|
||
if phase == phaseCommit || phase == phaseCleanup {
|
||
result, err := coordinator.executeSingleItem(ctx, phase)
|
||
if err == nil && result.Committed+result.Cleaned > 0 {
|
||
continue
|
||
}
|
||
} else {
|
||
_, _ = coordinator.executeStage(ctx, phase)
|
||
}
|
||
timer := time.NewTimer(interval)
|
||
select {
|
||
case <-ctx.Done():
|
||
timer.Stop()
|
||
return
|
||
case <-timer.C:
|
||
}
|
||
}
|
||
}()
|
||
}
|
||
}
|
||
group.Wait()
|
||
}
|
||
|
||
type Metrics struct {
|
||
QueueStats domain.ObjectSyncQueueStats
|
||
QueueSampleAt int64
|
||
Phases map[string]PhaseMetrics
|
||
Cycles int64
|
||
Errors int64
|
||
Planned int64
|
||
Committed int64
|
||
Cleaned int64
|
||
Reconciled int64
|
||
Released int64
|
||
Sealed int64
|
||
Backfilled int64
|
||
LastCycle int64
|
||
LastOK int64
|
||
DurationMS int64
|
||
}
|
||
|
||
func (coordinator *Coordinator) Metrics() Metrics {
|
||
phases := make(map[string]PhaseMetrics, phaseCount)
|
||
for index, name := range phaseNames {
|
||
item := &coordinator.phases[index]
|
||
phases[name] = PhaseMetrics{Cycles: item.cycles.Load(), Errors: item.errors.Load(), Completed: item.completed.Load(), ItemErrors: item.itemErrors.Load(), InFlight: item.inFlight.Load(), DurationMS: item.durationMS.Load(), TotalDurationMS: item.totalDurationMS.Load()}
|
||
}
|
||
queue := coordinator.queueSnapshot.Load()
|
||
if queue == nil {
|
||
queue = &queueSnapshot{}
|
||
}
|
||
return Metrics{
|
||
QueueStats: queue.stats,
|
||
QueueSampleAt: queue.sampleAt,
|
||
Phases: phases,
|
||
Cycles: coordinator.cycles.Load(),
|
||
Errors: coordinator.errors.Load(),
|
||
Planned: coordinator.planned.Load(),
|
||
Committed: coordinator.committed.Load(),
|
||
Cleaned: coordinator.cleaned.Load(),
|
||
Reconciled: coordinator.reconciled.Load(),
|
||
Released: coordinator.released.Load(),
|
||
Sealed: coordinator.sealed.Load(),
|
||
Backfilled: coordinator.backfilled.Load(),
|
||
LastCycle: coordinator.lastCycle.Load(),
|
||
LastOK: coordinator.lastOK.Load(),
|
||
DurationMS: coordinator.durationMS.Load(),
|
||
}
|
||
}
|
||
|
||
func (coordinator *Coordinator) recordSystemAudit(ctx context.Context, event domain.ObjectSyncAuditEvent) {
|
||
if err := AppendAudit(ctx, coordinator.syncStore, event); err != nil {
|
||
coordinator.errors.Add(1)
|
||
fmt.Printf("[drosync-coordinator] audit persistence error for %s: %v\n", event.Action, err)
|
||
}
|
||
}
|
||
|
||
type ProcessResult struct {
|
||
Planned int `json:"planned"`
|
||
Committed int `json:"committed"`
|
||
Cleaned int `json:"cleaned"`
|
||
Reconciled int `json:"reconciled"`
|
||
Released int `json:"released"`
|
||
Sealed int `json:"sealed"`
|
||
Backfilled int `json:"backfilled"`
|
||
}
|
||
|
||
func (coordinator *Coordinator) ProcessOnce(ctx context.Context) (ProcessResult, error) {
|
||
type outcome struct {
|
||
result ProcessResult
|
||
err error
|
||
}
|
||
results := make(chan outcome, phaseCount)
|
||
for phase := range phaseCount {
|
||
go func() { value, err := coordinator.executeStage(ctx, phase); results <- outcome{value, err} }()
|
||
}
|
||
result := ProcessResult{}
|
||
var failures []error
|
||
for range phaseCount {
|
||
value := <-results
|
||
result.Planned += value.result.Planned
|
||
result.Committed += value.result.Committed
|
||
result.Cleaned += value.result.Cleaned
|
||
result.Reconciled += value.result.Reconciled
|
||
result.Released += value.result.Released
|
||
result.Sealed += value.result.Sealed
|
||
result.Backfilled += value.result.Backfilled
|
||
if value.err != nil {
|
||
failures = append(failures, value.err)
|
||
}
|
||
}
|
||
return result, errors.Join(failures...)
|
||
}
|
||
|
||
type auditSealSigningPayload struct {
|
||
OrganizationID string `json:"organizationId"`
|
||
DayUTC string `json:"dayUtc"`
|
||
EventCount int64 `json:"eventCount"`
|
||
FirstEventHash string `json:"firstEventHash,omitempty"`
|
||
LastEventHash string `json:"lastEventHash,omitempty"`
|
||
AggregateHash string `json:"aggregateHash"`
|
||
SigningKeyID string `json:"signingKeyId"`
|
||
}
|
||
|
||
type auditSealDocument struct {
|
||
auditSealSigningPayload
|
||
Signature string `json:"signature"`
|
||
}
|
||
|
||
func validateAuditEventHash(event domain.ObjectSyncAuditEvent) error {
|
||
unsigned := event
|
||
unsigned.EventHash = ""
|
||
payload, err := json.Marshal(unsigned)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
sum := sha256.Sum256(append([]byte(event.PreviousHash), payload...))
|
||
if !strings.EqualFold(event.EventHash, hex.EncodeToString(sum[:])) {
|
||
return fmt.Errorf("audit event %s hash mismatch", event.ID)
|
||
}
|
||
return nil
|
||
}
|
||
|
||
func (coordinator *Coordinator) sealPreviousAuditDay(ctx context.Context, now time.Time) (int, error) {
|
||
dayStart := now.UTC().Truncate(24 * time.Hour).Add(-24 * time.Hour)
|
||
dayEnd := dayStart.Add(24 * time.Hour)
|
||
dayUTC := dayStart.Format("2006-01-02")
|
||
organizations, err := coordinator.store.ListOrganizations(ctx)
|
||
if err != nil {
|
||
return 0, err
|
||
}
|
||
sealed := 0
|
||
for _, organization := range organizations {
|
||
existing, err := coordinator.syncStore.GetObjectSyncAuditSeal(ctx, organization.ID, dayUTC)
|
||
if err != nil {
|
||
return sealed, err
|
||
}
|
||
if existing != nil {
|
||
continue
|
||
}
|
||
ok, err := coordinator.sealAuditDay(ctx, organization.ID, dayUTC, dayStart, dayEnd)
|
||
if err != nil {
|
||
return sealed, err
|
||
}
|
||
if ok {
|
||
sealed++
|
||
}
|
||
}
|
||
return sealed, nil
|
||
}
|
||
|
||
func (coordinator *Coordinator) sealAuditDay(ctx context.Context, organizationID, dayUTC string, dayStart, dayEnd time.Time) (bool, error) {
|
||
resourceSum := sha256.Sum256([]byte("audit-seal\x00" + organizationID + "\x00" + dayUTC))
|
||
resourceHash := hex.EncodeToString(resourceSum[:])
|
||
leaseOwner := coordinator.owner + ":" + dayUTC
|
||
lease, acquired, err := coordinator.syncStore.AcquireObjectSyncLease(ctx, domain.ObjectSyncLeaseRecord{
|
||
ID: "osl_seal_" + resourceHash,
|
||
OrganizationID: organizationID,
|
||
ResourceHash: resourceHash,
|
||
ResourceType: "audit_seal",
|
||
KeyOrPrefix: dayUTC,
|
||
OwnerSessionID: leaseOwner,
|
||
OwnerItemID: resourceHash,
|
||
}, claimTTL)
|
||
if err != nil {
|
||
return false, err
|
||
}
|
||
if !acquired {
|
||
return false, nil
|
||
}
|
||
defer func() {
|
||
if err := coordinator.syncStore.ReleaseObjectSyncLease(context.Background(), resourceHash, leaseOwner, resourceHash, lease.FencingToken); err != nil {
|
||
coordinator.recordAuxiliaryFailure(phaseAudit, "release audit lease", err)
|
||
}
|
||
}()
|
||
existing, err := coordinator.syncStore.GetObjectSyncAuditSeal(ctx, organizationID, dayUTC)
|
||
if err != nil || existing != nil {
|
||
return false, err
|
||
}
|
||
|
||
hasher := sha256.New()
|
||
var count int64
|
||
var firstHash string
|
||
var lastHash string
|
||
cursor := ""
|
||
for {
|
||
page, err := coordinator.syncStore.ListObjectSyncAuditRange(
|
||
ctx,
|
||
organizationID,
|
||
dayStart.Format(time.RFC3339Nano),
|
||
dayEnd.Format(time.RFC3339Nano),
|
||
domain.ObjectSyncListQuery{Cursor: cursor, Limit: 1000},
|
||
)
|
||
if err != nil {
|
||
return false, err
|
||
}
|
||
for _, event := range page.Items {
|
||
if err := validateAuditEventHash(event); err != nil {
|
||
return false, err
|
||
}
|
||
if count > 0 && event.PreviousHash != lastHash {
|
||
return false, fmt.Errorf("audit chain discontinuity before event %s", event.ID)
|
||
}
|
||
if count == 0 {
|
||
firstHash = event.EventHash
|
||
}
|
||
lastHash = event.EventHash
|
||
_, _ = io.WriteString(hasher, event.EventHash+"\n")
|
||
count++
|
||
}
|
||
if page.NextCursor == "" {
|
||
break
|
||
}
|
||
cursor = page.NextCursor
|
||
}
|
||
aggregateHash := hex.EncodeToString(hasher.Sum(nil))
|
||
signingPayload := auditSealSigningPayload{
|
||
OrganizationID: organizationID,
|
||
DayUTC: dayUTC,
|
||
EventCount: count,
|
||
FirstEventHash: firstHash,
|
||
LastEventHash: lastHash,
|
||
AggregateHash: aggregateHash,
|
||
SigningKeyID: coordinator.signingID,
|
||
}
|
||
signingBytes, err := json.Marshal(signingPayload)
|
||
if err != nil {
|
||
return false, err
|
||
}
|
||
signature := base64.StdEncoding.EncodeToString(ed25519.Sign(coordinator.signingKey, signingBytes))
|
||
documentBytes, err := json.MarshalIndent(auditSealDocument{
|
||
auditSealSigningPayload: signingPayload,
|
||
Signature: signature,
|
||
}, "", " ")
|
||
if err != nil {
|
||
return false, err
|
||
}
|
||
documentBytes = append(documentBytes, '\n')
|
||
|
||
spaces, err := coordinator.syncStore.ListObjectSyncSpaces(ctx, organizationID)
|
||
if err != nil {
|
||
return false, err
|
||
}
|
||
if len(spaces) == 0 {
|
||
if count == 0 {
|
||
return false, nil
|
||
}
|
||
return false, fmt.Errorf("organization %s has audit events but no sync space for audit sealing", organizationID)
|
||
}
|
||
space := spaces[0]
|
||
connection, err := coordinator.store.GetStorageConnection(ctx, space.ConnectionID)
|
||
if err != nil || connection == nil {
|
||
return false, fmt.Errorf("audit seal storage connection is unavailable")
|
||
}
|
||
objectKey := connection.InternalStorageKey(path.Join(".drosync-internal/audit", organizationID, dayUTC+".json"))
|
||
object, err := coordinator.publisher.PutInternal(
|
||
ctx,
|
||
*connection,
|
||
space.Bucket,
|
||
objectKey,
|
||
bytes.NewReader(documentBytes),
|
||
int64(len(documentBytes)),
|
||
"application/json",
|
||
map[string]string{
|
||
"drosync-audit-day": dayUTC,
|
||
"drosync-audit-hash": aggregateHash,
|
||
"drosync-signing-key": coordinator.signingID,
|
||
},
|
||
)
|
||
if IsPreconditionFailure(err) {
|
||
object, err = coordinator.publisher.Head(ctx, *connection, space.Bucket, objectKey)
|
||
if err == nil && (!object.Exists || object.SizeBytes != int64(len(documentBytes))) {
|
||
err = fmt.Errorf("existing audit seal object does not match the expected size")
|
||
}
|
||
}
|
||
if err != nil {
|
||
return false, err
|
||
}
|
||
recordIDHash := sha256.Sum256([]byte(organizationID + "\x00" + dayUTC))
|
||
record := domain.ObjectSyncAuditSealRecord{
|
||
ID: "osas_" + hex.EncodeToString(recordIDHash[:12]),
|
||
OrganizationID: organizationID,
|
||
DayUTC: dayUTC,
|
||
EventCount: count,
|
||
FirstEventHash: firstHash,
|
||
LastEventHash: lastHash,
|
||
AggregateHash: aggregateHash,
|
||
Signature: signature,
|
||
SigningKeyID: coordinator.signingID,
|
||
ObjectKey: objectKey,
|
||
ObjectETag: object.ETag,
|
||
ObjectVersionID: object.VersionID,
|
||
CreatedAt: nowISO(),
|
||
PurgeAt: time.Now().UTC().Add(versionRetention),
|
||
}
|
||
if err := coordinator.syncStore.InsertObjectSyncAuditSeal(ctx, record); err != nil {
|
||
return false, err
|
||
}
|
||
return true, nil
|
||
}
|
||
|
||
func (coordinator *Coordinator) reconcileSpace(ctx context.Context, reconcile *domain.ObjectSyncReconcileRecord) error {
|
||
token, id := reconcile.CoordinatorOwner, reconcile.ID
|
||
if token == "" {
|
||
return errClaimLost
|
||
}
|
||
requireClaim := func() error {
|
||
return requireOwnedClaim(ctx, func(callCtx context.Context) (bool, error) {
|
||
return coordinator.syncStore.RenewObjectSyncReconcileClaim(callCtx, id, token, claimTTL)
|
||
})
|
||
}
|
||
space, err := coordinator.syncStore.GetObjectSyncSpace(ctx, reconcile.SyncSpaceID)
|
||
if err != nil || space == nil || space.OrganizationID != reconcile.OrganizationID {
|
||
return fmt.Errorf("sync space is unavailable")
|
||
}
|
||
connection, err := coordinator.store.GetStorageConnection(ctx, space.ConnectionID)
|
||
if err != nil || connection == nil {
|
||
return fmt.Errorf("storage connection is unavailable")
|
||
}
|
||
var scanned int64
|
||
var drifted int64
|
||
cursor := ""
|
||
for {
|
||
page, err := coordinator.syncStore.ListObjectSyncHeads(ctx, reconcile.OrganizationID, reconcile.SyncSpaceID, cursor, 1000)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
for _, head := range page.Items {
|
||
remote, err := coordinator.publisher.Head(ctx, *connection, space.Bucket, head.Key)
|
||
if err != nil {
|
||
return fmt.Errorf("head %s: %w", head.Key, err)
|
||
}
|
||
if err := requireClaim(); err != nil {
|
||
return err
|
||
}
|
||
headDrifted := !remote.Exists || strings.Trim(remote.ETag, `"`) != strings.Trim(head.RemoteETag, `"`)
|
||
if headDrifted {
|
||
drifted++
|
||
}
|
||
scanned++
|
||
// The page was read before the slow HEADs above. A commit or digest
|
||
// attestation that replaced this receipt since then is not what the
|
||
// verdict was computed against, so the store skips it.
|
||
if _, err := coordinator.syncStore.SetObjectSyncHeadDrifted(ctx, head, headDrifted, nowISO()); err != nil {
|
||
return err
|
||
}
|
||
}
|
||
reconcile.Scanned = scanned
|
||
reconcile.Drifted = drifted
|
||
reconcile.UpdatedAt = nowISO()
|
||
if err := requireClaim(); err != nil {
|
||
return err
|
||
}
|
||
reconcile.CoordinatorLeaseUntil = time.Now().UTC().Add(claimTTL)
|
||
saved, err := coordinator.syncStore.SaveClaimedObjectSyncReconcile(ctx, *reconcile, token)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
if !saved {
|
||
return errClaimLost
|
||
}
|
||
if page.NextCursor == "" {
|
||
break
|
||
}
|
||
cursor = page.NextCursor
|
||
}
|
||
if err := requireClaim(); err != nil {
|
||
return err
|
||
}
|
||
space.Enabled = false
|
||
space.Mode = "blocked"
|
||
if drifted > 0 {
|
||
space.Capability.State = domain.ObjectSyncCapabilityDrifted
|
||
} else {
|
||
switch {
|
||
case space.Capability.ConditionalCreate &&
|
||
space.Capability.ConditionalUpdate &&
|
||
space.Capability.ConditionalMultipart &&
|
||
space.Capability.ExclusivePrefixConfirmed:
|
||
space.Capability.State = domain.ObjectSyncCapabilityUpdateEnabled
|
||
case space.Capability.ConditionalCreate:
|
||
space.Capability.State = domain.ObjectSyncCapabilityCreateOnly
|
||
default:
|
||
space.Capability.State = domain.ObjectSyncCapabilityUnsupported
|
||
}
|
||
}
|
||
space.Policy.AllowConditionalUpdate = space.Capability.State == domain.ObjectSyncCapabilityUpdateEnabled
|
||
if space.Capability.State != domain.ObjectSyncCapabilityUpdateEnabled {
|
||
space.Policy.AllowForceOverwrite = false
|
||
}
|
||
space.UpdatedAt = nowISO()
|
||
if err := coordinator.syncStore.UpsertObjectSyncSpace(ctx, *space); err != nil {
|
||
return err
|
||
}
|
||
reconcile.Status = "completed"
|
||
reconcile.CompletedAt = nowISO()
|
||
reconcile.UpdatedAt = reconcile.CompletedAt
|
||
reconcile.CoordinatorOwner = ""
|
||
reconcile.CoordinatorLeaseUntil = time.Time{}
|
||
saved, err := coordinator.syncStore.SaveClaimedObjectSyncReconcile(ctx, *reconcile, token)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
if !saved {
|
||
return errClaimLost
|
||
}
|
||
coordinator.recordSystemAudit(ctx, domain.ObjectSyncAuditEvent{
|
||
OrganizationID: reconcile.OrganizationID,
|
||
ActorType: "system",
|
||
ActorID: coordinator.owner,
|
||
Action: "space.reconcile.completed",
|
||
Target: map[string]any{
|
||
"reconcileId": reconcile.ID,
|
||
"spaceId": reconcile.SyncSpaceID,
|
||
},
|
||
After: map[string]any{"scanned": scanned, "drifted": drifted},
|
||
Result: "success",
|
||
})
|
||
return nil
|
||
}
|
||
|
||
func (coordinator *Coordinator) cleanupStaging(ctx context.Context, item *domain.ObjectSyncItemRecord) error {
|
||
claimOwner := item.CoordinatorOwner
|
||
if claimOwner == "" {
|
||
return errClaimLost
|
||
}
|
||
session, err := coordinator.syncStore.GetObjectSyncSession(ctx, item.SessionID)
|
||
if err != nil || session == nil {
|
||
return fmt.Errorf("session is unavailable")
|
||
}
|
||
space, err := coordinator.syncStore.GetObjectSyncSpace(ctx, session.SyncSpaceID)
|
||
if err != nil || space == nil {
|
||
return fmt.Errorf("sync space is unavailable")
|
||
}
|
||
connection, err := coordinator.store.GetStorageConnection(ctx, space.ConnectionID)
|
||
if err != nil || connection == nil {
|
||
return fmt.Errorf("storage connection is unavailable")
|
||
}
|
||
if item.UploadID != "" {
|
||
if err := coordinator.publisher.AbortMultipart(ctx, *connection, space.Bucket, item.StagingKey, item.UploadID); err != nil && !isNotFound(err) {
|
||
return fmt.Errorf("abort staging multipart: %w", err)
|
||
}
|
||
}
|
||
if item.StagingKey != item.TargetKey {
|
||
if err := coordinator.publisher.Delete(ctx, *connection, space.Bucket, item.StagingKey); err != nil {
|
||
return err
|
||
}
|
||
}
|
||
if item.Status == domain.ObjectSyncItemUploading || item.Status == domain.ObjectSyncItemUploaded {
|
||
item.Status = domain.ObjectSyncItemExpired
|
||
}
|
||
item.StagingKey = ""
|
||
item.UploadID = ""
|
||
item.UploadedParts = nil
|
||
item.CoordinatorOwner = ""
|
||
item.CoordinatorLeaseUntil = time.Time{}
|
||
item.UpdatedAt = nowISO()
|
||
if err := coordinator.saveClaimedItem(ctx, *item, claimOwner); err != nil {
|
||
return err
|
||
}
|
||
if item.LeaseID != "" {
|
||
if err := coordinator.syncStore.ReleaseObjectSyncLease(ctx, item.LeaseID, item.SessionID, item.ID, item.FencingToken); err != nil {
|
||
coordinator.recordAuxiliaryFailure(phaseCleanup, "release cleaned lease", err)
|
||
}
|
||
}
|
||
return nil
|
||
}
|
||
|
||
func nowISO() string {
|
||
return time.Now().UTC().Format(time.RFC3339Nano)
|
||
}
|
||
|
||
func firstNonEmpty(values ...string) string {
|
||
for _, value := range values {
|
||
if strings.TrimSpace(value) != "" {
|
||
return value
|
||
}
|
||
}
|
||
return ""
|
||
}
|
||
|
||
func (coordinator *Coordinator) planSession(ctx context.Context, session *domain.ObjectSyncSessionRecord) error {
|
||
token := session.CoordinatorOwner
|
||
if token == "" {
|
||
return errClaimLost
|
||
}
|
||
space, err := coordinator.syncStore.GetObjectSyncSpace(ctx, session.SyncSpaceID)
|
||
if err != nil || space == nil {
|
||
return fmt.Errorf("sync space is unavailable")
|
||
}
|
||
connection, err := coordinator.store.GetStorageConnection(ctx, space.ConnectionID)
|
||
if err != nil || connection == nil {
|
||
return fmt.Errorf("storage connection is unavailable")
|
||
}
|
||
manifestCount := session.Counts.Manifest
|
||
cursor := ""
|
||
for {
|
||
page, err := coordinator.syncStore.ListObjectSyncItems(ctx, domain.ObjectSyncListQuery{SessionID: session.ID, Cursor: cursor, Limit: 1000})
|
||
if err != nil {
|
||
return err
|
||
}
|
||
type observed struct {
|
||
remote ObjectMetadata
|
||
head *domain.ObjectSyncObjectHeadRecord
|
||
}
|
||
facts := make([]observed, len(page.Items))
|
||
group, queryCtx := errgroup.WithContext(ctx)
|
||
group.SetLimit(planConcurrency)
|
||
for index, item := range page.Items {
|
||
if item.Status != domain.ObjectSyncItemDiscovered {
|
||
continue
|
||
}
|
||
group.Go(func() error {
|
||
head, err := coordinator.syncStore.GetObjectSyncHead(queryCtx, session.OrganizationID, space.ID, item.TargetKey)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
var remote ObjectMetadata
|
||
// Formal publication receipts are authoritative for upload-once planning.
|
||
// Only new paths need a collision check against object storage.
|
||
if head == nil {
|
||
remote, err = coordinator.publisher.Head(queryCtx, *connection, space.Bucket, item.TargetKey)
|
||
if err != nil {
|
||
return fmt.Errorf("head %s: %w", item.TargetKey, err)
|
||
}
|
||
}
|
||
facts[index] = observed{remote, head}
|
||
return nil
|
||
})
|
||
}
|
||
if err := group.Wait(); err != nil {
|
||
return err
|
||
}
|
||
updates := make([]domain.ObjectSyncItemRecord, 0, len(page.Items))
|
||
for index, item := range page.Items {
|
||
if item.Status != domain.ObjectSyncItemDiscovered {
|
||
continue
|
||
}
|
||
remote, head := facts[index].remote, facts[index].head
|
||
item.ExpectedBase = domain.ObjectSyncRemoteBase{Exists: remote.Exists, ETag: remote.ETag, SizeBytes: remote.SizeBytes, LastModified: remote.LastModified, VersionID: remote.VersionID}
|
||
switch {
|
||
case head != nil && len(head.MD5Sum) != 32:
|
||
item.Action = "deny_archived_hash_change"
|
||
item.Status = domain.ObjectSyncItemDenied
|
||
item.LastError = "已归档对象缺少 MD5 回执,暂停上传;旧 SHA256 回执退出内容比较和清理支持,保留云端原文件"
|
||
case head != nil && head.MD5Sum == item.MD5Sum && head.SizeBytes == item.SizeBytes:
|
||
item.Action = "unchanged"
|
||
item.Status = domain.ObjectSyncItemUnchanged
|
||
item.ExpectedBase = domain.ObjectSyncRemoteBase{Exists: true, ETag: head.RemoteETag, SizeBytes: head.SizeBytes, VersionID: head.VersionID, PlatformVersion: head.PlatformVersion}
|
||
case head != nil:
|
||
item.Action = "deny_archived_hash_change"
|
||
item.Status = domain.ObjectSyncItemDenied
|
||
item.LastError = fmt.Sprintf("同名文件内容冲突,暂停上传:%s;云端已提交 MD5Sum=%s,本地 MD5Sum=%s;保留云端原文件", item.TargetKey, head.MD5Sum, item.MD5Sum)
|
||
case remote.Exists:
|
||
item.Action = "deny_existing_untracked"
|
||
item.Status = domain.ObjectSyncItemDenied
|
||
item.LastError = "目标路径已有对象但缺少可信提交记录,暂停上传;保留云端原文件"
|
||
default:
|
||
item.Action = "create"
|
||
item.Status = domain.ObjectSyncItemPlanned
|
||
}
|
||
item.UpdatedAt = nowISO()
|
||
updates = append(updates, item)
|
||
}
|
||
owned, err := coordinator.syncStore.RenewObjectSyncSessionClaim(ctx, session.ID, token, claimTTL)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
if !owned {
|
||
return errClaimLost
|
||
}
|
||
if err := ctx.Err(); err != nil {
|
||
return err
|
||
}
|
||
if err := coordinator.syncStore.UpsertObjectSyncItems(ctx, updates); err != nil {
|
||
return err
|
||
}
|
||
if page.NextCursor == "" {
|
||
break
|
||
}
|
||
cursor = page.NextCursor
|
||
}
|
||
summaries, err := coordinator.syncStore.SummarizeObjectSyncItems(ctx, []string{session.ID})
|
||
if err != nil {
|
||
return err
|
||
}
|
||
session.Counts = summaries[session.ID].Counts
|
||
session.Counts.Manifest = manifestCount
|
||
session.Status = domain.ObjectSyncSessionPlanned
|
||
session.UpdatedAt = nowISO()
|
||
session.CoordinatorOwner = ""
|
||
session.CoordinatorLeaseUntil = time.Time{}
|
||
saved, err := coordinator.syncStore.SaveClaimedObjectSyncSession(ctx, *session, token)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
if !saved {
|
||
return errClaimLost
|
||
}
|
||
return nil
|
||
}
|
||
|
||
func conflictKey(original, itemID string, now time.Time) string {
|
||
shortID := strings.TrimPrefix(itemID, "osi_")
|
||
if len(shortID) > 10 {
|
||
shortID = shortID[:10]
|
||
}
|
||
return fmt.Sprintf("%s.__conflict__%s_%s", original, now.UTC().Format("20060102T150405Z"), shortID)
|
||
}
|
||
|
||
func (coordinator *Coordinator) commitItem(ctx context.Context, item *domain.ObjectSyncItemRecord) error {
|
||
return coordinator.commitItemWithObservation(ctx, item, nil)
|
||
}
|
||
|
||
func (coordinator *Coordinator) commitItemWithObservation(ctx context.Context, item *domain.ObjectSyncItemRecord, observed *ObjectMetadata) error {
|
||
claimOwner := item.CoordinatorOwner
|
||
if claimOwner == "" {
|
||
return errClaimLost
|
||
}
|
||
session, err := coordinator.syncStore.GetObjectSyncSession(ctx, item.SessionID)
|
||
if err != nil || session == nil {
|
||
return fmt.Errorf("session is unavailable")
|
||
}
|
||
space, err := coordinator.syncStore.GetObjectSyncSpace(ctx, session.SyncSpaceID)
|
||
if err != nil || space == nil {
|
||
return fmt.Errorf("sync space is unavailable")
|
||
}
|
||
connection, err := coordinator.store.GetStorageConnection(ctx, space.ConnectionID)
|
||
if err != nil || connection == nil {
|
||
return fmt.Errorf("storage connection is unavailable")
|
||
}
|
||
renewed, err := coordinator.syncStore.RenewObjectSyncLease(ctx, item.LeaseID, session.ID, item.ID, item.FencingToken, claimTTL)
|
||
if err != nil || !renewed {
|
||
return fmt.Errorf("lease is stale; publish rejected by fencing token")
|
||
}
|
||
sourceKey := item.StagingKey
|
||
preserveSource := false
|
||
if item.Action == "force_overwrite" {
|
||
sourceKey = item.ForceSourceKey
|
||
preserveSource = true
|
||
}
|
||
metadata := map[string]string{
|
||
"drosync-session": session.ID,
|
||
"drosync-item": item.ID,
|
||
"drosync-md5sum": item.MD5Sum,
|
||
"drosync-fence": fmt.Sprintf("%d", item.FencingToken),
|
||
}
|
||
if preserveSource {
|
||
metadata["drosync-force-source"] = sourceKey
|
||
metadata["drosync-force-actor"] = item.ForceActorID
|
||
}
|
||
targetKey := item.TargetKey
|
||
logicalTarget := item.LogicalTargetKey
|
||
keptBoth := !preserveSource && logicalTarget != "" && logicalTarget != targetKey
|
||
if keptBoth {
|
||
metadata["drosync-logical-target"] = logicalTarget
|
||
}
|
||
// A previous worker may have published and then lost its response or claim.
|
||
// This bounded HEAD avoids copying those bytes again; existence alone is not
|
||
// a receipt. Every operation marker, content hash and size must agree.
|
||
var current ObjectMetadata
|
||
if observed != nil {
|
||
if err := connection.CheckStoragePath(space.Bucket, targetKey, false); err != nil {
|
||
return err
|
||
}
|
||
if preserveSource || sourceKey != targetKey || item.Action != "create" {
|
||
return fmt.Errorf("observation cannot authorize a copy or overwrite")
|
||
}
|
||
current = *observed
|
||
} else {
|
||
current, err = coordinator.publisher.Head(ctx, *connection, space.Bucket, targetKey)
|
||
}
|
||
if err != nil {
|
||
return fmt.Errorf("verify publication target: %w", err)
|
||
}
|
||
ifMatch, ifNoneMatch := "", ""
|
||
switch item.Action {
|
||
case "create":
|
||
ifNoneMatch = "*"
|
||
case "update":
|
||
if space.Capability.State != domain.ObjectSyncCapabilityUpdateEnabled || !space.Policy.AllowConditionalUpdate {
|
||
return fmt.Errorf("conditional update gate is no longer enabled")
|
||
}
|
||
ifMatch = item.ExpectedBase.ETag
|
||
case "force_overwrite":
|
||
if space.Capability.State != domain.ObjectSyncCapabilityUpdateEnabled ||
|
||
!space.Capability.ExclusivePrefixConfirmed ||
|
||
!space.Policy.AllowConditionalUpdate ||
|
||
!space.Policy.AllowForceOverwrite {
|
||
return fmt.Errorf("force overwrite gate is no longer enabled")
|
||
}
|
||
if current.Exists {
|
||
ifMatch = current.ETag
|
||
} else {
|
||
ifNoneMatch = "*"
|
||
}
|
||
default:
|
||
return fmt.Errorf("unsupported commit action %q", item.Action)
|
||
}
|
||
if keptBoth {
|
||
ifMatch, ifNoneMatch = "", "*"
|
||
}
|
||
directUpload := !preserveSource && sourceKey == targetKey
|
||
if directUpload && !publicationMatches(current, item.SizeBytes, metadata) {
|
||
return fmt.Errorf("direct upload target is missing or its bound identity/hash/size does not match")
|
||
}
|
||
published := current
|
||
if !publicationMatches(current, item.SizeBytes, metadata) {
|
||
staging, headErr := coordinator.publisher.Head(ctx, *connection, space.Bucket, sourceKey)
|
||
if headErr != nil {
|
||
return headErr
|
||
}
|
||
if !staging.Exists || staging.SizeBytes != item.SizeBytes {
|
||
return fmt.Errorf("staging object is missing or size changed")
|
||
}
|
||
var confirmedConflict bool
|
||
published, err, confirmedConflict = coordinator.publishAndConfirm(ctx, *connection, space.Bucket, sourceKey, targetKey, item.SizeBytes, staging.ETag, ifMatch, ifNoneMatch, metadata)
|
||
if confirmedConflict && !preserveSource && !keptBoth {
|
||
logicalTarget = targetKey
|
||
targetKey = conflictKey(targetKey, item.ID, time.Now())
|
||
metadata["drosync-logical-target"] = logicalTarget
|
||
// Persist the sole conflict destination before any remote write. A
|
||
// restart must inspect the same object, never invent another suffix.
|
||
item.TargetKey, item.LogicalTargetKey = targetKey, logicalTarget
|
||
item.UpdatedAt = nowISO()
|
||
if saveErr := coordinator.saveClaimedItem(ctx, *item, claimOwner); saveErr != nil {
|
||
return saveErr
|
||
}
|
||
published, err, _ = coordinator.publishAndConfirm(ctx, *connection, space.Bucket, sourceKey, targetKey, item.SizeBytes, staging.ETag, "", "*", metadata)
|
||
keptBoth = err == nil
|
||
}
|
||
if err != nil {
|
||
return err
|
||
}
|
||
}
|
||
if err := ctx.Err(); err != nil {
|
||
return err
|
||
}
|
||
owned, claimErr := coordinator.syncStore.RenewObjectSyncItemClaim(ctx, item.ID, claimOwner, claimTTL)
|
||
if claimErr != nil {
|
||
return claimErr
|
||
}
|
||
if !owned {
|
||
return errClaimLost
|
||
}
|
||
previousHead, err := coordinator.syncStore.GetObjectSyncHead(ctx, session.OrganizationID, space.ID, targetKey)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
platformVersion := int64(1)
|
||
if previousHead != nil {
|
||
platformVersion = previousHead.PlatformVersion + 1
|
||
// Head/version writes may have succeeded before the item receipt was
|
||
// saved. Reuse that exact publication instead of creating version N+1.
|
||
if previousHead.ItemID == item.ID && previousHead.SessionID == session.ID &&
|
||
previousHead.RemoteETag == published.ETag && previousHead.VersionID == published.VersionID &&
|
||
previousHead.MD5Sum == item.MD5Sum && previousHead.SizeBytes == published.SizeBytes && previousHead.PlatformVersion > 0 {
|
||
platformVersion = previousHead.PlatformVersion
|
||
}
|
||
}
|
||
now := time.Now().UTC()
|
||
head := domain.ObjectSyncObjectHeadRecord{
|
||
OrganizationID: session.OrganizationID,
|
||
SyncSpaceID: space.ID,
|
||
Key: targetKey,
|
||
PlatformVersion: platformVersion,
|
||
RemoteETag: published.ETag,
|
||
VersionID: published.VersionID,
|
||
SizeBytes: published.SizeBytes,
|
||
MD5Sum: item.MD5Sum,
|
||
SessionID: session.ID,
|
||
ItemID: item.ID,
|
||
UpdatedAt: now.Format(time.RFC3339Nano),
|
||
}
|
||
if err := coordinator.syncStore.UpsertObjectSyncHead(ctx, head); err != nil {
|
||
return err
|
||
}
|
||
if err := coordinator.syncStore.InsertObjectSyncVersion(ctx, domain.ObjectSyncVersionRecord{
|
||
ID: fmt.Sprintf("osv_%s_%d", item.ID, platformVersion),
|
||
OrganizationID: session.OrganizationID,
|
||
SyncSpaceID: space.ID,
|
||
Key: targetKey,
|
||
LogicalTargetKey: logicalTarget,
|
||
SourceKey: item.ForceSourceKey,
|
||
PlatformVersion: platformVersion,
|
||
RemoteETag: published.ETag,
|
||
VersionID: published.VersionID,
|
||
SizeBytes: published.SizeBytes,
|
||
MD5Sum: item.MD5Sum,
|
||
SessionID: session.ID,
|
||
ItemID: item.ID,
|
||
ActorID: firstNonEmpty(item.ForceActorID, session.DeviceID),
|
||
CommittedAt: now.Format(time.RFC3339Nano),
|
||
PurgeAt: now.Add(versionRetention),
|
||
}); err != nil {
|
||
return err
|
||
}
|
||
item.TargetKey = targetKey
|
||
item.LogicalTargetKey = logicalTarget
|
||
item.LastError = ""
|
||
item.CommittedVersion = domain.ObjectSyncRemoteBase{
|
||
Exists: true,
|
||
ETag: published.ETag,
|
||
SizeBytes: published.SizeBytes,
|
||
VersionID: published.VersionID,
|
||
PlatformVersion: platformVersion,
|
||
}
|
||
if keptBoth {
|
||
item.Status = domain.ObjectSyncItemKeptBoth
|
||
} else {
|
||
item.Status = domain.ObjectSyncItemCommitted
|
||
}
|
||
item.CoordinatorOwner = ""
|
||
item.CoordinatorLeaseUntil = time.Time{}
|
||
item.UpdatedAt = now.Format(time.RFC3339Nano)
|
||
if err := coordinator.saveClaimedItem(ctx, *item, claimOwner); err != nil {
|
||
return err
|
||
}
|
||
if !preserveSource && !directUpload {
|
||
// Publication already committed. Keep staging metadata for the cleanup
|
||
// worker on failure, and expose the cleanup failure without undoing it.
|
||
if err := coordinator.publisher.Delete(ctx, *connection, space.Bucket, sourceKey); err != nil {
|
||
coordinator.recordAuxiliaryFailure(phaseCleanup, "delete published staging", err)
|
||
}
|
||
}
|
||
if err := coordinator.syncStore.ReleaseObjectSyncLease(ctx, item.LeaseID, session.ID, item.ID, item.FencingToken); err != nil {
|
||
coordinator.recordAuxiliaryFailure(phaseCommit, "release committed lease", err)
|
||
}
|
||
coordinator.recordSystemAudit(ctx, domain.ObjectSyncAuditEvent{
|
||
OrganizationID: session.OrganizationID,
|
||
ActorType: "system",
|
||
ActorID: coordinator.owner,
|
||
DeviceID: session.DeviceID,
|
||
SessionID: session.ID,
|
||
ItemID: item.ID,
|
||
Action: "item." + item.Action + ".committed",
|
||
Target: map[string]any{
|
||
"key": item.TargetKey,
|
||
"sourceKey": item.ForceSourceKey,
|
||
},
|
||
After: map[string]any{
|
||
"etag": item.CommittedVersion.ETag,
|
||
"versionId": item.CommittedVersion.VersionID,
|
||
"platformVersion": item.CommittedVersion.PlatformVersion,
|
||
"sizeBytes": item.CommittedVersion.SizeBytes,
|
||
},
|
||
Result: "success",
|
||
})
|
||
return nil
|
||
}
|
||
|
||
func (coordinator *Coordinator) publishRelease(ctx context.Context, release *domain.ObjectSyncReleaseRecord) error {
|
||
token, id := release.CoordinatorOwner, release.ID
|
||
if token == "" {
|
||
return errClaimLost
|
||
}
|
||
requireClaim := func() error {
|
||
return requireOwnedClaim(ctx, func(callCtx context.Context) (bool, error) {
|
||
return coordinator.syncStore.RenewObjectSyncReleaseClaim(callCtx, id, token, claimTTL)
|
||
})
|
||
}
|
||
temp, err := os.CreateTemp("", "drosync-release-*.ndjson")
|
||
if err != nil {
|
||
return err
|
||
}
|
||
tempName := temp.Name()
|
||
defer os.Remove(tempName)
|
||
hasher := sha256.New()
|
||
writer := bufio.NewWriter(io.MultiWriter(temp, hasher))
|
||
var objectCount int64
|
||
var totalBytes int64
|
||
var firstSession *domain.ObjectSyncSessionRecord
|
||
for _, sessionID := range release.SessionIDs {
|
||
session, err := coordinator.syncStore.GetObjectSyncSession(ctx, sessionID)
|
||
if err != nil || session == nil || session.OrganizationID != release.OrganizationID {
|
||
_ = temp.Close()
|
||
return fmt.Errorf("release session %s is unavailable", sessionID)
|
||
}
|
||
if release.SyncSpaceID == "" {
|
||
release.SyncSpaceID = session.SyncSpaceID
|
||
} else if session.SyncSpaceID != release.SyncSpaceID {
|
||
_ = temp.Close()
|
||
return fmt.Errorf("release session %s belongs to a different sync space", sessionID)
|
||
}
|
||
if firstSession == nil {
|
||
firstSession = session
|
||
}
|
||
cursor := ""
|
||
for {
|
||
page, err := coordinator.syncStore.ListObjectSyncItems(ctx, domain.ObjectSyncListQuery{SessionID: session.ID, Cursor: cursor, Limit: 1000})
|
||
if err != nil {
|
||
_ = temp.Close()
|
||
return err
|
||
}
|
||
for _, item := range page.Items {
|
||
if item.Status != domain.ObjectSyncItemCommitted && item.Status != domain.ObjectSyncItemKeptBoth && item.Status != domain.ObjectSyncItemUnchanged {
|
||
continue
|
||
}
|
||
line := map[string]any{
|
||
"sessionId": item.SessionID,
|
||
"itemId": item.ID,
|
||
"logicalTargetKey": item.LogicalTargetKey,
|
||
"key": item.TargetKey,
|
||
"sizeBytes": item.SizeBytes,
|
||
"md5sum": item.MD5Sum,
|
||
"remote": item.CommittedVersion,
|
||
"status": item.Status,
|
||
}
|
||
payload, _ := json.Marshal(line)
|
||
if _, err := writer.Write(append(payload, '\n')); err != nil {
|
||
_ = temp.Close()
|
||
return err
|
||
}
|
||
objectCount++
|
||
totalBytes += item.SizeBytes
|
||
}
|
||
if page.NextCursor == "" {
|
||
break
|
||
}
|
||
cursor = page.NextCursor
|
||
}
|
||
}
|
||
if err := writer.Flush(); err != nil {
|
||
_ = temp.Close()
|
||
return err
|
||
}
|
||
if err := temp.Close(); err != nil {
|
||
return err
|
||
}
|
||
if firstSession == nil {
|
||
return fmt.Errorf("release has no sessions")
|
||
}
|
||
space, err := coordinator.syncStore.GetObjectSyncSpace(ctx, release.SyncSpaceID)
|
||
if err != nil || space == nil {
|
||
return fmt.Errorf("release sync space is unavailable")
|
||
}
|
||
connection, err := coordinator.store.GetStorageConnection(ctx, space.ConnectionID)
|
||
if err != nil || connection == nil {
|
||
return fmt.Errorf("release storage connection is unavailable")
|
||
}
|
||
info, err := os.Stat(tempName)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
reader, err := os.Open(tempName)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
defer reader.Close()
|
||
manifestKey := connection.InternalStorageKey(path.Join(".drosync-internal/releases", release.OrganizationID, release.ID+".ndjson"))
|
||
digest := hex.EncodeToString(hasher.Sum(nil))
|
||
if err := requireClaim(); err != nil {
|
||
return err
|
||
}
|
||
if _, err := coordinator.publisher.PutInternal(ctx, *connection, space.Bucket, manifestKey, reader, info.Size(), "application/x-ndjson", map[string]string{
|
||
"drosync-release": release.ID,
|
||
"drosync-sha256": digest,
|
||
}); err != nil {
|
||
return err
|
||
}
|
||
if err := requireClaim(); err != nil {
|
||
return err
|
||
}
|
||
release.ManifestKey = manifestKey
|
||
release.ManifestSHA256 = digest
|
||
release.ObjectCount = objectCount
|
||
release.TotalBytes = totalBytes
|
||
release.Status = "published"
|
||
release.PublishedAt = nowISO()
|
||
release.CoordinatorOwner = ""
|
||
release.CoordinatorLeaseUntil = time.Time{}
|
||
saved, err := coordinator.syncStore.SaveClaimedObjectSyncRelease(ctx, *release, token)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
if !saved {
|
||
return errClaimLost
|
||
}
|
||
coordinator.recordSystemAudit(ctx, domain.ObjectSyncAuditEvent{
|
||
OrganizationID: release.OrganizationID,
|
||
ActorType: "system",
|
||
ActorID: coordinator.owner,
|
||
Action: "release.published",
|
||
Target: map[string]any{"releaseId": release.ID, "manifestKey": release.ManifestKey},
|
||
After: map[string]any{
|
||
"manifestSha256": release.ManifestSHA256,
|
||
"objectCount": release.ObjectCount,
|
||
"totalBytes": release.TotalBytes,
|
||
},
|
||
Result: "success",
|
||
})
|
||
return nil
|
||
}
|