forked from longtao.wu/droboticflow
Retire confirmed missing staging resources while rejecting ambiguous errors, noncanonical targets and malformed or already-retired Mongo keys. Validate Mongo 8.0.17 and real BOS default 64 MiB ranges, including replacement between ranges, and preserve regression and exact-resource cleanup evidence without deploying services or changing the client release. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
1145 lines
38 KiB
Go
1145 lines
38 KiB
Go
package objectsync
|
||
|
||
import (
|
||
"bufio"
|
||
"bytes"
|
||
"context"
|
||
"crypto/ed25519"
|
||
"crypto/sha256"
|
||
"encoding/base64"
|
||
"encoding/hex"
|
||
"encoding/json"
|
||
"errors"
|
||
"fmt"
|
||
"io"
|
||
"os"
|
||
"path"
|
||
"strings"
|
||
"sync"
|
||
"sync/atomic"
|
||
"time"
|
||
|
||
"droboticflow/internal/domain"
|
||
"droboticflow/internal/store"
|
||
|
||
"golang.org/x/sync/errgroup"
|
||
)
|
||
|
||
const (
|
||
claimTTL = 10 * time.Minute
|
||
versionRetention = 365 * 24 * time.Hour
|
||
stagingRetention = 24 * time.Hour
|
||
)
|
||
|
||
type Coordinator struct {
|
||
store store.Store
|
||
syncStore store.ObjectSyncStore
|
||
publisher Publisher
|
||
owner string
|
||
signingKey ed25519.PrivateKey
|
||
signingID string
|
||
cycles atomic.Int64
|
||
errors atomic.Int64
|
||
planned atomic.Int64
|
||
committed atomic.Int64
|
||
cleaned atomic.Int64
|
||
reconciled atomic.Int64
|
||
released atomic.Int64
|
||
sealed atomic.Int64
|
||
backfilled atomic.Int64
|
||
lastCycle atomic.Int64
|
||
lastOK atomic.Int64
|
||
durationMS atomic.Int64
|
||
phases [phaseCount]phaseCounters
|
||
renewInterval time.Duration
|
||
queueSnapshot atomic.Pointer[queueSnapshot]
|
||
}
|
||
|
||
func NewCoordinator(base store.Store, publisher Publisher, owner string) (*Coordinator, error) {
|
||
syncStore, ok := store.AsObjectSyncStore(base)
|
||
if !ok {
|
||
return nil, fmt.Errorf("store does not implement object sync persistence")
|
||
}
|
||
if publisher == nil {
|
||
return nil, fmt.Errorf("object publisher is required")
|
||
}
|
||
if strings.TrimSpace(owner) == "" {
|
||
owner = fmt.Sprintf("coordinator-%d", time.Now().UnixNano())
|
||
}
|
||
return &Coordinator{store: base, syncStore: syncStore, publisher: publisher, owner: owner}, nil
|
||
}
|
||
|
||
func (coordinator *Coordinator) ConfigureAuditSigning(encodedPrivateKey, keyID string) error {
|
||
encodedPrivateKey = strings.TrimSpace(encodedPrivateKey)
|
||
if encodedPrivateKey == "" {
|
||
coordinator.signingKey = nil
|
||
coordinator.signingID = ""
|
||
return nil
|
||
}
|
||
raw, err := base64.StdEncoding.DecodeString(encodedPrivateKey)
|
||
if err != nil {
|
||
raw, err = base64.RawStdEncoding.DecodeString(encodedPrivateKey)
|
||
}
|
||
if err != nil {
|
||
return fmt.Errorf("decode audit signing key: %w", err)
|
||
}
|
||
switch len(raw) {
|
||
case ed25519.SeedSize:
|
||
coordinator.signingKey = ed25519.NewKeyFromSeed(raw)
|
||
case ed25519.PrivateKeySize:
|
||
coordinator.signingKey = ed25519.PrivateKey(append([]byte(nil), raw...))
|
||
default:
|
||
return fmt.Errorf("audit signing key must be a %d-byte Ed25519 seed or %d-byte private key", ed25519.SeedSize, ed25519.PrivateKeySize)
|
||
}
|
||
coordinator.signingID = strings.TrimSpace(keyID)
|
||
if coordinator.signingID == "" {
|
||
return fmt.Errorf("audit signing key id is required")
|
||
}
|
||
return nil
|
||
}
|
||
|
||
func (coordinator *Coordinator) AuditSigningConfigured() bool {
|
||
return len(coordinator.signingKey) == ed25519.PrivateKeySize && coordinator.signingID != ""
|
||
}
|
||
|
||
// Each phase owns its own loop: slow planning, cleanup or audit cannot stop
|
||
// subsequent commit batches, and each phase has bounded internal parallelism.
|
||
func (coordinator *Coordinator) Run(ctx context.Context, interval time.Duration) {
|
||
if interval <= 0 {
|
||
interval = 2 * time.Second
|
||
}
|
||
var group sync.WaitGroup
|
||
group.Add(1)
|
||
go func() { defer group.Done(); coordinator.runQueueSampler(ctx) }()
|
||
for phase := range phaseCount {
|
||
workers := 1
|
||
if phase == phaseCommit {
|
||
workers = commitConcurrency
|
||
}
|
||
if phase == phaseCleanup {
|
||
workers = cleanupConcurrency
|
||
}
|
||
for range workers {
|
||
group.Add(1)
|
||
go func() {
|
||
defer group.Done()
|
||
for ctx.Err() == nil {
|
||
if phase == phaseCommit || phase == phaseCleanup {
|
||
result, err := coordinator.executeSingleItem(ctx, phase)
|
||
if err == nil && result.Committed+result.Cleaned > 0 {
|
||
continue
|
||
}
|
||
} else {
|
||
_, _ = coordinator.executeStage(ctx, phase)
|
||
}
|
||
timer := time.NewTimer(interval)
|
||
select {
|
||
case <-ctx.Done():
|
||
timer.Stop()
|
||
return
|
||
case <-timer.C:
|
||
}
|
||
}
|
||
}()
|
||
}
|
||
}
|
||
group.Wait()
|
||
}
|
||
|
||
type Metrics struct {
|
||
QueueStats domain.ObjectSyncQueueStats
|
||
QueueSampleAt int64
|
||
Phases map[string]PhaseMetrics
|
||
Cycles int64
|
||
Errors int64
|
||
Planned int64
|
||
Committed int64
|
||
Cleaned int64
|
||
Reconciled int64
|
||
Released int64
|
||
Sealed int64
|
||
Backfilled int64
|
||
LastCycle int64
|
||
LastOK int64
|
||
DurationMS int64
|
||
}
|
||
|
||
func (coordinator *Coordinator) Metrics() Metrics {
|
||
phases := make(map[string]PhaseMetrics, phaseCount)
|
||
for index, name := range phaseNames {
|
||
item := &coordinator.phases[index]
|
||
phases[name] = PhaseMetrics{Cycles: item.cycles.Load(), Errors: item.errors.Load(), Completed: item.completed.Load(), ItemErrors: item.itemErrors.Load(), InFlight: item.inFlight.Load(), DurationMS: item.durationMS.Load(), TotalDurationMS: item.totalDurationMS.Load()}
|
||
}
|
||
queue := coordinator.queueSnapshot.Load()
|
||
if queue == nil {
|
||
queue = &queueSnapshot{}
|
||
}
|
||
return Metrics{
|
||
QueueStats: queue.stats,
|
||
QueueSampleAt: queue.sampleAt,
|
||
Phases: phases,
|
||
Cycles: coordinator.cycles.Load(),
|
||
Errors: coordinator.errors.Load(),
|
||
Planned: coordinator.planned.Load(),
|
||
Committed: coordinator.committed.Load(),
|
||
Cleaned: coordinator.cleaned.Load(),
|
||
Reconciled: coordinator.reconciled.Load(),
|
||
Released: coordinator.released.Load(),
|
||
Sealed: coordinator.sealed.Load(),
|
||
Backfilled: coordinator.backfilled.Load(),
|
||
LastCycle: coordinator.lastCycle.Load(),
|
||
LastOK: coordinator.lastOK.Load(),
|
||
DurationMS: coordinator.durationMS.Load(),
|
||
}
|
||
}
|
||
|
||
func (coordinator *Coordinator) recordSystemAudit(ctx context.Context, event domain.ObjectSyncAuditEvent) {
|
||
if err := AppendAudit(ctx, coordinator.syncStore, event); err != nil {
|
||
coordinator.errors.Add(1)
|
||
fmt.Printf("[drosync-coordinator] audit persistence error for %s: %v\n", event.Action, err)
|
||
}
|
||
}
|
||
|
||
type ProcessResult struct {
|
||
Planned int `json:"planned"`
|
||
Committed int `json:"committed"`
|
||
Cleaned int `json:"cleaned"`
|
||
Reconciled int `json:"reconciled"`
|
||
Released int `json:"released"`
|
||
Sealed int `json:"sealed"`
|
||
Backfilled int `json:"backfilled"`
|
||
}
|
||
|
||
func (coordinator *Coordinator) ProcessOnce(ctx context.Context) (ProcessResult, error) {
|
||
type outcome struct {
|
||
result ProcessResult
|
||
err error
|
||
}
|
||
results := make(chan outcome, phaseCount)
|
||
for phase := range phaseCount {
|
||
go func() { value, err := coordinator.executeStage(ctx, phase); results <- outcome{value, err} }()
|
||
}
|
||
result := ProcessResult{}
|
||
var failures []error
|
||
for range phaseCount {
|
||
value := <-results
|
||
result.Planned += value.result.Planned
|
||
result.Committed += value.result.Committed
|
||
result.Cleaned += value.result.Cleaned
|
||
result.Reconciled += value.result.Reconciled
|
||
result.Released += value.result.Released
|
||
result.Sealed += value.result.Sealed
|
||
result.Backfilled += value.result.Backfilled
|
||
if value.err != nil {
|
||
failures = append(failures, value.err)
|
||
}
|
||
}
|
||
return result, errors.Join(failures...)
|
||
}
|
||
|
||
type auditSealSigningPayload struct {
|
||
OrganizationID string `json:"organizationId"`
|
||
DayUTC string `json:"dayUtc"`
|
||
EventCount int64 `json:"eventCount"`
|
||
FirstEventHash string `json:"firstEventHash,omitempty"`
|
||
LastEventHash string `json:"lastEventHash,omitempty"`
|
||
AggregateHash string `json:"aggregateHash"`
|
||
SigningKeyID string `json:"signingKeyId"`
|
||
}
|
||
|
||
type auditSealDocument struct {
|
||
auditSealSigningPayload
|
||
Signature string `json:"signature"`
|
||
}
|
||
|
||
func validateAuditEventHash(event domain.ObjectSyncAuditEvent) error {
|
||
unsigned := event
|
||
unsigned.EventHash = ""
|
||
payload, err := json.Marshal(unsigned)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
sum := sha256.Sum256(append([]byte(event.PreviousHash), payload...))
|
||
if !strings.EqualFold(event.EventHash, hex.EncodeToString(sum[:])) {
|
||
return fmt.Errorf("audit event %s hash mismatch", event.ID)
|
||
}
|
||
return nil
|
||
}
|
||
|
||
func (coordinator *Coordinator) sealPreviousAuditDay(ctx context.Context, now time.Time) (int, error) {
|
||
dayStart := now.UTC().Truncate(24 * time.Hour).Add(-24 * time.Hour)
|
||
dayEnd := dayStart.Add(24 * time.Hour)
|
||
dayUTC := dayStart.Format("2006-01-02")
|
||
organizations, err := coordinator.store.ListOrganizations(ctx)
|
||
if err != nil {
|
||
return 0, err
|
||
}
|
||
sealed := 0
|
||
for _, organization := range organizations {
|
||
existing, err := coordinator.syncStore.GetObjectSyncAuditSeal(ctx, organization.ID, dayUTC)
|
||
if err != nil {
|
||
return sealed, err
|
||
}
|
||
if existing != nil {
|
||
continue
|
||
}
|
||
ok, err := coordinator.sealAuditDay(ctx, organization.ID, dayUTC, dayStart, dayEnd)
|
||
if err != nil {
|
||
return sealed, err
|
||
}
|
||
if ok {
|
||
sealed++
|
||
}
|
||
}
|
||
return sealed, nil
|
||
}
|
||
|
||
func (coordinator *Coordinator) sealAuditDay(ctx context.Context, organizationID, dayUTC string, dayStart, dayEnd time.Time) (bool, error) {
|
||
resourceSum := sha256.Sum256([]byte("audit-seal\x00" + organizationID + "\x00" + dayUTC))
|
||
resourceHash := hex.EncodeToString(resourceSum[:])
|
||
leaseOwner := coordinator.owner + ":" + dayUTC
|
||
lease, acquired, err := coordinator.syncStore.AcquireObjectSyncLease(ctx, domain.ObjectSyncLeaseRecord{
|
||
ID: "osl_seal_" + resourceHash,
|
||
OrganizationID: organizationID,
|
||
ResourceHash: resourceHash,
|
||
ResourceType: "audit_seal",
|
||
KeyOrPrefix: dayUTC,
|
||
OwnerSessionID: leaseOwner,
|
||
OwnerItemID: resourceHash,
|
||
}, claimTTL)
|
||
if err != nil {
|
||
return false, err
|
||
}
|
||
if !acquired {
|
||
return false, nil
|
||
}
|
||
defer func() {
|
||
if err := coordinator.syncStore.ReleaseObjectSyncLease(context.Background(), resourceHash, leaseOwner, resourceHash, lease.FencingToken); err != nil {
|
||
coordinator.recordAuxiliaryFailure(phaseAudit, "release audit lease", err)
|
||
}
|
||
}()
|
||
existing, err := coordinator.syncStore.GetObjectSyncAuditSeal(ctx, organizationID, dayUTC)
|
||
if err != nil || existing != nil {
|
||
return false, err
|
||
}
|
||
|
||
hasher := sha256.New()
|
||
var count int64
|
||
var firstHash string
|
||
var lastHash string
|
||
cursor := ""
|
||
for {
|
||
page, err := coordinator.syncStore.ListObjectSyncAuditRange(
|
||
ctx,
|
||
organizationID,
|
||
dayStart.Format(time.RFC3339Nano),
|
||
dayEnd.Format(time.RFC3339Nano),
|
||
domain.ObjectSyncListQuery{Cursor: cursor, Limit: 1000},
|
||
)
|
||
if err != nil {
|
||
return false, err
|
||
}
|
||
for _, event := range page.Items {
|
||
if err := validateAuditEventHash(event); err != nil {
|
||
return false, err
|
||
}
|
||
if count > 0 && event.PreviousHash != lastHash {
|
||
return false, fmt.Errorf("audit chain discontinuity before event %s", event.ID)
|
||
}
|
||
if count == 0 {
|
||
firstHash = event.EventHash
|
||
}
|
||
lastHash = event.EventHash
|
||
_, _ = io.WriteString(hasher, event.EventHash+"\n")
|
||
count++
|
||
}
|
||
if page.NextCursor == "" {
|
||
break
|
||
}
|
||
cursor = page.NextCursor
|
||
}
|
||
aggregateHash := hex.EncodeToString(hasher.Sum(nil))
|
||
signingPayload := auditSealSigningPayload{
|
||
OrganizationID: organizationID,
|
||
DayUTC: dayUTC,
|
||
EventCount: count,
|
||
FirstEventHash: firstHash,
|
||
LastEventHash: lastHash,
|
||
AggregateHash: aggregateHash,
|
||
SigningKeyID: coordinator.signingID,
|
||
}
|
||
signingBytes, err := json.Marshal(signingPayload)
|
||
if err != nil {
|
||
return false, err
|
||
}
|
||
signature := base64.StdEncoding.EncodeToString(ed25519.Sign(coordinator.signingKey, signingBytes))
|
||
documentBytes, err := json.MarshalIndent(auditSealDocument{
|
||
auditSealSigningPayload: signingPayload,
|
||
Signature: signature,
|
||
}, "", " ")
|
||
if err != nil {
|
||
return false, err
|
||
}
|
||
documentBytes = append(documentBytes, '\n')
|
||
|
||
spaces, err := coordinator.syncStore.ListObjectSyncSpaces(ctx, organizationID)
|
||
if err != nil {
|
||
return false, err
|
||
}
|
||
if len(spaces) == 0 {
|
||
if count == 0 {
|
||
return false, nil
|
||
}
|
||
return false, fmt.Errorf("organization %s has audit events but no sync space for audit sealing", organizationID)
|
||
}
|
||
space := spaces[0]
|
||
connection, err := coordinator.store.GetStorageConnection(ctx, space.ConnectionID)
|
||
if err != nil || connection == nil {
|
||
return false, fmt.Errorf("audit seal storage connection is unavailable")
|
||
}
|
||
objectKey := connection.InternalStorageKey(path.Join(".drosync-internal/audit", organizationID, dayUTC+".json"))
|
||
object, err := coordinator.publisher.PutInternal(
|
||
ctx,
|
||
*connection,
|
||
space.Bucket,
|
||
objectKey,
|
||
bytes.NewReader(documentBytes),
|
||
int64(len(documentBytes)),
|
||
"application/json",
|
||
map[string]string{
|
||
"drosync-audit-day": dayUTC,
|
||
"drosync-audit-hash": aggregateHash,
|
||
"drosync-signing-key": coordinator.signingID,
|
||
},
|
||
)
|
||
if IsPreconditionFailure(err) {
|
||
object, err = coordinator.publisher.Head(ctx, *connection, space.Bucket, objectKey)
|
||
if err == nil && (!object.Exists || object.SizeBytes != int64(len(documentBytes))) {
|
||
err = fmt.Errorf("existing audit seal object does not match the expected size")
|
||
}
|
||
}
|
||
if err != nil {
|
||
return false, err
|
||
}
|
||
recordIDHash := sha256.Sum256([]byte(organizationID + "\x00" + dayUTC))
|
||
record := domain.ObjectSyncAuditSealRecord{
|
||
ID: "osas_" + hex.EncodeToString(recordIDHash[:12]),
|
||
OrganizationID: organizationID,
|
||
DayUTC: dayUTC,
|
||
EventCount: count,
|
||
FirstEventHash: firstHash,
|
||
LastEventHash: lastHash,
|
||
AggregateHash: aggregateHash,
|
||
Signature: signature,
|
||
SigningKeyID: coordinator.signingID,
|
||
ObjectKey: objectKey,
|
||
ObjectETag: object.ETag,
|
||
ObjectVersionID: object.VersionID,
|
||
CreatedAt: nowISO(),
|
||
PurgeAt: time.Now().UTC().Add(versionRetention),
|
||
}
|
||
if err := coordinator.syncStore.InsertObjectSyncAuditSeal(ctx, record); err != nil {
|
||
return false, err
|
||
}
|
||
return true, nil
|
||
}
|
||
|
||
func (coordinator *Coordinator) reconcileSpace(ctx context.Context, reconcile *domain.ObjectSyncReconcileRecord) error {
|
||
token, id := reconcile.CoordinatorOwner, reconcile.ID
|
||
if token == "" {
|
||
return errClaimLost
|
||
}
|
||
requireClaim := func() error {
|
||
return requireOwnedClaim(ctx, func(callCtx context.Context) (bool, error) {
|
||
return coordinator.syncStore.RenewObjectSyncReconcileClaim(callCtx, id, token, claimTTL)
|
||
})
|
||
}
|
||
space, err := coordinator.syncStore.GetObjectSyncSpace(ctx, reconcile.SyncSpaceID)
|
||
if err != nil || space == nil || space.OrganizationID != reconcile.OrganizationID {
|
||
return fmt.Errorf("sync space is unavailable")
|
||
}
|
||
connection, err := coordinator.store.GetStorageConnection(ctx, space.ConnectionID)
|
||
if err != nil || connection == nil {
|
||
return fmt.Errorf("storage connection is unavailable")
|
||
}
|
||
var scanned int64
|
||
var drifted int64
|
||
cursor := ""
|
||
for {
|
||
page, err := coordinator.syncStore.ListObjectSyncHeads(ctx, reconcile.OrganizationID, reconcile.SyncSpaceID, cursor, 1000)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
for _, head := range page.Items {
|
||
remote, err := coordinator.publisher.Head(ctx, *connection, space.Bucket, head.Key)
|
||
if err != nil {
|
||
return fmt.Errorf("head %s: %w", head.Key, err)
|
||
}
|
||
if err := requireClaim(); err != nil {
|
||
return err
|
||
}
|
||
headDrifted := !remote.Exists || strings.Trim(remote.ETag, `"`) != strings.Trim(head.RemoteETag, `"`)
|
||
if headDrifted {
|
||
drifted++
|
||
}
|
||
scanned++
|
||
// The page was read before the slow HEADs above. A commit or digest
|
||
// attestation that replaced this receipt since then is not what the
|
||
// verdict was computed against, so the store skips it.
|
||
if _, err := coordinator.syncStore.SetObjectSyncHeadDrifted(ctx, head, headDrifted, nowISO()); err != nil {
|
||
return err
|
||
}
|
||
}
|
||
reconcile.Scanned = scanned
|
||
reconcile.Drifted = drifted
|
||
reconcile.UpdatedAt = nowISO()
|
||
if err := requireClaim(); err != nil {
|
||
return err
|
||
}
|
||
reconcile.CoordinatorLeaseUntil = time.Now().UTC().Add(claimTTL)
|
||
saved, err := coordinator.syncStore.SaveClaimedObjectSyncReconcile(ctx, *reconcile, token)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
if !saved {
|
||
return errClaimLost
|
||
}
|
||
if page.NextCursor == "" {
|
||
break
|
||
}
|
||
cursor = page.NextCursor
|
||
}
|
||
if err := requireClaim(); err != nil {
|
||
return err
|
||
}
|
||
space.Enabled = false
|
||
space.Mode = "blocked"
|
||
if drifted > 0 {
|
||
space.Capability.State = domain.ObjectSyncCapabilityDrifted
|
||
} else {
|
||
switch {
|
||
case space.Capability.ConditionalCreate &&
|
||
space.Capability.ConditionalUpdate &&
|
||
space.Capability.ConditionalMultipart &&
|
||
space.Capability.ExclusivePrefixConfirmed:
|
||
space.Capability.State = domain.ObjectSyncCapabilityUpdateEnabled
|
||
case space.Capability.ConditionalCreate:
|
||
space.Capability.State = domain.ObjectSyncCapabilityCreateOnly
|
||
default:
|
||
space.Capability.State = domain.ObjectSyncCapabilityUnsupported
|
||
}
|
||
}
|
||
space.Policy.AllowConditionalUpdate = space.Capability.State == domain.ObjectSyncCapabilityUpdateEnabled
|
||
if space.Capability.State != domain.ObjectSyncCapabilityUpdateEnabled {
|
||
space.Policy.AllowForceOverwrite = false
|
||
}
|
||
space.UpdatedAt = nowISO()
|
||
if err := coordinator.syncStore.UpsertObjectSyncSpace(ctx, *space); err != nil {
|
||
return err
|
||
}
|
||
reconcile.Status = "completed"
|
||
reconcile.CompletedAt = nowISO()
|
||
reconcile.UpdatedAt = reconcile.CompletedAt
|
||
reconcile.CoordinatorOwner = ""
|
||
reconcile.CoordinatorLeaseUntil = time.Time{}
|
||
saved, err := coordinator.syncStore.SaveClaimedObjectSyncReconcile(ctx, *reconcile, token)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
if !saved {
|
||
return errClaimLost
|
||
}
|
||
coordinator.recordSystemAudit(ctx, domain.ObjectSyncAuditEvent{
|
||
OrganizationID: reconcile.OrganizationID,
|
||
ActorType: "system",
|
||
ActorID: coordinator.owner,
|
||
Action: "space.reconcile.completed",
|
||
Target: map[string]any{
|
||
"reconcileId": reconcile.ID,
|
||
"spaceId": reconcile.SyncSpaceID,
|
||
},
|
||
After: map[string]any{"scanned": scanned, "drifted": drifted},
|
||
Result: "success",
|
||
})
|
||
return nil
|
||
}
|
||
|
||
func (coordinator *Coordinator) cleanupStaging(ctx context.Context, item *domain.ObjectSyncItemRecord) error {
|
||
claimOwner := item.CoordinatorOwner
|
||
if claimOwner == "" {
|
||
return errClaimLost
|
||
}
|
||
if err := validateCleanupObjectKey(item.StagingKey); err != nil {
|
||
return err
|
||
}
|
||
if err := validateCleanupObjectKey(item.TargetKey); err != nil {
|
||
return err
|
||
}
|
||
session, err := coordinator.syncStore.GetObjectSyncSession(ctx, item.SessionID)
|
||
if err != nil || session == nil {
|
||
return fmt.Errorf("session is unavailable")
|
||
}
|
||
space, err := coordinator.syncStore.GetObjectSyncSpace(ctx, session.SyncSpaceID)
|
||
if err != nil || space == nil {
|
||
return fmt.Errorf("sync space is unavailable")
|
||
}
|
||
connection, err := coordinator.store.GetStorageConnection(ctx, space.ConnectionID)
|
||
if err != nil || connection == nil {
|
||
return fmt.Errorf("storage connection is unavailable")
|
||
}
|
||
if item.UploadID != "" {
|
||
if err := coordinator.publisher.AbortMultipart(ctx, *connection, space.Bucket, item.StagingKey, item.UploadID); err != nil {
|
||
return fmt.Errorf("abort staging multipart: %w", err)
|
||
}
|
||
}
|
||
if item.StagingKey != item.TargetKey {
|
||
if err := coordinator.publisher.Delete(ctx, *connection, space.Bucket, item.StagingKey); err != nil {
|
||
return err
|
||
}
|
||
}
|
||
if item.Status == domain.ObjectSyncItemUploading || item.Status == domain.ObjectSyncItemUploaded {
|
||
item.Status = domain.ObjectSyncItemExpired
|
||
}
|
||
item.StagingKey = ""
|
||
item.UploadID = ""
|
||
item.UploadedParts = nil
|
||
item.CoordinatorOwner = ""
|
||
item.CoordinatorLeaseUntil = time.Time{}
|
||
item.UpdatedAt = nowISO()
|
||
if err := coordinator.saveClaimedItem(ctx, *item, claimOwner); err != nil {
|
||
return err
|
||
}
|
||
if item.LeaseID != "" {
|
||
if err := coordinator.syncStore.ReleaseObjectSyncLease(ctx, item.LeaseID, item.SessionID, item.ID, item.FencingToken); err != nil {
|
||
coordinator.recordAuxiliaryFailure(phaseCleanup, "release cleaned lease", err)
|
||
}
|
||
}
|
||
return nil
|
||
}
|
||
|
||
func nowISO() string {
|
||
return time.Now().UTC().Format(time.RFC3339Nano)
|
||
}
|
||
|
||
func firstNonEmpty(values ...string) string {
|
||
for _, value := range values {
|
||
if strings.TrimSpace(value) != "" {
|
||
return value
|
||
}
|
||
}
|
||
return ""
|
||
}
|
||
|
||
func (coordinator *Coordinator) planSession(ctx context.Context, session *domain.ObjectSyncSessionRecord) error {
|
||
token := session.CoordinatorOwner
|
||
if token == "" {
|
||
return errClaimLost
|
||
}
|
||
space, err := coordinator.syncStore.GetObjectSyncSpace(ctx, session.SyncSpaceID)
|
||
if err != nil || space == nil {
|
||
return fmt.Errorf("sync space is unavailable")
|
||
}
|
||
connection, err := coordinator.store.GetStorageConnection(ctx, space.ConnectionID)
|
||
if err != nil || connection == nil {
|
||
return fmt.Errorf("storage connection is unavailable")
|
||
}
|
||
manifestCount := session.Counts.Manifest
|
||
cursor := ""
|
||
for {
|
||
page, err := coordinator.syncStore.ListObjectSyncItems(ctx, domain.ObjectSyncListQuery{SessionID: session.ID, Cursor: cursor, Limit: 1000})
|
||
if err != nil {
|
||
return err
|
||
}
|
||
type observed struct {
|
||
remote ObjectMetadata
|
||
head *domain.ObjectSyncObjectHeadRecord
|
||
}
|
||
facts := make([]observed, len(page.Items))
|
||
group, queryCtx := errgroup.WithContext(ctx)
|
||
group.SetLimit(planConcurrency)
|
||
for index, item := range page.Items {
|
||
if item.Status != domain.ObjectSyncItemDiscovered {
|
||
continue
|
||
}
|
||
group.Go(func() error {
|
||
head, err := coordinator.syncStore.GetObjectSyncHead(queryCtx, session.OrganizationID, space.ID, item.TargetKey)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
var remote ObjectMetadata
|
||
// Formal publication receipts are authoritative for upload-once planning.
|
||
// Only new paths need a collision check against object storage.
|
||
if head == nil {
|
||
remote, err = coordinator.publisher.Head(queryCtx, *connection, space.Bucket, item.TargetKey)
|
||
if err != nil {
|
||
return fmt.Errorf("head %s: %w", item.TargetKey, err)
|
||
}
|
||
}
|
||
facts[index] = observed{remote, head}
|
||
return nil
|
||
})
|
||
}
|
||
if err := group.Wait(); err != nil {
|
||
return err
|
||
}
|
||
updates := make([]domain.ObjectSyncItemRecord, 0, len(page.Items))
|
||
for index, item := range page.Items {
|
||
if item.Status != domain.ObjectSyncItemDiscovered {
|
||
continue
|
||
}
|
||
remote, head := facts[index].remote, facts[index].head
|
||
item.ExpectedBase = domain.ObjectSyncRemoteBase{Exists: remote.Exists, ETag: remote.ETag, SizeBytes: remote.SizeBytes, LastModified: remote.LastModified, VersionID: remote.VersionID}
|
||
switch {
|
||
case head != nil && len(head.MD5Sum) != 32:
|
||
item.Action = "deny_archived_hash_change"
|
||
item.Status = domain.ObjectSyncItemDenied
|
||
item.LastError = "已归档对象缺少 MD5 回执,暂停上传;旧 SHA256 回执退出内容比较和清理支持,保留云端原文件"
|
||
case head != nil && head.MD5Sum == item.MD5Sum && head.SizeBytes == item.SizeBytes:
|
||
item.Action = "unchanged"
|
||
item.Status = domain.ObjectSyncItemUnchanged
|
||
item.ExpectedBase = domain.ObjectSyncRemoteBase{Exists: true, ETag: head.RemoteETag, SizeBytes: head.SizeBytes, VersionID: head.VersionID, PlatformVersion: head.PlatformVersion}
|
||
case head != nil:
|
||
item.Action = "deny_archived_hash_change"
|
||
item.Status = domain.ObjectSyncItemDenied
|
||
item.LastError = fmt.Sprintf("同名文件内容冲突,暂停上传:%s;云端已提交 MD5Sum=%s,本地 MD5Sum=%s;保留云端原文件", item.TargetKey, head.MD5Sum, item.MD5Sum)
|
||
case remote.Exists:
|
||
item.Action = "deny_existing_untracked"
|
||
item.Status = domain.ObjectSyncItemDenied
|
||
item.LastError = "目标路径已有对象但缺少可信提交记录,暂停上传;保留云端原文件"
|
||
default:
|
||
item.Action = "create"
|
||
item.Status = domain.ObjectSyncItemPlanned
|
||
}
|
||
item.UpdatedAt = nowISO()
|
||
updates = append(updates, item)
|
||
}
|
||
owned, err := coordinator.syncStore.RenewObjectSyncSessionClaim(ctx, session.ID, token, claimTTL)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
if !owned {
|
||
return errClaimLost
|
||
}
|
||
if err := ctx.Err(); err != nil {
|
||
return err
|
||
}
|
||
if err := coordinator.syncStore.UpsertObjectSyncItems(ctx, updates); err != nil {
|
||
return err
|
||
}
|
||
if page.NextCursor == "" {
|
||
break
|
||
}
|
||
cursor = page.NextCursor
|
||
}
|
||
summaries, err := coordinator.syncStore.SummarizeObjectSyncItems(ctx, []string{session.ID})
|
||
if err != nil {
|
||
return err
|
||
}
|
||
session.Counts = summaries[session.ID].Counts
|
||
session.Counts.Manifest = manifestCount
|
||
session.Status = domain.ObjectSyncSessionPlanned
|
||
session.UpdatedAt = nowISO()
|
||
session.CoordinatorOwner = ""
|
||
session.CoordinatorLeaseUntil = time.Time{}
|
||
saved, err := coordinator.syncStore.SaveClaimedObjectSyncSession(ctx, *session, token)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
if !saved {
|
||
return errClaimLost
|
||
}
|
||
return nil
|
||
}
|
||
|
||
func conflictKey(original, itemID string, now time.Time) string {
|
||
shortID := strings.TrimPrefix(itemID, "osi_")
|
||
if len(shortID) > 10 {
|
||
shortID = shortID[:10]
|
||
}
|
||
return fmt.Sprintf("%s.__conflict__%s_%s", original, now.UTC().Format("20060102T150405Z"), shortID)
|
||
}
|
||
|
||
func (coordinator *Coordinator) commitItem(ctx context.Context, item *domain.ObjectSyncItemRecord) error {
|
||
return coordinator.commitItemWithObservation(ctx, item, nil)
|
||
}
|
||
|
||
func (coordinator *Coordinator) commitItemWithObservation(ctx context.Context, item *domain.ObjectSyncItemRecord, observed *ObjectMetadata) error {
|
||
claimOwner := item.CoordinatorOwner
|
||
if claimOwner == "" {
|
||
return errClaimLost
|
||
}
|
||
session, err := coordinator.syncStore.GetObjectSyncSession(ctx, item.SessionID)
|
||
if err != nil || session == nil {
|
||
return fmt.Errorf("session is unavailable")
|
||
}
|
||
space, err := coordinator.syncStore.GetObjectSyncSpace(ctx, session.SyncSpaceID)
|
||
if err != nil || space == nil {
|
||
return fmt.Errorf("sync space is unavailable")
|
||
}
|
||
connection, err := coordinator.store.GetStorageConnection(ctx, space.ConnectionID)
|
||
if err != nil || connection == nil {
|
||
return fmt.Errorf("storage connection is unavailable")
|
||
}
|
||
renewed, err := coordinator.syncStore.RenewObjectSyncLease(ctx, item.LeaseID, session.ID, item.ID, item.FencingToken, claimTTL)
|
||
if err != nil || !renewed {
|
||
return fmt.Errorf("lease is stale; publish rejected by fencing token")
|
||
}
|
||
sourceKey := item.StagingKey
|
||
preserveSource := false
|
||
if item.Action == "force_overwrite" {
|
||
sourceKey = item.ForceSourceKey
|
||
preserveSource = true
|
||
}
|
||
metadata := map[string]string{
|
||
"drosync-session": session.ID,
|
||
"drosync-item": item.ID,
|
||
"drosync-md5sum": item.MD5Sum,
|
||
"drosync-fence": fmt.Sprintf("%d", item.FencingToken),
|
||
}
|
||
if preserveSource {
|
||
metadata["drosync-force-source"] = sourceKey
|
||
metadata["drosync-force-actor"] = item.ForceActorID
|
||
}
|
||
targetKey := item.TargetKey
|
||
logicalTarget := item.LogicalTargetKey
|
||
keptBoth := !preserveSource && logicalTarget != "" && logicalTarget != targetKey
|
||
if keptBoth {
|
||
metadata["drosync-logical-target"] = logicalTarget
|
||
}
|
||
// A previous worker may have published and then lost its response or claim.
|
||
// This bounded HEAD avoids copying those bytes again; existence alone is not
|
||
// a receipt. Every operation marker, content hash and size must agree.
|
||
var current ObjectMetadata
|
||
if observed != nil {
|
||
if err := connection.CheckStoragePath(space.Bucket, targetKey, false); err != nil {
|
||
return err
|
||
}
|
||
if preserveSource || sourceKey != targetKey || item.Action != "create" {
|
||
return fmt.Errorf("observation cannot authorize a copy or overwrite")
|
||
}
|
||
current = *observed
|
||
} else {
|
||
current, err = coordinator.publisher.Head(ctx, *connection, space.Bucket, targetKey)
|
||
}
|
||
if err != nil {
|
||
return fmt.Errorf("verify publication target: %w", err)
|
||
}
|
||
ifMatch, ifNoneMatch := "", ""
|
||
switch item.Action {
|
||
case "create":
|
||
ifNoneMatch = "*"
|
||
case "update":
|
||
if space.Capability.State != domain.ObjectSyncCapabilityUpdateEnabled || !space.Policy.AllowConditionalUpdate {
|
||
return fmt.Errorf("conditional update gate is no longer enabled")
|
||
}
|
||
ifMatch = item.ExpectedBase.ETag
|
||
case "force_overwrite":
|
||
if space.Capability.State != domain.ObjectSyncCapabilityUpdateEnabled ||
|
||
!space.Capability.ExclusivePrefixConfirmed ||
|
||
!space.Policy.AllowConditionalUpdate ||
|
||
!space.Policy.AllowForceOverwrite {
|
||
return fmt.Errorf("force overwrite gate is no longer enabled")
|
||
}
|
||
if current.Exists {
|
||
ifMatch = current.ETag
|
||
} else {
|
||
ifNoneMatch = "*"
|
||
}
|
||
default:
|
||
return fmt.Errorf("unsupported commit action %q", item.Action)
|
||
}
|
||
if keptBoth {
|
||
ifMatch, ifNoneMatch = "", "*"
|
||
}
|
||
directUpload := !preserveSource && sourceKey == targetKey
|
||
if directUpload && !publicationMatches(current, item.SizeBytes, metadata) {
|
||
return fmt.Errorf("direct upload target is missing or its bound identity/hash/size does not match")
|
||
}
|
||
published := current
|
||
if !publicationMatches(current, item.SizeBytes, metadata) {
|
||
staging, headErr := coordinator.publisher.Head(ctx, *connection, space.Bucket, sourceKey)
|
||
if headErr != nil {
|
||
return headErr
|
||
}
|
||
if !staging.Exists || staging.SizeBytes != item.SizeBytes {
|
||
return fmt.Errorf("staging object is missing or size changed")
|
||
}
|
||
var confirmedConflict bool
|
||
published, err, confirmedConflict = coordinator.publishAndConfirm(ctx, *connection, space.Bucket, sourceKey, targetKey, item.SizeBytes, staging.ETag, ifMatch, ifNoneMatch, metadata)
|
||
if confirmedConflict && !preserveSource && !keptBoth {
|
||
logicalTarget = targetKey
|
||
targetKey = conflictKey(targetKey, item.ID, time.Now())
|
||
metadata["drosync-logical-target"] = logicalTarget
|
||
// Persist the sole conflict destination before any remote write. A
|
||
// restart must inspect the same object, never invent another suffix.
|
||
item.TargetKey, item.LogicalTargetKey = targetKey, logicalTarget
|
||
item.UpdatedAt = nowISO()
|
||
if saveErr := coordinator.saveClaimedItem(ctx, *item, claimOwner); saveErr != nil {
|
||
return saveErr
|
||
}
|
||
published, err, _ = coordinator.publishAndConfirm(ctx, *connection, space.Bucket, sourceKey, targetKey, item.SizeBytes, staging.ETag, "", "*", metadata)
|
||
keptBoth = err == nil
|
||
}
|
||
if err != nil {
|
||
return err
|
||
}
|
||
}
|
||
if err := ctx.Err(); err != nil {
|
||
return err
|
||
}
|
||
owned, claimErr := coordinator.syncStore.RenewObjectSyncItemClaim(ctx, item.ID, claimOwner, claimTTL)
|
||
if claimErr != nil {
|
||
return claimErr
|
||
}
|
||
if !owned {
|
||
return errClaimLost
|
||
}
|
||
previousHead, err := coordinator.syncStore.GetObjectSyncHead(ctx, session.OrganizationID, space.ID, targetKey)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
platformVersion := int64(1)
|
||
if previousHead != nil {
|
||
platformVersion = previousHead.PlatformVersion + 1
|
||
// Head/version writes may have succeeded before the item receipt was
|
||
// saved. Reuse that exact publication instead of creating version N+1.
|
||
if previousHead.ItemID == item.ID && previousHead.SessionID == session.ID &&
|
||
previousHead.RemoteETag == published.ETag && previousHead.VersionID == published.VersionID &&
|
||
previousHead.MD5Sum == item.MD5Sum && previousHead.SizeBytes == published.SizeBytes && previousHead.PlatformVersion > 0 {
|
||
platformVersion = previousHead.PlatformVersion
|
||
}
|
||
}
|
||
now := time.Now().UTC()
|
||
head := domain.ObjectSyncObjectHeadRecord{
|
||
OrganizationID: session.OrganizationID,
|
||
SyncSpaceID: space.ID,
|
||
Key: targetKey,
|
||
PlatformVersion: platformVersion,
|
||
RemoteETag: published.ETag,
|
||
VersionID: published.VersionID,
|
||
SizeBytes: published.SizeBytes,
|
||
MD5Sum: item.MD5Sum,
|
||
SessionID: session.ID,
|
||
ItemID: item.ID,
|
||
UpdatedAt: now.Format(time.RFC3339Nano),
|
||
}
|
||
if err := coordinator.syncStore.UpsertObjectSyncHead(ctx, head); err != nil {
|
||
return err
|
||
}
|
||
if err := coordinator.syncStore.InsertObjectSyncVersion(ctx, domain.ObjectSyncVersionRecord{
|
||
ID: fmt.Sprintf("osv_%s_%d", item.ID, platformVersion),
|
||
OrganizationID: session.OrganizationID,
|
||
SyncSpaceID: space.ID,
|
||
Key: targetKey,
|
||
LogicalTargetKey: logicalTarget,
|
||
SourceKey: item.ForceSourceKey,
|
||
PlatformVersion: platformVersion,
|
||
RemoteETag: published.ETag,
|
||
VersionID: published.VersionID,
|
||
SizeBytes: published.SizeBytes,
|
||
MD5Sum: item.MD5Sum,
|
||
SessionID: session.ID,
|
||
ItemID: item.ID,
|
||
ActorID: firstNonEmpty(item.ForceActorID, session.DeviceID),
|
||
CommittedAt: now.Format(time.RFC3339Nano),
|
||
PurgeAt: now.Add(versionRetention),
|
||
}); err != nil {
|
||
return err
|
||
}
|
||
item.TargetKey = targetKey
|
||
item.LogicalTargetKey = logicalTarget
|
||
item.LastError = ""
|
||
item.CommittedVersion = domain.ObjectSyncRemoteBase{
|
||
Exists: true,
|
||
ETag: published.ETag,
|
||
SizeBytes: published.SizeBytes,
|
||
VersionID: published.VersionID,
|
||
PlatformVersion: platformVersion,
|
||
}
|
||
if keptBoth {
|
||
item.Status = domain.ObjectSyncItemKeptBoth
|
||
} else {
|
||
item.Status = domain.ObjectSyncItemCommitted
|
||
}
|
||
item.CoordinatorOwner = ""
|
||
item.CoordinatorLeaseUntil = time.Time{}
|
||
item.UpdatedAt = now.Format(time.RFC3339Nano)
|
||
if err := coordinator.saveClaimedItem(ctx, *item, claimOwner); err != nil {
|
||
return err
|
||
}
|
||
if !preserveSource && !directUpload {
|
||
// Publication already committed. Keep staging metadata for the cleanup
|
||
// worker on failure, and expose the cleanup failure without undoing it.
|
||
if err := coordinator.publisher.Delete(ctx, *connection, space.Bucket, sourceKey); err != nil {
|
||
coordinator.recordAuxiliaryFailure(phaseCleanup, "delete published staging", err)
|
||
}
|
||
}
|
||
if err := coordinator.syncStore.ReleaseObjectSyncLease(ctx, item.LeaseID, session.ID, item.ID, item.FencingToken); err != nil {
|
||
coordinator.recordAuxiliaryFailure(phaseCommit, "release committed lease", err)
|
||
}
|
||
coordinator.recordSystemAudit(ctx, domain.ObjectSyncAuditEvent{
|
||
OrganizationID: session.OrganizationID,
|
||
ActorType: "system",
|
||
ActorID: coordinator.owner,
|
||
DeviceID: session.DeviceID,
|
||
SessionID: session.ID,
|
||
ItemID: item.ID,
|
||
Action: "item." + item.Action + ".committed",
|
||
Target: map[string]any{
|
||
"key": item.TargetKey,
|
||
"sourceKey": item.ForceSourceKey,
|
||
},
|
||
After: map[string]any{
|
||
"etag": item.CommittedVersion.ETag,
|
||
"versionId": item.CommittedVersion.VersionID,
|
||
"platformVersion": item.CommittedVersion.PlatformVersion,
|
||
"sizeBytes": item.CommittedVersion.SizeBytes,
|
||
},
|
||
Result: "success",
|
||
})
|
||
return nil
|
||
}
|
||
|
||
func (coordinator *Coordinator) publishRelease(ctx context.Context, release *domain.ObjectSyncReleaseRecord) error {
|
||
token, id := release.CoordinatorOwner, release.ID
|
||
if token == "" {
|
||
return errClaimLost
|
||
}
|
||
requireClaim := func() error {
|
||
return requireOwnedClaim(ctx, func(callCtx context.Context) (bool, error) {
|
||
return coordinator.syncStore.RenewObjectSyncReleaseClaim(callCtx, id, token, claimTTL)
|
||
})
|
||
}
|
||
temp, err := os.CreateTemp("", "drosync-release-*.ndjson")
|
||
if err != nil {
|
||
return err
|
||
}
|
||
tempName := temp.Name()
|
||
defer os.Remove(tempName)
|
||
hasher := sha256.New()
|
||
writer := bufio.NewWriter(io.MultiWriter(temp, hasher))
|
||
var objectCount int64
|
||
var totalBytes int64
|
||
var firstSession *domain.ObjectSyncSessionRecord
|
||
for _, sessionID := range release.SessionIDs {
|
||
session, err := coordinator.syncStore.GetObjectSyncSession(ctx, sessionID)
|
||
if err != nil || session == nil || session.OrganizationID != release.OrganizationID {
|
||
_ = temp.Close()
|
||
return fmt.Errorf("release session %s is unavailable", sessionID)
|
||
}
|
||
if release.SyncSpaceID == "" {
|
||
release.SyncSpaceID = session.SyncSpaceID
|
||
} else if session.SyncSpaceID != release.SyncSpaceID {
|
||
_ = temp.Close()
|
||
return fmt.Errorf("release session %s belongs to a different sync space", sessionID)
|
||
}
|
||
if firstSession == nil {
|
||
firstSession = session
|
||
}
|
||
cursor := ""
|
||
for {
|
||
page, err := coordinator.syncStore.ListObjectSyncItems(ctx, domain.ObjectSyncListQuery{SessionID: session.ID, Cursor: cursor, Limit: 1000})
|
||
if err != nil {
|
||
_ = temp.Close()
|
||
return err
|
||
}
|
||
for _, item := range page.Items {
|
||
if item.Status != domain.ObjectSyncItemCommitted && item.Status != domain.ObjectSyncItemKeptBoth && item.Status != domain.ObjectSyncItemUnchanged {
|
||
continue
|
||
}
|
||
line := map[string]any{
|
||
"sessionId": item.SessionID,
|
||
"itemId": item.ID,
|
||
"logicalTargetKey": item.LogicalTargetKey,
|
||
"key": item.TargetKey,
|
||
"sizeBytes": item.SizeBytes,
|
||
"md5sum": item.MD5Sum,
|
||
"remote": item.CommittedVersion,
|
||
"status": item.Status,
|
||
}
|
||
payload, _ := json.Marshal(line)
|
||
if _, err := writer.Write(append(payload, '\n')); err != nil {
|
||
_ = temp.Close()
|
||
return err
|
||
}
|
||
objectCount++
|
||
totalBytes += item.SizeBytes
|
||
}
|
||
if page.NextCursor == "" {
|
||
break
|
||
}
|
||
cursor = page.NextCursor
|
||
}
|
||
}
|
||
if err := writer.Flush(); err != nil {
|
||
_ = temp.Close()
|
||
return err
|
||
}
|
||
if err := temp.Close(); err != nil {
|
||
return err
|
||
}
|
||
if firstSession == nil {
|
||
return fmt.Errorf("release has no sessions")
|
||
}
|
||
space, err := coordinator.syncStore.GetObjectSyncSpace(ctx, release.SyncSpaceID)
|
||
if err != nil || space == nil {
|
||
return fmt.Errorf("release sync space is unavailable")
|
||
}
|
||
connection, err := coordinator.store.GetStorageConnection(ctx, space.ConnectionID)
|
||
if err != nil || connection == nil {
|
||
return fmt.Errorf("release storage connection is unavailable")
|
||
}
|
||
info, err := os.Stat(tempName)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
reader, err := os.Open(tempName)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
defer reader.Close()
|
||
manifestKey := connection.InternalStorageKey(path.Join(".drosync-internal/releases", release.OrganizationID, release.ID+".ndjson"))
|
||
digest := hex.EncodeToString(hasher.Sum(nil))
|
||
if err := requireClaim(); err != nil {
|
||
return err
|
||
}
|
||
if _, err := coordinator.publisher.PutInternal(ctx, *connection, space.Bucket, manifestKey, reader, info.Size(), "application/x-ndjson", map[string]string{
|
||
"drosync-release": release.ID,
|
||
"drosync-sha256": digest,
|
||
}); err != nil {
|
||
return err
|
||
}
|
||
if err := requireClaim(); err != nil {
|
||
return err
|
||
}
|
||
release.ManifestKey = manifestKey
|
||
release.ManifestSHA256 = digest
|
||
release.ObjectCount = objectCount
|
||
release.TotalBytes = totalBytes
|
||
release.Status = "published"
|
||
release.PublishedAt = nowISO()
|
||
release.CoordinatorOwner = ""
|
||
release.CoordinatorLeaseUntil = time.Time{}
|
||
saved, err := coordinator.syncStore.SaveClaimedObjectSyncRelease(ctx, *release, token)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
if !saved {
|
||
return errClaimLost
|
||
}
|
||
coordinator.recordSystemAudit(ctx, domain.ObjectSyncAuditEvent{
|
||
OrganizationID: release.OrganizationID,
|
||
ActorType: "system",
|
||
ActorID: coordinator.owner,
|
||
Action: "release.published",
|
||
Target: map[string]any{"releaseId": release.ID, "manifestKey": release.ManifestKey},
|
||
After: map[string]any{
|
||
"manifestSha256": release.ManifestSHA256,
|
||
"objectCount": release.ObjectCount,
|
||
"totalBytes": release.TotalBytes,
|
||
},
|
||
Result: "success",
|
||
})
|
||
return nil
|
||
}
|