forked from longtao.wu/droboticflow
419 lines
14 KiB
Go
419 lines
14 KiB
Go
package objectsync
|
|
|
|
import (
|
|
"context"
|
|
"crypto/ed25519"
|
|
"crypto/sha256"
|
|
"encoding/base64"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"io"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"droboticflow/internal/domain"
|
|
"droboticflow/internal/store"
|
|
|
|
"github.com/baidubce/bce-sdk-go/bce"
|
|
)
|
|
|
|
type fakePublisher struct {
|
|
mu sync.Mutex
|
|
objects map[string]ObjectMetadata
|
|
precondition map[string]bool
|
|
publishedKeys []string
|
|
}
|
|
|
|
type itemListCountingStore struct {
|
|
*store.MemoryStore
|
|
itemLists int
|
|
}
|
|
|
|
func (dataStore *itemListCountingStore) ListObjectSyncItems(ctx context.Context, query domain.ObjectSyncListQuery) (domain.ObjectSyncItemPage, error) {
|
|
dataStore.itemLists++
|
|
return dataStore.MemoryStore.ListObjectSyncItems(ctx, query)
|
|
}
|
|
|
|
func (publisher *fakePublisher) AbortMultipart(_ context.Context, _ domain.StorageConnectionRecord, _, _, _ string) error {
|
|
return nil
|
|
}
|
|
|
|
func (publisher *fakePublisher) Head(_ context.Context, _ domain.StorageConnectionRecord, bucket, key string) (ObjectMetadata, error) {
|
|
publisher.mu.Lock()
|
|
defer publisher.mu.Unlock()
|
|
item, ok := publisher.objects[bucket+"/"+key]
|
|
if !ok {
|
|
return ObjectMetadata{Exists: false}, nil
|
|
}
|
|
return item, nil
|
|
}
|
|
|
|
func (publisher *fakePublisher) PublishConditional(
|
|
_ context.Context,
|
|
_ domain.StorageConnectionRecord,
|
|
bucket, sourceKey, targetKey string,
|
|
sizeBytes int64,
|
|
_, _, _ string,
|
|
metadata map[string]string,
|
|
) (ObjectMetadata, error) {
|
|
publisher.mu.Lock()
|
|
defer publisher.mu.Unlock()
|
|
if publisher.precondition[targetKey] {
|
|
delete(publisher.precondition, targetKey)
|
|
return ObjectMetadata{}, &bce.BceServiceError{StatusCode: 412}
|
|
}
|
|
source := publisher.objects[bucket+"/"+sourceKey]
|
|
result := ObjectMetadata{Exists: true, ETag: "published-" + targetKey, SizeBytes: sizeBytes, UserMeta: metadata}
|
|
if source.VersionID != "" {
|
|
result.VersionID = "version-" + targetKey
|
|
}
|
|
publisher.objects[bucket+"/"+targetKey] = result
|
|
publisher.publishedKeys = append(publisher.publishedKeys, targetKey)
|
|
return result, nil
|
|
}
|
|
|
|
func (publisher *fakePublisher) PutInternal(
|
|
_ context.Context,
|
|
_ domain.StorageConnectionRecord,
|
|
bucket, key string,
|
|
reader io.Reader,
|
|
size int64,
|
|
_ string,
|
|
_ map[string]string,
|
|
) (ObjectMetadata, error) {
|
|
publisher.mu.Lock()
|
|
defer publisher.mu.Unlock()
|
|
payload, _ := io.ReadAll(reader)
|
|
if int64(len(payload)) != size {
|
|
return ObjectMetadata{}, io.ErrUnexpectedEOF
|
|
}
|
|
result := ObjectMetadata{Exists: true, ETag: "manifest", SizeBytes: size}
|
|
publisher.objects[bucket+"/"+key] = result
|
|
return result, nil
|
|
}
|
|
|
|
func (publisher *fakePublisher) Delete(_ context.Context, _ domain.StorageConnectionRecord, bucket, key string) error {
|
|
publisher.mu.Lock()
|
|
defer publisher.mu.Unlock()
|
|
delete(publisher.objects, bucket+"/"+key)
|
|
return nil
|
|
}
|
|
|
|
func coordinatorFixture(t *testing.T) (*Coordinator, *store.MemoryStore, *fakePublisher) {
|
|
t.Helper()
|
|
dataStore := store.NewMemoryStore()
|
|
connection := domain.StorageConnectionRecord{
|
|
ID: "bos",
|
|
OrganizationID: domain.DefaultOrganizationID,
|
|
Name: "BOS",
|
|
Provider: domain.StorageProviderBOS,
|
|
}
|
|
if err := dataStore.UpsertStorageConnection(t.Context(), connection); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
syncStore, _ := store.AsObjectSyncStore(dataStore)
|
|
if err := syncStore.UpsertObjectSyncSpace(t.Context(), domain.ObjectSyncSpaceRecord{
|
|
ID: "space",
|
|
OrganizationID: domain.DefaultOrganizationID,
|
|
ConnectionID: connection.ID,
|
|
Bucket: "bucket",
|
|
RootPrefix: "incoming",
|
|
Mode: "create_only",
|
|
Enabled: true,
|
|
Policy: domain.DefaultObjectSyncSpacePolicy(),
|
|
Capability: domain.ObjectSyncCapabilityRecord{
|
|
State: domain.ObjectSyncCapabilityCreateOnly,
|
|
ConditionalCreate: true,
|
|
},
|
|
}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
publisher := &fakePublisher{objects: map[string]ObjectMetadata{}, precondition: map[string]bool{}}
|
|
coordinator, err := NewCoordinator(dataStore, publisher, "test-owner")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return coordinator, dataStore, publisher
|
|
}
|
|
|
|
func testAuditEvent(t *testing.T, id, previousHash, occurredAt string) domain.ObjectSyncAuditEvent {
|
|
t.Helper()
|
|
event := domain.ObjectSyncAuditEvent{
|
|
ID: id,
|
|
EventKey: id,
|
|
OrganizationID: domain.DefaultOrganizationID,
|
|
ActorType: "user",
|
|
ActorID: "tester",
|
|
Action: "test.audit",
|
|
Result: "success",
|
|
PreviousHash: previousHash,
|
|
OccurredAt: occurredAt,
|
|
PurgeAt: time.Now().UTC().Add(365 * 24 * time.Hour),
|
|
}
|
|
unsigned := event
|
|
unsigned.EventHash = ""
|
|
payload, err := json.Marshal(unsigned)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
sum := sha256.Sum256(append([]byte(event.PreviousHash), payload...))
|
|
event.EventHash = hex.EncodeToString(sum[:])
|
|
return event
|
|
}
|
|
|
|
func TestCoordinatorSealsPreviousAuditDay(t *testing.T) {
|
|
coordinator, dataStore, publisher := coordinatorFixture(t)
|
|
syncStore, _ := store.AsObjectSyncStore(dataStore)
|
|
now := time.Date(2026, time.July, 28, 12, 0, 0, 0, time.UTC)
|
|
first := testAuditEvent(t, "audit-1", "", "2026-07-27T01:00:00Z")
|
|
second := testAuditEvent(t, "audit-2", first.EventHash, "2026-07-27T02:00:00Z")
|
|
if err := syncStore.AppendObjectSyncAudit(t.Context(), first); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := syncStore.AppendObjectSyncAudit(t.Context(), second); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
seed := make([]byte, ed25519.SeedSize)
|
|
for index := range seed {
|
|
seed[index] = byte(index + 1)
|
|
}
|
|
if err := coordinator.ConfigureAuditSigning(base64.StdEncoding.EncodeToString(seed), "test-key"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
count, err := coordinator.sealPreviousAuditDay(t.Context(), now)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if count != 1 {
|
|
t.Fatalf("expected one audit seal, got %d", count)
|
|
}
|
|
seal, err := syncStore.GetObjectSyncAuditSeal(t.Context(), domain.DefaultOrganizationID, "2026-07-27")
|
|
if err != nil || seal == nil {
|
|
t.Fatalf("expected persisted audit seal, got %#v, %v", seal, err)
|
|
}
|
|
if seal.EventCount != 2 || seal.FirstEventHash != first.EventHash || seal.LastEventHash != second.EventHash {
|
|
t.Fatalf("unexpected seal contents: %#v", seal)
|
|
}
|
|
payload, err := json.Marshal(auditSealSigningPayload{
|
|
OrganizationID: seal.OrganizationID,
|
|
DayUTC: seal.DayUTC,
|
|
EventCount: seal.EventCount,
|
|
FirstEventHash: seal.FirstEventHash,
|
|
LastEventHash: seal.LastEventHash,
|
|
AggregateHash: seal.AggregateHash,
|
|
SigningKeyID: seal.SigningKeyID,
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
signature, err := base64.StdEncoding.DecodeString(seal.Signature)
|
|
if err != nil || !ed25519.Verify(ed25519.NewKeyFromSeed(seed).Public().(ed25519.PublicKey), payload, signature) {
|
|
t.Fatal("audit seal signature did not verify")
|
|
}
|
|
if _, ok := publisher.objects["bucket/"+seal.ObjectKey]; !ok {
|
|
t.Fatalf("expected immutable seal object %s", seal.ObjectKey)
|
|
}
|
|
secondCount, err := coordinator.sealPreviousAuditDay(t.Context(), now)
|
|
if err != nil || secondCount != 0 {
|
|
t.Fatalf("expected idempotent second sealing pass, got %d, %v", secondCount, err)
|
|
}
|
|
}
|
|
|
|
func TestCoordinatorForceOverwritePreservesConflictSource(t *testing.T) {
|
|
coordinator, dataStore, publisher := coordinatorFixture(t)
|
|
syncStore, _ := store.AsObjectSyncStore(dataStore)
|
|
space, err := syncStore.GetObjectSyncSpace(t.Context(), "space")
|
|
if err != nil || space == nil {
|
|
t.Fatalf("space = %#v, %v", space, err)
|
|
}
|
|
space.Capability.State = domain.ObjectSyncCapabilityUpdateEnabled
|
|
space.Capability.ConditionalCreate = true
|
|
space.Capability.ConditionalUpdate = true
|
|
space.Capability.ConditionalMultipart = true
|
|
space.Capability.ExclusivePrefixConfirmed = true
|
|
space.Policy.AllowConditionalUpdate = true
|
|
space.Policy.AllowForceOverwrite = true
|
|
if err := syncStore.UpsertObjectSyncSpace(t.Context(), *space); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
session := domain.ObjectSyncSessionRecord{
|
|
ID: "session-force",
|
|
OrganizationID: domain.DefaultOrganizationID,
|
|
SyncSpaceID: space.ID,
|
|
DeviceID: "device-1",
|
|
Status: domain.ObjectSyncSessionConflicted,
|
|
CreatedAt: nowISO(),
|
|
UpdatedAt: nowISO(),
|
|
}
|
|
if err := syncStore.UpsertObjectSyncSession(t.Context(), session); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
item := domain.ObjectSyncItemRecord{
|
|
ID: "item-force",
|
|
OrganizationID: domain.DefaultOrganizationID,
|
|
SessionID: session.ID,
|
|
ClientItemID: "client-force",
|
|
TargetKey: "incoming/file.bin",
|
|
LogicalTargetKey: "incoming/file.bin",
|
|
SizeBytes: 4,
|
|
MD5Sum: "abcd",
|
|
Action: "force_overwrite",
|
|
Status: domain.ObjectSyncItemCommitting,
|
|
ForceSourceKey: "incoming/file.bin.__conflict__20260728T000000Z_source",
|
|
ForceActorID: "admin-1",
|
|
CreatedAt: nowISO(),
|
|
UpdatedAt: nowISO(),
|
|
}
|
|
lease, acquired, err := syncStore.AcquireObjectSyncLease(t.Context(), domain.ObjectSyncLeaseRecord{
|
|
OrganizationID: session.OrganizationID,
|
|
ResourceHash: "force-resource",
|
|
ResourceType: "object",
|
|
OwnerSessionID: session.ID,
|
|
OwnerItemID: item.ID,
|
|
}, claimTTL)
|
|
if err != nil || !acquired {
|
|
t.Fatalf("lease = %#v, acquired=%t, err=%v", lease, acquired, err)
|
|
}
|
|
item.LeaseID = lease.ID
|
|
item.FencingToken = lease.FencingToken
|
|
if err := syncStore.UpsertObjectSyncItems(t.Context(), []domain.ObjectSyncItemRecord{item}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
sourceObject := "bucket/" + item.ForceSourceKey
|
|
targetObject := "bucket/" + item.TargetKey
|
|
publisher.objects[sourceObject] = ObjectMetadata{Exists: true, ETag: "conflict-etag", SizeBytes: item.SizeBytes}
|
|
publisher.objects[targetObject] = ObjectMetadata{Exists: true, ETag: "current-etag", SizeBytes: item.SizeBytes}
|
|
claimed, err := syncStore.ClaimObjectSyncItem(t.Context(), coordinator.owner, claimTTL)
|
|
if err != nil || claimed == nil {
|
|
t.Fatalf("claim failed: %v", err)
|
|
}
|
|
if err := coordinator.commitItem(t.Context(), claimed); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, exists := publisher.objects[sourceObject]; !exists {
|
|
t.Fatal("force overwrite deleted the preserved conflict source")
|
|
}
|
|
if publisher.objects[targetObject].ETag != "published-"+item.TargetKey {
|
|
t.Fatalf("target was not published: %#v", publisher.objects[targetObject])
|
|
}
|
|
saved, err := syncStore.GetObjectSyncItem(t.Context(), item.ID)
|
|
if err != nil || saved == nil || saved.Status != domain.ObjectSyncItemCommitted {
|
|
t.Fatalf("saved force item = %#v, %v", saved, err)
|
|
}
|
|
}
|
|
|
|
func TestCoordinatorPlansCreateWithoutEmbeddingManifest(t *testing.T) {
|
|
coordinator, dataStore, _ := coordinatorFixture(t)
|
|
syncStore, _ := store.AsObjectSyncStore(dataStore)
|
|
now := time.Now().UTC()
|
|
if err := syncStore.UpsertObjectSyncSession(t.Context(), domain.ObjectSyncSessionRecord{
|
|
ID: "session",
|
|
OrganizationID: domain.DefaultOrganizationID,
|
|
SyncSpaceID: "space",
|
|
Status: domain.ObjectSyncSessionPlanning,
|
|
Counts: domain.ObjectSyncSessionCounts{Manifest: 1},
|
|
UpdatedAt: now.Format(time.RFC3339Nano),
|
|
}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := syncStore.UpsertObjectSyncItems(t.Context(), []domain.ObjectSyncItemRecord{{
|
|
ID: "item",
|
|
OrganizationID: domain.DefaultOrganizationID,
|
|
SessionID: "session",
|
|
TargetKey: "incoming/file.bin",
|
|
SizeBytes: 4,
|
|
MD5Sum: strings.Repeat("a", 32),
|
|
Status: domain.ObjectSyncItemDiscovered,
|
|
}}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
result, err := coordinator.ProcessOnce(t.Context())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if result.Planned != 1 {
|
|
t.Fatalf("planned = %d", result.Planned)
|
|
}
|
|
item, _ := syncStore.GetObjectSyncItem(t.Context(), "item")
|
|
if item.Status != domain.ObjectSyncItemPlanned || item.Action != "create" {
|
|
t.Fatalf("unexpected planned item: %#v", item)
|
|
}
|
|
}
|
|
|
|
func TestCoordinatorKeepsBothOnConditionalConflict(t *testing.T) {
|
|
coordinator, dataStore, publisher := coordinatorFixture(t)
|
|
counted := &itemListCountingStore{MemoryStore: dataStore}
|
|
coordinator, err := NewCoordinator(counted, publisher, "test-owner")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
syncStore, _ := store.AsObjectSyncStore(dataStore)
|
|
now := time.Now().UTC()
|
|
session := domain.ObjectSyncSessionRecord{
|
|
ID: "session",
|
|
OrganizationID: domain.DefaultOrganizationID,
|
|
SyncSpaceID: "space",
|
|
DeviceID: "device",
|
|
Status: domain.ObjectSyncSessionRunning,
|
|
Counts: domain.ObjectSyncSessionCounts{Manifest: 1, Create: 1},
|
|
UpdatedAt: now.Format(time.RFC3339Nano),
|
|
}
|
|
if err := syncStore.UpsertObjectSyncSession(t.Context(), session); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
lease, acquired, err := syncStore.AcquireObjectSyncLease(t.Context(), domain.ObjectSyncLeaseRecord{
|
|
ID: "lease",
|
|
OrganizationID: domain.DefaultOrganizationID,
|
|
ResourceHash: "resource",
|
|
ResourceType: "object",
|
|
ConnectionID: "bos",
|
|
Bucket: "bucket",
|
|
KeyOrPrefix: "incoming/file.bin",
|
|
OwnerSessionID: session.ID,
|
|
OwnerItemID: "item",
|
|
}, time.Minute)
|
|
if err != nil || !acquired {
|
|
t.Fatalf("lease: acquired=%v err=%v", acquired, err)
|
|
}
|
|
item := domain.ObjectSyncItemRecord{
|
|
ID: "item",
|
|
OrganizationID: domain.DefaultOrganizationID,
|
|
SessionID: session.ID,
|
|
TargetKey: "incoming/file.bin",
|
|
SizeBytes: 4,
|
|
MD5Sum: strings.Repeat("b", 32),
|
|
Action: "create",
|
|
Status: domain.ObjectSyncItemCommitting,
|
|
StagingKey: ".drosync-internal/staging/session/item",
|
|
LeaseID: lease.ID,
|
|
FencingToken: lease.FencingToken,
|
|
}
|
|
if err := syncStore.UpsertObjectSyncItems(t.Context(), []domain.ObjectSyncItemRecord{item}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
publisher.objects["bucket/"+item.StagingKey] = ObjectMetadata{Exists: true, ETag: "staging", SizeBytes: 4}
|
|
publisher.precondition[item.TargetKey] = true
|
|
result, err := coordinator.ProcessOnce(t.Context())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if result.Committed != 1 {
|
|
t.Fatalf("committed = %d", result.Committed)
|
|
}
|
|
saved, _ := syncStore.GetObjectSyncItem(t.Context(), item.ID)
|
|
if saved.Status != domain.ObjectSyncItemKeptBoth {
|
|
t.Fatalf("status = %s, error=%s", saved.Status, saved.LastError)
|
|
}
|
|
if saved.LogicalTargetKey != item.TargetKey || !strings.Contains(saved.TargetKey, ".__conflict__") {
|
|
t.Fatalf("unexpected keep-both keys: logical=%q target=%q", saved.LogicalTargetKey, saved.TargetKey)
|
|
}
|
|
if saved.CommittedVersion.ETag == "" {
|
|
t.Fatal("committed version metadata missing")
|
|
}
|
|
if counted.itemLists != 0 {
|
|
t.Fatalf("committing one item enumerated the entire session %d times", counted.itemLists)
|
|
}
|
|
}
|