forked from longtao.wu/droboticflow
257 lines
12 KiB
Go
257 lines
12 KiB
Go
package store
|
|
|
|
import (
|
|
"fmt"
|
|
"testing"
|
|
"time"
|
|
|
|
"droboticflow/internal/domain"
|
|
"go.mongodb.org/mongo-driver/bson"
|
|
"go.mongodb.org/mongo-driver/mongo/integration/mtest"
|
|
)
|
|
|
|
var progressStatuses = []domain.ObjectSyncItemStatus{
|
|
domain.ObjectSyncItemCommitted, domain.ObjectSyncItemKeptBoth,
|
|
domain.ObjectSyncItemUnchanged, domain.ObjectSyncItemSkipped, domain.ObjectSyncItemDenied,
|
|
domain.ObjectSyncItemFailed, domain.ObjectSyncItemAborted, domain.ObjectSyncItemExpired,
|
|
domain.ObjectSyncItemCommitting,
|
|
}
|
|
|
|
func assertProgressSummary(t *testing.T, summary domain.ObjectSyncItemSummary) {
|
|
t.Helper()
|
|
if summary.TotalItems != 900 || summary.TerminalItems != 800 || summary.Counts.Manifest != 900 ||
|
|
summary.Counts.Committed != 200 || summary.Counts.KeptBoth != 100 || summary.Counts.Failed != 300 ||
|
|
summary.Counts.Unchanged != 100 || summary.Counts.Skipped != 100 || summary.Counts.Denied != 100 ||
|
|
summary.Counts.Create != 900 {
|
|
t.Fatalf("incorrect progress summary: %#v", summary)
|
|
}
|
|
}
|
|
|
|
func TestMemoryObjectSyncProgressCountsRetriesAndAllTerminalStates(t *testing.T) {
|
|
dataStore := NewMemoryStore()
|
|
items := make([]domain.ObjectSyncItemRecord, 0, 900)
|
|
for _, status := range progressStatuses {
|
|
for index := 0; index < 100; index++ {
|
|
items = append(items, domain.ObjectSyncItemRecord{ID: fmt.Sprintf("%s-%d", status, index), SessionID: "session", Status: status, Action: "create"})
|
|
}
|
|
}
|
|
// A retry of the same write must not inflate counts.
|
|
for index := 0; index < 2; index++ {
|
|
if err := dataStore.UpsertObjectSyncItems(t.Context(), items); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
if err := dataStore.UpsertObjectSyncItems(t.Context(), []domain.ObjectSyncItemRecord{{ID: "other", SessionID: "other", Status: domain.ObjectSyncItemCommitted}}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
summaries, err := dataStore.SummarizeObjectSyncItems(t.Context(), []string{"session", "empty"})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
assertProgressSummary(t, summaries["session"])
|
|
if len(summaries) != 2 || summaries["empty"].TotalItems != 0 {
|
|
t.Fatalf("wrong session isolation: %#v", summaries)
|
|
}
|
|
items[0].Status = domain.ObjectSyncItemCommitting
|
|
if err := dataStore.UpsertObjectSyncItems(t.Context(), items[:1]); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
summaries, err = dataStore.SummarizeObjectSyncItems(t.Context(), []string{"session"})
|
|
if err != nil || summaries["session"].TerminalItems != 799 || summaries["session"].Counts.Committed != 199 {
|
|
t.Fatalf("summary did not reflect state transition: %#v, %v", summaries, err)
|
|
}
|
|
}
|
|
|
|
func TestMongoObjectSyncProgressUsesGroupedResults(t *testing.T) {
|
|
mt := mtest.New(t, mtest.NewOptions().ClientType(mtest.Mock))
|
|
mt.Run("mixed terminal and pending states", func(mt *mtest.T) {
|
|
dataStore := NewMongoStore(mt.DB)
|
|
groups := make([]bson.D, 0, len(progressStatuses))
|
|
for _, status := range progressStatuses {
|
|
groups = append(groups, bson.D{{Key: "_id", Value: bson.M{"sessionId": "session", "status": status, "action": "create"}}, {Key: "count", Value: int64(100)}})
|
|
}
|
|
mt.AddMockResponses(mtest.CreateCursorResponse(0, mt.DB.Name()+".objectSyncItems", mtest.FirstBatch, groups...))
|
|
summaries, err := dataStore.ReconcileObjectSyncItems(mt.Context(), []string{"session", "empty"})
|
|
if err != nil {
|
|
mt.Fatal(err)
|
|
}
|
|
assertProgressSummary(mt.T, summaries["session"])
|
|
if len(summaries) != 2 || summaries["empty"].TotalItems != 0 {
|
|
mt.Fatalf("unexpected summaries: %#v", summaries)
|
|
}
|
|
command := mt.GetStartedEvent()
|
|
if command == nil || command.CommandName != "aggregate" {
|
|
mt.Fatal("progress enumerated item documents instead of aggregating")
|
|
}
|
|
})
|
|
mt.Run("aggregation errors propagate", func(mt *mtest.T) {
|
|
mt.AddMockResponses(mtest.CreateCommandErrorResponse(mtest.CommandError{Code: 169, Message: "summary unavailable"}))
|
|
summaries, err := NewMongoStore(mt.DB).ReconcileObjectSyncItems(mt.Context(), []string{"session"})
|
|
if err == nil || summaries != nil {
|
|
mt.Fatalf("error was silently treated as completion: %#v, %v", summaries, err)
|
|
}
|
|
})
|
|
}
|
|
|
|
func progressFixture(session string, page int, revision int64, dirty bool, summary domain.ObjectSyncItemSummary) bson.D {
|
|
return bson.D{{Key: "_id", Value: progressPageID(session, page)}, {Key: "sessionId", Value: session}, {Key: "manifestPage", Value: page}, {Key: "revision", Value: revision}, {Key: "dirty", Value: dirty}, {Key: "validUntil", Value: time.Now().Add(time.Minute)}, {Key: "summary", Value: summary}}
|
|
}
|
|
|
|
func TestMongoObjectSyncProgressHotCacheAvoidsItemQueries(t *testing.T) {
|
|
mt := mtest.New(t, mtest.NewOptions().ClientType(mtest.Mock))
|
|
mt.Run("fresh cached pages", func(mt *mtest.T) {
|
|
dataStore := NewMongoStore(mt.DB)
|
|
complete := addObjectSyncItemCount(domain.ObjectSyncItemSummary{}, domain.ObjectSyncItemCommitted, "create", 1000)
|
|
pending := addObjectSyncItemCount(domain.ObjectSyncItemSummary{}, domain.ObjectSyncItemCommitting, "update", 20)
|
|
mt.AddMockResponses(mtest.CreateCursorResponse(0, mt.DB.Name()+".objectSyncProgress", mtest.FirstBatch,
|
|
progressFixture("s", -1, 0, false, domain.ObjectSyncItemSummary{}), progressFixture("s", 1, 2, false, complete), progressFixture("s", 2, 5, false, pending)))
|
|
result, err := dataStore.SummarizeObjectSyncItems(mt.Context(), []string{"s"})
|
|
if err != nil || result["s"].TotalItems != 1020 || result["s"].TerminalItems != 1000 || result["s"].Counts.Update != 20 {
|
|
mt.Fatalf("bad cached summary: %#v, %v", result, err)
|
|
}
|
|
for _, event := range mt.GetAllStartedEvents() {
|
|
if event.CommandName != "find" || event.Command.Lookup("find").StringValue() != "objectSyncProgress" {
|
|
mt.Fatalf("hot polling queried item data: %s", event.Command)
|
|
}
|
|
}
|
|
})
|
|
}
|
|
|
|
func TestMongoObjectSyncProgressDirtyPageRefreshUsesRevisionCAS(t *testing.T) {
|
|
for _, matched := range []int32{0, 1} {
|
|
mt := mtest.New(t, mtest.NewOptions().ClientType(mtest.Mock))
|
|
mt.Run(fmt.Sprintf("matched=%d", matched), func(mt *mtest.T) {
|
|
dataStore := NewMongoStore(mt.DB)
|
|
mt.AddMockResponses(
|
|
mtest.CreateCursorResponse(0, mt.DB.Name()+".objectSyncProgress", mtest.FirstBatch,
|
|
progressFixture("s", -1, 0, false, domain.ObjectSyncItemSummary{}), progressFixture("s", 7, 42, true, domain.ObjectSyncItemSummary{})),
|
|
mtest.CreateCursorResponse(0, mt.DB.Name()+".objectSyncItems", mtest.FirstBatch, bson.D{{Key: "_id", Value: bson.M{"sessionId": "s", "manifestPage": 7, "status": "committed", "action": "create"}}, {Key: "count", Value: int64(1000)}}),
|
|
mtest.CreateSuccessResponse(bson.E{Key: "n", Value: matched}, bson.E{Key: "nModified", Value: matched}),
|
|
)
|
|
result, err := dataStore.SummarizeObjectSyncItems(mt.Context(), []string{"s"})
|
|
if err != nil || result["s"].TotalItems != 1000 {
|
|
mt.Fatalf("refresh failed: %#v, %v", result, err)
|
|
}
|
|
wantTerminal := int64(1000)
|
|
if matched == 0 {
|
|
wantTerminal = 0
|
|
}
|
|
if result["s"].TerminalItems != wantTerminal {
|
|
mt.Fatalf("raced refresh reported completion: %#v", result)
|
|
}
|
|
var checked bool
|
|
for _, event := range mt.GetAllStartedEvents() {
|
|
if event.CommandName == "aggregate" {
|
|
assertProgressCoveredQuery(mt, event.Command, true)
|
|
}
|
|
if event.CommandName == "update" {
|
|
updates, _ := event.Command.Lookup("updates").Array().Values()
|
|
query := updates[0].Document().Lookup("q").Document()
|
|
if query.Lookup("revision").Int64() != 42 || query.Lookup("_id").StringValue() != "s:7" {
|
|
mt.Fatalf("cache write omitted CAS: %s", event.Command)
|
|
}
|
|
checked = true
|
|
}
|
|
}
|
|
if !checked {
|
|
mt.Fatal("no cache CAS write")
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestMongoObjectSyncProgressInvalidationDeduplicatesManifestPages(t *testing.T) {
|
|
mt := mtest.New(t, mtest.NewOptions().ClientType(mtest.Mock))
|
|
mt.Run("one revision per touched page", func(mt *mtest.T) {
|
|
mt.AddMockResponses(mtest.CreateSuccessResponse(bson.E{Key: "n", Value: int32(2)}, bson.E{Key: "nModified", Value: int32(2)}))
|
|
err := NewMongoStore(mt.DB).invalidateObjectSyncProgress(mt.Context(), []domain.ObjectSyncItemRecord{
|
|
{SessionID: "s", ManifestPage: 1}, {SessionID: "s", ManifestPage: 1}, {SessionID: "s", ManifestPage: 2},
|
|
})
|
|
if err != nil {
|
|
mt.Fatal(err)
|
|
}
|
|
updates, err := mt.GetStartedEvent().Command.Lookup("updates").Array().Values()
|
|
if err != nil || len(updates) != 2 {
|
|
mt.Fatalf("duplicated invalidation writes: %v %v", updates, err)
|
|
}
|
|
for _, update := range updates {
|
|
mutation := update.Document().Lookup("u").Document()
|
|
if mutation.Lookup("$inc").Document().Lookup("revision").Int64() != 1 || !mutation.Lookup("$set").Document().Lookup("dirty").Boolean() {
|
|
mt.Fatalf("invalid invalidation: %s", mutation)
|
|
}
|
|
}
|
|
})
|
|
}
|
|
|
|
func TestMongoObjectSyncProgressColdFailureDoesNotClaimCompletion(t *testing.T) {
|
|
mt := mtest.New(t, mtest.NewOptions().ClientType(mtest.Mock))
|
|
mt.Run("bootstrap database failure", func(mt *mtest.T) {
|
|
mt.AddMockResponses(mtest.CreateCursorResponse(0, mt.DB.Name()+".objectSyncProgress", mtest.FirstBatch), mtest.CreateCommandErrorResponse(mtest.CommandError{Code: 169, Message: "unavailable"}))
|
|
result, err := NewMongoStore(mt.DB).SummarizeObjectSyncItems(mt.Context(), []string{"s"})
|
|
if err == nil || result != nil {
|
|
mt.Fatalf("bootstrap failure looked complete: %#v %v", result, err)
|
|
}
|
|
for _, event := range mt.GetAllStartedEvents() {
|
|
if event.CommandName == "aggregate" {
|
|
assertProgressCoveredQuery(mt, event.Command, false)
|
|
}
|
|
}
|
|
})
|
|
}
|
|
|
|
func assertProgressCoveredQuery(mt *mtest.T, command bson.Raw, withState bool) {
|
|
mt.Helper()
|
|
if command.Lookup("hint").StringValue() != "sessionId_page_status_action" {
|
|
mt.Fatalf("missing covering index hint: %s", command)
|
|
}
|
|
stages, err := command.Lookup("pipeline").Array().Values()
|
|
if err != nil || len(stages) != 3 {
|
|
mt.Fatalf("unexpected progress pipeline: %s", command)
|
|
}
|
|
projection := stages[1].Document().Lookup("$project").Document()
|
|
keys := []string{"sessionId", "manifestPage"}
|
|
if withState {
|
|
keys = append(keys, "status", "action")
|
|
}
|
|
for _, key := range keys {
|
|
if projection.Lookup(key).Int32() != 1 {
|
|
mt.Fatalf("missing covered projection %s: %s", key, command)
|
|
}
|
|
}
|
|
if projection.Lookup("_id").Int32() != 0 {
|
|
mt.Fatal("progress query fetches item documents")
|
|
}
|
|
}
|
|
|
|
func TestMongoObjectSyncProgressExpiryAndSessionIsolation(t *testing.T) {
|
|
mt := mtest.New(t, mtest.NewOptions().ClientType(mtest.Mock))
|
|
mt.Run("fresh sessions remain isolated", func(mt *mtest.T) {
|
|
one := addObjectSyncItemCount(domain.ObjectSyncItemSummary{}, domain.ObjectSyncItemCommitted, "create", 3)
|
|
two := addObjectSyncItemCount(domain.ObjectSyncItemSummary{}, domain.ObjectSyncItemFailed, "update", 7)
|
|
mt.AddMockResponses(mtest.CreateCursorResponse(0, mt.DB.Name()+".objectSyncProgress", mtest.FirstBatch,
|
|
progressFixture("a", -1, 0, false, domain.ObjectSyncItemSummary{}), progressFixture("a", 1, 1, false, one),
|
|
progressFixture("b", -1, 0, false, domain.ObjectSyncItemSummary{}), progressFixture("b", 1, 1, false, two)))
|
|
result, err := NewMongoStore(mt.DB).SummarizeObjectSyncItems(mt.Context(), []string{"a", "b"})
|
|
if err != nil || len(result) != 2 || result["a"] != one || result["b"] != two {
|
|
mt.Fatalf("mixed cached sessions: %#v %v", result, err)
|
|
}
|
|
})
|
|
mt.Run("expired clean page is revalidated", func(mt *mtest.T) {
|
|
stale := progressFixture("s", 1, 8, false, addObjectSyncItemCount(domain.ObjectSyncItemSummary{}, domain.ObjectSyncItemCommitted, "create", 1000))
|
|
for i := range stale {
|
|
if stale[i].Key == "validUntil" {
|
|
stale[i].Value = time.Now().Add(-time.Second)
|
|
}
|
|
}
|
|
mt.AddMockResponses(
|
|
mtest.CreateCursorResponse(0, mt.DB.Name()+".objectSyncProgress", mtest.FirstBatch, progressFixture("s", -1, 0, false, domain.ObjectSyncItemSummary{}), stale),
|
|
mtest.CreateCursorResponse(0, mt.DB.Name()+".objectSyncItems", mtest.FirstBatch, bson.D{{Key: "_id", Value: bson.M{"sessionId": "s", "manifestPage": 1, "status": "committing", "action": "create"}}, {Key: "count", Value: int64(1000)}}),
|
|
mtest.CreateSuccessResponse(bson.E{Key: "n", Value: int32(1)}, bson.E{Key: "nModified", Value: int32(1)}),
|
|
)
|
|
result, err := NewMongoStore(mt.DB).SummarizeObjectSyncItems(mt.Context(), []string{"s"})
|
|
if err != nil || result["s"].TotalItems != 1000 || result["s"].TerminalItems != 0 {
|
|
mt.Fatalf("expired cache trusted stale completion: %#v %v", result, err)
|
|
}
|
|
})
|
|
}
|