forked from longtao.wu/droboticflow
351 lines
13 KiB
Go
351 lines
13 KiB
Go
package store
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"sort"
|
|
"time"
|
|
|
|
"droboticflow/internal/domain"
|
|
"go.mongodb.org/mongo-driver/bson"
|
|
"go.mongodb.org/mongo-driver/mongo"
|
|
"go.mongodb.org/mongo-driver/mongo/options"
|
|
)
|
|
|
|
func emptyObjectSyncSummaries(sessionIDs []string) map[string]domain.ObjectSyncItemSummary {
|
|
result := make(map[string]domain.ObjectSyncItemSummary, len(sessionIDs))
|
|
for _, id := range sessionIDs {
|
|
result[id] = domain.ObjectSyncItemSummary{}
|
|
}
|
|
return result
|
|
}
|
|
|
|
func addObjectSyncItemCount(summary domain.ObjectSyncItemSummary, status domain.ObjectSyncItemStatus, action string, count int64) domain.ObjectSyncItemSummary {
|
|
summary.TotalItems += count
|
|
summary.Counts.Manifest += count
|
|
switch status {
|
|
case domain.ObjectSyncItemCommitted:
|
|
summary.Counts.Committed += count
|
|
case domain.ObjectSyncItemKeptBoth:
|
|
summary.Counts.Committed += count
|
|
summary.Counts.KeptBoth += count
|
|
case domain.ObjectSyncItemUnchanged:
|
|
summary.Counts.Unchanged += count
|
|
case domain.ObjectSyncItemSkipped:
|
|
summary.Counts.Skipped += count
|
|
case domain.ObjectSyncItemDenied:
|
|
summary.Counts.Denied += count
|
|
case domain.ObjectSyncItemFailed, domain.ObjectSyncItemAborted, domain.ObjectSyncItemExpired:
|
|
summary.Counts.Failed += count
|
|
}
|
|
switch status {
|
|
case domain.ObjectSyncItemCommitted, domain.ObjectSyncItemKeptBoth,
|
|
domain.ObjectSyncItemUnchanged, domain.ObjectSyncItemSkipped, domain.ObjectSyncItemDenied,
|
|
domain.ObjectSyncItemFailed, domain.ObjectSyncItemAborted, domain.ObjectSyncItemExpired:
|
|
summary.TerminalItems += count
|
|
}
|
|
switch action {
|
|
case "create":
|
|
summary.Counts.Create += count
|
|
case "update":
|
|
summary.Counts.Update += count
|
|
}
|
|
return summary
|
|
}
|
|
|
|
func (store *MemoryStore) SummarizeObjectSyncItems(_ context.Context, sessionIDs []string) (map[string]domain.ObjectSyncItemSummary, error) {
|
|
result := emptyObjectSyncSummaries(sessionIDs)
|
|
if len(result) == 0 {
|
|
return result, nil
|
|
}
|
|
store.mu.Lock()
|
|
defer store.mu.Unlock()
|
|
for _, item := range store.objectSyncItems {
|
|
if summary, ok := result[item.SessionID]; ok {
|
|
result[item.SessionID] = addObjectSyncItemCount(summary, item.Status, item.Action, 1)
|
|
}
|
|
}
|
|
return result, nil
|
|
}
|
|
|
|
func (store *MongoStore) ReconcileObjectSyncItems(ctx context.Context, sessionIDs []string) (map[string]domain.ObjectSyncItemSummary, error) {
|
|
result := emptyObjectSyncSummaries(sessionIDs)
|
|
if len(result) == 0 {
|
|
return result, nil
|
|
}
|
|
// Only state and action are needed. Paths, checksums and multipart records
|
|
// never leave MongoDB while polling progress.
|
|
cursor, err := store.objectSyncItems.Aggregate(ctx, mongo.Pipeline{
|
|
{{Key: "$match", Value: bson.M{"sessionId": bson.M{"$in": sessionIDs}}}},
|
|
{{Key: "$project", Value: bson.M{"_id": 0, "sessionId": 1, "status": 1, "action": 1}}},
|
|
{{Key: "$group", Value: bson.M{
|
|
"_id": bson.M{"sessionId": "$sessionId", "status": "$status", "action": "$action"},
|
|
"count": bson.M{"$sum": 1},
|
|
}}},
|
|
})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer cursor.Close(ctx)
|
|
for cursor.Next(ctx) {
|
|
var group struct {
|
|
Key struct {
|
|
SessionID string `bson:"sessionId"`
|
|
Status domain.ObjectSyncItemStatus `bson:"status"`
|
|
Action string `bson:"action"`
|
|
} `bson:"_id"`
|
|
Count int64 `bson:"count"`
|
|
}
|
|
if err := cursor.Decode(&group); err != nil {
|
|
return nil, err
|
|
}
|
|
result[group.Key.SessionID] = addObjectSyncItemCount(result[group.Key.SessionID], group.Key.Status, group.Key.Action, group.Count)
|
|
}
|
|
if err := cursor.Err(); err != nil {
|
|
return nil, err
|
|
}
|
|
return result, nil
|
|
}
|
|
|
|
// Reconciliation is authoritative and must be used before marking a session
|
|
// complete. The progress cache below is a disposable polling optimization.
|
|
func (store *MemoryStore) ReconcileObjectSyncItems(ctx context.Context, sessionIDs []string) (map[string]domain.ObjectSyncItemSummary, error) {
|
|
return store.SummarizeObjectSyncItems(ctx, sessionIDs)
|
|
}
|
|
|
|
const objectSyncProgressTTL = 60 * time.Second
|
|
const objectSyncProgressRetention = 30 * 24 * time.Hour
|
|
|
|
type objectSyncProgressPage struct {
|
|
ID string `bson:"_id"`
|
|
SessionID string `bson:"sessionId"`
|
|
ManifestPage int `bson:"manifestPage"`
|
|
Revision int64 `bson:"revision"`
|
|
Dirty bool `bson:"dirty"`
|
|
ValidUntil time.Time `bson:"validUntil"`
|
|
Summary domain.ObjectSyncItemSummary `bson:"summary"`
|
|
OrganizationID string `bson:"organizationId"`
|
|
PurgeAt time.Time `bson:"purgeAt"`
|
|
}
|
|
|
|
func progressPageID(sessionID string, page int) string { return fmt.Sprintf("%s:%d", sessionID, page) }
|
|
|
|
// Mark before AND after writes. The revision CAS prevents a refresh racing a
|
|
// writer from clearing that writer's invalidation. TTL bounds a crashed writer
|
|
// which never reaches its second invalidation; completion never trusts cache.
|
|
func (store *MongoStore) invalidateObjectSyncProgress(ctx context.Context, items []domain.ObjectSyncItemRecord) error {
|
|
now := time.Now().UTC()
|
|
pages := make(map[string]domain.ObjectSyncItemRecord)
|
|
for _, item := range items {
|
|
pages[progressPageID(item.SessionID, item.ManifestPage)] = item
|
|
}
|
|
writes := make([]mongo.WriteModel, 0, len(pages))
|
|
for id, item := range pages {
|
|
writes = append(writes, mongo.NewUpdateOneModel().SetFilter(bson.M{"_id": id}).SetUpdate(bson.M{
|
|
"$inc": bson.M{"revision": int64(1)},
|
|
"$set": bson.M{"dirty": true, "purgeAt": now.Add(objectSyncProgressRetention)},
|
|
"$setOnInsert": bson.M{"sessionId": item.SessionID, "manifestPage": item.ManifestPage, "organizationId": item.OrganizationID},
|
|
}).SetUpsert(true))
|
|
}
|
|
if len(writes) == 0 {
|
|
return nil
|
|
}
|
|
_, err := store.objectSyncProgress.BulkWrite(ctx, writes, options.BulkWrite().SetOrdered(false))
|
|
return err
|
|
}
|
|
|
|
func (store *MongoStore) readObjectSyncProgressPages(ctx context.Context, ids []string) ([]objectSyncProgressPage, error) {
|
|
cursor, err := store.objectSyncProgress.Find(ctx, bson.M{"sessionId": bson.M{"$in": ids}})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return decodeMany[objectSyncProgressPage](ctx, cursor)
|
|
}
|
|
|
|
// A sentinel (page -1) records that all existing manifest pages were discovered.
|
|
// Refreshing it once per minute also discovers out-of-band rows and recovers
|
|
// from interrupted cache initialization without relying on old wire formats.
|
|
func (store *MongoStore) bootstrapObjectSyncProgress(ctx context.Context, sessionIDs []string, now time.Time) error {
|
|
cursor, err := store.objectSyncItems.Aggregate(ctx, mongo.Pipeline{
|
|
{{Key: "$match", Value: bson.M{"sessionId": bson.M{"$in": sessionIDs}}}},
|
|
{{Key: "$project", Value: bson.M{"_id": 0, "sessionId": 1, "manifestPage": 1}}},
|
|
{{Key: "$group", Value: bson.M{"_id": bson.M{"sessionId": "$sessionId", "manifestPage": "$manifestPage"}}}},
|
|
}, options.Aggregate().SetHint("sessionId_page_status_action"))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer cursor.Close(ctx)
|
|
identities := make(map[string]string, len(sessionIDs))
|
|
for _, id := range sessionIDs {
|
|
session, err := store.GetObjectSyncSession(ctx, id)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if session != nil {
|
|
identities[id] = session.OrganizationID
|
|
}
|
|
}
|
|
writes := make([]mongo.WriteModel, 0)
|
|
for cursor.Next(ctx) {
|
|
var row struct {
|
|
Key struct {
|
|
SessionID string `bson:"sessionId"`
|
|
ManifestPage int `bson:"manifestPage"`
|
|
} `bson:"_id"`
|
|
}
|
|
if err := cursor.Decode(&row); err != nil {
|
|
return err
|
|
}
|
|
page := row.Key
|
|
writes = append(writes, mongo.NewUpdateOneModel().SetFilter(bson.M{"_id": progressPageID(page.SessionID, page.ManifestPage)}).SetUpdate(bson.M{"$setOnInsert": bson.M{
|
|
"sessionId": page.SessionID, "manifestPage": page.ManifestPage, "revision": int64(0), "dirty": true, "organizationId": identities[page.SessionID], "purgeAt": now.Add(objectSyncProgressRetention),
|
|
}}).SetUpsert(true))
|
|
}
|
|
if err := cursor.Err(); err != nil {
|
|
return err
|
|
}
|
|
if len(writes) > 0 {
|
|
if _, err := store.objectSyncProgress.BulkWrite(ctx, writes, options.BulkWrite().SetOrdered(false)); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
// Write sentinels only after every discovered page exists; a crash earlier
|
|
// leaves initialization incomplete and the next poll repeats discovery.
|
|
writes = writes[:0]
|
|
for _, id := range sessionIDs {
|
|
writes = append(writes, mongo.NewUpdateOneModel().SetFilter(bson.M{"_id": progressPageID(id, -1)}).SetUpdate(bson.M{
|
|
"$set": bson.M{"validUntil": now.Add(objectSyncProgressTTL), "purgeAt": now.Add(objectSyncProgressRetention)},
|
|
"$setOnInsert": bson.M{"sessionId": id, "manifestPage": -1, "revision": int64(0), "dirty": false, "organizationId": identities[id]},
|
|
}).SetUpsert(true))
|
|
}
|
|
_, err = store.objectSyncProgress.BulkWrite(ctx, writes, options.BulkWrite().SetOrdered(false))
|
|
return err
|
|
}
|
|
|
|
func mergeObjectSyncSummary(a, b domain.ObjectSyncItemSummary) domain.ObjectSyncItemSummary {
|
|
a.TotalItems += b.TotalItems
|
|
a.TerminalItems += b.TerminalItems
|
|
a.Counts.Manifest += b.Counts.Manifest
|
|
a.Counts.Create += b.Counts.Create
|
|
a.Counts.Update += b.Counts.Update
|
|
a.Counts.KeptBoth += b.Counts.KeptBoth
|
|
a.Counts.Skipped += b.Counts.Skipped
|
|
a.Counts.Denied += b.Counts.Denied
|
|
a.Counts.Failed += b.Counts.Failed
|
|
a.Counts.Committed += b.Counts.Committed
|
|
a.Counts.Unchanged += b.Counts.Unchanged
|
|
return a
|
|
}
|
|
|
|
func progressPageFresh(page objectSyncProgressPage, now time.Time) bool {
|
|
return !page.Dirty && page.ValidUntil.After(now)
|
|
}
|
|
|
|
func (store *MongoStore) SummarizeObjectSyncItems(ctx context.Context, sessionIDs []string) (map[string]domain.ObjectSyncItemSummary, error) {
|
|
result := emptyObjectSyncSummaries(sessionIDs)
|
|
if len(result) == 0 {
|
|
return result, nil
|
|
}
|
|
now := time.Now().UTC()
|
|
pages, err := store.readObjectSyncProgressPages(ctx, sessionIDs)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
initialized := make(map[string]bool)
|
|
for _, page := range pages {
|
|
if page.ManifestPage == -1 && progressPageFresh(page, now) {
|
|
initialized[page.SessionID] = true
|
|
}
|
|
}
|
|
missing := make([]string, 0)
|
|
for id := range result {
|
|
if !initialized[id] {
|
|
missing = append(missing, id)
|
|
}
|
|
}
|
|
if len(missing) > 0 {
|
|
if err := store.bootstrapObjectSyncProgress(ctx, missing, now); err != nil {
|
|
return nil, err
|
|
}
|
|
pages, err = store.readObjectSyncProgressPages(ctx, sessionIDs)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
dirty := make(map[string]objectSyncProgressPage)
|
|
filtersBySession := make(map[string][]int)
|
|
for _, page := range pages {
|
|
if page.ManifestPage < 0 {
|
|
continue
|
|
}
|
|
if progressPageFresh(page, now) {
|
|
result[page.SessionID] = mergeObjectSyncSummary(result[page.SessionID], page.Summary)
|
|
} else {
|
|
dirty[page.ID] = page
|
|
filtersBySession[page.SessionID] = append(filtersBySession[page.SessionID], page.ManifestPage)
|
|
}
|
|
}
|
|
if len(dirty) == 0 {
|
|
return result, nil
|
|
}
|
|
// One grouped query for dirty pages, not N round trips and not a full scan
|
|
// for every item transition. The covering index starts sessionId/page.
|
|
filters := make(bson.A, 0, len(filtersBySession))
|
|
sessions := make([]string, 0, len(filtersBySession))
|
|
for id := range filtersBySession {
|
|
sessions = append(sessions, id)
|
|
}
|
|
sort.Strings(sessions)
|
|
for _, id := range sessions {
|
|
filters = append(filters, bson.M{"sessionId": id, "manifestPage": bson.M{"$in": filtersBySession[id]}})
|
|
}
|
|
cursor, err := store.objectSyncItems.Aggregate(ctx, mongo.Pipeline{
|
|
{{Key: "$match", Value: bson.M{"$or": filters}}},
|
|
{{Key: "$project", Value: bson.M{"_id": 0, "sessionId": 1, "manifestPage": 1, "status": 1, "action": 1}}},
|
|
{{Key: "$group", Value: bson.M{"_id": bson.M{"sessionId": "$sessionId", "manifestPage": "$manifestPage", "status": "$status", "action": "$action"}, "count": bson.M{"$sum": 1}}}},
|
|
}, options.Aggregate().SetHint("sessionId_page_status_action"))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer cursor.Close(ctx)
|
|
summaries := make(map[string]domain.ObjectSyncItemSummary, len(dirty))
|
|
for cursor.Next(ctx) {
|
|
var group struct {
|
|
Key struct {
|
|
SessionID string `bson:"sessionId"`
|
|
ManifestPage int `bson:"manifestPage"`
|
|
Status domain.ObjectSyncItemStatus `bson:"status"`
|
|
Action string `bson:"action"`
|
|
} `bson:"_id"`
|
|
Count int64 `bson:"count"`
|
|
}
|
|
if err := cursor.Decode(&group); err != nil {
|
|
return nil, err
|
|
}
|
|
id := progressPageID(group.Key.SessionID, group.Key.ManifestPage)
|
|
summaries[id] = addObjectSyncItemCount(summaries[id], group.Key.Status, group.Key.Action, group.Count)
|
|
}
|
|
if err := cursor.Err(); err != nil {
|
|
return nil, err
|
|
}
|
|
writes := make([]mongo.WriteModel, 0, len(dirty))
|
|
for id, page := range dirty {
|
|
summary := summaries[id]
|
|
result[page.SessionID] = mergeObjectSyncSummary(result[page.SessionID], summary)
|
|
writes = append(writes, mongo.NewUpdateOneModel().SetFilter(bson.M{"_id": id, "revision": page.Revision}).SetUpdate(bson.M{"$set": bson.M{"dirty": false, "summary": summary, "validUntil": now.Add(objectSyncProgressTTL)}}))
|
|
}
|
|
refreshed, err := store.objectSyncProgress.BulkWrite(ctx, writes, options.BulkWrite().SetOrdered(false))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if refreshed.MatchedCount != int64(len(writes)) {
|
|
// A writer raced this refresh. Do not publish a possibly stale
|
|
// all-terminal hint; the next poll observes its dirty revision.
|
|
for id, summary := range result {
|
|
summary.TerminalItems = 0
|
|
result[id] = summary
|
|
}
|
|
}
|
|
return result, nil
|
|
}
|