droboticflow/internal/store/object_sync_progress.go

351 lines
13 KiB
Go

package store
import (
"context"
"fmt"
"sort"
"time"
"droboticflow/internal/domain"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
)
func emptyObjectSyncSummaries(sessionIDs []string) map[string]domain.ObjectSyncItemSummary {
result := make(map[string]domain.ObjectSyncItemSummary, len(sessionIDs))
for _, id := range sessionIDs {
result[id] = domain.ObjectSyncItemSummary{}
}
return result
}
func addObjectSyncItemCount(summary domain.ObjectSyncItemSummary, status domain.ObjectSyncItemStatus, action string, count int64) domain.ObjectSyncItemSummary {
summary.TotalItems += count
summary.Counts.Manifest += count
switch status {
case domain.ObjectSyncItemCommitted:
summary.Counts.Committed += count
case domain.ObjectSyncItemKeptBoth:
summary.Counts.Committed += count
summary.Counts.KeptBoth += count
case domain.ObjectSyncItemUnchanged:
summary.Counts.Unchanged += count
case domain.ObjectSyncItemSkipped:
summary.Counts.Skipped += count
case domain.ObjectSyncItemDenied:
summary.Counts.Denied += count
case domain.ObjectSyncItemFailed, domain.ObjectSyncItemAborted, domain.ObjectSyncItemExpired:
summary.Counts.Failed += count
}
switch status {
case domain.ObjectSyncItemCommitted, domain.ObjectSyncItemKeptBoth,
domain.ObjectSyncItemUnchanged, domain.ObjectSyncItemSkipped, domain.ObjectSyncItemDenied,
domain.ObjectSyncItemFailed, domain.ObjectSyncItemAborted, domain.ObjectSyncItemExpired:
summary.TerminalItems += count
}
switch action {
case "create":
summary.Counts.Create += count
case "update":
summary.Counts.Update += count
}
return summary
}
func (store *MemoryStore) SummarizeObjectSyncItems(_ context.Context, sessionIDs []string) (map[string]domain.ObjectSyncItemSummary, error) {
result := emptyObjectSyncSummaries(sessionIDs)
if len(result) == 0 {
return result, nil
}
store.mu.Lock()
defer store.mu.Unlock()
for _, item := range store.objectSyncItems {
if summary, ok := result[item.SessionID]; ok {
result[item.SessionID] = addObjectSyncItemCount(summary, item.Status, item.Action, 1)
}
}
return result, nil
}
func (store *MongoStore) ReconcileObjectSyncItems(ctx context.Context, sessionIDs []string) (map[string]domain.ObjectSyncItemSummary, error) {
result := emptyObjectSyncSummaries(sessionIDs)
if len(result) == 0 {
return result, nil
}
// Only state and action are needed. Paths, checksums and multipart records
// never leave MongoDB while polling progress.
cursor, err := store.objectSyncItems.Aggregate(ctx, mongo.Pipeline{
{{Key: "$match", Value: bson.M{"sessionId": bson.M{"$in": sessionIDs}}}},
{{Key: "$project", Value: bson.M{"_id": 0, "sessionId": 1, "status": 1, "action": 1}}},
{{Key: "$group", Value: bson.M{
"_id": bson.M{"sessionId": "$sessionId", "status": "$status", "action": "$action"},
"count": bson.M{"$sum": 1},
}}},
})
if err != nil {
return nil, err
}
defer cursor.Close(ctx)
for cursor.Next(ctx) {
var group struct {
Key struct {
SessionID string `bson:"sessionId"`
Status domain.ObjectSyncItemStatus `bson:"status"`
Action string `bson:"action"`
} `bson:"_id"`
Count int64 `bson:"count"`
}
if err := cursor.Decode(&group); err != nil {
return nil, err
}
result[group.Key.SessionID] = addObjectSyncItemCount(result[group.Key.SessionID], group.Key.Status, group.Key.Action, group.Count)
}
if err := cursor.Err(); err != nil {
return nil, err
}
return result, nil
}
// Reconciliation is authoritative and must be used before marking a session
// complete. The progress cache below is a disposable polling optimization.
func (store *MemoryStore) ReconcileObjectSyncItems(ctx context.Context, sessionIDs []string) (map[string]domain.ObjectSyncItemSummary, error) {
return store.SummarizeObjectSyncItems(ctx, sessionIDs)
}
const objectSyncProgressTTL = 60 * time.Second
const objectSyncProgressRetention = 30 * 24 * time.Hour
type objectSyncProgressPage struct {
ID string `bson:"_id"`
SessionID string `bson:"sessionId"`
ManifestPage int `bson:"manifestPage"`
Revision int64 `bson:"revision"`
Dirty bool `bson:"dirty"`
ValidUntil time.Time `bson:"validUntil"`
Summary domain.ObjectSyncItemSummary `bson:"summary"`
OrganizationID string `bson:"organizationId"`
PurgeAt time.Time `bson:"purgeAt"`
}
func progressPageID(sessionID string, page int) string { return fmt.Sprintf("%s:%d", sessionID, page) }
// Mark before AND after writes. The revision CAS prevents a refresh racing a
// writer from clearing that writer's invalidation. TTL bounds a crashed writer
// which never reaches its second invalidation; completion never trusts cache.
func (store *MongoStore) invalidateObjectSyncProgress(ctx context.Context, items []domain.ObjectSyncItemRecord) error {
now := time.Now().UTC()
pages := make(map[string]domain.ObjectSyncItemRecord)
for _, item := range items {
pages[progressPageID(item.SessionID, item.ManifestPage)] = item
}
writes := make([]mongo.WriteModel, 0, len(pages))
for id, item := range pages {
writes = append(writes, mongo.NewUpdateOneModel().SetFilter(bson.M{"_id": id}).SetUpdate(bson.M{
"$inc": bson.M{"revision": int64(1)},
"$set": bson.M{"dirty": true, "purgeAt": now.Add(objectSyncProgressRetention)},
"$setOnInsert": bson.M{"sessionId": item.SessionID, "manifestPage": item.ManifestPage, "organizationId": item.OrganizationID},
}).SetUpsert(true))
}
if len(writes) == 0 {
return nil
}
_, err := store.objectSyncProgress.BulkWrite(ctx, writes, options.BulkWrite().SetOrdered(false))
return err
}
func (store *MongoStore) readObjectSyncProgressPages(ctx context.Context, ids []string) ([]objectSyncProgressPage, error) {
cursor, err := store.objectSyncProgress.Find(ctx, bson.M{"sessionId": bson.M{"$in": ids}})
if err != nil {
return nil, err
}
return decodeMany[objectSyncProgressPage](ctx, cursor)
}
// A sentinel (page -1) records that all existing manifest pages were discovered.
// Refreshing it once per minute also discovers out-of-band rows and recovers
// from interrupted cache initialization without relying on old wire formats.
func (store *MongoStore) bootstrapObjectSyncProgress(ctx context.Context, sessionIDs []string, now time.Time) error {
cursor, err := store.objectSyncItems.Aggregate(ctx, mongo.Pipeline{
{{Key: "$match", Value: bson.M{"sessionId": bson.M{"$in": sessionIDs}}}},
{{Key: "$project", Value: bson.M{"_id": 0, "sessionId": 1, "manifestPage": 1}}},
{{Key: "$group", Value: bson.M{"_id": bson.M{"sessionId": "$sessionId", "manifestPage": "$manifestPage"}}}},
}, options.Aggregate().SetHint("sessionId_page_status_action"))
if err != nil {
return err
}
defer cursor.Close(ctx)
identities := make(map[string]string, len(sessionIDs))
for _, id := range sessionIDs {
session, err := store.GetObjectSyncSession(ctx, id)
if err != nil {
return err
}
if session != nil {
identities[id] = session.OrganizationID
}
}
writes := make([]mongo.WriteModel, 0)
for cursor.Next(ctx) {
var row struct {
Key struct {
SessionID string `bson:"sessionId"`
ManifestPage int `bson:"manifestPage"`
} `bson:"_id"`
}
if err := cursor.Decode(&row); err != nil {
return err
}
page := row.Key
writes = append(writes, mongo.NewUpdateOneModel().SetFilter(bson.M{"_id": progressPageID(page.SessionID, page.ManifestPage)}).SetUpdate(bson.M{"$setOnInsert": bson.M{
"sessionId": page.SessionID, "manifestPage": page.ManifestPage, "revision": int64(0), "dirty": true, "organizationId": identities[page.SessionID], "purgeAt": now.Add(objectSyncProgressRetention),
}}).SetUpsert(true))
}
if err := cursor.Err(); err != nil {
return err
}
if len(writes) > 0 {
if _, err := store.objectSyncProgress.BulkWrite(ctx, writes, options.BulkWrite().SetOrdered(false)); err != nil {
return err
}
}
// Write sentinels only after every discovered page exists; a crash earlier
// leaves initialization incomplete and the next poll repeats discovery.
writes = writes[:0]
for _, id := range sessionIDs {
writes = append(writes, mongo.NewUpdateOneModel().SetFilter(bson.M{"_id": progressPageID(id, -1)}).SetUpdate(bson.M{
"$set": bson.M{"validUntil": now.Add(objectSyncProgressTTL), "purgeAt": now.Add(objectSyncProgressRetention)},
"$setOnInsert": bson.M{"sessionId": id, "manifestPage": -1, "revision": int64(0), "dirty": false, "organizationId": identities[id]},
}).SetUpsert(true))
}
_, err = store.objectSyncProgress.BulkWrite(ctx, writes, options.BulkWrite().SetOrdered(false))
return err
}
func mergeObjectSyncSummary(a, b domain.ObjectSyncItemSummary) domain.ObjectSyncItemSummary {
a.TotalItems += b.TotalItems
a.TerminalItems += b.TerminalItems
a.Counts.Manifest += b.Counts.Manifest
a.Counts.Create += b.Counts.Create
a.Counts.Update += b.Counts.Update
a.Counts.KeptBoth += b.Counts.KeptBoth
a.Counts.Skipped += b.Counts.Skipped
a.Counts.Denied += b.Counts.Denied
a.Counts.Failed += b.Counts.Failed
a.Counts.Committed += b.Counts.Committed
a.Counts.Unchanged += b.Counts.Unchanged
return a
}
func progressPageFresh(page objectSyncProgressPage, now time.Time) bool {
return !page.Dirty && page.ValidUntil.After(now)
}
func (store *MongoStore) SummarizeObjectSyncItems(ctx context.Context, sessionIDs []string) (map[string]domain.ObjectSyncItemSummary, error) {
result := emptyObjectSyncSummaries(sessionIDs)
if len(result) == 0 {
return result, nil
}
now := time.Now().UTC()
pages, err := store.readObjectSyncProgressPages(ctx, sessionIDs)
if err != nil {
return nil, err
}
initialized := make(map[string]bool)
for _, page := range pages {
if page.ManifestPage == -1 && progressPageFresh(page, now) {
initialized[page.SessionID] = true
}
}
missing := make([]string, 0)
for id := range result {
if !initialized[id] {
missing = append(missing, id)
}
}
if len(missing) > 0 {
if err := store.bootstrapObjectSyncProgress(ctx, missing, now); err != nil {
return nil, err
}
pages, err = store.readObjectSyncProgressPages(ctx, sessionIDs)
if err != nil {
return nil, err
}
}
dirty := make(map[string]objectSyncProgressPage)
filtersBySession := make(map[string][]int)
for _, page := range pages {
if page.ManifestPage < 0 {
continue
}
if progressPageFresh(page, now) {
result[page.SessionID] = mergeObjectSyncSummary(result[page.SessionID], page.Summary)
} else {
dirty[page.ID] = page
filtersBySession[page.SessionID] = append(filtersBySession[page.SessionID], page.ManifestPage)
}
}
if len(dirty) == 0 {
return result, nil
}
// One grouped query for dirty pages, not N round trips and not a full scan
// for every item transition. The covering index starts sessionId/page.
filters := make(bson.A, 0, len(filtersBySession))
sessions := make([]string, 0, len(filtersBySession))
for id := range filtersBySession {
sessions = append(sessions, id)
}
sort.Strings(sessions)
for _, id := range sessions {
filters = append(filters, bson.M{"sessionId": id, "manifestPage": bson.M{"$in": filtersBySession[id]}})
}
cursor, err := store.objectSyncItems.Aggregate(ctx, mongo.Pipeline{
{{Key: "$match", Value: bson.M{"$or": filters}}},
{{Key: "$project", Value: bson.M{"_id": 0, "sessionId": 1, "manifestPage": 1, "status": 1, "action": 1}}},
{{Key: "$group", Value: bson.M{"_id": bson.M{"sessionId": "$sessionId", "manifestPage": "$manifestPage", "status": "$status", "action": "$action"}, "count": bson.M{"$sum": 1}}}},
}, options.Aggregate().SetHint("sessionId_page_status_action"))
if err != nil {
return nil, err
}
defer cursor.Close(ctx)
summaries := make(map[string]domain.ObjectSyncItemSummary, len(dirty))
for cursor.Next(ctx) {
var group struct {
Key struct {
SessionID string `bson:"sessionId"`
ManifestPage int `bson:"manifestPage"`
Status domain.ObjectSyncItemStatus `bson:"status"`
Action string `bson:"action"`
} `bson:"_id"`
Count int64 `bson:"count"`
}
if err := cursor.Decode(&group); err != nil {
return nil, err
}
id := progressPageID(group.Key.SessionID, group.Key.ManifestPage)
summaries[id] = addObjectSyncItemCount(summaries[id], group.Key.Status, group.Key.Action, group.Count)
}
if err := cursor.Err(); err != nil {
return nil, err
}
writes := make([]mongo.WriteModel, 0, len(dirty))
for id, page := range dirty {
summary := summaries[id]
result[page.SessionID] = mergeObjectSyncSummary(result[page.SessionID], summary)
writes = append(writes, mongo.NewUpdateOneModel().SetFilter(bson.M{"_id": id, "revision": page.Revision}).SetUpdate(bson.M{"$set": bson.M{"dirty": false, "summary": summary, "validUntil": now.Add(objectSyncProgressTTL)}}))
}
refreshed, err := store.objectSyncProgress.BulkWrite(ctx, writes, options.BulkWrite().SetOrdered(false))
if err != nil {
return nil, err
}
if refreshed.MatchedCount != int64(len(writes)) {
// A writer raced this refresh. Do not publish a possibly stale
// all-terminal hint; the next poll observes its dirty revision.
for id, summary := range result {
summary.TerminalItems = 0
result[id] = summary
}
}
return result, nil
}