droboticflow/internal/store/object_sync_progress_integration_test.go

170 lines
6.7 KiB
Go

package store
import (
"context"
"encoding/json"
"fmt"
"net/url"
"os"
"sort"
"strconv"
"sync/atomic"
"testing"
"time"
"droboticflow/internal/domain"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/event"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
)
// Opt-in, localhost-only capacity probe. It owns and drops only its unique test
// database. Run with DROSYNC_BENCHMARK_MONGO_URI=mongodb://127.0.0.1:28322.
func TestObjectSyncProgressRealMongo(t *testing.T) {
uri := os.Getenv("DROSYNC_BENCHMARK_MONGO_URI")
if uri == "" {
t.Skip("set DROSYNC_BENCHMARK_MONGO_URI for the local capacity probe")
}
u, err := url.Parse(uri)
if err != nil || (u.Hostname() != "localhost" && u.Hostname() != "127.0.0.1") || u.User != nil {
t.Fatal("benchmark requires unauthenticated localhost URI")
}
n := 100000
if raw := os.Getenv("DROSYNC_BENCHMARK_FILES"); raw != "" {
n, err = strconv.Atoi(raw)
if err != nil || n < 1000 || n > 1000000 {
t.Fatal("benchmark files must be 1000..1000000")
}
}
ctx, cancel := context.WithTimeout(t.Context(), 5*time.Minute)
defer cancel()
var itemAggregates atomic.Int64
client, err := mongo.Connect(ctx, options.Client().ApplyURI(uri).SetMonitor(&event.CommandMonitor{Started: func(_ context.Context, e *event.CommandStartedEvent) {
if e.CommandName == "aggregate" && e.Command.Lookup("aggregate").StringValue() == "objectSyncItems" {
itemAggregates.Add(1)
}
}}))
if err != nil {
t.Fatal(err)
}
defer client.Disconnect(context.Background())
db := client.Database(fmt.Sprintf("drosync_capacity_%d", time.Now().UnixNano()))
defer db.Drop(context.Background())
s := NewMongoStore(db)
if err = s.EnsureIndexes(ctx); err != nil {
t.Fatal(err)
}
session := domain.ObjectSyncSessionRecord{ID: "capacity", OrganizationID: "local-fixture", Status: domain.ObjectSyncSessionRunning, ManifestFinalPage: (n + 999) / 1000, Counts: domain.ObjectSyncSessionCounts{Manifest: int64(n)}}
if err = s.UpsertObjectSyncSession(ctx, session); err != nil {
t.Fatal(err)
}
for start := 0; start < n; start += 1000 {
rows := make([]any, 0, 1000)
for i := start; i < start+1000 && i < n; i++ {
status := domain.ObjectSyncItemCommitting
if i%4 != 0 {
status = domain.ObjectSyncItemCommitted
}
rows = append(rows, domain.ObjectSyncItemRecord{ID: fmt.Sprintf("item-%09d", i), OrganizationID: session.OrganizationID, SessionID: session.ID, ClientItemID: fmt.Sprintf("client-%09d", i), ManifestPage: i/1000 + 1, RelativePath: fmt.Sprintf("2026_10_03/task/episode-%09d/data.mcap", i), TargetKey: fmt.Sprintf("capture/%09d/data.mcap", i), SizeBytes: 4096, MD5Sum: "aabbccddeeff0011223344556677889900aabbccddeeff0011223344556677889900", Action: "create", Status: status, PurgeAt: time.Now().Add(24 * time.Hour)})
}
if _, err = s.objectSyncItems.InsertMany(ctx, rows); err != nil {
t.Fatal(err)
}
}
sample := func(f func() error) []float64 {
v := make([]float64, 0, 5)
for i := 0; i < 5; i++ {
start := time.Now()
if err := f(); err != nil {
t.Fatal(err)
}
v = append(v, float64(time.Since(start).Nanoseconds())/1e6)
}
return v
}
exact := sample(func() error {
summary, e := s.ReconcileObjectSyncItems(ctx, []string{session.ID})
if e == nil && summary[session.ID].TotalItems != int64(n) {
t.Fatal("wrong authoritative total")
}
return e
})
coldStart := time.Now()
summary, err := s.SummarizeObjectSyncItems(ctx, []string{session.ID})
coldMS := float64(time.Since(coldStart).Nanoseconds()) / 1e6
if err != nil || summary[session.ID].TotalItems != int64(n) {
t.Fatalf("cold cache: %#v %v", summary, err)
}
before := itemAggregates.Load()
warm := sample(func() error { _, e := s.SummarizeObjectSyncItems(ctx, []string{session.ID}); return e })
warmItemAggregates := itemAggregates.Load() - before
if warmItemAggregates != 0 {
t.Fatalf("hot polls scanned items %d times", warmItemAggregates)
}
item, err := s.GetObjectSyncItem(ctx, "item-000000000")
if err != nil {
t.Fatal(err)
}
item.Status = domain.ObjectSyncItemCommitted
if err = s.UpsertObjectSyncItems(ctx, []domain.ObjectSyncItemRecord{*item}); err != nil {
t.Fatal(err)
}
dirtyStart := time.Now()
changed, err := s.SummarizeObjectSyncItems(ctx, []string{session.ID})
dirtyMS := float64(time.Since(dirtyStart).Nanoseconds()) / 1e6
if err != nil {
t.Fatal(err)
}
reconciled, err := s.ReconcileObjectSyncItems(ctx, []string{session.ID})
if err != nil {
t.Fatal(err)
}
if changed[session.ID] != reconciled[session.ID] {
t.Fatalf("dirty page mismatch: %#v %#v", changed, reconciled)
}
// An independent daemon's subsequent poll sees the same materialized summary.
other, err := NewMongoStore(db).SummarizeObjectSyncItems(ctx, []string{session.ID})
if err != nil || other[session.ID] != reconciled[session.ID] {
t.Fatalf("cross process cache mismatch: %#v %v", other, err)
}
var explain bson.M
pipeline := mongo.Pipeline{
{{Key: "$match", Value: bson.M{"$or": bson.A{bson.M{"sessionId": session.ID, "manifestPage": bson.M{"$in": []int{1}}}}}}},
{{Key: "$project", Value: bson.M{"_id": 0, "sessionId": 1, "manifestPage": 1, "status": 1, "action": 1}}},
{{Key: "$group", Value: bson.M{"_id": bson.M{"sessionId": "$sessionId", "manifestPage": "$manifestPage", "status": "$status", "action": "$action"}, "count": bson.M{"$sum": 1}}}},
}
if err = db.RunCommand(ctx, bson.D{{Key: "explain", Value: bson.D{{Key: "aggregate", Value: "objectSyncItems"}, {Key: "pipeline", Value: pipeline}, {Key: "cursor", Value: bson.M{}}, {Key: "hint", Value: "sessionId_page_status_action"}}}, {Key: "verbosity", Value: "executionStats"}}).Decode(&explain); err != nil {
t.Fatal(err)
}
var findStats func(any) bson.M
findStats = func(value any) bson.M {
switch v := value.(type) {
case bson.M:
if _, ok := v["totalKeysExamined"]; ok {
return v
}
for _, child := range v {
if found := findStats(child); found != nil {
return found
}
}
case bson.A:
for _, child := range v {
if found := findStats(child); found != nil {
return found
}
}
}
return nil
}
stats := findStats(explain)
if stats == nil {
t.Fatal("aggregate explain did not expose execution statistics")
}
median := func(v []float64) float64 { w := append([]float64(nil), v...); sort.Float64s(w); return w[len(w)/2] }
result := map[string]any{"records": n, "manifestPageSize": 1000, "authoritativeSamplesMs": exact, "authoritativeMedianMs": median(exact), "coldSummaryMs": coldMS, "warmSamplesMs": warm, "warmMedianMs": median(warm), "warmItemAggregations": warmItemAggregates, "oneDirtyPageMs": dirtyMS, "dirtyPageExplain": stats, "correctness": "cold, hot, dirty, and separate-store summaries match authoritative counts"}
encoded, _ := json.Marshal(result)
fmt.Printf("DROSYNC_PROGRESS_BENCHMARK_JSON=%s\n", encoded)
}