forked from longtao.wu/droboticflow
170 lines
6.7 KiB
Go
170 lines
6.7 KiB
Go
package store
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"net/url"
|
|
"os"
|
|
"sort"
|
|
"strconv"
|
|
"sync/atomic"
|
|
"testing"
|
|
"time"
|
|
|
|
"droboticflow/internal/domain"
|
|
"go.mongodb.org/mongo-driver/bson"
|
|
"go.mongodb.org/mongo-driver/event"
|
|
"go.mongodb.org/mongo-driver/mongo"
|
|
"go.mongodb.org/mongo-driver/mongo/options"
|
|
)
|
|
|
|
// Opt-in, localhost-only capacity probe. It owns and drops only its unique test
|
|
// database. Run with DROSYNC_BENCHMARK_MONGO_URI=mongodb://127.0.0.1:28322.
|
|
func TestObjectSyncProgressRealMongo(t *testing.T) {
|
|
uri := os.Getenv("DROSYNC_BENCHMARK_MONGO_URI")
|
|
if uri == "" {
|
|
t.Skip("set DROSYNC_BENCHMARK_MONGO_URI for the local capacity probe")
|
|
}
|
|
u, err := url.Parse(uri)
|
|
if err != nil || (u.Hostname() != "localhost" && u.Hostname() != "127.0.0.1") || u.User != nil {
|
|
t.Fatal("benchmark requires unauthenticated localhost URI")
|
|
}
|
|
n := 100000
|
|
if raw := os.Getenv("DROSYNC_BENCHMARK_FILES"); raw != "" {
|
|
n, err = strconv.Atoi(raw)
|
|
if err != nil || n < 1000 || n > 1000000 {
|
|
t.Fatal("benchmark files must be 1000..1000000")
|
|
}
|
|
}
|
|
ctx, cancel := context.WithTimeout(t.Context(), 5*time.Minute)
|
|
defer cancel()
|
|
var itemAggregates atomic.Int64
|
|
client, err := mongo.Connect(ctx, options.Client().ApplyURI(uri).SetMonitor(&event.CommandMonitor{Started: func(_ context.Context, e *event.CommandStartedEvent) {
|
|
if e.CommandName == "aggregate" && e.Command.Lookup("aggregate").StringValue() == "objectSyncItems" {
|
|
itemAggregates.Add(1)
|
|
}
|
|
}}))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer client.Disconnect(context.Background())
|
|
db := client.Database(fmt.Sprintf("drosync_capacity_%d", time.Now().UnixNano()))
|
|
defer db.Drop(context.Background())
|
|
s := NewMongoStore(db)
|
|
if err = s.EnsureIndexes(ctx); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
session := domain.ObjectSyncSessionRecord{ID: "capacity", OrganizationID: "local-fixture", Status: domain.ObjectSyncSessionRunning, ManifestFinalPage: (n + 999) / 1000, Counts: domain.ObjectSyncSessionCounts{Manifest: int64(n)}}
|
|
if err = s.UpsertObjectSyncSession(ctx, session); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for start := 0; start < n; start += 1000 {
|
|
rows := make([]any, 0, 1000)
|
|
for i := start; i < start+1000 && i < n; i++ {
|
|
status := domain.ObjectSyncItemCommitting
|
|
if i%4 != 0 {
|
|
status = domain.ObjectSyncItemCommitted
|
|
}
|
|
rows = append(rows, domain.ObjectSyncItemRecord{ID: fmt.Sprintf("item-%09d", i), OrganizationID: session.OrganizationID, SessionID: session.ID, ClientItemID: fmt.Sprintf("client-%09d", i), ManifestPage: i/1000 + 1, RelativePath: fmt.Sprintf("2026_10_03/task/episode-%09d/data.mcap", i), TargetKey: fmt.Sprintf("capture/%09d/data.mcap", i), SizeBytes: 4096, MD5Sum: "aabbccddeeff0011223344556677889900aabbccddeeff0011223344556677889900", Action: "create", Status: status, PurgeAt: time.Now().Add(24 * time.Hour)})
|
|
}
|
|
if _, err = s.objectSyncItems.InsertMany(ctx, rows); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
sample := func(f func() error) []float64 {
|
|
v := make([]float64, 0, 5)
|
|
for i := 0; i < 5; i++ {
|
|
start := time.Now()
|
|
if err := f(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
v = append(v, float64(time.Since(start).Nanoseconds())/1e6)
|
|
}
|
|
return v
|
|
}
|
|
exact := sample(func() error {
|
|
summary, e := s.ReconcileObjectSyncItems(ctx, []string{session.ID})
|
|
if e == nil && summary[session.ID].TotalItems != int64(n) {
|
|
t.Fatal("wrong authoritative total")
|
|
}
|
|
return e
|
|
})
|
|
coldStart := time.Now()
|
|
summary, err := s.SummarizeObjectSyncItems(ctx, []string{session.ID})
|
|
coldMS := float64(time.Since(coldStart).Nanoseconds()) / 1e6
|
|
if err != nil || summary[session.ID].TotalItems != int64(n) {
|
|
t.Fatalf("cold cache: %#v %v", summary, err)
|
|
}
|
|
before := itemAggregates.Load()
|
|
warm := sample(func() error { _, e := s.SummarizeObjectSyncItems(ctx, []string{session.ID}); return e })
|
|
warmItemAggregates := itemAggregates.Load() - before
|
|
if warmItemAggregates != 0 {
|
|
t.Fatalf("hot polls scanned items %d times", warmItemAggregates)
|
|
}
|
|
item, err := s.GetObjectSyncItem(ctx, "item-000000000")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
item.Status = domain.ObjectSyncItemCommitted
|
|
if err = s.UpsertObjectSyncItems(ctx, []domain.ObjectSyncItemRecord{*item}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
dirtyStart := time.Now()
|
|
changed, err := s.SummarizeObjectSyncItems(ctx, []string{session.ID})
|
|
dirtyMS := float64(time.Since(dirtyStart).Nanoseconds()) / 1e6
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
reconciled, err := s.ReconcileObjectSyncItems(ctx, []string{session.ID})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if changed[session.ID] != reconciled[session.ID] {
|
|
t.Fatalf("dirty page mismatch: %#v %#v", changed, reconciled)
|
|
}
|
|
// An independent daemon's subsequent poll sees the same materialized summary.
|
|
other, err := NewMongoStore(db).SummarizeObjectSyncItems(ctx, []string{session.ID})
|
|
if err != nil || other[session.ID] != reconciled[session.ID] {
|
|
t.Fatalf("cross process cache mismatch: %#v %v", other, err)
|
|
}
|
|
var explain bson.M
|
|
pipeline := mongo.Pipeline{
|
|
{{Key: "$match", Value: bson.M{"$or": bson.A{bson.M{"sessionId": session.ID, "manifestPage": bson.M{"$in": []int{1}}}}}}},
|
|
{{Key: "$project", Value: bson.M{"_id": 0, "sessionId": 1, "manifestPage": 1, "status": 1, "action": 1}}},
|
|
{{Key: "$group", Value: bson.M{"_id": bson.M{"sessionId": "$sessionId", "manifestPage": "$manifestPage", "status": "$status", "action": "$action"}, "count": bson.M{"$sum": 1}}}},
|
|
}
|
|
if err = db.RunCommand(ctx, bson.D{{Key: "explain", Value: bson.D{{Key: "aggregate", Value: "objectSyncItems"}, {Key: "pipeline", Value: pipeline}, {Key: "cursor", Value: bson.M{}}, {Key: "hint", Value: "sessionId_page_status_action"}}}, {Key: "verbosity", Value: "executionStats"}}).Decode(&explain); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var findStats func(any) bson.M
|
|
findStats = func(value any) bson.M {
|
|
switch v := value.(type) {
|
|
case bson.M:
|
|
if _, ok := v["totalKeysExamined"]; ok {
|
|
return v
|
|
}
|
|
for _, child := range v {
|
|
if found := findStats(child); found != nil {
|
|
return found
|
|
}
|
|
}
|
|
case bson.A:
|
|
for _, child := range v {
|
|
if found := findStats(child); found != nil {
|
|
return found
|
|
}
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
stats := findStats(explain)
|
|
if stats == nil {
|
|
t.Fatal("aggregate explain did not expose execution statistics")
|
|
}
|
|
median := func(v []float64) float64 { w := append([]float64(nil), v...); sort.Float64s(w); return w[len(w)/2] }
|
|
result := map[string]any{"records": n, "manifestPageSize": 1000, "authoritativeSamplesMs": exact, "authoritativeMedianMs": median(exact), "coldSummaryMs": coldMS, "warmSamplesMs": warm, "warmMedianMs": median(warm), "warmItemAggregations": warmItemAggregates, "oneDirtyPageMs": dirtyMS, "dirtyPageExplain": stats, "correctness": "cold, hot, dirty, and separate-store summaries match authoritative counts"}
|
|
encoded, _ := json.Marshal(result)
|
|
fmt.Printf("DROSYNC_PROGRESS_BENCHMARK_JSON=%s\n", encoded)
|
|
}
|