forked from longtao.wu/droboticflow
135 lines
4.4 KiB
Go
135 lines
4.4 KiB
Go
package api
|
|
|
|
import (
|
|
"context"
|
|
"crypto/rand"
|
|
"encoding/json"
|
|
"os"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"droboticflow/internal/domain"
|
|
"droboticflow/internal/testsupport"
|
|
"go.mongodb.org/mongo-driver/bson"
|
|
"go.mongodb.org/mongo-driver/mongo"
|
|
"go.mongodb.org/mongo-driver/mongo/options"
|
|
"go.mongodb.org/mongo-driver/mongo/writeconcern"
|
|
)
|
|
|
|
// A bounded diagnostic, not HTTP capacity evidence. The outer runner retains
|
|
// the existing node/production health guards. Never accepts a production URI.
|
|
func TestQueryReplicaColdWriteProbeLocal(t *testing.T) {
|
|
if os.Getenv("QUERY_REPLICA_COLD_WRITE_PROBE") != "1" {
|
|
t.Skip("explicit isolated cold-write diagnostic only")
|
|
}
|
|
if os.Getenv("QUERY_REPLICA_CLUSTER_TEST") != "1" {
|
|
t.Fatal("owned synthetic cluster required")
|
|
}
|
|
pace := isolatedComparisonInt(t, "QUERY_COLD_PACE_KIB", 0, 0, 128)
|
|
if pace != 0 && pace != 128 {
|
|
t.Fatal("only unpaced / 128KiB per second diagnostics")
|
|
}
|
|
ctx, cancel := context.WithTimeout(t.Context(), 3*time.Minute)
|
|
defer cancel()
|
|
c, err := mongo.Connect(ctx, options.Client().ApplyURI(testsupport.QueryReplicaTargetURI(t)).SetMaxPoolSize(6).SetWriteConcern(writeconcern.New(writeconcern.WMajority(), writeconcern.J(true))))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer c.Disconnect(context.Background())
|
|
var marker bson.M
|
|
if err = c.Database("query25_synthetic").Collection("marker").FindOne(ctx, bson.M{"_id": "query25-confirmed"}).Decode(&marker); err != nil || marker["value"] != "preserve-query25-v1" {
|
|
t.Fatal("owned target marker absent", err)
|
|
}
|
|
db := c.Database(domain.NewID("cold_write_synthetic"))
|
|
defer func() {
|
|
cleanup, stop := context.WithTimeout(context.Background(), 30*time.Second)
|
|
defer stop()
|
|
if e := db.Drop(cleanup); e != nil {
|
|
t.Error("synthetic cleanup", e)
|
|
}
|
|
}()
|
|
if _, err = db.Collection("control").InsertOne(ctx, bson.M{"_id": "heartbeat", "at": time.Now()}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var mu sync.Mutex
|
|
samples := []bson.M{}
|
|
record := func(v bson.M) { mu.Lock(); samples = append(samples, v); mu.Unlock() }
|
|
status := func() {
|
|
short, stop := context.WithTimeout(ctx, 3*time.Second)
|
|
defer stop()
|
|
var s bson.M
|
|
if e := c.Database("admin").RunCommand(short, bson.M{"serverStatus": 1}).Decode(&s); e == nil {
|
|
wt, _ := s["wiredTiger"].(bson.M)
|
|
record(bson.M{"kind": "storage", "at": time.Now().UTC(), "log": wt["log"], "checkpoint": wt["checkpoint"], "cache": wt["cache"]})
|
|
}
|
|
}
|
|
status()
|
|
stopHeartbeat := make(chan struct{})
|
|
done := make(chan struct{})
|
|
go func() {
|
|
defer close(done)
|
|
tick := time.NewTicker(time.Second)
|
|
defer tick.Stop()
|
|
for {
|
|
select {
|
|
case <-stopHeartbeat:
|
|
return
|
|
case <-ctx.Done():
|
|
return
|
|
case <-tick.C:
|
|
short, stop := context.WithTimeout(ctx, 3*time.Second)
|
|
start := time.Now()
|
|
_, e := db.Collection("control").UpdateOne(short, bson.M{"_id": "heartbeat"}, bson.M{"$set": bson.M{"at": time.Now()}})
|
|
stop()
|
|
record(bson.M{"kind": "heartbeat", "at": time.Now().UTC(), "ms": float64(time.Since(start).Microseconds()) / 1000, "failed": e != nil})
|
|
status()
|
|
}
|
|
}
|
|
}()
|
|
var stopOnce sync.Once
|
|
stop := func() { stopOnce.Do(func() { close(stopHeartbeat); <-done }) }
|
|
defer stop()
|
|
started := time.Now()
|
|
completed := 0
|
|
for batch := 0; batch < 32 && ctx.Err() == nil; batch++ {
|
|
rows := []any{}
|
|
for i := 0; i < 8; i++ {
|
|
payload := make([]byte, 32768)
|
|
if _, e := rand.Read(payload); e != nil {
|
|
t.Fatal(e)
|
|
}
|
|
rows = append(rows, bson.M{"_id": batch*8 + i, "payload": payload})
|
|
}
|
|
start := time.Now()
|
|
_, e := db.Collection("facts").InsertMany(ctx, rows)
|
|
record(bson.M{"kind": "batch", "at": time.Now().UTC(), "ms": float64(time.Since(start).Microseconds()) / 1000, "failed": e != nil})
|
|
if e != nil {
|
|
break
|
|
}
|
|
completed++
|
|
if pace > 0 {
|
|
due := started.Add(time.Duration(completed) * 2 * time.Second)
|
|
if wait := time.Until(due); wait > 0 {
|
|
select {
|
|
case <-ctx.Done():
|
|
case <-time.After(wait):
|
|
}
|
|
}
|
|
}
|
|
}
|
|
stop()
|
|
status()
|
|
report := bson.M{"kind": "cold-write-probe", "paceKiB": pace, "payloadBytes": 8 << 20, "completedBatches": completed, "majority": true, "journal": true, "heartbeatTimeoutMs": 3000, "seconds": time.Since(started).Seconds(), "samples": samples, "capacityAccepted": false}
|
|
raw, err := json.MarshalIndent(report, "", " ")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err = os.WriteFile(os.Getenv("QUERY_COMPARE_REPORT"), raw, 0600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if completed != 32 {
|
|
t.Fatal("bounded diagnostic incomplete", completed, ctx.Err())
|
|
}
|
|
}
|