droboticflow/internal/api/query_replica_isolated_test.go
d-robotics fc7786190f
Some checks are pending
web-ci / checks (push) Waiting to run
web-ci / pool-search-mongo (push) Waiting to run
web-ci / work-availability (push) Blocked by required conditions
web-ci / query-serving (push) Blocked by required conditions
⚡ Publish query reports and coalesce task pool reads
2026-11-13 22:31:44 -08:00

1024 lines
38 KiB
Go

package api
import (
"context"
"crypto/sha256"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"net/http/httptest"
"os"
"os/exec"
"reflect"
"runtime"
"sort"
"strconv"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
"droboticflow/internal/domain"
"droboticflow/internal/productionstats"
"droboticflow/internal/queryserving"
"droboticflow/internal/store"
"droboticflow/internal/testsupport"
valkey "github.com/valkey-io/valkey-go"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/event"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
"go.mongodb.org/mongo-driver/mongo/readconcern"
"go.mongodb.org/mongo-driver/mongo/readpref"
)
// This harness has no production URI input. Both DBs are newly named synthetic
// fixtures, and the target must carry the independently verified node4 marker.
// HTTP uses the actual session, ACL and count/report handlers. Business recovery
// schedulers are deliberately not duplicated in the two benchmark handlers.
func TestQueryReplicaIsolatedHTTPComparisonLocal(t *testing.T) {
local := os.Getenv("QUERY_PERIODIC_HTTP_LOCAL") == "1"
if os.Getenv("QUERY_REPLICA_ISOLATED_E2E") != "1" && !local {
t.Skip("explicit node4 synthetic comparison only")
}
if local {
verifyPeriodicLocalHTTPFixture(t)
} else if os.Getenv("QUERY_REPLICA_CLUSTER_TEST") != "1" {
t.Fatal("cluster synthetic target must be explicitly selected")
}
t.Setenv("ANNOTATION_SOURCE_VALIDATION_MODE", "advisory")
t.Setenv("QUERY_SERVING_NAMESPACE", "isolated-http-comparison")
published := os.Getenv("QUERY_PUBLISHED_RESULTS") == "1"
cacheURL := os.Getenv("QUERY_COMPARE_VALKEY_URL")
if published && cacheURL == "" {
t.Fatal("published result comparison requires explicit isolated Valkey")
}
t.Setenv("QUERY_SERVING_READS", "1")
t.Setenv("QUERY_VALKEY_URL", cacheURL)
ctx := t.Context()
n := isolatedComparisonInt(t, "QUERY_COMPARE_TASKS", 26218, 20, 30000)
waves := isolatedComparisonInt(t, "QUERY_COMPARE_WAVES", 2, 1, 100)
maxActors := isolatedComparisonInt(t, "QUERY_COMPARE_MAX_ACTORS", 81, 81, 500)
maxInflight := isolatedComparisonInt(t, "QUERY_COMPARE_CPU_MAX_INFLIGHT", 0, 0, 8)
if maxInflight > 0 && !local {
t.Fatal("supplemental matched CPU comparison is local-only")
}
minPool := isolatedComparisonInt(t, "QUERY_COMPARE_MIN_POOL", 0, 0, 100)
compression := os.Getenv("QUERY_COMPARE_WIRE_COMPRESSION")
if compression != "" && compression != "zstd" {
t.Fatal("unsupported comparison compression")
}
var mu sync.Mutex
var phase string
inflight := map[int64]string{}
metrics := map[string]*isolatedCommands{}
var targetStatsMu sync.Mutex
var factTrace []map[string]any
factPending := map[int64]int{}
if path := os.Getenv("QUERY_COMPARE_DIAGNOSTIC_REPORT"); local && path != "" {
defer func() {
targetStatsMu.Lock()
defer targetStatsMu.Unlock()
raw, _ := json.MarshalIndent(factTrace, "", " ")
if err := os.WriteFile(path, raw, 0600); err != nil {
t.Error(err)
}
}()
}
var workforceAggregations atomic.Int64
targetCommands := map[string]int64{}
targetMaxMillis := map[string]float64{}
targetMonitor := &event.CommandMonitor{Started: func(ctx context.Context, e *event.CommandStartedEvent) {
name, _ := e.Command.Lookup("aggregate").StringValueOK()
if name == "queryWorkforceFacts" {
workforceAggregations.Add(1)
}
if !local {
return
}
if name != "queryWorkforceFacts" {
return
}
targetStatsMu.Lock()
defer targetStatsMu.Unlock()
if len(factTrace) >= 512 {
return
}
maxTime, present := e.Command.Lookup("maxTimeMS").AsInt64OK()
remaining := int64(-1)
if deadline, ok := ctx.Deadline(); ok {
remaining = time.Until(deadline).Milliseconds()
}
factPending[e.RequestID] = len(factTrace)
factTrace = append(factTrace, map[string]any{"appTimeUTC": time.Now().UTC().Format(time.RFC3339Nano), "wireMaxTimeMS": maxTime, "wireMaxTimePresent": present, "contextRemainingMS": remaining})
}, Succeeded: func(_ context.Context, e *event.CommandSucceededEvent) {
targetStatsMu.Lock()
defer targetStatsMu.Unlock()
targetCommands[e.CommandName]++
ms := float64(e.DurationNanos) / 1e6
targetMaxMillis[e.CommandName] = max(targetMaxMillis[e.CommandName], ms)
if i, ok := factPending[e.RequestID]; ok {
factTrace[i]["durationMS"] = ms
factTrace[i]["failed"] = false
delete(factPending, e.RequestID)
}
}, Failed: func(_ context.Context, e *event.CommandFailedEvent) {
targetStatsMu.Lock()
defer targetStatsMu.Unlock()
if i, ok := factPending[e.RequestID]; ok {
factTrace[i]["durationMS"] = float64(e.DurationNanos) / 1e6
factTrace[i]["failed"] = true
factTrace[i]["maxTimeFailure"] = strings.Contains(e.Failure, "MaxTimeMS")
delete(factPending, e.RequestID)
}
}}
// Separate foreground report aggregation from legitimate background builds
// and paginated detail aggregates when checking finished-result fallback.
var apiWorkforceAggregations atomic.Int64
apiTargetMonitor := &event.CommandMonitor{Started: func(ctx context.Context, e *event.CommandStartedEvent) {
name, _ := e.Command.Lookup("aggregate").StringValueOK()
if name == "queryWorkforceFacts" {
apiWorkforceAggregations.Add(1)
}
targetMonitor.Started(ctx, e)
}, Succeeded: targetMonitor.Succeeded, Failed: targetMonitor.Failed}
monitor := &event.CommandMonitor{
Started: func(_ context.Context, e *event.CommandStartedEvent) {
mu.Lock()
defer mu.Unlock()
if phase != "" {
m := metrics[phase]
m.Commands[e.CommandName]++
if m.Collections == nil {
m.Collections = map[string]int64{}
}
value := e.Command.Lookup(e.CommandName)
if e.CommandName == "getMore" {
value = e.Command.Lookup("collection")
}
if collection, ok := value.StringValueOK(); ok {
m.Collections[e.CommandName+":"+collection]++
}
m.RequestBytes += int64(len(e.Command))
inflight[e.RequestID] = phase
}
},
Succeeded: func(_ context.Context, e *event.CommandSucceededEvent) {
mu.Lock()
defer mu.Unlock()
if name := inflight[e.RequestID]; name != "" {
metrics[name].ReplyBytes += int64(len(e.Reply))
delete(inflight, e.RequestID)
}
},
Failed: func(_ context.Context, e *event.CommandFailedEvent) {
mu.Lock()
delete(inflight, e.RequestID)
mu.Unlock()
},
}
connect := func(uri string, monitor *event.CommandMonitor) (*mongo.Database, *store.MongoStore) {
opts := options.Client().ApplyURI(uri).SetMonitor(monitor).SetServerSelectionTimeout(5 * time.Second).SetMinPoolSize(uint64(minPool))
if compression != "" {
opts.SetCompressors([]string{compression})
}
c, err := mongo.Connect(ctx, opts)
if err != nil {
t.Fatal(err)
}
db := c.Database(domain.NewID("query_http_synthetic"))
t.Cleanup(func() {
clean, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
if local && os.Getenv("QUERY_COMPARE_KEEP_FAILED_FIXTURES") == "1" && t.Failed() {
t.Logf("retained owned diagnostic fixture %s", db.Name())
} else if err := db.Drop(clean); err != nil {
t.Errorf("synthetic cleanup %s: %v", db.Name(), err)
}
_ = c.Disconnect(clean)
})
return db, store.NewMongoStore(db)
}
sdb, old := connect("mongodb://127.0.0.1:27119/?directConnection=true", monitor)
db, _ := connect(testsupport.QueryReplicaTargetURI(t), targetMonitor)
if !local {
var marker bson.M
if err := db.Client().Database("query25_synthetic").Collection("marker").FindOne(ctx, bson.M{"_id": "query25-confirmed"}).Decode(&marker); err != nil || marker["value"] != "preserve-query25-v1" {
t.Fatalf("owned 25Gi target identity not confirmed: %v", err)
}
}
newReader := store.NewMongoStore(sdb)
newReader.EnableQuerySourceTracking(true)
if err := old.EnsureIndexes(ctx); err != nil {
t.Fatal(err)
}
fixture := seedIsolatedHTTPComparison(t, sdb, old, n)
annotationDigest := isolatedAnnotationDigest(t, sdb)
peer := func(uri, name string, size uint64, monitor *event.CommandMonitor, majority bool) *store.MongoStore {
opts := options.Client().ApplyURI(uri).SetMaxPoolSize(size).SetMonitor(monitor).SetServerSelectionTimeout(5 * time.Second)
if compression != "" {
opts.SetCompressors([]string{compression})
}
if majority {
opts.SetReadPreference(readpref.Primary()).SetReadConcern(readconcern.Majority())
}
client, err := mongo.Connect(ctx, opts)
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() {
stop, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
_ = client.Disconnect(stop)
})
return store.NewMongoStore(client.Database(name))
}
workerSource := peer("mongodb://127.0.0.1:27119/?directConnection=true", sdb.Name(), 4, monitor, false)
workerTarget := peer(testsupport.QueryReplicaTargetURI(t), db.Name(), 4, targetMonitor, true)
q, err := store.NewQueryReplica(workerSource, workerTarget)
if err != nil {
t.Fatal(err)
}
for _, fn := range []func(context.Context) error{q.VerifyTopology, q.EnsureSourceIndexes, q.EnsureTargetIndexes, q.Acquire} {
if err := fn(ctx); err != nil {
t.Fatal(err)
}
}
start := time.Now()
q.BatchSize = isolatedComparisonInt(t, "QUERY_COMPARE_BOOTSTRAP_BATCH", 200, 20, 200)
bootstrapRestarts := 0
t.Logf("bootstrap starting source=%s target=%s batch=%d", sdb.Name(), db.Name(), q.BatchSize)
for i := 0; ; i++ {
ready, err := q.RunOnce(ctx, true)
if err != nil {
if errors.Is(err, store.ErrQueryReplicaLease) && bootstrapRestarts < 3 {
// Reacquire only our own/expired lease, never another live owner.
// Durable checkpoints, not a fresh database, resume slow commits.
if err := q.Acquire(ctx); err != nil {
t.Fatal("bootstrap lease recovery", err)
}
bootstrapRestarts++
t.Logf("bootstrap lease recovered count=%d", bootstrapRestarts)
continue
}
t.Fatal("bootstrap", err)
}
if ready {
break
}
if i > n/q.BatchSize+100 {
t.Fatal("bootstrap did not converge")
}
if i%10 == 0 {
t.Logf("bootstrap pass=%d seconds=%.1f", i, time.Since(start).Seconds())
}
}
t.Logf("bootstrap tasks=%d seconds=%.3f source=%s target=%s", n, time.Since(start).Seconds(), sdb.Name(), db.Name())
q.BatchSize = 200
builderTarget := peer(testsupport.QueryReplicaTargetURI(t), db.Name(), 100, targetMonitor, true)
if err := builderTarget.VerifyQueryReplicaTarget(ctx); err != nil {
t.Fatal(err)
}
closeBuilderCache, e := builderTarget.ConfigureQueryServingFromEnvironment()
if e != nil {
t.Fatal(e)
}
t.Cleanup(closeBuilderCache)
if published {
cache, ok := builderTarget.SharedQueryCache().(*queryserving.ManagedValkey)
if !ok {
t.Fatal("missing dedicated shared cache")
}
until := time.Now().Add(5 * time.Second)
for !cache.Ready() && time.Now().Before(until) {
time.Sleep(20 * time.Millisecond)
}
if !cache.Ready() {
t.Fatal("dedicated cache not ready")
}
}
if err := builderTarget.ConfigureQueryRefreshMode(os.Getenv("QUERY_REPLICA_REFRESH_MODE")); err != nil {
t.Fatal(err)
}
if err := builderTarget.EnsureQueryReadModelIndexes(ctx); err != nil {
t.Fatal(err)
}
workerCtx, stopWorkers := context.WithCancel(ctx)
defer stopWorkers()
builder := store.NewQueryReadBuilder(builderTarget)
workerErrors := make(chan error, 2)
var workers sync.WaitGroup
workers.Add(2)
go func() {
defer workers.Done()
if err := builder.Run(workerCtx); err != nil && workerCtx.Err() == nil {
workerErrors <- err
}
}()
// The measured new phase includes every poll, reconcile and acknowledgement.
projectorEnabled := make(chan struct{})
go func() {
defer workers.Done()
select {
case <-projectorEnabled:
case <-workerCtx.Done():
return
}
interval := time.Second
if os.Getenv("QUERY_REPLICA_REFRESH_MODE") == "periodic" {
interval = store.QueryPeriodicSyncInterval
}
for workerCtx.Err() == nil {
if _, err := q.RunOnce(workerCtx, false); err != nil && workerCtx.Err() == nil {
workerErrors <- err
return
}
if err := q.WaitNextPass(workerCtx, interval); err != nil && workerCtx.Err() == nil {
workerErrors <- err
return
}
}
}()
defer func() { stopWorkers(); workers.Wait(); builder.Close() }()
materializeStarted := time.Now()
readyCtx, cancel := context.WithTimeout(ctx, 4*time.Minute)
err = builder.WaitReady(readyCtx, fixture.org, fixture.projects)
cancel()
if err != nil {
t.Fatal("query materialization", err)
}
materializeSeconds := time.Since(materializeStarted).Seconds()
// The projector is intentionally paused for the authority measurement. A
// long initial build must not leave that fixture's watermark expired before
// the first paired HTTP read, even though its source has had no changes.
if err := q.Acquire(ctx); err != nil {
t.Fatal(err)
}
if err := q.PublishReady(ctx); err != nil {
t.Fatal(err)
}
// Use the actual API configuration entry point (20-connection, majority
// target client), separate from projector and builder connection pools.
t.Setenv("QUERY_REPLICA_REPORTS", "1")
t.Setenv("QUERY_REPLICA_POOLS", "0")
t.Setenv("QUERY_REPLICA_COUNTS", "1")
t.Setenv("QUERY_SOURCE_TRACKING", "1")
uri := testsupport.QueryReplicaTargetURI(t)
if compression != "" {
uri += "&compressors=" + compression
}
t.Setenv("QUERY_READ_MONGO_URI", uri)
t.Setenv("QUERY_READ_MONGO_DB", db.Name())
t.Setenv("QUERY_SERVING_READS", "1")
t.Setenv("QUERY_VALKEY_URL", cacheURL)
t.Setenv("QUERY_TYPESENSE_URL", "")
closeReplica, err := newReader.ConfigureQueryReplicaFromEnvironment(ctx, apiTargetMonitor)
if err != nil {
t.Fatal(err)
}
t.Cleanup(closeReplica)
serve := func(s store.Store) *httptest.Server {
h := &server{store: s, requireSessionAuth: true, authLimiter: newRateLimiter(20, time.Minute), externalWriteLimiter: newRateLimiter(120, time.Minute), dashboardResponses: dashboardResponseCache{now: time.Now}}
ts := httptest.NewServer(h.routes())
t.Cleanup(ts.Close)
return ts
}
before, after := serve(old), serve(newReader)
client := &http.Client{Timeout: 25 * time.Second}
request := func(url, path string, actor int) (int, []byte, error) {
r, err := http.NewRequestWithContext(ctx, http.MethodGet, url+path, nil)
if err != nil {
return 0, nil, err
}
r.AddCookie(&http.Cookie{Name: sessionCookieName, Value: fixture.tokens[actor]})
if published {
r.Header.Set("Accept", "application/vnd.droboticflow.prepared+json")
}
resp, err := client.Do(r)
if err != nil {
return 0, nil, err
}
defer resp.Body.Close()
body, err := io.ReadAll(resp.Body)
return resp.StatusCode, body, err
}
paths := []string{
"/api/human-item-counts?pluginId=annotation-qa&stageIds=semantic_review",
"/api/dashboard/production?from=2026-11-01&to=2026-11-12",
"/api/dashboard/workforce?from=2026-11-01T00:00:00Z&to=2026-11-13T00:00:00Z&groupBy=member",
"/api/workforce-returns?from=2026-11-01T00:00:00Z&to=2026-11-13T00:00:00Z&limit=20&offset=0",
}
publicationWarmSeconds := float64(0)
publicationPending := 0
if published {
warmStart := time.Now()
deadline := materializeStarted.Add(4 * time.Minute)
pending := map[int]bool{}
for actor := 0; actor < maxActors; actor++ {
pending[actor] = true
}
for len(pending) > 0 {
for actor := range pending {
if !time.Now().Before(deadline) {
t.Fatalf("published responses did not become ready within original materialization deadline: %d", len(pending))
}
status, _, err := request(after.URL, paths[2], actor)
if err != nil {
t.Fatal(err)
}
if status == 200 {
delete(pending, actor)
} else if status == http.StatusAccepted {
publicationPending++
} else {
t.Fatalf("publication prewarm status %d", status)
}
}
if len(pending) > 0 {
time.Sleep(time.Second)
}
}
publicationWarmSeconds = time.Since(warmStart).Seconds()
t.Logf("published response startup seconds=%.3f pending_responses=%d", publicationWarmSeconds, publicationPending)
}
// First reads use the existing parity requests, with no extra query load.
// Exact business JSON equality; only generated report timestamps are volatile.
firstReads := map[string]any{}
for i, path := range paths {
start := time.Now()
a, ab, ae := request(before.URL, path, 0)
oldMs := float64(time.Since(start).Microseconds()) / 1000
start = time.Now()
b, bb, be := request(after.URL, path, 0)
firstReads[strings.Split(path, "?")[0]] = map[string]any{"oldMs": oldMs, "queryMs": float64(time.Since(start).Microseconds()) / 1000}
if ae != nil || be != nil || a != 200 || b != 200 {
t.Fatalf("route %d: old=%d/%v %.300s new=%d/%v %.300s", i, a, ae, ab, b, be, bb)
}
if !reflect.DeepEqual(isolatedBusinessJSON(t, ab), isolatedBusinessJSON(t, bb)) {
t.Fatalf("route %d business results differ: old %.500s new %.500s", i, ab, bb)
}
if i == 0 {
t.Logf("counter parity %s", ab)
}
}
// Keep output evidence independently identifiable rather than parsing t.Log.
report := map[string]any{"tasks": n, "projects": len(fixture.projects), "sourceDatabase": sdb.Name(), "targetDatabase": db.Name(), "cases": []any{}, "parity": true, "httpSessionAuth": true, "sourceMinimumPoolBothVariants": minPool, "apiQueryMaximumPool": 20, "projectorMaximumPool": 4, "builderMaximumPool": 100, "wireCompressionBothVariants": compression, "sourceReplyBytesKind": "logical BSON; not compressed wire bytes", "cache": "none; existing API snapshot/response caches unchanged"}
report["refreshMode"] = os.Getenv("QUERY_REPLICA_REFRESH_MODE")
report["localDocker"] = local
report["targetMetricsIncludeAPI"] = true
report["supplementalMaxInFlight"] = maxInflight
cases := []any{}
accepted := true
expectedResults := map[string]any{}
for _, mode := range []string{"old", "query"} {
url := before.URL
if mode == "query" {
url = after.URL
// Reacquire after the baseline measurement's idle period.
if err := q.Acquire(ctx); err != nil {
t.Fatal(err)
}
if _, err := q.RunOnce(ctx, false); err != nil {
t.Fatal(err)
}
close(projectorEnabled)
}
for _, concurrency := range []int{81, 200, 500} {
if concurrency > maxActors {
continue
}
name := fmt.Sprintf("%s-%d", mode, concurrency)
// Each variant pays the same session lookup cost; do not let the old
// phase warm a shared authentication cache for the query phase.
principalCacheMu.Lock()
principalCache = map[string]principalCacheEntry{}
principalCacheMu.Unlock()
scansBefore := isolatedSourceScans(t, sdb)
cpuBefore := isolatedSourceCPU(t)
mu.Lock()
phase = name
metrics[name] = &isolatedCommands{Commands: map[string]int64{}}
mu.Unlock()
t.Log("phase", name, time.Now().UTC().Format(time.RFC3339Nano))
started := time.Now()
latency := make([]float64, 0, concurrency*waves)
byRoute := map[string][]float64{}
errors := map[string]int{}
var firstError string
var responses int64
matchedResults, mismatches := 0, 0
var slots chan struct{}
if maxInflight > 0 {
slots = make(chan struct{}, maxInflight)
}
waveReports := []any{}
for wave := 0; wave < waves; wave++ {
offsets := map[string]int{}
for path, samples := range byRoute {
offsets[path] = len(samples)
}
var wg sync.WaitGroup
var results sync.Mutex
barrier := make(chan struct{})
for actor := 0; actor < concurrency; actor++ {
wg.Add(1)
go func(actor int) {
defer wg.Done()
<-barrier
if slots != nil {
slots <- struct{}{}
defer func() { <-slots }()
}
path := paths[2+(actor+wave)%2]
if actor%81 == 0 {
path = paths[1]
} else if actor%81 >= 61 {
path = paths[0]
}
s := time.Now()
status, body, err := request(url, path, actor)
results.Lock()
defer results.Unlock()
ms := float64(time.Since(s).Microseconds()) / 1000
latency = append(latency, ms)
key := strings.Split(path, "?")[0]
byRoute[key] = append(byRoute[key], ms)
responses += int64(len(body))
if status == 200 && err == nil {
id := fmt.Sprintf("%d/%d/%s", wave, actor, path)
value := isolatedBusinessJSON(t, body)
if mode == "old" {
expectedResults[id] = value
} else if expected, ok := expectedResults[id]; ok {
matchedResults++
if !reflect.DeepEqual(expected, value) {
mismatches++
}
}
}
if status != 200 || err != nil {
errors[fmt.Sprintf("%d/%v", status, err)]++
if firstError == "" {
firstError = fmt.Sprintf("%s: %d/%v %.300s", key, status, err, body)
}
}
}(actor)
}
close(barrier)
wg.Wait()
waveRoutes := map[string]any{}
for path, samples := range byRoute {
values := append([]float64(nil), samples[offsets[path]:]...)
if len(values) == 0 {
continue
}
sort.Float64s(values)
waveRoutes[path] = map[string]any{"requests": len(values), "p95Ms": values[(len(values)*95+99)/100-1], "maxMs": values[len(values)-1]}
}
waveReports = append(waveReports, map[string]any{"wave": wave + 1, "routes": waveRoutes})
}
mu.Lock()
phase = ""
original := metrics[name]
m := &isolatedCommands{Commands: map[string]int64{}, Collections: map[string]int64{}, RequestBytes: original.RequestBytes, ReplyBytes: original.ReplyBytes}
for collection, count := range original.Collections {
m.Collections[collection] = count
}
for cmd, count := range original.Commands {
m.Commands[cmd] = count
}
mu.Unlock()
cpu := isolatedSourceCPU(t) - cpuBefore
scans := isolatedSourceScans(t, sdb)
for key, before := range scansBefore {
scans[key] -= before
}
sort.Float64s(latency)
routes := map[string]any{}
for path, samples := range byRoute {
sort.Float64s(samples)
p95 := samples[(len(samples)*95+99)/100-1]
routes[path] = map[string]any{"requests": len(samples), "p95Ms": p95, "maxMs": samples[len(samples)-1]}
if mode == "query" && p95 > 2000 {
t.Errorf("%s %s p95 exceeds preliminary 2s bound", name, path)
}
}
result := map[string]any{"name": name, "seconds": time.Since(started).Seconds(), "requests": len(latency), "p50Ms": latency[(len(latency)-1)/2], "p95Ms": latency[(len(latency)-1)*95/100], "errors": errors, "firstError": firstError, "routes": routes, "responseBytes": responses, "sourceCPUSeconds": cpu, "sourceScans": scans, "source": m}
result["waves"] = waveReports
result["matchedBusinessResults"], result["businessMismatches"] = matchedResults, mismatches
if mode == "query" && mismatches > 0 {
t.Errorf("%d per-actor business results differed", mismatches)
accepted = false
}
cases = append(cases, result)
encoded, _ := json.Marshal(result)
t.Log(string(encoded))
if len(errors) != 0 {
accepted = false
t.Errorf("%s has HTTP failures: %s", name, firstError)
break
}
if mode == "query" && latency[(len(latency)-1)*95/100] > 2000 {
t.Errorf("%s p95 exceeds preliminary 2s bound", name)
}
}
}
if published {
// Separate list migration from count/report rollout; formal operations
// and personal pages keep their authoritative path.
newReader.EnableQueryReplicaPoolReads(true)
poolCases := []any{}
expectedPool := map[string]any{}
for _, mode := range []string{"old", "query"} {
url := before.URL
if mode == "query" {
url = after.URL
}
for _, pool := range []string{"available", "available&search=part0001", "available&search=absent-capture", "mine", "reviewed"} {
principalCacheMu.Lock()
principalCache = map[string]principalCacheEntry{}
principalCacheMu.Unlock()
latencies := []float64{}
var guard sync.Mutex
var wg sync.WaitGroup
for actor := 61; actor < 81; actor++ {
wg.Add(1)
go func(actor int) {
defer wg.Done()
path := "/api/human-items?pluginId=annotation-qa&stageId=semantic_review&limit=20&pool=" + pool
start := time.Now()
status, body, err := request(url, path, actor)
guard.Lock()
defer guard.Unlock()
latencies = append(latencies, float64(time.Since(start).Microseconds())/1000)
if status != 200 || err != nil {
t.Errorf("pool %s/%s/%d: status=%d err=%v %.200s", mode, pool, actor, status, err, body)
return
}
key := fmt.Sprintf("%s/%d", pool, actor)
value := isolatedBusinessJSON(t, body)
if mode == "old" {
expectedPool[key] = value
} else if !reflect.DeepEqual(expectedPool[key], value) {
t.Errorf("pool business mismatch %s", key)
}
}(actor)
}
wg.Wait()
sort.Float64s(latencies)
p95 := latencies[(len(latencies)-1)*95/100]
poolCases = append(poolCases, map[string]any{"mode": mode, "pool": pool, "requests": len(latencies), "p95Ms": p95})
if mode == "query" && p95 > 2000 {
t.Errorf("%s pool p95 exceeds 2s: %f", pool, p95)
}
}
}
report["poolListCases"] = poolCases
newReader.EnableQueryReplicaPoolReads(false)
}
if published {
opts, e := valkey.ParseURL(cacheURL)
if e != nil {
t.Fatal(e)
}
vc, e := valkey.NewClient(opts)
if e != nil {
t.Fatal(e)
}
defer vc.Close()
if e = vc.Do(ctx, vc.B().Flushdb().Sync().Build()).Error(); e != nil {
t.Fatal(e)
}
beforeAggregate := apiWorkforceAggregations.Load()
fallback := map[string]float64{}
for _, path := range paths {
now := time.Now()
beforeReport := apiWorkforceAggregations.Load()
status, _, e := request(after.URL, path, 0)
if e != nil || status != 200 {
t.Fatalf("cache loss fallback status=%d err=%v", status, e)
}
if strings.HasPrefix(path, "/api/dashboard/workforce?") && apiWorkforceAggregations.Load() != beforeReport {
t.Error("cache loss reran report aggregation")
}
fallback[strings.Split(path, "?")[0]] = float64(time.Since(now).Microseconds()) / 1000
}
report["cacheLossFallbackMs"] = fallback
report["cacheLossDetailAndReportAggregations"] = apiWorkforceAggregations.Load() - beforeAggregate
// A separate API client with an unreachable loopback cache must still
// serve durable products; this does not stop any cluster service.
t.Setenv("QUERY_VALKEY_URL", "redis://127.0.0.1:1/0")
blocked := store.NewMongoStore(sdb)
finish, e := blocked.ConfigureQueryReplicaFromEnvironment(ctx, apiTargetMonitor)
if e != nil {
t.Fatal(e)
}
defer finish()
unavailable := serve(blocked)
unavailableMs := map[string]float64{}
for _, path := range paths {
now := time.Now()
status, _, e := request(unavailable.URL, path, 0)
if e != nil || status != 200 {
t.Fatalf("unavailable cache fallback status=%d err=%v", status, e)
}
unavailableMs[strings.Split(path, "?")[0]] = float64(time.Since(now).Microseconds()) / 1000
}
report["cacheUnavailableFallbackMs"] = unavailableMs
}
if published {
report["cache"] = "dedicated Valkey with majority-committed durable response fallback"
}
report["publishedResults"] = published
report["publicationPrewarmSeconds"] = publicationWarmSeconds
report["publicationPendingResponses"] = publicationPending
report["materializationSeconds"] = materializeSeconds
report["firstReads"] = firstReads
report["cases"] = cases
report["errorFree"] = accepted
report["capacityAccepted"] = false
report["bootstrapLeaseRecoveries"] = bootstrapRestarts
report["queryServing"] = newReader.QueryServingStatistics()
if newReader.QueryServingStatistics()["report_hits"] == 0 {
t.Error("production read did not use its background materialized report")
}
if actual := isolatedAnnotationDigest(t, sdb); actual != annotationDigest {
t.Fatal("read projection changed task status, version or annotation")
}
report["annotationsAndVersionsPreserved"] = true
targetStatsMu.Lock()
report["targetWorkerCommands"] = targetCommands
report["targetWorkerMaxRoundTripMs"] = targetMaxMillis
targetStatsMu.Unlock()
select {
case err := <-workerErrors:
t.Fatal("background worker failed", err)
default:
}
if path := os.Getenv("QUERY_COMPARE_REPORT"); path != "" {
targetStatsMu.Lock()
data, err := json.MarshalIndent(report, "", " ")
targetStatsMu.Unlock()
if err != nil {
t.Fatal(err)
}
if err := os.WriteFile(path, data, 0600); err != nil {
t.Fatal(err)
}
}
}
func isolatedAnnotationDigest(t *testing.T, db *mongo.Database) string {
t.Helper()
cur, err := db.Collection("humanItems").Find(t.Context(), bson.M{}, options.Find().SetProjection(bson.M{"_id": 0, "id": 1, "version": 1, "status": 1, "assignedUserId": 1, "subject.payload.annotationDraft": 1}).SetSort(bson.D{{Key: "id", Value: 1}}))
if err != nil {
t.Fatal(err)
}
defer cur.Close(t.Context())
hash := sha256.New()
for cur.Next(t.Context()) {
var row bson.M
if err := cur.Decode(&row); err != nil {
t.Fatal(err)
}
data, err := json.Marshal(row)
if err != nil {
t.Fatal(err)
}
_, _ = hash.Write(data)
}
if err := cur.Err(); err != nil {
t.Fatal(err)
}
return fmt.Sprintf("%x", hash.Sum(nil))
}
func isolatedSourceScans(t *testing.T, db *mongo.Database) map[string]int64 {
t.Helper()
var stats struct {
Network struct {
BytesIn int64 `bson:"bytesIn"`
BytesOut int64 `bson:"bytesOut"`
PhysicalBytesIn int64 `bson:"physicalBytesIn"`
PhysicalBytesOut int64 `bson:"physicalBytesOut"`
} `bson:"network"`
Metrics struct {
QueryExecutor struct {
Scanned int64 `bson:"scanned"`
ScannedObjects int64 `bson:"scannedObjects"`
} `bson:"queryExecutor"`
} `bson:"metrics"`
}
if err := db.Client().Database("admin").RunCommand(t.Context(), bson.M{"serverStatus": 1}).Decode(&stats); err != nil {
t.Fatal("synthetic source scan counters", err)
}
return map[string]int64{"indexEntries": stats.Metrics.QueryExecutor.Scanned, "documents": stats.Metrics.QueryExecutor.ScannedObjects, "networkLogicalBytesIn": stats.Network.BytesIn, "networkLogicalBytesOut": stats.Network.BytesOut, "networkPhysicalBytesIn": stats.Network.PhysicalBytesIn, "networkPhysicalBytesOut": stats.Network.PhysicalBytesOut}
}
func isolatedSourceCPU(t *testing.T) float64 {
t.Helper()
if os.Getenv("QUERY_COMPARE_IN_CLUSTER") == "1" {
if runtime.GOOS != "linux" || (os.Getenv("NODE_NAME") != "10.189.16.4" && os.Getenv("NODE_NAME") != "10.189.16.33") || os.Getenv("POD_NAME") != "node4-query-fixture" {
t.Fatal("isolated source process identity rejected")
}
raw, err := os.ReadFile("/synthetic/mongod.pid")
if err != nil {
t.Fatal(err)
}
pid := strings.TrimSpace(string(raw))
if _, err := strconv.ParseUint(pid, 10, 32); err != nil {
t.Fatal("invalid synthetic source pid")
}
command, err := os.ReadFile("/proc/" + pid + "/cmdline")
if err != nil || !strings.Contains(string(command), "--dbpath\x00/synthetic/source\x00") || !strings.Contains(string(command), "--port\x0027119\x00") {
t.Fatal("source PID does not own the synthetic database")
}
stat, err := os.ReadFile("/proc/" + pid + "/stat")
if err != nil {
t.Fatal(err)
}
fields := strings.Fields(string(stat))
if len(fields) < 15 || fields[1] != "(mongod)" {
t.Fatal("unexpected synthetic process counters")
}
clock, err := exec.Command("getconf", "CLK_TCK").Output()
if err != nil {
t.Fatal(err)
}
hz, err := strconv.ParseFloat(strings.TrimSpace(string(clock)), 64)
if err != nil || hz <= 0 {
t.Fatal("invalid process clock")
}
user, e1 := strconv.ParseFloat(fields[13], 64)
system, e2 := strconv.ParseFloat(fields[14], 64)
if e1 != nil || e2 != nil {
t.Fatal("invalid process CPU counters")
}
return (user + system) / hz
}
raw, err := exec.Command("docker", "exec", "fanyu-query-replica-local-source-1", "cat", "/sys/fs/cgroup/cpu.stat").Output()
if err != nil {
t.Fatal("isolated source cgroup unavailable", err)
}
for _, line := range strings.Split(string(raw), "\n") {
fields := strings.Fields(line)
if len(fields) == 2 && fields[0] == "usage_usec" {
value, err := strconv.ParseFloat(fields[1], 64)
if err != nil {
t.Fatal(err)
}
return value / 1e6
}
}
t.Fatal("isolated source CPU counter missing")
return 0
}
type isolatedCommands struct {
Collections map[string]int64 `json:"collections"`
Commands map[string]int64 `json:"commands"`
RequestBytes int64 `json:"requestBytes"`
ReplyBytes int64 `json:"replyBytes"`
}
type isolatedHTTPFixture struct {
org string
projects []string
tokens []string
}
func isolatedComparisonInt(t *testing.T, key string, fallback, min, max int) int {
t.Helper()
if os.Getenv(key) == "" {
return fallback
}
n, err := strconv.Atoi(os.Getenv(key))
if err != nil || n < min || n > max {
t.Fatalf("invalid %s", key)
}
return n
}
func isolatedBusinessJSON(t *testing.T, data []byte) any {
t.Helper()
var result any
if err := json.Unmarshal(data, &result); err != nil {
t.Fatal(err)
}
// Remove only named generation timestamps, never business dates or versions.
var walk func(any)
walk = func(v any) {
switch x := v.(type) {
case map[string]any:
delete(x, "asOf")
delete(x, "generatedAt")
for _, child := range x {
walk(child)
}
case []any:
for _, child := range x {
walk(child)
}
}
}
walk(result)
return result
}
func seedIsolatedHTTPComparison(t *testing.T, db *mongo.Database, s *store.MongoStore, n int) isolatedHTTPFixture {
t.Helper()
ctx := t.Context()
f := isolatedHTTPFixture{org: "query-synthetic-org"}
must := func(err error) {
t.Helper()
if err != nil {
t.Fatal(err)
}
}
must(s.UpsertOrganization(ctx, domain.OrganizationRecord{ID: f.org, Name: "Synthetic only", Slug: "query-synthetic", Status: domain.OrganizationStatusActive}))
for i := 0; i < 3; i++ {
must(s.UpsertSupplier(ctx, domain.SupplierRecord{ID: fmt.Sprintf("v%d", i), OrganizationID: f.org, Status: "active", Name: fmt.Sprintf("Synthetic supplier %d", i)}))
}
for i := 0; i < 20; i++ {
id := fmt.Sprintf("p%02d", i)
f.projects = append(f.projects, id)
must(s.UpsertProject(ctx, domain.ProjectRecord{ID: id, OrganizationID: f.org, Name: "Synthetic " + id, Status: domain.ProjectStatusActive}))
must(s.UpsertHumanBatch(ctx, domain.HumanBatchRecord{ID: "b" + id, RunID: "batch-run-" + id, GateNodeID: "gate", OrganizationID: f.org, ProjectID: id, PluginID: "annotation-qa", Status: domain.HumanBatchActive, Stages: domain.DefaultFanyuV5HumanStages()}))
}
_, err := s.SaveProductionStatsScope(ctx, f.org, productionstats.Scope{ProjectIDs: f.projects, Revision: 1}, 0)
must(err)
rows := map[string][]any{}
push := func(c string, d any) {
if c == "annotationTasks" {
raw, err := bson.Marshal(d)
must(err)
var doc bson.M
must(bson.Unmarshal(raw, &doc))
doc["sourceKey"] = doc["id"]
d = doc
}
rows[c] = append(rows[c], d)
}
flush := func() {
for c, docs := range rows {
if len(docs) > 0 {
_, err := db.Collection(c).InsertMany(ctx, docs)
must(err)
}
}
rows = map[string][]any{}
}
for i := 0; i < 500; i++ {
role := domain.UserRoleAnnotator
if i%81 == 0 {
role = domain.UserRoleAdmin
} else if i%81 >= 73 {
role = domain.UserRoleAcceptanceReviewer
} else if i%81 >= 61 {
role = domain.UserRoleQualityReviewer
}
uid, vendor := fmt.Sprintf("u%03d", i), fmt.Sprintf("v%d", i%3)
push("users", domain.UserRecord{ID: uid, OrganizationID: f.org, Role: role, Name: uid, Phone: fmt.Sprintf("138%08d", i), Status: domain.UserStatusActive, SupplierID: vendor})
token, err := randomSessionToken()
must(err)
f.tokens = append(f.tokens, token)
push("sessions", domain.SessionRecord{ID: "session-" + uid, UserID: uid, OrganizationID: f.org, TokenHash: sessionTokenHash(token), ExpiresAt: time.Now().Add(3 * time.Hour).UTC().Format(time.RFC3339), ExpiresAtTime: time.Now().Add(3 * time.Hour)})
for _, p := range f.projects {
push("projectMembers", domain.ProjectMemberRecord{ID: p + uid, OrganizationID: f.org, ProjectID: p, UserID: uid, SupplierID: vendor})
ar := domain.AnnotationRoleAnnotator
if role == domain.UserRoleQualityReviewer {
ar = domain.AnnotationRoleQualityReviewer
}
push("annotationMemberships", domain.ProjectAnnotationMembershipRecord{ID: p + uid, OrganizationID: f.org, ProjectID: p, UserID: uid, WorkgroupID: vendor, VendorID: vendor, AssignmentRole: ar})
}
}
flush()
for i := 0; i < n; i++ {
id := fmt.Sprintf("t%06d", i)
p, vendor, uid := f.projects[i%len(f.projects)], fmt.Sprintf("v%d", (i%60+1)%3), fmt.Sprintf("u%03d", i%60+1)
at := fmt.Sprintf("2026-11-%02dT10:00:00Z", i%10+1)
run := domain.RunRecord{ID: "r" + id, OrganizationID: f.org, ProjectID: p, TaskID: "c" + id, Status: domain.RunStatusWaitingForHuman, CaptureDirectories: []string{"SYNTHETIC/" + id}, CreatedAt: at}
task := domain.AnnotationTaskRecord{ID: id, OrganizationID: f.org, ProjectID: p, RunID: run.ID, ArtifactID: "SYNTHETIC/" + id + "/part0001", SourceVideo: "bos://synthetic.invalid/video.mp4", ClipEnd: 60, CurrentRevision: 1, Status: domain.AnnotationTaskStatusPendingQuality, SupplierID: vendor, WorkgroupID: vendor, VendorID: vendor, TemplateVersion: 5, CreatedAt: at, UpdatedAt: at}
item := domain.HumanItemRecord{ID: "i" + id, OrganizationID: f.org, ProjectID: p, BatchID: "b" + p, RunID: run.ID, LegacyTaskID: id, PluginID: "annotation-qa", CurrentStageID: domain.FanyuV5StageSemanticReview, Status: domain.HumanItemReady, SupplierID: vendor, WorkgroupID: vendor, VendorID: vendor, LatestAuthorUserID: uid, CurrentRevision: 1, Version: 1, CreatedAt: at, UpdatedAt: at, Subject: domain.HumanSubjectRef{ItemKey: task.ArtifactID, Payload: map[string]any{"durationSeconds": 60, "annotationDraft": strings.Repeat("synthetic protected annotation;", 128)}}}
push("runs", run)
push("captureRecords", domain.CaptureRecord{ID: run.TaskID, OrganizationID: f.org, ProjectID: p, Prefix: "SYNTHETIC/" + id, CaptureID: "SYNTHETIC/" + id, RunID: run.ID})
push("annotationTasks", task)
push("humanItems", item)
push("annotationRevisions", domain.AnnotationRevisionRecord{ID: "rev" + id, OrganizationID: f.org, TaskID: id, Revision: 1, AuthorUserID: uid, CountsAsOutput: true})
push("sliceRevisions", domain.SliceRevisionRecord{ID: "slice" + id, OrganizationID: f.org, TaskID: id, Revision: 1, Segments: []domain.SliceSegment{{ID: "s", Start: 0, End: 60}}})
push("annotationWorklogEvents", domain.AnnotationWorklogEventRecord{ID: "event" + id, OrganizationID: f.org, ProjectID: p, TaskID: id, UserID: uid, SupplierID: vendor, EventType: "combined_submitted", Revision: 1, OutputDelta: 1, CreatedAt: at})
if i%6 == 0 {
push("reviewOperations", domain.ReviewOperation{ID: "op" + id, OrganizationID: f.org, ProjectID: p, ItemID: item.ID, TaskID: id, StageID: "quality_review", Status: "completed", Decision: "rejected", CreatedAt: at, CompletedAt: at, ReviewRound: 1, Draft: domain.ReviewDraft{Problems: []domain.ReviewProblem{{ID: "problem", Scope: "slice", SliceID: "s", Start: 0, End: 10, TimeBasis: "capture-seconds", Text: "Synthetic return detail must survive"}}}, Effects: []domain.ReviewDecisionEffect{{Review: &domain.AnnotationReviewRecord{ID: "review" + id, TaskID: id, Revision: 1}}, {BeforeTask: &task}, {BeforeItem: &item, Item: &item}}})
}
if (i+1)%500 == 0 {
flush()
}
}
flush()
return f
}