droboticflow/internal/store/query_cold_write_local_test.go

292 lines
10 KiB
Go

package store
import (
"bytes"
"compress/gzip"
"context"
"crypto/rand"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"os"
"reflect"
"sort"
"strings"
"sync/atomic"
"testing"
"time"
"droboticflow/internal/domain"
"droboticflow/internal/outputsnapshot"
"droboticflow/internal/workforcestats"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/event"
"golang.org/x/time/rate"
)
func TestQueryBuildSliceReadsGroupEqualVersionSetsLocal(t *testing.T) {
if os.Getenv("QUERY_REPLICA_LOCAL_TEST") != "1" {
t.Skip("owned synthetic replica required")
}
var reads atomic.Int32
m := &event.CommandMonitor{Started: func(_ context.Context, e *event.CommandStartedEvent) {
if e.CommandName == "find" {
if n, _ := e.Command.Lookup("find").StringValueOK(); n == "sliceRevisions" {
reads.Add(1)
}
}
}}
s, _, _ := newLocalReplica(t, m, nil)
tasks, events, slices := []any{}, []any{}, []any{}
for i := 0; i < 1200; i++ {
id := fmt.Sprintf("version-task-%04d", i)
v := i%2 + 1
tasks = append(tasks, bson.M{"id": id, "organizationId": "o", "projectId": "p"})
events = append(events, bson.M{"id": "event-" + id, "organizationId": "o", "projectId": "p", "taskId": id, "eventType": "combined_submitted", "revision": v, "createdAt": "2026-11-01T10:00:00Z"})
for r := 1; r <= 3; r++ {
slices = append(slices, bson.M{"id": fmt.Sprintf("%s/%d", id, r), "organizationId": "o", "taskId": id, "revision": r, "segments": bson.A{bson.M{"start": 0, "end": r}}})
}
}
for _, x := range []struct {
c mongoCollection
rows []any
}{{s.annotationTasks, tasks}, {s.annotationWorklogEvents, events}, {s.sliceRevisions, slices}} {
if _, err := x.c.InsertMany(t.Context(), x.rows); err != nil {
t.Fatal(err)
}
}
q := workforcestats.Query{Mode: domain.RunModeAll}
reads.Store(0)
old, err := s.LoadWorkforceStats(t.Context(), "o", q)
if err != nil {
t.Fatal(err)
}
oldReads := reads.Load()
reads.Store(0)
ctx := context.WithValue(t.Context(), workforceBuildBudgetKey{}, &workforceBuildBudget{})
next, err := s.LoadWorkforceStats(ctx, "o", q)
if err != nil {
t.Fatal(err)
}
if reads.Load() != 2 || oldReads != 6 {
t.Fatalf("background slice queries not merged: old=%d new=%d", oldReads, reads.Load())
}
less := func(rows []domain.SliceRevisionRecord) {
sort.Slice(rows, func(i, j int) bool {
if rows[i].TaskID == rows[j].TaskID {
return rows[i].Revision < rows[j].Revision
}
return rows[i].TaskID < rows[j].TaskID
})
}
less(old.Slices)
less(next.Slices)
if len(next.Slices) != 1200 || !reflect.DeepEqual(old.Slices, next.Slices) || !reflect.DeepEqual(workforcestats.Build(old, q), workforcestats.Build(next, q)) {
t.Fatal("grouping changed exact versions or historical statistics")
}
}
func TestQueryBuildSmallAndLegacyBlobChunksLocal(t *testing.T) {
if os.Getenv("QUERY_REPLICA_LOCAL_TEST") != "1" {
t.Skip("owned synthetic replica required")
}
_, target, _ := newLocalReplica(t, nil, nil)
data := make([]byte, 300<<10)
if _, err := rand.Read(data); err != nil {
t.Fatal(err)
}
value := map[string]string{"payload": hex.EncodeToString(data)}
h, err := target.stageQueryModel(t.Context(), "small", value, time.Now())
if err != nil {
t.Fatal(err)
}
if h.ChunkBytes != queryModelBuildChunkBytes || h.Chunks < 2 {
t.Fatal("new construction still emits large bursts", h)
}
var got map[string]string
if err = target.loadQueryModel(t.Context(), "small", h, &got); err != nil || !reflect.DeepEqual(got, value) {
t.Fatal("small chunk round trip", err)
}
var raw bytes.Buffer
zw := gzip.NewWriter(&raw)
if err = json.NewEncoder(zw).Encode(value); err != nil {
t.Fatal(err)
}
if err = zw.Close(); err != nil {
t.Fatal(err)
}
sum := sha256.Sum256(raw.Bytes())
legacy := QueryModelHead{Generation: "legacy", Bytes: raw.Len(), Chunks: (raw.Len() + queryModelChunkBytes - 1) / queryModelChunkBytes, Digest: hex.EncodeToString(sum[:])}
for i := 0; i < legacy.Chunks; i++ {
if _, err = target.queryData().InsertOne(t.Context(), bson.M{"_id": queryModelBlobKey("old", legacy.Generation, i), "data": raw.Bytes()[i*queryModelChunkBytes : min((i+1)*queryModelChunkBytes, raw.Len())]}); err != nil {
t.Fatal(err)
}
}
got = nil
if err = target.loadQueryModel(t.Context(), "old", legacy, &got); err != nil || !reflect.DeepEqual(got, value) {
t.Fatal("old manifest without chunkBytes lost compatibility", err)
}
}
func TestQueryBuildSearchUsesBoundedPacedBatchesLocal(t *testing.T) {
if os.Getenv("QUERY_REPLICA_LOCAL_TEST") != "1" {
t.Skip("owned synthetic replica required")
}
var maxBytes atomic.Int32
monitor := &event.CommandMonitor{Started: func(_ context.Context, e *event.CommandStartedEvent) {
if e.CommandName != "insert" {
return
}
name, _ := e.Command.Lookup("insert").StringValueOK()
if name != "queryReadModelData" {
return
}
values, _ := e.Command.Lookup("documents").Array().Values()
size := 0
for _, v := range values {
size += len(v.Document())
}
for before := maxBytes.Load(); int32(size) > before; before = maxBytes.Load() {
if maxBytes.CompareAndSwap(before, int32(size)) {
break
}
}
}}
_, target, _ := newLocalReplica(t, nil, monitor)
for i := 0; i < 300; i++ {
if err := target.UpsertHumanItem(t.Context(), domain.HumanItemRecord{ID: fmt.Sprint(i), OrganizationID: "o", PluginID: "annotation-qa", ProjectID: "p", Status: domain.HumanItemReady, Subject: domain.HumanSubjectRef{ItemKey: strings.Repeat("synthetic-name-", 100) + fmt.Sprint(i)}}); err != nil {
t.Fatal(err)
}
}
l, ok, err := target.acquireQueryModelLease(t.Context())
if err != nil || !ok {
t.Fatal(ok, err)
}
var inspected atomic.Int32
target.queryBuildWrites = &queryBuildWriteGate{limiter: rate.NewLimiter(rate.Inf, queryBuildBatchBytes), inspect: func(context.Context) (bool, error) { inspected.Add(1); return false, nil }}
if err = target.buildQuerySearchModel(t.Context(), l, "o"); err != nil {
t.Fatal(err)
}
st, err := target.querySearchState(t.Context())
if err != nil {
t.Fatal(err)
}
if st.Heads[queryModelKey("search", "o", nil)].Rows != 300 || inspected.Load() == 0 || maxBytes.Load() > queryBuildBatchBytes {
t.Fatalf("search writer bypassed budget or lost rows: rows=%d inspect=%d maxBytes=%d", st.Heads[queryModelKey("search", "o", nil)].Rows, inspected.Load(), maxBytes.Load())
}
}
func TestQueryBuildShardStagingChecksLeaseWithoutHeartbeatWritesLocal(t *testing.T) {
if os.Getenv("QUERY_REPLICA_LOCAL_TEST") != "1" {
t.Skip("owned synthetic replica required")
}
var renews atomic.Int32
monitor := &event.CommandMonitor{Started: func(_ context.Context, e *event.CommandStartedEvent) {
if e.CommandName == "update" {
if v, ok := e.Command.Lookup("update").StringValueOK(); ok && v == outputsnapshot.StateCollection {
renews.Add(1)
}
}
}}
_, target, _ := newLocalReplica(t, nil, monitor)
now := time.Now().UTC().Truncate(time.Millisecond)
l, ok, err := target.AcquireOutputSnapshot(t.Context(), "cold-stage", now)
if err != nil || !ok {
t.Fatal(ok, err)
}
before := renews.Load()
for i := 0; i < 3; i++ {
ok, err := target.StageOutputSnapshotShard(t.Context(), l, i, []byte{byte(i)}, now.Add(time.Second))
if err != nil || !ok {
t.Fatal(ok, err)
}
}
if got := renews.Load() - before; got != 0 {
t.Fatalf("staging redundantly persisted %d heartbeats", got)
}
st, err := target.OutputSnapshotState(t.Context())
if err != nil || !st.Until.Equal(now.Add(outputsnapshot.LeaseDuration)) {
t.Fatalf("staging changed the existing lease: %+v %v", st, err)
}
if ok, err := target.StageOutputSnapshotShard(t.Context(), l, 3, []byte("expired"), now.Add(outputsnapshot.LeaseDuration+time.Second)); err != nil || ok {
t.Fatal("expired writer staged", ok, err)
}
}
func TestQueryBuildShardWaitsForCheckpointWithoutHoldingWriteTransactionLocal(t *testing.T) {
if os.Getenv("QUERY_REPLICA_LOCAL_TEST") != "1" {
t.Skip("owned synthetic replica required")
}
var inserts atomic.Int32
monitor := &event.CommandMonitor{Started: func(_ context.Context, e *event.CommandStartedEvent) {
if e.CommandName == "insert" {
if v, ok := e.Command.Lookup("insert").StringValueOK(); ok && v == outputsnapshot.SlotsCollection {
inserts.Add(1)
}
}
}}
_, target, _ := newLocalReplica(t, nil, monitor)
l, ok, err := target.AcquireOutputSnapshot(t.Context(), "checkpoint", time.Now())
if err != nil || !ok {
t.Fatal(ok, err)
}
var busy atomic.Bool
busy.Store(true)
entered := make(chan struct{}, 1)
target.queryBuildWrites = &queryBuildWriteGate{limiter: rate.NewLimiter(rate.Inf, queryBuildBatchBytes), inspect: func(context.Context) (bool, error) {
select {
case entered <- struct{}{}:
default:
}
return busy.Load(), nil
}}
done := make(chan error, 1)
go func() {
ok, e := target.StageOutputSnapshotShard(t.Context(), l, 0, []byte("complete"), time.Now())
if e == nil && !ok {
e = context.Canceled
}
done <- e
}()
<-entered
if inserts.Load() != 0 {
t.Fatal("data write entered checkpoint IO queue")
}
// The independent heartbeat remains available while the data producer waits.
if ok, e := target.RenewOutputSnapshot(t.Context(), l, time.Now()); e != nil || !ok {
t.Fatal("checkpoint pause blocked control writes", ok, e)
}
busy.Store(false)
select {
case e := <-done:
if e != nil {
t.Fatal(e)
}
case <-time.After(3 * time.Second):
t.Fatal("producer did not resume")
}
if inserts.Load() != 1 {
t.Fatal("resumed staging was duplicated", inserts.Load())
}
}
func TestQueryBuildShardRechecksExpiryAfterCheckpointWaitLocal(t *testing.T) {
if os.Getenv("QUERY_REPLICA_LOCAL_TEST") != "1" {
t.Skip("owned synthetic replica required")
}
_, target, _ := newLocalReplica(t, nil, nil)
// The lease is valid at admission but expires while waiting. A timestamp
// captured before that wait must not authorize the subsequent staging.
now := time.Now()
l, ok, err := target.AcquireOutputSnapshot(t.Context(), "expiring", now.Add(-outputsnapshot.LeaseDuration+500*time.Millisecond))
if err != nil || !ok {
t.Fatal(ok, err)
}
calls := 0
target.queryBuildWrites = &queryBuildWriteGate{limiter: rate.NewLimiter(rate.Inf, queryBuildBatchBytes), inspect: func(context.Context) (bool, error) { calls++; return calls == 1, nil }}
if ok, err = target.StageOutputSnapshotShard(t.Context(), l, 0, []byte("late"), now); err != nil || ok {
t.Fatal("pre-wait time authorized an expired owner", ok, err)
}
}