forked from longtao.wu/droboticflow
292 lines
10 KiB
Go
292 lines
10 KiB
Go
package store
|
|
|
|
import (
|
|
"bytes"
|
|
"compress/gzip"
|
|
"context"
|
|
"crypto/rand"
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"fmt"
|
|
"os"
|
|
"reflect"
|
|
"sort"
|
|
"strings"
|
|
"sync/atomic"
|
|
"testing"
|
|
"time"
|
|
|
|
"droboticflow/internal/domain"
|
|
"droboticflow/internal/outputsnapshot"
|
|
"droboticflow/internal/workforcestats"
|
|
"go.mongodb.org/mongo-driver/bson"
|
|
"go.mongodb.org/mongo-driver/event"
|
|
"golang.org/x/time/rate"
|
|
)
|
|
|
|
func TestQueryBuildSliceReadsGroupEqualVersionSetsLocal(t *testing.T) {
|
|
if os.Getenv("QUERY_REPLICA_LOCAL_TEST") != "1" {
|
|
t.Skip("owned synthetic replica required")
|
|
}
|
|
var reads atomic.Int32
|
|
m := &event.CommandMonitor{Started: func(_ context.Context, e *event.CommandStartedEvent) {
|
|
if e.CommandName == "find" {
|
|
if n, _ := e.Command.Lookup("find").StringValueOK(); n == "sliceRevisions" {
|
|
reads.Add(1)
|
|
}
|
|
}
|
|
}}
|
|
s, _, _ := newLocalReplica(t, m, nil)
|
|
tasks, events, slices := []any{}, []any{}, []any{}
|
|
for i := 0; i < 1200; i++ {
|
|
id := fmt.Sprintf("version-task-%04d", i)
|
|
v := i%2 + 1
|
|
tasks = append(tasks, bson.M{"id": id, "organizationId": "o", "projectId": "p"})
|
|
events = append(events, bson.M{"id": "event-" + id, "organizationId": "o", "projectId": "p", "taskId": id, "eventType": "combined_submitted", "revision": v, "createdAt": "2026-11-01T10:00:00Z"})
|
|
for r := 1; r <= 3; r++ {
|
|
slices = append(slices, bson.M{"id": fmt.Sprintf("%s/%d", id, r), "organizationId": "o", "taskId": id, "revision": r, "segments": bson.A{bson.M{"start": 0, "end": r}}})
|
|
}
|
|
}
|
|
for _, x := range []struct {
|
|
c mongoCollection
|
|
rows []any
|
|
}{{s.annotationTasks, tasks}, {s.annotationWorklogEvents, events}, {s.sliceRevisions, slices}} {
|
|
if _, err := x.c.InsertMany(t.Context(), x.rows); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
q := workforcestats.Query{Mode: domain.RunModeAll}
|
|
reads.Store(0)
|
|
old, err := s.LoadWorkforceStats(t.Context(), "o", q)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
oldReads := reads.Load()
|
|
reads.Store(0)
|
|
ctx := context.WithValue(t.Context(), workforceBuildBudgetKey{}, &workforceBuildBudget{})
|
|
next, err := s.LoadWorkforceStats(ctx, "o", q)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if reads.Load() != 2 || oldReads != 6 {
|
|
t.Fatalf("background slice queries not merged: old=%d new=%d", oldReads, reads.Load())
|
|
}
|
|
less := func(rows []domain.SliceRevisionRecord) {
|
|
sort.Slice(rows, func(i, j int) bool {
|
|
if rows[i].TaskID == rows[j].TaskID {
|
|
return rows[i].Revision < rows[j].Revision
|
|
}
|
|
return rows[i].TaskID < rows[j].TaskID
|
|
})
|
|
}
|
|
less(old.Slices)
|
|
less(next.Slices)
|
|
if len(next.Slices) != 1200 || !reflect.DeepEqual(old.Slices, next.Slices) || !reflect.DeepEqual(workforcestats.Build(old, q), workforcestats.Build(next, q)) {
|
|
t.Fatal("grouping changed exact versions or historical statistics")
|
|
}
|
|
}
|
|
|
|
func TestQueryBuildSmallAndLegacyBlobChunksLocal(t *testing.T) {
|
|
if os.Getenv("QUERY_REPLICA_LOCAL_TEST") != "1" {
|
|
t.Skip("owned synthetic replica required")
|
|
}
|
|
_, target, _ := newLocalReplica(t, nil, nil)
|
|
data := make([]byte, 300<<10)
|
|
if _, err := rand.Read(data); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
value := map[string]string{"payload": hex.EncodeToString(data)}
|
|
h, err := target.stageQueryModel(t.Context(), "small", value, time.Now())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if h.ChunkBytes != queryModelBuildChunkBytes || h.Chunks < 2 {
|
|
t.Fatal("new construction still emits large bursts", h)
|
|
}
|
|
var got map[string]string
|
|
if err = target.loadQueryModel(t.Context(), "small", h, &got); err != nil || !reflect.DeepEqual(got, value) {
|
|
t.Fatal("small chunk round trip", err)
|
|
}
|
|
var raw bytes.Buffer
|
|
zw := gzip.NewWriter(&raw)
|
|
if err = json.NewEncoder(zw).Encode(value); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err = zw.Close(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
sum := sha256.Sum256(raw.Bytes())
|
|
legacy := QueryModelHead{Generation: "legacy", Bytes: raw.Len(), Chunks: (raw.Len() + queryModelChunkBytes - 1) / queryModelChunkBytes, Digest: hex.EncodeToString(sum[:])}
|
|
for i := 0; i < legacy.Chunks; i++ {
|
|
if _, err = target.queryData().InsertOne(t.Context(), bson.M{"_id": queryModelBlobKey("old", legacy.Generation, i), "data": raw.Bytes()[i*queryModelChunkBytes : min((i+1)*queryModelChunkBytes, raw.Len())]}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
got = nil
|
|
if err = target.loadQueryModel(t.Context(), "old", legacy, &got); err != nil || !reflect.DeepEqual(got, value) {
|
|
t.Fatal("old manifest without chunkBytes lost compatibility", err)
|
|
}
|
|
}
|
|
|
|
func TestQueryBuildSearchUsesBoundedPacedBatchesLocal(t *testing.T) {
|
|
if os.Getenv("QUERY_REPLICA_LOCAL_TEST") != "1" {
|
|
t.Skip("owned synthetic replica required")
|
|
}
|
|
var maxBytes atomic.Int32
|
|
monitor := &event.CommandMonitor{Started: func(_ context.Context, e *event.CommandStartedEvent) {
|
|
if e.CommandName != "insert" {
|
|
return
|
|
}
|
|
name, _ := e.Command.Lookup("insert").StringValueOK()
|
|
if name != "queryReadModelData" {
|
|
return
|
|
}
|
|
values, _ := e.Command.Lookup("documents").Array().Values()
|
|
size := 0
|
|
for _, v := range values {
|
|
size += len(v.Document())
|
|
}
|
|
for before := maxBytes.Load(); int32(size) > before; before = maxBytes.Load() {
|
|
if maxBytes.CompareAndSwap(before, int32(size)) {
|
|
break
|
|
}
|
|
}
|
|
}}
|
|
_, target, _ := newLocalReplica(t, nil, monitor)
|
|
for i := 0; i < 300; i++ {
|
|
if err := target.UpsertHumanItem(t.Context(), domain.HumanItemRecord{ID: fmt.Sprint(i), OrganizationID: "o", PluginID: "annotation-qa", ProjectID: "p", Status: domain.HumanItemReady, Subject: domain.HumanSubjectRef{ItemKey: strings.Repeat("synthetic-name-", 100) + fmt.Sprint(i)}}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
l, ok, err := target.acquireQueryModelLease(t.Context())
|
|
if err != nil || !ok {
|
|
t.Fatal(ok, err)
|
|
}
|
|
var inspected atomic.Int32
|
|
target.queryBuildWrites = &queryBuildWriteGate{limiter: rate.NewLimiter(rate.Inf, queryBuildBatchBytes), inspect: func(context.Context) (bool, error) { inspected.Add(1); return false, nil }}
|
|
if err = target.buildQuerySearchModel(t.Context(), l, "o"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
st, err := target.querySearchState(t.Context())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if st.Heads[queryModelKey("search", "o", nil)].Rows != 300 || inspected.Load() == 0 || maxBytes.Load() > queryBuildBatchBytes {
|
|
t.Fatalf("search writer bypassed budget or lost rows: rows=%d inspect=%d maxBytes=%d", st.Heads[queryModelKey("search", "o", nil)].Rows, inspected.Load(), maxBytes.Load())
|
|
}
|
|
}
|
|
|
|
func TestQueryBuildShardStagingChecksLeaseWithoutHeartbeatWritesLocal(t *testing.T) {
|
|
if os.Getenv("QUERY_REPLICA_LOCAL_TEST") != "1" {
|
|
t.Skip("owned synthetic replica required")
|
|
}
|
|
var renews atomic.Int32
|
|
monitor := &event.CommandMonitor{Started: func(_ context.Context, e *event.CommandStartedEvent) {
|
|
if e.CommandName == "update" {
|
|
if v, ok := e.Command.Lookup("update").StringValueOK(); ok && v == outputsnapshot.StateCollection {
|
|
renews.Add(1)
|
|
}
|
|
}
|
|
}}
|
|
_, target, _ := newLocalReplica(t, nil, monitor)
|
|
now := time.Now().UTC().Truncate(time.Millisecond)
|
|
l, ok, err := target.AcquireOutputSnapshot(t.Context(), "cold-stage", now)
|
|
if err != nil || !ok {
|
|
t.Fatal(ok, err)
|
|
}
|
|
before := renews.Load()
|
|
for i := 0; i < 3; i++ {
|
|
ok, err := target.StageOutputSnapshotShard(t.Context(), l, i, []byte{byte(i)}, now.Add(time.Second))
|
|
if err != nil || !ok {
|
|
t.Fatal(ok, err)
|
|
}
|
|
}
|
|
if got := renews.Load() - before; got != 0 {
|
|
t.Fatalf("staging redundantly persisted %d heartbeats", got)
|
|
}
|
|
st, err := target.OutputSnapshotState(t.Context())
|
|
if err != nil || !st.Until.Equal(now.Add(outputsnapshot.LeaseDuration)) {
|
|
t.Fatalf("staging changed the existing lease: %+v %v", st, err)
|
|
}
|
|
if ok, err := target.StageOutputSnapshotShard(t.Context(), l, 3, []byte("expired"), now.Add(outputsnapshot.LeaseDuration+time.Second)); err != nil || ok {
|
|
t.Fatal("expired writer staged", ok, err)
|
|
}
|
|
}
|
|
|
|
func TestQueryBuildShardWaitsForCheckpointWithoutHoldingWriteTransactionLocal(t *testing.T) {
|
|
if os.Getenv("QUERY_REPLICA_LOCAL_TEST") != "1" {
|
|
t.Skip("owned synthetic replica required")
|
|
}
|
|
var inserts atomic.Int32
|
|
monitor := &event.CommandMonitor{Started: func(_ context.Context, e *event.CommandStartedEvent) {
|
|
if e.CommandName == "insert" {
|
|
if v, ok := e.Command.Lookup("insert").StringValueOK(); ok && v == outputsnapshot.SlotsCollection {
|
|
inserts.Add(1)
|
|
}
|
|
}
|
|
}}
|
|
_, target, _ := newLocalReplica(t, nil, monitor)
|
|
l, ok, err := target.AcquireOutputSnapshot(t.Context(), "checkpoint", time.Now())
|
|
if err != nil || !ok {
|
|
t.Fatal(ok, err)
|
|
}
|
|
var busy atomic.Bool
|
|
busy.Store(true)
|
|
entered := make(chan struct{}, 1)
|
|
target.queryBuildWrites = &queryBuildWriteGate{limiter: rate.NewLimiter(rate.Inf, queryBuildBatchBytes), inspect: func(context.Context) (bool, error) {
|
|
select {
|
|
case entered <- struct{}{}:
|
|
default:
|
|
}
|
|
return busy.Load(), nil
|
|
}}
|
|
done := make(chan error, 1)
|
|
go func() {
|
|
ok, e := target.StageOutputSnapshotShard(t.Context(), l, 0, []byte("complete"), time.Now())
|
|
if e == nil && !ok {
|
|
e = context.Canceled
|
|
}
|
|
done <- e
|
|
}()
|
|
<-entered
|
|
if inserts.Load() != 0 {
|
|
t.Fatal("data write entered checkpoint IO queue")
|
|
}
|
|
// The independent heartbeat remains available while the data producer waits.
|
|
if ok, e := target.RenewOutputSnapshot(t.Context(), l, time.Now()); e != nil || !ok {
|
|
t.Fatal("checkpoint pause blocked control writes", ok, e)
|
|
}
|
|
busy.Store(false)
|
|
select {
|
|
case e := <-done:
|
|
if e != nil {
|
|
t.Fatal(e)
|
|
}
|
|
case <-time.After(3 * time.Second):
|
|
t.Fatal("producer did not resume")
|
|
}
|
|
if inserts.Load() != 1 {
|
|
t.Fatal("resumed staging was duplicated", inserts.Load())
|
|
}
|
|
}
|
|
|
|
func TestQueryBuildShardRechecksExpiryAfterCheckpointWaitLocal(t *testing.T) {
|
|
if os.Getenv("QUERY_REPLICA_LOCAL_TEST") != "1" {
|
|
t.Skip("owned synthetic replica required")
|
|
}
|
|
_, target, _ := newLocalReplica(t, nil, nil)
|
|
// The lease is valid at admission but expires while waiting. A timestamp
|
|
// captured before that wait must not authorize the subsequent staging.
|
|
now := time.Now()
|
|
l, ok, err := target.AcquireOutputSnapshot(t.Context(), "expiring", now.Add(-outputsnapshot.LeaseDuration+500*time.Millisecond))
|
|
if err != nil || !ok {
|
|
t.Fatal(ok, err)
|
|
}
|
|
calls := 0
|
|
target.queryBuildWrites = &queryBuildWriteGate{limiter: rate.NewLimiter(rate.Inf, queryBuildBatchBytes), inspect: func(context.Context) (bool, error) { calls++; return calls == 1, nil }}
|
|
if ok, err = target.StageOutputSnapshotShard(t.Context(), l, 0, []byte("late"), now); err != nil || ok {
|
|
t.Fatal("pre-wait time authorized an expired owner", ok, err)
|
|
}
|
|
}
|