droboticflow/internal/store/pool_count_performance_test.go

352 lines
16 KiB
Go

package store
import (
"context"
"droboticflow/internal/domain"
"fmt"
"net/url"
"os"
"reflect"
"sort"
"sync/atomic"
"testing"
"time"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/event"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
)
func testPoolCountPredicates(t *testing.T, s Store) {
t.Helper()
ctx := context.Background()
org, project := "pool-fixture-org", "pool-fixture-project"
legacy := domain.HumanBatchRecord{ID: "pool-legacy", OrganizationID: org, ProjectID: project, RunID: "pool-fixture-run", Stages: domain.DefaultFanyuV58HumanStages(), Status: domain.HumanBatchActive}
pooled := legacy
pooled.ID = "pool-shared"
pooled.ProcessingProfile = domain.ProcessingProfileRawHuman
exceptional := legacy
exceptional.ID = "pool-exception"
exceptional.Stages = []domain.HumanStageDefinition{{ID: "legacy-review-as-annotator", Kind: domain.HumanStageReview, RequiredRole: "cutter"}}
custom := legacy
custom.ID = "pool-custom"
custom.Stages = []domain.HumanStageDefinition{{ID: "custom-annotation", Kind: domain.HumanStageAnnotation, RequiredRole: "unknown-historical"}}
duplicate := legacy
duplicate.ID = "pool-duplicate-stage"
duplicate.Stages = []domain.HumanStageDefinition{{ID: "duplicate-id", Kind: domain.HumanStageReview, RequiredRole: "quality_reviewer"}, {ID: "duplicate-id", Kind: domain.HumanStageReview, RequiredRole: "annotator"}}
completed := legacy
completed.ID = "pool-completed"
completed.Status = domain.HumanBatchCompleted
foreign := legacy
foreign.ID = "pool-foreign-project"
foreign.ProjectID = "other-project"
for _, b := range []domain.HumanBatchRecord{legacy, pooled, exceptional, custom, completed, foreign, duplicate} {
if err := s.UpsertHumanBatch(ctx, b); err != nil {
t.Fatal(err)
}
}
base := domain.HumanItemRecord{ID: "candidate", OrganizationID: org, ProjectID: project, PluginID: "annotation-qa", BatchID: legacy.ID, RunID: legacy.RunID, LegacyTaskID: "legacy-task", CurrentStageID: domain.FanyuV58StageAnnotationCombined, Status: domain.HumanItemReady, CreatedAt: "2026-10-01T00:00:00Z"}
items := []domain.HumanItemRecord{}
add := func(id string, edit func(*domain.HumanItemRecord)) {
v := base
v.ID = id
v.Subject.ItemKey = id
if edit != nil {
edit(&v)
}
if v.LegacyTaskID != "" {
if v.LegacyTaskID == base.LegacyTaskID {
v.LegacyTaskID = id + "-task"
}
task := domain.AnnotationTaskRecord{ID: v.LegacyTaskID, OrganizationID: v.OrganizationID, ProjectID: v.ProjectID, RunID: v.RunID}
if err := s.UpsertAnnotationTask(ctx, task); err != nil {
t.Fatal(err)
}
completeSourceAdmissionTestProof(t, s, task)
}
items = append(items, v)
if err := s.UpsertHumanItem(ctx, v); err != nil {
t.Fatal(err)
}
}
add("legacy-public", nil)
add("legacy-owned", func(v *domain.HumanItemRecord) { v.SupplierID = "vendor" })
add("legacy-foreign", func(v *domain.HumanItemRecord) { v.SupplierID = "foreign" })
add("pool-public", func(v *domain.HumanItemRecord) { v.BatchID = pooled.ID })
add("pool-owned", func(v *domain.HumanItemRecord) { v.BatchID = pooled.ID; v.SupplierID = "vendor" })
add("pool-drain", func(v *domain.HumanItemRecord) {
v.BatchID = pooled.ID
v.SupplierID = "vendor"
v.LegacyTaskID = "drain-task"
v.Status = domain.HumanItemRework
})
add("pool-internal-public", func(v *domain.HumanItemRecord) { v.BatchID = pooled.ID; v.ProcessingScope = "internal" })
add("legacy-internal-public", func(v *domain.HumanItemRecord) { v.ProcessingScope = "internal" })
add("duplicate-stage", func(v *domain.HumanItemRecord) { v.BatchID = duplicate.ID; v.CurrentStageID = "duplicate-id" })
add("completed", func(v *domain.HumanItemRecord) { v.BatchID = completed.ID })
add("wrong-project-batch", func(v *domain.HumanItemRecord) { v.BatchID = foreign.ID })
add("missing-batch", func(v *domain.HumanItemRecord) { v.BatchID = "absent" })
add("assigned", func(v *domain.HumanItemRecord) { v.AssignedUserID = "actor" })
add("own-author", func(v *domain.HumanItemRecord) {
v.LatestAuthorUserID = "actor"
v.CurrentStageID = domain.FanyuV58StageQualityReview
})
add("exception", func(v *domain.HumanItemRecord) {
v.BatchID = exceptional.ID
v.CurrentStageID = "legacy-review-as-annotator"
})
add("custom", func(v *domain.HumanItemRecord) { v.BatchID = custom.ID; v.CurrentStageID = "custom-annotation" })
add("other-plugin", func(v *domain.HumanItemRecord) { v.PluginID = "another-plugin" })
add("other-org", func(v *domain.HumanItemRecord) { v.OrganizationID = "other-org" })
add("other-project", func(v *domain.HumanItemRecord) { v.ProjectID = "other-project" })
add("no-task", func(v *domain.HumanItemRecord) { v.LegacyTaskID = "" })
add("claimed-status", func(v *domain.HumanItemRecord) { v.Status = domain.HumanItemClaimed })
cases := []struct {
name string
scope AnnotationClaimMembership
want int
}{
{"legacy", AnnotationClaimMembership{SupplierID: "vendor", Legacy: true}, 3},
{"active-pool", AnnotationClaimMembership{SupplierID: "vendor", SupplierPool: true}, 3},
{"both", AnnotationClaimMembership{SupplierID: "vendor", Legacy: true, SupplierPool: true}, 6},
{"revoked-drain", AnnotationClaimMembership{SupplierID: "vendor", DrainTaskIDs: []string{"drain-task"}}, 1},
{"no-access", AnnotationClaimMembership{SupplierID: "vendor"}, 0},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
q := AnnotationClaimCountQuery{ProjectID: project, ActiveSupplierIDs: []string{"vendor"}, Memberships: []AnnotationClaimMembership{tc.scope}}
got, err := s.CountAnnotationClaimCandidates(ctx, org, q)
if err != nil || got != tc.want {
t.Fatalf("count=%d want=%d err=%v", got, tc.want, err)
}
})
}
q := domain.HumanItemQuery{ProjectID: project, BlindAnnotationActorID: "actor", Limit: 2, ListProjection: true}
ids := []string{}
for {
page, err := s.ListHumanItems(ctx, org, q)
if err != nil {
t.Fatal(err)
}
for _, item := range page {
ids = append(ids, item.ID)
}
if len(page) < q.Limit {
break
}
last := page[len(page)-1]
q.Cursor = &domain.HumanItemCursor{CreatedAt: last.CreatedAt, ID: last.ID}
}
sort.Strings(ids)
want := []string{"assigned", "exception", "other-plugin", "own-author"}
if !reflect.DeepEqual(ids, want) {
t.Fatalf("blind visibility=%v want=%v", ids, want)
}
// Scope changes take effect on the next call and never reuse actor authority.
q.Cursor = nil
q.BlindAnnotationActorID = "different-actor"
q.Limit = 0
page, err := s.ListHumanItems(ctx, org, q)
if err != nil {
t.Fatal(err)
}
for _, item := range page {
if item.ID == "assigned" || item.ID == "own-author" {
t.Fatalf("old actor visible: %s", item.ID)
}
}
}
func TestPoolCountPredicatesMemory(t *testing.T) { testPoolCountPredicates(t, NewMemoryStore()) }
func poolLocalMongo(t *testing.T, monitors ...*event.CommandMonitor) (context.Context, *MongoStore) {
t.Helper()
uri := os.Getenv("POOL_LOCAL_MONGO_URI")
if uri == "" {
t.Skip("POOL_LOCAL_MONGO_URI required")
}
parsed, err := url.Parse(uri)
if err != nil || parsed.Scheme != "mongodb" || (parsed.Host != "127.0.0.1:27018" && parsed.Host != "127.0.0.1:27038") {
t.Fatal("isolated local replica only")
}
ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second)
t.Cleanup(cancel)
clientOptions := options.Client().ApplyURI(uri)
if len(monitors) > 0 {
clientOptions.SetMonitor(monitors[0])
}
client, err := mongo.Connect(ctx, clientOptions)
if err != nil {
t.Fatal("local Mongo connect failed")
}
db := client.Database(fmt.Sprintf("pool_performance_test_%d", time.Now().UnixNano()))
t.Cleanup(func() { db.Drop(context.Background()); client.Disconnect(context.Background()) })
return ctx, NewMongoStore(db)
}
func TestPoolCountPredicatesMongo(t *testing.T) {
for _, indexed := range []bool{false, true} {
t.Run(fmt.Sprintf("indexed=%t", indexed), func(t *testing.T) {
ctx, s := poolLocalMongo(t)
if indexed {
if _, err := s.humanBatches.Indexes().CreateOne(ctx, blindAnnotationStageExceptionIndex()); err != nil {
t.Fatal(err)
}
}
testPoolCountPredicates(t, s)
})
}
}
func TestPoolPerformanceMongoBacklog(t *testing.T) {
ctx, s := poolLocalMongo(t)
batch := domain.HumanBatchRecord{ID: "normal-batch", OrganizationID: "perf-org", ProjectID: "perf-project", Stages: domain.DefaultFanyuV58HumanStages(), Status: domain.HumanBatchActive}
if err := s.UpsertHumanBatch(ctx, batch); err != nil {
t.Fatal(err)
}
docs := make([]interface{}, 12000)
batchDocs := make([]interface{}, len(docs))
for i := range docs {
currentBatch := batch
currentBatch.ID = fmt.Sprintf("normal-batch-%05d", i)
currentBatch.RunID = fmt.Sprintf("run-%05d", i)
batchDocs[i] = currentBatch
docs[i] = domain.HumanItemRecord{ID: fmt.Sprintf("backlog-%05d", i), OrganizationID: batch.OrganizationID, ProjectID: batch.ProjectID, PluginID: "annotation-qa", BatchID: currentBatch.ID, LegacyTaskID: fmt.Sprintf("task-%05d", i), CurrentStageID: domain.FanyuV58StageAnnotationCombined, Status: domain.HumanItemReady, Subject: domain.HumanSubjectRef{ItemKey: fmt.Sprintf("subject-%05d", i)}, CreatedAt: fmt.Sprintf("2026-10-01T%05d", i)}
}
if _, err := s.humanBatches.InsertMany(ctx, batchDocs); err != nil {
t.Fatal(err)
}
if _, err := s.humanItems.InsertMany(ctx, docs); err != nil {
t.Fatal(err)
}
q := domain.HumanItemQuery{PluginID: "annotation-qa", ProjectID: batch.ProjectID, UnassignedOnly: true, Statuses: []domain.HumanItemStatus{domain.HumanItemReady, domain.HumanItemRework, domain.HumanItemSubmitted}, ListProjection: true, Limit: 100}
explain := func(filter bson.M) bson.M {
var result bson.M
err := s.db.RunCommand(ctx, bson.D{{Key: "explain", Value: bson.D{{Key: "find", Value: "humanItems"}, {Key: "filter", Value: filter}, {Key: "sort", Value: bson.D{{Key: "createdAt", Value: -1}, {Key: "id", Value: -1}}}, {Key: "limit", Value: 100}}}, {Key: "verbosity", Value: "executionStats"}}).Decode(&result)
if err != nil {
t.Fatal(err)
}
stats, _ := result["executionStats"].(bson.M)
return bson.M{"nReturned": stats["nReturned"], "totalKeysExamined": stats["totalKeysExamined"], "totalDocsExamined": stats["totalDocsExamined"], "executionTimeMillis": stats["executionTimeMillis"]}
}
baseline := explain(humanItemMongoFilter(batch.OrganizationID, q))
if err := s.EnsureIndexes(ctx); err != nil {
t.Fatal(err)
}
indexed := explain(humanItemMongoFilter(batch.OrganizationID, q))
q.BlindAnnotationActorID = "actor"
visibility, err := s.blindAnnotationMongoVisibility(ctx, batch.OrganizationID, q)
if err != nil {
t.Fatal(err)
}
after := explain(bson.M{"$and": bson.A{humanItemMongoFilter(batch.OrganizationID, q), visibility}})
started := time.Now()
items, err := s.ListHumanItems(ctx, batch.OrganizationID, q)
if err != nil || len(items) != 0 {
t.Fatalf("items=%d err=%v", len(items), err)
}
t.Logf("12k backlog baseline=%v indexed=%v scoped=%v list=%s", baseline, indexed, after, time.Since(started))
countStarted := time.Now()
count, err := s.CountAnnotationClaimCandidates(ctx, batch.OrganizationID, AnnotationClaimCountQuery{ProjectID: batch.ProjectID, ActiveSupplierIDs: []string{"vendor"}, Memberships: []AnnotationClaimMembership{{SupplierID: "vendor", Legacy: true}}})
if err != nil || count != 12000 {
t.Fatalf("count=%d err=%v", count, err)
}
t.Logf("aggregate count=%d duration=%s", count, time.Since(countStarted))
if docs, ok := after["totalDocsExamined"].(int32); ok && docs > 1 {
t.Fatalf("private backlog examined %d documents", docs)
}
}
func TestPoolStageRoleCountAndVisibilityMatchesDomainMongo(t *testing.T) {
for _, indexed := range []bool{false, true} {
t.Run(fmt.Sprintf("indexed=%t", indexed), func(t *testing.T) {
ctx, s := poolLocalMongo(t)
if indexed {
if _, err := s.humanBatches.Indexes().CreateOne(ctx, blindAnnotationStageExceptionIndex()); err != nil {
t.Fatal(err)
}
}
testPoolStageRoleCountAndVisibility(t, ctx, s)
})
}
}
func testPoolStageRoleCountAndVisibility(t *testing.T, ctx context.Context, s *MongoStore) {
ids := []string{"split_annotation", domain.FanyuV5StageCutAdjustment, "split_quality_review", domain.FanyuV5StageCutReview, domain.FanyuV5StageSemanticAnnotation, "quality_review", domain.FanyuV5StageSemanticReview, "acceptance", "prepackage_acceptance", "custom-stage"}
roles := []string{"", "annotator", "cutter", "quality_reviewer", "acceptance_reviewer", "historical-unknown"}
org := "role-fixture-org"
for _, id := range ids {
for _, role := range roles {
t.Run(id+"/"+role, func(t *testing.T) {
key := id + "-" + role
stage := domain.HumanStageDefinition{ID: id, Kind: domain.HumanStageAnnotation, RequiredRole: role}
batch := domain.HumanBatchRecord{ID: key, OrganizationID: org, ProjectID: key, Status: domain.HumanBatchActive, Stages: []domain.HumanStageDefinition{stage}}
item := domain.HumanItemRecord{ID: key, BatchID: key, OrganizationID: org, ProjectID: key, PluginID: "annotation-qa", LegacyTaskID: key, CurrentStageID: id, Status: domain.HumanItemReady}
if err := s.UpsertHumanBatch(ctx, batch); err != nil {
t.Fatal(err)
}
if err := s.UpsertHumanItem(ctx, item); err != nil {
t.Fatal(err)
}
validatedTask := domain.AnnotationTaskRecord{ID: key, OrganizationID: org, ProjectID: key}
if err := s.UpsertAnnotationTask(ctx, validatedTask); err != nil {
t.Fatal(err)
}
completeSourceAdmissionTestProof(t, s, validatedTask)
want := 0
if domain.AnnotationRoleForHumanStage(stage) == domain.AnnotationRoleAnnotator {
want = 1
}
count, err := s.CountAnnotationClaimCandidates(ctx, org, AnnotationClaimCountQuery{ProjectID: key, ActiveSupplierIDs: []string{"vendor"}, Memberships: []AnnotationClaimMembership{{SupplierID: "vendor", Legacy: true}}})
if err != nil || count != want {
t.Fatalf("stage=%+v count=%d want=%d err=%v", stage, count, want, err)
}
batch.Stages[0].Kind = domain.HumanStageReview
if err := s.UpsertHumanBatch(ctx, batch); err != nil {
t.Fatal(err)
}
items, err := s.ListHumanItems(ctx, org, domain.HumanItemQuery{ProjectID: key, PluginID: "annotation-qa", BlindAnnotationActorID: "actor"})
if err != nil || len(items) != want {
t.Fatalf("stage=%+v visible=%d want=%d err=%v", batch.Stages[0], len(items), want, err)
}
})
}
}
}
// A normal unclaimed backlog must not be expanded into batch metadata or a
// distinct batch-ID reply before a private list can return an empty page.
func TestBlindAnnotationBacklogMetadataBounded(t *testing.T) {
var distinctCalls atomic.Int32
monitor := &event.CommandMonitor{Started: func(_ context.Context, e *event.CommandStartedEvent) {
if e.CommandName == "distinct" {
distinctCalls.Add(1)
}
}}
ctx, s := poolLocalMongo(t, monitor)
batches := make([]interface{}, 25000)
items := make([]interface{}, len(batches))
for i := range batches {
id := fmt.Sprintf("normal-%05d", i)
batches[i] = domain.HumanBatchRecord{ID: id, OrganizationID: "bounded-org", ProjectID: "p", Stages: domain.DefaultFanyuV58HumanStages()}
items[i] = domain.HumanItemRecord{ID: id, OrganizationID: "bounded-org", ProjectID: "p", BatchID: id, PluginID: "annotation-qa", CurrentStageID: domain.FanyuV58StageAnnotationCombined}
}
if _, err := s.humanBatches.InsertMany(ctx, batches); err != nil {
t.Fatal(err)
}
if _, err := s.humanItems.InsertMany(ctx, items); err != nil {
t.Fatal(err)
}
started := time.Now()
metadata, err := s.blindAnnotationExceptionalBatches(ctx, "bounded-org")
if err != nil || len(metadata) != 0 {
t.Fatalf("normal backlog metadata=%d err=%v", len(metadata), err)
}
result, err := s.ListHumanItems(ctx, "bounded-org", domain.HumanItemQuery{ProjectID: "p", PluginID: "annotation-qa", BlindAnnotationActorID: "actor", Limit: 100, ListProjection: true})
if err != nil || len(result) != 0 {
t.Fatalf("private list=%d err=%v", len(result), err)
}
if got := distinctCalls.Load(); got != 0 {
t.Fatalf("backlog distinct expansion returned: %d commands", got)
}
t.Logf("25k normal batches: exceptional metadata=%d list=%s", len(metadata), time.Since(started))
}