droboticflow/internal/store/platform_query_regression_test.go

283 lines
11 KiB
Go

package store
import (
"context"
"droboticflow/internal/domain"
"fmt"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/event"
"strings"
"sync"
"testing"
)
func TestPlatformClaimStageReadDoesNotFetchReviewBacklog(t *testing.T) {
// Exercise the operational/advisory path with real task scope; an orphan
// remains excluded even when expensive source audits are advisory.
t.Setenv("ANNOTATION_SOURCE_VALIDATION_MODE", "advisory")
var mu sync.Mutex
var query bson.M
ctx, s := poolLocalMongo(t, &event.CommandMonitor{Started: func(_ context.Context, e *event.CommandStartedEvent) {
if e.CommandName == "find" && e.Command.Lookup("find").StringValue() == "humanItems" {
mu.Lock()
defer mu.Unlock()
_ = bson.Unmarshal(e.Command, &query)
}
}})
if err := s.EnsureIndexes(ctx); err != nil {
t.Fatal(err)
}
const org, project = "closure-org", "closure-project"
batch := domain.HumanBatchRecord{ID: "batch", OrganizationID: org, ProjectID: project, Stages: domain.DefaultFanyuV58HumanStages(), Status: domain.HumanBatchActive}
if err := s.UpsertHumanBatch(ctx, batch); err != nil {
t.Fatal(err)
}
docs := make([]any, 1042)
tasks := make([]any, 0, 42)
for i := range docs {
stage := domain.FanyuV58StageQualityReview
if i < 42 {
stage = domain.FanyuV58StageAnnotationCombined
tasks = append(tasks, bson.M{"id": fmt.Sprintf("task-%d", i), "organizationId": org, "projectId": project, "runId": "run"})
}
docs[i] = bson.M{"id": fmt.Sprintf("item-%d", i), "organizationId": org, "projectId": project, "pluginId": "annotation-qa", "batchId": batch.ID, "runId": "run", "legacyTaskId": fmt.Sprintf("task-%d", i), "currentStageId": stage, "status": "ready", "assignedUserId": "", "supplierId": "supplier", "subject": bson.M{"itemKey": fmt.Sprintf("source-%d", i), "payload": bson.M{"large": strings.Repeat("x", 16<<10)}}}
}
docs = append(docs, bson.M{"id": "orphan", "organizationId": org, "projectId": project, "runId": "run", "pluginId": "annotation-qa", "batchId": batch.ID, "legacyTaskId": "missing", "currentStageId": domain.FanyuV58StageAnnotationCombined, "status": "ready", "assignedUserId": "", "supplierId": "supplier"})
if _, err := s.annotationTasks.InsertMany(ctx, tasks); err != nil {
t.Fatal(err)
}
if _, err := s.humanItems.InsertMany(ctx, docs); err != nil {
t.Fatal(err)
}
n, err := s.CountAnnotationClaimCandidates(ctx, org, AnnotationClaimCountQuery{ProjectID: project, ActiveSupplierIDs: []string{"supplier"}, Memberships: []AnnotationClaimMembership{{SupplierID: "supplier", Legacy: true}}})
if err != nil || n != 42 {
t.Fatal("wrong inventory", n, err)
}
mu.Lock()
captured := query
mu.Unlock()
find := bson.D{{Key: "find", Value: "humanItems"}, {Key: "filter", Value: captured["filter"]}, {Key: "projection", Value: captured["projection"]}}
if h := captured["hint"]; h != nil {
find = append(find, bson.E{Key: "hint", Value: h})
}
var result bson.M
if err := s.db.RunCommand(ctx, bson.D{{Key: "explain", Value: find}, {Key: "verbosity", Value: "executionStats"}}).Decode(&result); err != nil {
t.Fatal(err)
}
stats := deletionDoc(result["executionStats"])
examined := supplierNumber(stats["totalDocsExamined"])
t.Logf("claim candidates=%d examined=%d keys=%v", n, examined, stats["totalKeysExamined"])
if examined > 50 {
t.Fatal("annotation inventory fetched unrelated review payloads", examined)
}
}
func TestPlatformWorkerRunCountsDoNotFetchRunPayloads(t *testing.T) {
var mu sync.Mutex
var query bson.M
ctx, s := poolLocalMongo(t, &event.CommandMonitor{Started: func(_ context.Context, e *event.CommandStartedEvent) {
if (e.CommandName == "aggregate" && e.Command.Lookup("aggregate").StringValue() == "runs") || (e.CommandName == "find" && e.Command.Lookup("find").StringValue() == "runs") {
mu.Lock()
defer mu.Unlock()
_ = bson.Unmarshal(e.Command, &query)
}
}})
if err := s.EnsureIndexes(ctx); err != nil {
t.Fatal(err)
}
if _, err := s.runs.Indexes().CreateOne(ctx, PlatformReadIndexes()["runs"][0]); err != nil {
t.Fatal(err)
}
docs := make([]any, 2300)
for i := range docs {
status := "completed"
if i < 300 {
status = "queued"
}
docs[i] = bson.M{"id": fmt.Sprintf("run-%d", i), "workflowId": fmt.Sprintf("workflow-%d", i%7), "status": status, "logs": strings.Repeat("x", 16<<10)}
}
if _, err := s.runs.InsertMany(ctx, docs); err != nil {
t.Fatal(err)
}
n, err := s.CountWorkerRuns(ctx)
if err != nil || n.QueuedTotal != 300 {
t.Fatal("wrong worker counts", n, err)
}
mu.Lock()
captured := query
mu.Unlock()
command := bson.D{{Key: "aggregate", Value: "runs"}, {Key: "pipeline", Value: captured["pipeline"]}, {Key: "cursor", Value: bson.M{}}}
if _, ok := captured["find"]; ok {
command = bson.D{{Key: "find", Value: "runs"}, {Key: "filter", Value: captured["filter"]}, {Key: "projection", Value: captured["projection"]}}
}
if h := captured["hint"]; h != nil {
command = append(command, bson.E{Key: "hint", Value: h})
}
var result bson.M
if err := s.db.RunCommand(ctx, bson.D{{Key: "explain", Value: command}, {Key: "verbosity", Value: "executionStats"}}).Decode(&result); err != nil {
t.Fatal(err)
}
stats := deletionDoc(result["executionStats"])
if stats == nil {
for _, raw := range deletionArray(result["stages"]) {
if cursor := deletionDoc(deletionDoc(raw)["$cursor"]); cursor != nil {
stats = deletionDoc(cursor["executionStats"])
}
}
}
t.Logf("worker queued=%d examined=%v keys=%v", n.QueuedTotal, stats["totalDocsExamined"], stats["totalKeysExamined"])
if stats == nil || supplierNumber(stats["totalDocsExamined"]) != 0 {
t.Fatalf("worker polling fetched large run bodies: docs=%v", stats["totalDocsExamined"])
}
}
// A broad text match must not build and sort every foreign join before the
// first page can be returned. The original matcher still decides each row.
func TestPlatformBroadPoolSearchBoundsFirstPageJoins(t *testing.T) {
var mu sync.Mutex
var joined []bson.M
ctx, s := poolLocalMongo(t, &event.CommandMonitor{Started: func(_ context.Context, e *event.CommandStartedEvent) {
if e.CommandName != "aggregate" || e.Command.Lookup("aggregate").StringValue() != "humanItems" {
return
}
var q bson.M
if bson.Unmarshal(e.Command, &q) != nil {
return
}
raw, _ := bson.Marshal(q)
if strings.Contains(string(raw), "$lookup") {
mu.Lock()
joined = append(joined, q)
mu.Unlock()
}
}})
if err := s.EnsureIndexes(ctx); err != nil {
t.Fatal(err)
}
docs := make([]any, 5000)
for i := range docs {
docs[i] = bson.M{"id": fmt.Sprintf("pool-%05d", i), "organizationId": "broad-org", "pluginId": "annotation-qa", "currentStageId": "prepackage_acceptance", "status": "ready", "assignedUserId": "", "createdAt": fmt.Sprintf("2026-11-%05d", i), "subject": bson.M{"itemKey": fmt.Sprintf("FYLX_BROAD_%05d/part0001", i)}}
}
if _, err := s.humanItems.InsertMany(ctx, docs); err != nil {
t.Fatal(err)
}
rows, err := s.ListHumanItems(ctx, "broad-org", domain.HumanItemQuery{PluginID: "annotation-qa", CurrentStageID: "prepackage_acceptance", Search: "FYLX_BROAD", Limit: 25, IdentityProjection: true})
if err != nil || len(rows) != 25 {
t.Fatal("first page", len(rows), err)
}
if rows[0].ID != "pool-04999" || rows[24].ID != "pool-04975" {
t.Fatal("order changed")
}
mu.Lock()
queries := joined
mu.Unlock()
if len(queries) == 0 {
t.Fatal("no actual join query observed")
}
for _, q := range queries {
var result bson.M
cmd := poolTestExplainCommand(q)
for _, key := range []string{"$db", "lsid", "$clusterTime", "readConcern", "$readPreference"} {
delete(q, key)
}
cmd = poolTestExplainCommand(q)
if err := s.db.RunCommand(ctx, cmd).Decode(&result); err != nil {
t.Fatal(err)
}
var examined int64
if stats := deletionDoc(result["executionStats"]); stats != nil {
examined = int64(supplierNumber(stats["totalDocsExamined"]))
}
for _, raw := range deletionArray(result["stages"]) {
if c := deletionDoc(deletionDoc(raw)["$cursor"]); c != nil {
examined = int64(supplierNumber(deletionDoc(c["executionStats"])["totalDocsExamined"]))
}
}
t.Logf("first-page joins=%d root documents=%d", len(queries), examined)
if examined > 250 {
t.Fatal("first-page join materialized whole inventory", examined)
}
}
}
func TestPlatformRecoveryLookupIndexesPreserveLegacyReceipts(t *testing.T) {
var mu sync.Mutex
reads := map[string]bson.M{}
ctx, s := poolLocalMongo(t, &event.CommandMonitor{Started: func(_ context.Context, e *event.CommandStartedEvent) {
if e.CommandName != "find" {
return
}
name := e.Command.Lookup("find").StringValue()
if name != "reviewOperations" && name != "reviewSessions" && name != "captureRecords" {
return
}
var q bson.M
if bson.Unmarshal(e.Command, &q) != nil {
return
}
mu.Lock()
reads[name] = q
mu.Unlock()
}})
ops, sessions, captures := []any{}, []any{}, []any{}
for i := 0; i < 3000; i++ {
ops = append(ops, bson.M{"_id": fmt.Sprintf("legacy-receipt-%d", i), "organizationId": "recovery-org", "userId": "reviewer", "operationId": fmt.Sprintf("op-%d", i), "itemId": "item", "taskId": "task", "status": "completed"})
sessions = append(sessions, bson.M{"_id": fmt.Sprintf("session-%d", i), "revision": 1})
captures = append(captures, bson.M{"id": fmt.Sprintf("capture-%d", i), "runId": fmt.Sprintf("run-%d", i)})
}
for name, docs := range map[string][]any{"reviewOperations": ops, "reviewSessions": sessions, "captureRecords": captures} {
if _, err := s.db.Collection(name).InsertMany(ctx, docs); err != nil {
t.Fatal(err)
}
}
explainLast := func(name string) int64 {
mu.Lock()
q := reads[name]
mu.Unlock()
for _, k := range []string{"$db", "lsid", "$clusterTime", "readConcern", "$readPreference"} {
delete(q, k)
}
var result bson.M
if err := s.db.RunCommand(ctx, poolTestExplainCommand(q)).Decode(&result); err != nil {
t.Fatal(err)
}
return supplierNumber(deletionDoc(result["executionStats"])["totalDocsExamined"])
}
for _, indexed := range []bool{false, true} {
if indexed {
for name, models := range PlatformReadIndexes() {
if _, err := s.db.Collection(name).Indexes().CreateMany(ctx, models); err != nil {
t.Fatal(err)
}
}
}
op, err := s.FindReviewOperation(ctx, "recovery-org", "reviewer", "item", "op-2999")
if err != nil || op == nil || op.ID != "legacy-receipt-2999" {
t.Fatal("legacy item receipt lookup changed", err)
}
itemDocs := explainLast("reviewOperations")
alias, err := s.FindReviewOperation(ctx, "recovery-org", "reviewer", "task", "op-2999")
if err != nil || alias == nil || alias.ID != op.ID {
t.Fatal("task alias receipt lookup changed", err)
}
if denied, err := s.FindReviewOperation(ctx, "foreign-org", "reviewer", "item", "op-2999"); err != nil || denied != nil {
t.Fatal("receipt tenant scope widened", err)
}
if q, err := s.GetReviewContinuation(ctx, "queue-with-no-pending-admission"); err != nil || q != nil {
t.Fatal("missing queue recovery changed", err)
}
sessionDocs := explainLast("reviewSessions")
capture, err := s.GetCaptureRecordByRunID(ctx, "run-2999")
if err != nil || capture == nil || capture.ID != "capture-2999" {
t.Fatal("capture run relation changed", err)
}
captureDocs := explainLast("captureRecords")
t.Logf("indexed=%t receiptDocs=%d sessionDocs=%d captureDocs=%d", indexed, itemDocs, sessionDocs, captureDocs)
if indexed && (itemDocs > 1 || sessionDocs > 0 || captureDocs > 1) {
t.Fatal("recovery lookup still scans unrelated records")
}
}
reviewContinuationAdmissionContract(t, s)
}