diff --git a/internal/store/human_pool_search_scope_test.go b/internal/store/human_pool_search_scope_test.go new file mode 100644 index 00000000..992a7926 --- /dev/null +++ b/internal/store/human_pool_search_scope_test.go @@ -0,0 +1,135 @@ +package store + +import ( + "context" + "droboticflow/internal/domain" + "fmt" + "net/url" + "os" + "sync" + "sync/atomic" + "testing" + "time" + + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/event" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" +) + +// A migrated organization may have tens of thousands of unrelated captures. +// Every reverse lookup must be restricted to references from the base pool, +// rather than paying for a tenant-wide validity/regex scan on each keystroke. +func TestHumanPoolSearchMongoReferenceScope(t *testing.T) { + uri := os.Getenv("POOL_LOCAL_MONGO_URI") + if uri == "" { + t.Skip("isolated local Mongo opt-in") + } + u, parseErr := url.Parse(uri) + if parseErr != nil || u.Scheme != "mongodb" || (u.Host != "127.0.0.1:27018" && u.Host != "127.0.0.1:27017") { + t.Fatal("local Mongo only") + } + ctx, cancel := context.WithTimeout(context.Background(), 45*time.Second) + defer cancel() + var measuring atomic.Bool + var mu sync.Mutex + var queries []bson.M + monitor := &event.CommandMonitor{Started: func(_ context.Context, e *event.CommandStartedEvent) { + if !measuring.Load() || e.CommandName != "find" { + return + } + name := e.Command.Lookup("find").StringValue() + if name == "humanItems" { + return + } + var filter bson.M + if err := bson.Unmarshal(e.Command.Lookup("filter").Document(), &filter); err != nil { + return + } + mu.Lock() + queries = append(queries, bson.M{"collection": name, "filter": filter}) + mu.Unlock() + }} + client, err := mongo.Connect(ctx, options.Client().ApplyURI(uri).SetMonitor(monitor)) + if err != nil { + t.Fatal(err) + } + defer client.Disconnect(context.Background()) + db := client.Database(fmt.Sprintf("pool_search_scope_%d", time.Now().UnixNano())) + defer db.Drop(context.Background()) + s := NewMongoStore(db) + const org = "scope-org" + for _, name := range []string{"humanItems", "runs", "captureRecords", "annotationTasks", "projects"} { + if _, err := db.Collection(name).Indexes().CreateOne(ctx, mongo.IndexModel{Keys: bson.D{{Key: "id", Value: 1}}, Options: options.Index().SetUnique(true)}); err != nil { + t.Fatal(err) + } + } + if err := s.UpsertProject(ctx, domain.ProjectRecord{ID: "p", OrganizationID: org, Name: "Project"}); err != nil { + t.Fatal(err) + } + if err := s.UpsertRun(ctx, domain.RunRecord{ID: "r", OrganizationID: org, TaskID: "capture"}); err != nil { + t.Fatal(err) + } + if err := s.UpsertCaptureRecord(ctx, domain.CaptureRecord{ID: "capture", OrganizationID: org, CaptureID: "CAPTURE_NEEDLE"}); err != nil { + t.Fatal(err) + } + if err := s.UpsertAnnotationTask(ctx, domain.AnnotationTaskRecord{ID: "task", OrganizationID: org, ArtifactID: "camera_left_part0001.mp4"}); err != nil { + t.Fatal(err) + } + if err := s.UpsertHumanItem(ctx, domain.HumanItemRecord{ID: "item", OrganizationID: org, ProjectID: "p", RunID: "r", LegacyTaskID: "task", CurrentStageID: "quality_review", Status: domain.HumanItemReady, Subject: domain.HumanSubjectRef{ItemKey: "camera_left_part0001.mp4"}}); err != nil { + t.Fatal(err) + } + // Unrelated same-tenant data must not broaden a lookup. + for n := 0; n < 40; n++ { + if err := s.UpsertCaptureRecord(ctx, domain.CaptureRecord{ID: fmt.Sprint("unrelated-", n), OrganizationID: org, CaptureID: "CAPTURE_NEEDLE"}); err != nil { + t.Fatal(err) + } + } + for _, search := range []string{"CAPTURE_NEEDLE/part0001", "no-such-capture"} { + queries = nil + measuring.Store(true) + got, err := s.ListHumanItems(ctx, org, domain.HumanItemQuery{Search: search, StageIDs: []string{"quality_review"}, PoolCandidateProjection: true, Limit: 200}) + measuring.Store(false) + if err != nil { + t.Fatal(err) + } + want := 0 + if search != "no-such-capture" { + want = 1 + } + if len(got) != want { + t.Fatalf("search=%s rows=%d want=%d", search, len(got), want) + } + if len(queries) == 0 { + t.Fatal("reverse lookup path was not exercised") + } + for _, query := range queries { + if !poolTestHasIDScope(query["filter"]) { + t.Fatalf("unbounded reverse lookup: %v", query) + } + } + } +} + +func poolTestHasIDScope(v any) bool { + switch value := v.(type) { + case bson.M: + if id, ok := value["id"].(bson.M); ok { + if _, ok := id["$in"]; ok { + return true + } + } + for _, child := range value { + if poolTestHasIDScope(child) { + return true + } + } + case bson.A: + for _, child := range value { + if poolTestHasIDScope(child) { + return true + } + } + } + return false +} diff --git a/internal/store/pool_search_metadata.go b/internal/store/pool_search_metadata.go index 706a78de..2861da0d 100644 --- a/internal/store/pool_search_metadata.go +++ b/internal/store/pool_search_metadata.go @@ -139,6 +139,14 @@ func (s *MongoStore) humanPoolSearchCandidates(ctx context.Context, org string, if err != mongo.ErrNoDocuments { return nil, err } + // Reverse lookups must stay within references reachable from the base pool. + // A migrated tenant-wide validity/regex query still FETCHes every source + // document; an index on organization alone does not make that query cheap. + const maxReferences = 4096 + refs, broad, err := s.humanPoolSearchReferenceScope(ctx, org, base, maxReferences) + if err != nil || broad { + return nil, err + } token := strings.ToLower(words[0]) if i := strings.LastIndex(token, "/part"); i > 0 { token = token[:i] @@ -148,7 +156,6 @@ func (s *MongoStore) humanPoolSearchCandidates(ctx context.Context, org string, return bson.M{"$or": bson.A{bson.M{"$expr": bson.M{"$not": bson.A{poolSearchValidExpression(collection)}}}, bson.M{poolSearchField + ".text": bson.M{"$regex": pattern}}}} } match := matchFor("captureRecords") - const maxReferences = 4096 ids := func(collection string, extra bson.M) ([]string, bool, error) { f := bson.M{"organizationId": normalizedOrganizationID(org), "$and": bson.A{match}} if extra != nil { @@ -171,7 +178,7 @@ func (s *MongoStore) humanPoolSearchCandidates(ctx context.Context, org string, } return out, len(out) > maxReferences, cur.Err() } - captureIDs, broad, err := ids("captureRecords", nil) + captureIDs, broad, err := ids("captureRecords", bson.M{"id": bson.M{"$in": refs["captureRecords"]}}) if err != nil || broad { return nil, err } @@ -182,17 +189,17 @@ func (s *MongoStore) humanPoolSearchCandidates(ctx context.Context, org string, runMatch = bson.M{"$or": bson.A{runMatch, bson.M{"taskId": bson.M{"$in": captureIDs}}}} } match = runMatch - runIDs, broad, err := ids("runs", nil) + runIDs, broad, err := ids("runs", bson.M{"id": bson.M{"$in": refs["runs"]}}) if err != nil || broad { return nil, err } match = matchFor("annotationTasks") - taskIDs, broad, err := ids("annotationTasks", nil) + taskIDs, broad, err := ids("annotationTasks", bson.M{"id": bson.M{"$in": refs["annotationTasks"]}}) if err != nil || broad { return nil, err } match = matchFor("projects") - projectIDs, broad, err := ids("projects", nil) + projectIDs, broad, err := ids("projects", bson.M{"id": bson.M{"$in": refs["projects"]}}) if err != nil || broad { return nil, err } @@ -205,6 +212,106 @@ func (s *MongoStore) humanPoolSearchCandidates(ctx context.Context, org string, return bson.M{"$or": branches}, nil } +// The bounded projection contains no customer payload. Overflow falls back to +// the existing full search, never a truncated list. This only narrows candidate +// sources; the original final matcher and current permissions remain in force. +func (s *MongoStore) humanPoolSearchReferenceScope(ctx context.Context, org string, base bson.M, limit int) (map[string][]string, bool, error) { + sets := map[string]map[string]bool{} + for _, name := range []string{"runs", "annotationTasks", "projects", "captureRecords"} { + sets[name] = map[string]bool{} + } + read := func(collection string, filter, projection bson.M, visit func(bson.M)) (bool, error) { + cur, err := s.db.Collection(collection).Find(ctx, filter, options.Find().SetProjection(projection).SetLimit(int64(limit+1)).SetMaxTime(8*time.Second)) + if err != nil { + return false, err + } + defer cur.Close(ctx) + count := 0 + for cur.Next(ctx) { + count++ + if count > limit { + return true, nil + } + var row bson.M + if err := cur.Decode(&row); err != nil { + return false, err + } + visit(row) + } + return false, cur.Err() + } + add := func(collection string, value any) { + if id, ok := value.(string); ok && id != "" { + sets[collection][id] = true + } + } + values := func(collection string) []string { + out := make([]string, 0, len(sets[collection])) + for id := range sets[collection] { + out = append(out, id) + } + sort.Strings(out) + return out + } + // Decide overflow in Mongo so a broad no-hit query does not transfer a + // backlog of references just to select the compatibility path. + withinLimit := bson.M{"$lte": bson.A{"$count", limit}} + rootPipeline := mongo.Pipeline{ + bson.D{{Key: "$match", Value: base}}, + bson.D{{Key: "$limit", Value: int64(limit + 1)}}, + bson.D{{Key: "$group", Value: bson.M{ + "_id": nil, "count": bson.M{"$sum": 1}, + "runs": bson.M{"$addToSet": bson.M{"$ifNull": bson.A{"$runId", ""}}}, + "annotationTasks": bson.M{"$addToSet": bson.M{"$ifNull": bson.A{"$legacyTaskId", ""}}}, + "projects": bson.M{"$addToSet": bson.M{"$ifNull": bson.A{"$projectId", ""}}}, + }}}, + bson.D{{Key: "$project", Value: bson.M{ + "_id": 0, "count": 1, + "runs": bson.M{"$cond": bson.A{withinLimit, "$runs", bson.A{}}}, + "annotationTasks": bson.M{"$cond": bson.A{withinLimit, "$annotationTasks", bson.A{}}}, + "projects": bson.M{"$cond": bson.A{withinLimit, "$projects", bson.A{}}}, + }}}, + } + cur, err := s.humanItems.Aggregate(ctx, rootPipeline, options.Aggregate().SetAllowDiskUse(false).SetMaxTime(8*time.Second)) + if err != nil { + return nil, false, err + } + defer cur.Close(ctx) + if cur.Next(ctx) { + var row struct { + Count int `bson:"count"` + Runs []string `bson:"runs"` + Tasks []string `bson:"annotationTasks"` + Projects []string `bson:"projects"` + } + if err := cur.Decode(&row); err != nil { + return nil, false, err + } + if row.Count > limit { + return nil, true, nil + } + for name, ids := range map[string][]string{"runs": row.Runs, "annotationTasks": row.Tasks, "projects": row.Projects} { + for _, id := range ids { + add(name, id) + } + } + } + if err := cur.Err(); err != nil { + return nil, false, err + } + broad, err := read("runs", bson.M{"organizationId": normalizedOrganizationID(org), "id": bson.M{"$in": values("runs")}}, bson.M{"_id": 0, "taskId": 1}, func(row bson.M) { + add("captureRecords", row["taskId"]) + }) + if err != nil || broad { + return nil, broad, err + } + refs := map[string][]string{} + for name := range sets { + refs[name] = values(name) + } + return refs, false, nil +} + func poolSearchSourceValue(d bson.M, path string) any { if path == "__runReferences" { return poolRunReferences(d)