⚡ Scope reverse pool searches to reachable source records
Some checks failed
web-ci / checks (push) Has been cancelled
web-ci / pool-search-mongo (push) Has been cancelled
web-ci / checks (pull_request) Has been cancelled
web-ci / pool-search-mongo (pull_request) Has been cancelled

This commit is contained in:
Codex 2026-11-09 08:20:38 -08:00
parent bfaa8d6a2f
commit 7a1231c7c6
2 changed files with 247 additions and 5 deletions

View File

@ -0,0 +1,135 @@
package store
import (
"context"
"droboticflow/internal/domain"
"fmt"
"net/url"
"os"
"sync"
"sync/atomic"
"testing"
"time"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/event"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
)
// A migrated organization may have tens of thousands of unrelated captures.
// Every reverse lookup must be restricted to references from the base pool,
// rather than paying for a tenant-wide validity/regex scan on each keystroke.
func TestHumanPoolSearchMongoReferenceScope(t *testing.T) {
uri := os.Getenv("POOL_LOCAL_MONGO_URI")
if uri == "" {
t.Skip("isolated local Mongo opt-in")
}
u, parseErr := url.Parse(uri)
if parseErr != nil || u.Scheme != "mongodb" || (u.Host != "127.0.0.1:27018" && u.Host != "127.0.0.1:27017") {
t.Fatal("local Mongo only")
}
ctx, cancel := context.WithTimeout(context.Background(), 45*time.Second)
defer cancel()
var measuring atomic.Bool
var mu sync.Mutex
var queries []bson.M
monitor := &event.CommandMonitor{Started: func(_ context.Context, e *event.CommandStartedEvent) {
if !measuring.Load() || e.CommandName != "find" {
return
}
name := e.Command.Lookup("find").StringValue()
if name == "humanItems" {
return
}
var filter bson.M
if err := bson.Unmarshal(e.Command.Lookup("filter").Document(), &filter); err != nil {
return
}
mu.Lock()
queries = append(queries, bson.M{"collection": name, "filter": filter})
mu.Unlock()
}}
client, err := mongo.Connect(ctx, options.Client().ApplyURI(uri).SetMonitor(monitor))
if err != nil {
t.Fatal(err)
}
defer client.Disconnect(context.Background())
db := client.Database(fmt.Sprintf("pool_search_scope_%d", time.Now().UnixNano()))
defer db.Drop(context.Background())
s := NewMongoStore(db)
const org = "scope-org"
for _, name := range []string{"humanItems", "runs", "captureRecords", "annotationTasks", "projects"} {
if _, err := db.Collection(name).Indexes().CreateOne(ctx, mongo.IndexModel{Keys: bson.D{{Key: "id", Value: 1}}, Options: options.Index().SetUnique(true)}); err != nil {
t.Fatal(err)
}
}
if err := s.UpsertProject(ctx, domain.ProjectRecord{ID: "p", OrganizationID: org, Name: "Project"}); err != nil {
t.Fatal(err)
}
if err := s.UpsertRun(ctx, domain.RunRecord{ID: "r", OrganizationID: org, TaskID: "capture"}); err != nil {
t.Fatal(err)
}
if err := s.UpsertCaptureRecord(ctx, domain.CaptureRecord{ID: "capture", OrganizationID: org, CaptureID: "CAPTURE_NEEDLE"}); err != nil {
t.Fatal(err)
}
if err := s.UpsertAnnotationTask(ctx, domain.AnnotationTaskRecord{ID: "task", OrganizationID: org, ArtifactID: "camera_left_part0001.mp4"}); err != nil {
t.Fatal(err)
}
if err := s.UpsertHumanItem(ctx, domain.HumanItemRecord{ID: "item", OrganizationID: org, ProjectID: "p", RunID: "r", LegacyTaskID: "task", CurrentStageID: "quality_review", Status: domain.HumanItemReady, Subject: domain.HumanSubjectRef{ItemKey: "camera_left_part0001.mp4"}}); err != nil {
t.Fatal(err)
}
// Unrelated same-tenant data must not broaden a lookup.
for n := 0; n < 40; n++ {
if err := s.UpsertCaptureRecord(ctx, domain.CaptureRecord{ID: fmt.Sprint("unrelated-", n), OrganizationID: org, CaptureID: "CAPTURE_NEEDLE"}); err != nil {
t.Fatal(err)
}
}
for _, search := range []string{"CAPTURE_NEEDLE/part0001", "no-such-capture"} {
queries = nil
measuring.Store(true)
got, err := s.ListHumanItems(ctx, org, domain.HumanItemQuery{Search: search, StageIDs: []string{"quality_review"}, PoolCandidateProjection: true, Limit: 200})
measuring.Store(false)
if err != nil {
t.Fatal(err)
}
want := 0
if search != "no-such-capture" {
want = 1
}
if len(got) != want {
t.Fatalf("search=%s rows=%d want=%d", search, len(got), want)
}
if len(queries) == 0 {
t.Fatal("reverse lookup path was not exercised")
}
for _, query := range queries {
if !poolTestHasIDScope(query["filter"]) {
t.Fatalf("unbounded reverse lookup: %v", query)
}
}
}
}
func poolTestHasIDScope(v any) bool {
switch value := v.(type) {
case bson.M:
if id, ok := value["id"].(bson.M); ok {
if _, ok := id["$in"]; ok {
return true
}
}
for _, child := range value {
if poolTestHasIDScope(child) {
return true
}
}
case bson.A:
for _, child := range value {
if poolTestHasIDScope(child) {
return true
}
}
}
return false
}

View File

@ -139,6 +139,14 @@ func (s *MongoStore) humanPoolSearchCandidates(ctx context.Context, org string,
if err != mongo.ErrNoDocuments {
return nil, err
}
// Reverse lookups must stay within references reachable from the base pool.
// A migrated tenant-wide validity/regex query still FETCHes every source
// document; an index on organization alone does not make that query cheap.
const maxReferences = 4096
refs, broad, err := s.humanPoolSearchReferenceScope(ctx, org, base, maxReferences)
if err != nil || broad {
return nil, err
}
token := strings.ToLower(words[0])
if i := strings.LastIndex(token, "/part"); i > 0 {
token = token[:i]
@ -148,7 +156,6 @@ func (s *MongoStore) humanPoolSearchCandidates(ctx context.Context, org string,
return bson.M{"$or": bson.A{bson.M{"$expr": bson.M{"$not": bson.A{poolSearchValidExpression(collection)}}}, bson.M{poolSearchField + ".text": bson.M{"$regex": pattern}}}}
}
match := matchFor("captureRecords")
const maxReferences = 4096
ids := func(collection string, extra bson.M) ([]string, bool, error) {
f := bson.M{"organizationId": normalizedOrganizationID(org), "$and": bson.A{match}}
if extra != nil {
@ -171,7 +178,7 @@ func (s *MongoStore) humanPoolSearchCandidates(ctx context.Context, org string,
}
return out, len(out) > maxReferences, cur.Err()
}
captureIDs, broad, err := ids("captureRecords", nil)
captureIDs, broad, err := ids("captureRecords", bson.M{"id": bson.M{"$in": refs["captureRecords"]}})
if err != nil || broad {
return nil, err
}
@ -182,17 +189,17 @@ func (s *MongoStore) humanPoolSearchCandidates(ctx context.Context, org string,
runMatch = bson.M{"$or": bson.A{runMatch, bson.M{"taskId": bson.M{"$in": captureIDs}}}}
}
match = runMatch
runIDs, broad, err := ids("runs", nil)
runIDs, broad, err := ids("runs", bson.M{"id": bson.M{"$in": refs["runs"]}})
if err != nil || broad {
return nil, err
}
match = matchFor("annotationTasks")
taskIDs, broad, err := ids("annotationTasks", nil)
taskIDs, broad, err := ids("annotationTasks", bson.M{"id": bson.M{"$in": refs["annotationTasks"]}})
if err != nil || broad {
return nil, err
}
match = matchFor("projects")
projectIDs, broad, err := ids("projects", nil)
projectIDs, broad, err := ids("projects", bson.M{"id": bson.M{"$in": refs["projects"]}})
if err != nil || broad {
return nil, err
}
@ -205,6 +212,106 @@ func (s *MongoStore) humanPoolSearchCandidates(ctx context.Context, org string,
return bson.M{"$or": branches}, nil
}
// The bounded projection contains no customer payload. Overflow falls back to
// the existing full search, never a truncated list. This only narrows candidate
// sources; the original final matcher and current permissions remain in force.
func (s *MongoStore) humanPoolSearchReferenceScope(ctx context.Context, org string, base bson.M, limit int) (map[string][]string, bool, error) {
sets := map[string]map[string]bool{}
for _, name := range []string{"runs", "annotationTasks", "projects", "captureRecords"} {
sets[name] = map[string]bool{}
}
read := func(collection string, filter, projection bson.M, visit func(bson.M)) (bool, error) {
cur, err := s.db.Collection(collection).Find(ctx, filter, options.Find().SetProjection(projection).SetLimit(int64(limit+1)).SetMaxTime(8*time.Second))
if err != nil {
return false, err
}
defer cur.Close(ctx)
count := 0
for cur.Next(ctx) {
count++
if count > limit {
return true, nil
}
var row bson.M
if err := cur.Decode(&row); err != nil {
return false, err
}
visit(row)
}
return false, cur.Err()
}
add := func(collection string, value any) {
if id, ok := value.(string); ok && id != "" {
sets[collection][id] = true
}
}
values := func(collection string) []string {
out := make([]string, 0, len(sets[collection]))
for id := range sets[collection] {
out = append(out, id)
}
sort.Strings(out)
return out
}
// Decide overflow in Mongo so a broad no-hit query does not transfer a
// backlog of references just to select the compatibility path.
withinLimit := bson.M{"$lte": bson.A{"$count", limit}}
rootPipeline := mongo.Pipeline{
bson.D{{Key: "$match", Value: base}},
bson.D{{Key: "$limit", Value: int64(limit + 1)}},
bson.D{{Key: "$group", Value: bson.M{
"_id": nil, "count": bson.M{"$sum": 1},
"runs": bson.M{"$addToSet": bson.M{"$ifNull": bson.A{"$runId", ""}}},
"annotationTasks": bson.M{"$addToSet": bson.M{"$ifNull": bson.A{"$legacyTaskId", ""}}},
"projects": bson.M{"$addToSet": bson.M{"$ifNull": bson.A{"$projectId", ""}}},
}}},
bson.D{{Key: "$project", Value: bson.M{
"_id": 0, "count": 1,
"runs": bson.M{"$cond": bson.A{withinLimit, "$runs", bson.A{}}},
"annotationTasks": bson.M{"$cond": bson.A{withinLimit, "$annotationTasks", bson.A{}}},
"projects": bson.M{"$cond": bson.A{withinLimit, "$projects", bson.A{}}},
}}},
}
cur, err := s.humanItems.Aggregate(ctx, rootPipeline, options.Aggregate().SetAllowDiskUse(false).SetMaxTime(8*time.Second))
if err != nil {
return nil, false, err
}
defer cur.Close(ctx)
if cur.Next(ctx) {
var row struct {
Count int `bson:"count"`
Runs []string `bson:"runs"`
Tasks []string `bson:"annotationTasks"`
Projects []string `bson:"projects"`
}
if err := cur.Decode(&row); err != nil {
return nil, false, err
}
if row.Count > limit {
return nil, true, nil
}
for name, ids := range map[string][]string{"runs": row.Runs, "annotationTasks": row.Tasks, "projects": row.Projects} {
for _, id := range ids {
add(name, id)
}
}
}
if err := cur.Err(); err != nil {
return nil, false, err
}
broad, err := read("runs", bson.M{"organizationId": normalizedOrganizationID(org), "id": bson.M{"$in": values("runs")}}, bson.M{"_id": 0, "taskId": 1}, func(row bson.M) {
add("captureRecords", row["taskId"])
})
if err != nil || broad {
return nil, broad, err
}
refs := map[string][]string{}
for name := range sets {
refs[name] = values(name)
}
return refs, false, nil
}
func poolSearchSourceValue(d bson.M, path string) any {
if path == "__runReferences" {
return poolRunReferences(d)