forked from longtao.wu/droboticflow
⚡ Scope reverse pool searches to reachable source records
This commit is contained in:
parent
bfaa8d6a2f
commit
7a1231c7c6
135
internal/store/human_pool_search_scope_test.go
Normal file
135
internal/store/human_pool_search_scope_test.go
Normal file
@ -0,0 +1,135 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"droboticflow/internal/domain"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"os"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
"go.mongodb.org/mongo-driver/event"
|
||||
"go.mongodb.org/mongo-driver/mongo"
|
||||
"go.mongodb.org/mongo-driver/mongo/options"
|
||||
)
|
||||
|
||||
// A migrated organization may have tens of thousands of unrelated captures.
|
||||
// Every reverse lookup must be restricted to references from the base pool,
|
||||
// rather than paying for a tenant-wide validity/regex scan on each keystroke.
|
||||
func TestHumanPoolSearchMongoReferenceScope(t *testing.T) {
|
||||
uri := os.Getenv("POOL_LOCAL_MONGO_URI")
|
||||
if uri == "" {
|
||||
t.Skip("isolated local Mongo opt-in")
|
||||
}
|
||||
u, parseErr := url.Parse(uri)
|
||||
if parseErr != nil || u.Scheme != "mongodb" || (u.Host != "127.0.0.1:27018" && u.Host != "127.0.0.1:27017") {
|
||||
t.Fatal("local Mongo only")
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 45*time.Second)
|
||||
defer cancel()
|
||||
var measuring atomic.Bool
|
||||
var mu sync.Mutex
|
||||
var queries []bson.M
|
||||
monitor := &event.CommandMonitor{Started: func(_ context.Context, e *event.CommandStartedEvent) {
|
||||
if !measuring.Load() || e.CommandName != "find" {
|
||||
return
|
||||
}
|
||||
name := e.Command.Lookup("find").StringValue()
|
||||
if name == "humanItems" {
|
||||
return
|
||||
}
|
||||
var filter bson.M
|
||||
if err := bson.Unmarshal(e.Command.Lookup("filter").Document(), &filter); err != nil {
|
||||
return
|
||||
}
|
||||
mu.Lock()
|
||||
queries = append(queries, bson.M{"collection": name, "filter": filter})
|
||||
mu.Unlock()
|
||||
}}
|
||||
client, err := mongo.Connect(ctx, options.Client().ApplyURI(uri).SetMonitor(monitor))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer client.Disconnect(context.Background())
|
||||
db := client.Database(fmt.Sprintf("pool_search_scope_%d", time.Now().UnixNano()))
|
||||
defer db.Drop(context.Background())
|
||||
s := NewMongoStore(db)
|
||||
const org = "scope-org"
|
||||
for _, name := range []string{"humanItems", "runs", "captureRecords", "annotationTasks", "projects"} {
|
||||
if _, err := db.Collection(name).Indexes().CreateOne(ctx, mongo.IndexModel{Keys: bson.D{{Key: "id", Value: 1}}, Options: options.Index().SetUnique(true)}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := s.UpsertProject(ctx, domain.ProjectRecord{ID: "p", OrganizationID: org, Name: "Project"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.UpsertRun(ctx, domain.RunRecord{ID: "r", OrganizationID: org, TaskID: "capture"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.UpsertCaptureRecord(ctx, domain.CaptureRecord{ID: "capture", OrganizationID: org, CaptureID: "CAPTURE_NEEDLE"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.UpsertAnnotationTask(ctx, domain.AnnotationTaskRecord{ID: "task", OrganizationID: org, ArtifactID: "camera_left_part0001.mp4"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.UpsertHumanItem(ctx, domain.HumanItemRecord{ID: "item", OrganizationID: org, ProjectID: "p", RunID: "r", LegacyTaskID: "task", CurrentStageID: "quality_review", Status: domain.HumanItemReady, Subject: domain.HumanSubjectRef{ItemKey: "camera_left_part0001.mp4"}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Unrelated same-tenant data must not broaden a lookup.
|
||||
for n := 0; n < 40; n++ {
|
||||
if err := s.UpsertCaptureRecord(ctx, domain.CaptureRecord{ID: fmt.Sprint("unrelated-", n), OrganizationID: org, CaptureID: "CAPTURE_NEEDLE"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
for _, search := range []string{"CAPTURE_NEEDLE/part0001", "no-such-capture"} {
|
||||
queries = nil
|
||||
measuring.Store(true)
|
||||
got, err := s.ListHumanItems(ctx, org, domain.HumanItemQuery{Search: search, StageIDs: []string{"quality_review"}, PoolCandidateProjection: true, Limit: 200})
|
||||
measuring.Store(false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := 0
|
||||
if search != "no-such-capture" {
|
||||
want = 1
|
||||
}
|
||||
if len(got) != want {
|
||||
t.Fatalf("search=%s rows=%d want=%d", search, len(got), want)
|
||||
}
|
||||
if len(queries) == 0 {
|
||||
t.Fatal("reverse lookup path was not exercised")
|
||||
}
|
||||
for _, query := range queries {
|
||||
if !poolTestHasIDScope(query["filter"]) {
|
||||
t.Fatalf("unbounded reverse lookup: %v", query)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func poolTestHasIDScope(v any) bool {
|
||||
switch value := v.(type) {
|
||||
case bson.M:
|
||||
if id, ok := value["id"].(bson.M); ok {
|
||||
if _, ok := id["$in"]; ok {
|
||||
return true
|
||||
}
|
||||
}
|
||||
for _, child := range value {
|
||||
if poolTestHasIDScope(child) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
case bson.A:
|
||||
for _, child := range value {
|
||||
if poolTestHasIDScope(child) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@ -139,6 +139,14 @@ func (s *MongoStore) humanPoolSearchCandidates(ctx context.Context, org string,
|
||||
if err != mongo.ErrNoDocuments {
|
||||
return nil, err
|
||||
}
|
||||
// Reverse lookups must stay within references reachable from the base pool.
|
||||
// A migrated tenant-wide validity/regex query still FETCHes every source
|
||||
// document; an index on organization alone does not make that query cheap.
|
||||
const maxReferences = 4096
|
||||
refs, broad, err := s.humanPoolSearchReferenceScope(ctx, org, base, maxReferences)
|
||||
if err != nil || broad {
|
||||
return nil, err
|
||||
}
|
||||
token := strings.ToLower(words[0])
|
||||
if i := strings.LastIndex(token, "/part"); i > 0 {
|
||||
token = token[:i]
|
||||
@ -148,7 +156,6 @@ func (s *MongoStore) humanPoolSearchCandidates(ctx context.Context, org string,
|
||||
return bson.M{"$or": bson.A{bson.M{"$expr": bson.M{"$not": bson.A{poolSearchValidExpression(collection)}}}, bson.M{poolSearchField + ".text": bson.M{"$regex": pattern}}}}
|
||||
}
|
||||
match := matchFor("captureRecords")
|
||||
const maxReferences = 4096
|
||||
ids := func(collection string, extra bson.M) ([]string, bool, error) {
|
||||
f := bson.M{"organizationId": normalizedOrganizationID(org), "$and": bson.A{match}}
|
||||
if extra != nil {
|
||||
@ -171,7 +178,7 @@ func (s *MongoStore) humanPoolSearchCandidates(ctx context.Context, org string,
|
||||
}
|
||||
return out, len(out) > maxReferences, cur.Err()
|
||||
}
|
||||
captureIDs, broad, err := ids("captureRecords", nil)
|
||||
captureIDs, broad, err := ids("captureRecords", bson.M{"id": bson.M{"$in": refs["captureRecords"]}})
|
||||
if err != nil || broad {
|
||||
return nil, err
|
||||
}
|
||||
@ -182,17 +189,17 @@ func (s *MongoStore) humanPoolSearchCandidates(ctx context.Context, org string,
|
||||
runMatch = bson.M{"$or": bson.A{runMatch, bson.M{"taskId": bson.M{"$in": captureIDs}}}}
|
||||
}
|
||||
match = runMatch
|
||||
runIDs, broad, err := ids("runs", nil)
|
||||
runIDs, broad, err := ids("runs", bson.M{"id": bson.M{"$in": refs["runs"]}})
|
||||
if err != nil || broad {
|
||||
return nil, err
|
||||
}
|
||||
match = matchFor("annotationTasks")
|
||||
taskIDs, broad, err := ids("annotationTasks", nil)
|
||||
taskIDs, broad, err := ids("annotationTasks", bson.M{"id": bson.M{"$in": refs["annotationTasks"]}})
|
||||
if err != nil || broad {
|
||||
return nil, err
|
||||
}
|
||||
match = matchFor("projects")
|
||||
projectIDs, broad, err := ids("projects", nil)
|
||||
projectIDs, broad, err := ids("projects", bson.M{"id": bson.M{"$in": refs["projects"]}})
|
||||
if err != nil || broad {
|
||||
return nil, err
|
||||
}
|
||||
@ -205,6 +212,106 @@ func (s *MongoStore) humanPoolSearchCandidates(ctx context.Context, org string,
|
||||
return bson.M{"$or": branches}, nil
|
||||
}
|
||||
|
||||
// The bounded projection contains no customer payload. Overflow falls back to
|
||||
// the existing full search, never a truncated list. This only narrows candidate
|
||||
// sources; the original final matcher and current permissions remain in force.
|
||||
func (s *MongoStore) humanPoolSearchReferenceScope(ctx context.Context, org string, base bson.M, limit int) (map[string][]string, bool, error) {
|
||||
sets := map[string]map[string]bool{}
|
||||
for _, name := range []string{"runs", "annotationTasks", "projects", "captureRecords"} {
|
||||
sets[name] = map[string]bool{}
|
||||
}
|
||||
read := func(collection string, filter, projection bson.M, visit func(bson.M)) (bool, error) {
|
||||
cur, err := s.db.Collection(collection).Find(ctx, filter, options.Find().SetProjection(projection).SetLimit(int64(limit+1)).SetMaxTime(8*time.Second))
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
defer cur.Close(ctx)
|
||||
count := 0
|
||||
for cur.Next(ctx) {
|
||||
count++
|
||||
if count > limit {
|
||||
return true, nil
|
||||
}
|
||||
var row bson.M
|
||||
if err := cur.Decode(&row); err != nil {
|
||||
return false, err
|
||||
}
|
||||
visit(row)
|
||||
}
|
||||
return false, cur.Err()
|
||||
}
|
||||
add := func(collection string, value any) {
|
||||
if id, ok := value.(string); ok && id != "" {
|
||||
sets[collection][id] = true
|
||||
}
|
||||
}
|
||||
values := func(collection string) []string {
|
||||
out := make([]string, 0, len(sets[collection]))
|
||||
for id := range sets[collection] {
|
||||
out = append(out, id)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
// Decide overflow in Mongo so a broad no-hit query does not transfer a
|
||||
// backlog of references just to select the compatibility path.
|
||||
withinLimit := bson.M{"$lte": bson.A{"$count", limit}}
|
||||
rootPipeline := mongo.Pipeline{
|
||||
bson.D{{Key: "$match", Value: base}},
|
||||
bson.D{{Key: "$limit", Value: int64(limit + 1)}},
|
||||
bson.D{{Key: "$group", Value: bson.M{
|
||||
"_id": nil, "count": bson.M{"$sum": 1},
|
||||
"runs": bson.M{"$addToSet": bson.M{"$ifNull": bson.A{"$runId", ""}}},
|
||||
"annotationTasks": bson.M{"$addToSet": bson.M{"$ifNull": bson.A{"$legacyTaskId", ""}}},
|
||||
"projects": bson.M{"$addToSet": bson.M{"$ifNull": bson.A{"$projectId", ""}}},
|
||||
}}},
|
||||
bson.D{{Key: "$project", Value: bson.M{
|
||||
"_id": 0, "count": 1,
|
||||
"runs": bson.M{"$cond": bson.A{withinLimit, "$runs", bson.A{}}},
|
||||
"annotationTasks": bson.M{"$cond": bson.A{withinLimit, "$annotationTasks", bson.A{}}},
|
||||
"projects": bson.M{"$cond": bson.A{withinLimit, "$projects", bson.A{}}},
|
||||
}}},
|
||||
}
|
||||
cur, err := s.humanItems.Aggregate(ctx, rootPipeline, options.Aggregate().SetAllowDiskUse(false).SetMaxTime(8*time.Second))
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
defer cur.Close(ctx)
|
||||
if cur.Next(ctx) {
|
||||
var row struct {
|
||||
Count int `bson:"count"`
|
||||
Runs []string `bson:"runs"`
|
||||
Tasks []string `bson:"annotationTasks"`
|
||||
Projects []string `bson:"projects"`
|
||||
}
|
||||
if err := cur.Decode(&row); err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
if row.Count > limit {
|
||||
return nil, true, nil
|
||||
}
|
||||
for name, ids := range map[string][]string{"runs": row.Runs, "annotationTasks": row.Tasks, "projects": row.Projects} {
|
||||
for _, id := range ids {
|
||||
add(name, id)
|
||||
}
|
||||
}
|
||||
}
|
||||
if err := cur.Err(); err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
broad, err := read("runs", bson.M{"organizationId": normalizedOrganizationID(org), "id": bson.M{"$in": values("runs")}}, bson.M{"_id": 0, "taskId": 1}, func(row bson.M) {
|
||||
add("captureRecords", row["taskId"])
|
||||
})
|
||||
if err != nil || broad {
|
||||
return nil, broad, err
|
||||
}
|
||||
refs := map[string][]string{}
|
||||
for name := range sets {
|
||||
refs[name] = values(name)
|
||||
}
|
||||
return refs, false, nil
|
||||
}
|
||||
|
||||
func poolSearchSourceValue(d bson.M, path string) any {
|
||||
if path == "__runReferences" {
|
||||
return poolRunReferences(d)
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user