droboticflow/internal/store/query_search_model.go

296 lines
10 KiB
Go

package store
import (
"context"
"crypto/sha256"
"encoding/hex"
"errors"
"log"
"net/url"
"regexp"
"strings"
"time"
"unicode"
"droboticflow/internal/domain"
"droboticflow/internal/queryserving"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo/options"
)
type querySearchState struct {
Heads map[string]QueryModelHead `bson:"heads"`
Revisions map[string]int64 `bson:"searchRevisions"`
Global int64 `bson:"searchGlobalRevision"`
ObservedAt time.Time `bson:"observedAt"`
}
func (s *MongoStore) querySearchState(ctx context.Context) (querySearchState, error) {
var st querySearchState
err := s.queryControl().FindOne(ctx, bson.M{"_id": "global"}, options.FindOne().SetProjection(bson.M{"heads": 1, "searchRevisions": 1, "searchGlobalRevision": 1, "observedAt": 1})).Decode(&st)
return st, err
}
func querySafePath(value string) string {
u, err := url.Parse(value)
if err == nil && u.Scheme != "" && u.Host != "" {
u.RawQuery = ""
u.Fragment = ""
u.User = nil
return u.String()
}
return value
}
func searchRowText(row adminPoolRow) string {
// Both exact historical search corpora remain contiguous; adding fields in
// between could otherwise lose cross-field substring matches.
return adminPoolSearchText(row) + "\n" + humanPoolFallbackSearchText(row) + "\n" + queryCaptureFold(strings.Join(adminPoolCaptureValues(row), "\n"))
}
func (s *MongoStore) buildQuerySearchModel(ctx context.Context, l queryModelLease, org string) error {
ctx, stop := context.WithTimeout(ctx, 5*time.Minute)
defer stop()
key := queryModelKey("search", org, nil)
state, err := s.querySearchState(ctx)
if err != nil {
return err
}
h := QueryModelHead{Generation: domain.NewID("query-search"), AsOf: time.Now().UTC(), Revision: state.Revisions[key], GlobalRevision: state.Global}
started := time.Now()
defer func() {
log.Printf("query_search_build rows=%d elapsed_ms=%d", h.Rows, time.Since(started).Milliseconds())
}()
if s.queryServing != nil && s.queryServing.search != nil {
sum := sha256.Sum256([]byte(s.db.Name() + ":" + key + ":" + h.Generation))
h.Collection = "dfq_" + hex.EncodeToString(sum[:])
// Register intent first. Even a canceled/ambiguous create remains
// discoverable by this database's orphan collector.
if _, err = s.queryData().InsertOne(ctx, bson.M{"_id": "search_collection:" + h.Collection, "kind": "search_collection", "collection": h.Collection, "generation": h.Generation, "createdAt": time.Now().UTC()}); err != nil {
return err
}
if err = s.queryServing.search.Create(ctx, h.Collection); err != nil {
return err
}
}
complete := false
defer func() {
if !complete {
cleanup, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
_, _ = s.queryData().DeleteMany(cleanup, bson.M{"generation": h.Generation, "kind": bson.M{"$ne": "search_collection"}})
if h.Collection != "" {
_ = s.queryServing.search.Delete(cleanup, h.Collection)
}
}
}()
pipeline := humanPoolSearchPipeline(org, domain.HumanItemQuery{Search: "_build_names"}, bson.M{"organizationId": normalizedOrganizationID(org)})
cur, err := s.humanItems.Aggregate(ctx, pipeline, options.Aggregate().SetAllowDiskUse(true).SetBatchSize(200))
if err != nil {
return err
}
defer cur.Close(ctx)
for {
rows := []adminPoolRow{}
for len(rows) < 200 && cur.Next(ctx) {
var row adminPoolRow
if err = cur.Decode(&row); err != nil {
return err
}
rows = append(rows, row)
}
if err = cur.Err(); err != nil {
return err
}
if len(rows) == 0 {
break
}
if err = s.humanPoolSearchLegacyPayloads(ctx, org, domain.HumanItemQuery{Search: "_build_names"}, rows); err != nil {
return err
}
docs := make([]any, 0, len(rows))
searchDocs := make([]queryserving.SearchDocument, 0, len(rows))
batchBytes := 0
flush := func() error {
if len(docs) == 0 {
return nil
}
if err := s.paceQueryBuild(ctx, batchBytes); err != nil {
return err
}
if _, err := s.queryData().InsertMany(ctx, docs); err != nil {
return err
}
if h.Collection != "" {
if err := s.queryServing.search.Import(ctx, h.Collection, searchDocs); err != nil {
return err
}
}
h.Rows += len(docs)
docs, searchDocs, batchBytes = nil, nil, 0
return nil
}
for _, row := range rows {
for _, task := range row.Tasks {
if querySafePath(task.SourceVideo) != task.SourceVideo {
return errors.New("credential-bearing source requires legacy search")
}
}
text := searchRowText(row)
for _, value := range strings.Fields(text) {
if querySafePath(value) != value {
return errors.New("credential-bearing search field requires legacy search")
}
}
if len(text) > 64<<10 {
return errors.New("search names exceed materialization budget")
}
// Store only the conservative searchable text and identity. No
// task body, draft, review opinion, eligibility or credential.
id := queryserving.Key("row", h.Generation, row.ID)
doc := bson.M{"_id": id, "generation": h.Generation, "organizationId": normalizedOrganizationID(org), "itemId": row.ID, "text": text, "createdAt": time.Now().UTC()}
raw, e := bson.Marshal(doc)
if e != nil {
return e
}
if batchBytes+len(raw) > queryBuildBatchBytes {
if err = flush(); err != nil {
return err
}
}
docs = append(docs, doc)
batchBytes += len(raw)
searchDocs = append(searchDocs, queryserving.SearchDocument{ID: id, ItemID: row.ID, Grams: queryserving.IndexGrams(text)})
}
if err = flush(); err != nil {
return err
}
}
filter := queryLeaseFilter(l)
filter["searchGlobalRevision"] = state.Global
// Missing revision zero is equivalent to zero, without writing business data.
filter["$expr"] = bson.M{"$and": bson.A{bson.M{"$eq": bson.A{bson.M{"$ifNull": bson.A{"$searchRevisions." + key, int64(0)}}, h.Revision}}, bson.M{"$eq": bson.A{bson.M{"$ifNull": bson.A{"$searchGlobalRevision", int64(0)}}, h.GlobalRevision}}}}
delete(filter, "searchGlobalRevision")
res, err := s.queryControl().UpdateOne(ctx, filter, bson.M{"$set": bson.M{"heads." + key: h}})
if err != nil {
return err
}
if res.MatchedCount != 1 {
return errors.New("search source changed during build")
}
complete = true
return nil
}
// A complete, current catalog only narrows candidates. It never authorizes a
// row, changes a cursor, truncates history, or supplies cached task status.
func (s *MongoStore) querySearchCandidates(ctx context.Context, org, search, capture string) ([]string, bool) {
if s.queryServing == nil || search == "" && capture == "" {
return nil, false
}
short, cancel := context.WithTimeout(ctx, 300*time.Millisecond)
defer cancel()
st, err := s.querySearchState(short)
if err != nil {
return nil, false
}
key := queryModelKey("search", org, nil)
h, ok := st.Heads[key]
if !s.querySearchPublicationReady(st, h, ok, st.Revisions[key], time.Now()) {
return nil, false
}
cacheKey := queryserving.Key("search-candidates/v1", s.db.Name(), org, h.Generation, search, capture)
var ids []string
if queryserving.GetJSON(short, s.SharedQueryCache(), cacheKey, &ids) && queryCandidateBudget(ids) {
s.queryServing.stats.searchHits.Add(1)
return ids, true
}
if h.Collection != "" && s.queryServing.search != nil {
if out, err := s.queryServing.search.Candidates(short, h.Collection, search, queryCaptureFold(capture)); err == nil && queryCandidateBudget(out) {
s.queryServing.stats.searchHits.Add(1)
s.queryServing.stats.typesenseReads.Add(1)
queryserving.PutJSON(short, s.SharedQueryCache(), cacheKey, out, 5*time.Minute)
return out, true
}
}
clauses := bson.A{bson.M{"generation": h.Generation, "organizationId": normalizedOrganizationID(org)}}
for _, term := range []string{search, queryCaptureFold(capture)} {
if term != "" {
clauses = append(clauses, bson.M{"text": bson.M{"$regex": regexp.QuoteMeta(strings.ToLower(term))}})
}
}
cur, err := s.queryData().Find(short, bson.M{"$and": clauses}, options.Find().SetProjection(bson.M{"itemId": 1, "_id": 0}))
if err != nil {
return nil, false
}
defer cur.Close(short)
ids = []string{}
candidateBytes := 0
for cur.Next(short) {
var row struct {
ID string `bson:"itemId"`
}
if cur.Decode(&row) != nil || row.ID == "" {
return nil, false
}
ids = append(ids, row.ID)
candidateBytes += len(row.ID) + 16
if len(ids) > 100000 || candidateBytes > 4<<20 {
return nil, false
}
}
if cur.Err() != nil {
return nil, false
}
if !queryCandidateBudget(ids) {
return nil, false
}
s.queryServing.stats.mongoSearchReads.Add(1)
queryserving.PutJSON(short, s.SharedQueryCache(), cacheKey, ids, 5*time.Minute)
s.queryServing.stats.searchHits.Add(1)
return ids, true
}
// Keep the authoritative Mongo filter well below its BSON limit. An oversized
// candidate set is a miss, never a truncated inventory.
func queryCandidateBudget(ids []string) bool {
if len(ids) > 100000 {
return false
}
bytes := 0
for _, id := range ids {
if id == "" {
return false
}
bytes += len(id) + 16
if bytes > 4<<20 {
return false
}
}
return true
}
// Capture is exact EqualFold, whereas free search uses ToLower+Contains.
// Canonicalize each Unicode simple-fold cycle so sigma/final sigma and long-s
// cannot disappear from the conservative prefilter. Extra hits are still
// rejected by the original exact matcher.
func queryCaptureFold(value string) string {
var b strings.Builder
for _, r := range value {
smallest := r
for next := unicode.SimpleFold(r); next != r; next = unicode.SimpleFold(next) {
if next < smallest {
smallest = next
}
}
b.WriteRune(smallest)
}
return strings.ToLower(b.String())
}
// Leave room for joins/projections under Mongo's command size limit. Oversized
// optional acceleration falls through to the original query without truncation.
func queryCandidateFilter(filter any, ids []string) (bson.M, bool) {
joined := bson.M{"$and": bson.A{filter, bson.M{"id": bson.M{"$in": ids}}}}
encoded, err := bson.Marshal(joined)
return joined, err == nil && len(encoded) <= 12<<20
}