forked from longtao.wu/droboticflow
296 lines
10 KiB
Go
296 lines
10 KiB
Go
package store
|
|
|
|
import (
|
|
"context"
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"errors"
|
|
"log"
|
|
"net/url"
|
|
"regexp"
|
|
"strings"
|
|
"time"
|
|
"unicode"
|
|
|
|
"droboticflow/internal/domain"
|
|
"droboticflow/internal/queryserving"
|
|
"go.mongodb.org/mongo-driver/bson"
|
|
"go.mongodb.org/mongo-driver/mongo/options"
|
|
)
|
|
|
|
type querySearchState struct {
|
|
Heads map[string]QueryModelHead `bson:"heads"`
|
|
Revisions map[string]int64 `bson:"searchRevisions"`
|
|
Global int64 `bson:"searchGlobalRevision"`
|
|
ObservedAt time.Time `bson:"observedAt"`
|
|
}
|
|
|
|
func (s *MongoStore) querySearchState(ctx context.Context) (querySearchState, error) {
|
|
var st querySearchState
|
|
err := s.queryControl().FindOne(ctx, bson.M{"_id": "global"}, options.FindOne().SetProjection(bson.M{"heads": 1, "searchRevisions": 1, "searchGlobalRevision": 1, "observedAt": 1})).Decode(&st)
|
|
return st, err
|
|
}
|
|
func querySafePath(value string) string {
|
|
u, err := url.Parse(value)
|
|
if err == nil && u.Scheme != "" && u.Host != "" {
|
|
u.RawQuery = ""
|
|
u.Fragment = ""
|
|
u.User = nil
|
|
return u.String()
|
|
}
|
|
return value
|
|
}
|
|
func searchRowText(row adminPoolRow) string {
|
|
// Both exact historical search corpora remain contiguous; adding fields in
|
|
// between could otherwise lose cross-field substring matches.
|
|
return adminPoolSearchText(row) + "\n" + humanPoolFallbackSearchText(row) + "\n" + queryCaptureFold(strings.Join(adminPoolCaptureValues(row), "\n"))
|
|
}
|
|
|
|
func (s *MongoStore) buildQuerySearchModel(ctx context.Context, l queryModelLease, org string) error {
|
|
ctx, stop := context.WithTimeout(ctx, 5*time.Minute)
|
|
defer stop()
|
|
key := queryModelKey("search", org, nil)
|
|
state, err := s.querySearchState(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
h := QueryModelHead{Generation: domain.NewID("query-search"), AsOf: time.Now().UTC(), Revision: state.Revisions[key], GlobalRevision: state.Global}
|
|
started := time.Now()
|
|
defer func() {
|
|
log.Printf("query_search_build rows=%d elapsed_ms=%d", h.Rows, time.Since(started).Milliseconds())
|
|
}()
|
|
if s.queryServing != nil && s.queryServing.search != nil {
|
|
sum := sha256.Sum256([]byte(s.db.Name() + ":" + key + ":" + h.Generation))
|
|
h.Collection = "dfq_" + hex.EncodeToString(sum[:])
|
|
// Register intent first. Even a canceled/ambiguous create remains
|
|
// discoverable by this database's orphan collector.
|
|
if _, err = s.queryData().InsertOne(ctx, bson.M{"_id": "search_collection:" + h.Collection, "kind": "search_collection", "collection": h.Collection, "generation": h.Generation, "createdAt": time.Now().UTC()}); err != nil {
|
|
return err
|
|
}
|
|
if err = s.queryServing.search.Create(ctx, h.Collection); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
complete := false
|
|
defer func() {
|
|
if !complete {
|
|
cleanup, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
|
defer cancel()
|
|
_, _ = s.queryData().DeleteMany(cleanup, bson.M{"generation": h.Generation, "kind": bson.M{"$ne": "search_collection"}})
|
|
if h.Collection != "" {
|
|
_ = s.queryServing.search.Delete(cleanup, h.Collection)
|
|
}
|
|
}
|
|
}()
|
|
pipeline := humanPoolSearchPipeline(org, domain.HumanItemQuery{Search: "_build_names"}, bson.M{"organizationId": normalizedOrganizationID(org)})
|
|
cur, err := s.humanItems.Aggregate(ctx, pipeline, options.Aggregate().SetAllowDiskUse(true).SetBatchSize(200))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer cur.Close(ctx)
|
|
for {
|
|
rows := []adminPoolRow{}
|
|
for len(rows) < 200 && cur.Next(ctx) {
|
|
var row adminPoolRow
|
|
if err = cur.Decode(&row); err != nil {
|
|
return err
|
|
}
|
|
rows = append(rows, row)
|
|
}
|
|
if err = cur.Err(); err != nil {
|
|
return err
|
|
}
|
|
if len(rows) == 0 {
|
|
break
|
|
}
|
|
if err = s.humanPoolSearchLegacyPayloads(ctx, org, domain.HumanItemQuery{Search: "_build_names"}, rows); err != nil {
|
|
return err
|
|
}
|
|
docs := make([]any, 0, len(rows))
|
|
searchDocs := make([]queryserving.SearchDocument, 0, len(rows))
|
|
batchBytes := 0
|
|
flush := func() error {
|
|
if len(docs) == 0 {
|
|
return nil
|
|
}
|
|
if err := s.paceQueryBuild(ctx, batchBytes); err != nil {
|
|
return err
|
|
}
|
|
if _, err := s.queryData().InsertMany(ctx, docs); err != nil {
|
|
return err
|
|
}
|
|
if h.Collection != "" {
|
|
if err := s.queryServing.search.Import(ctx, h.Collection, searchDocs); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
h.Rows += len(docs)
|
|
docs, searchDocs, batchBytes = nil, nil, 0
|
|
return nil
|
|
}
|
|
for _, row := range rows {
|
|
for _, task := range row.Tasks {
|
|
if querySafePath(task.SourceVideo) != task.SourceVideo {
|
|
return errors.New("credential-bearing source requires legacy search")
|
|
}
|
|
}
|
|
text := searchRowText(row)
|
|
for _, value := range strings.Fields(text) {
|
|
if querySafePath(value) != value {
|
|
return errors.New("credential-bearing search field requires legacy search")
|
|
}
|
|
}
|
|
if len(text) > 64<<10 {
|
|
return errors.New("search names exceed materialization budget")
|
|
}
|
|
// Store only the conservative searchable text and identity. No
|
|
// task body, draft, review opinion, eligibility or credential.
|
|
id := queryserving.Key("row", h.Generation, row.ID)
|
|
doc := bson.M{"_id": id, "generation": h.Generation, "organizationId": normalizedOrganizationID(org), "itemId": row.ID, "text": text, "createdAt": time.Now().UTC()}
|
|
raw, e := bson.Marshal(doc)
|
|
if e != nil {
|
|
return e
|
|
}
|
|
if batchBytes+len(raw) > queryBuildBatchBytes {
|
|
if err = flush(); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
docs = append(docs, doc)
|
|
batchBytes += len(raw)
|
|
searchDocs = append(searchDocs, queryserving.SearchDocument{ID: id, ItemID: row.ID, Grams: queryserving.IndexGrams(text)})
|
|
}
|
|
if err = flush(); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
filter := queryLeaseFilter(l)
|
|
filter["searchGlobalRevision"] = state.Global
|
|
// Missing revision zero is equivalent to zero, without writing business data.
|
|
filter["$expr"] = bson.M{"$and": bson.A{bson.M{"$eq": bson.A{bson.M{"$ifNull": bson.A{"$searchRevisions." + key, int64(0)}}, h.Revision}}, bson.M{"$eq": bson.A{bson.M{"$ifNull": bson.A{"$searchGlobalRevision", int64(0)}}, h.GlobalRevision}}}}
|
|
delete(filter, "searchGlobalRevision")
|
|
res, err := s.queryControl().UpdateOne(ctx, filter, bson.M{"$set": bson.M{"heads." + key: h}})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if res.MatchedCount != 1 {
|
|
return errors.New("search source changed during build")
|
|
}
|
|
complete = true
|
|
return nil
|
|
}
|
|
|
|
// A complete, current catalog only narrows candidates. It never authorizes a
|
|
// row, changes a cursor, truncates history, or supplies cached task status.
|
|
func (s *MongoStore) querySearchCandidates(ctx context.Context, org, search, capture string) ([]string, bool) {
|
|
if s.queryServing == nil || search == "" && capture == "" {
|
|
return nil, false
|
|
}
|
|
short, cancel := context.WithTimeout(ctx, 300*time.Millisecond)
|
|
defer cancel()
|
|
st, err := s.querySearchState(short)
|
|
if err != nil {
|
|
return nil, false
|
|
}
|
|
key := queryModelKey("search", org, nil)
|
|
h, ok := st.Heads[key]
|
|
if !s.querySearchPublicationReady(st, h, ok, st.Revisions[key], time.Now()) {
|
|
return nil, false
|
|
}
|
|
cacheKey := queryserving.Key("search-candidates/v1", s.db.Name(), org, h.Generation, search, capture)
|
|
var ids []string
|
|
if queryserving.GetJSON(short, s.SharedQueryCache(), cacheKey, &ids) && queryCandidateBudget(ids) {
|
|
s.queryServing.stats.searchHits.Add(1)
|
|
return ids, true
|
|
}
|
|
if h.Collection != "" && s.queryServing.search != nil {
|
|
if out, err := s.queryServing.search.Candidates(short, h.Collection, search, queryCaptureFold(capture)); err == nil && queryCandidateBudget(out) {
|
|
s.queryServing.stats.searchHits.Add(1)
|
|
s.queryServing.stats.typesenseReads.Add(1)
|
|
queryserving.PutJSON(short, s.SharedQueryCache(), cacheKey, out, 5*time.Minute)
|
|
return out, true
|
|
}
|
|
}
|
|
clauses := bson.A{bson.M{"generation": h.Generation, "organizationId": normalizedOrganizationID(org)}}
|
|
for _, term := range []string{search, queryCaptureFold(capture)} {
|
|
if term != "" {
|
|
clauses = append(clauses, bson.M{"text": bson.M{"$regex": regexp.QuoteMeta(strings.ToLower(term))}})
|
|
}
|
|
}
|
|
cur, err := s.queryData().Find(short, bson.M{"$and": clauses}, options.Find().SetProjection(bson.M{"itemId": 1, "_id": 0}))
|
|
if err != nil {
|
|
return nil, false
|
|
}
|
|
defer cur.Close(short)
|
|
ids = []string{}
|
|
candidateBytes := 0
|
|
for cur.Next(short) {
|
|
var row struct {
|
|
ID string `bson:"itemId"`
|
|
}
|
|
if cur.Decode(&row) != nil || row.ID == "" {
|
|
return nil, false
|
|
}
|
|
ids = append(ids, row.ID)
|
|
candidateBytes += len(row.ID) + 16
|
|
if len(ids) > 100000 || candidateBytes > 4<<20 {
|
|
return nil, false
|
|
}
|
|
}
|
|
if cur.Err() != nil {
|
|
return nil, false
|
|
}
|
|
if !queryCandidateBudget(ids) {
|
|
return nil, false
|
|
}
|
|
s.queryServing.stats.mongoSearchReads.Add(1)
|
|
queryserving.PutJSON(short, s.SharedQueryCache(), cacheKey, ids, 5*time.Minute)
|
|
s.queryServing.stats.searchHits.Add(1)
|
|
return ids, true
|
|
}
|
|
|
|
// Keep the authoritative Mongo filter well below its BSON limit. An oversized
|
|
// candidate set is a miss, never a truncated inventory.
|
|
func queryCandidateBudget(ids []string) bool {
|
|
if len(ids) > 100000 {
|
|
return false
|
|
}
|
|
bytes := 0
|
|
for _, id := range ids {
|
|
if id == "" {
|
|
return false
|
|
}
|
|
bytes += len(id) + 16
|
|
if bytes > 4<<20 {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
|
|
// Capture is exact EqualFold, whereas free search uses ToLower+Contains.
|
|
// Canonicalize each Unicode simple-fold cycle so sigma/final sigma and long-s
|
|
// cannot disappear from the conservative prefilter. Extra hits are still
|
|
// rejected by the original exact matcher.
|
|
func queryCaptureFold(value string) string {
|
|
var b strings.Builder
|
|
for _, r := range value {
|
|
smallest := r
|
|
for next := unicode.SimpleFold(r); next != r; next = unicode.SimpleFold(next) {
|
|
if next < smallest {
|
|
smallest = next
|
|
}
|
|
}
|
|
b.WriteRune(smallest)
|
|
}
|
|
return strings.ToLower(b.String())
|
|
}
|
|
|
|
// Leave room for joins/projections under Mongo's command size limit. Oversized
|
|
// optional acceleration falls through to the original query without truncation.
|
|
func queryCandidateFilter(filter any, ids []string) (bson.M, bool) {
|
|
joined := bson.M{"$and": bson.A{filter, bson.M{"id": bson.M{"$in": ids}}}}
|
|
encoded, err := bson.Marshal(joined)
|
|
return joined, err == nil && len(encoded) <= 12<<20
|
|
}
|