forked from longtao.wu/droboticflow
136 lines
4.7 KiB
Go
136 lines
4.7 KiB
Go
package store
|
|
|
|
import (
|
|
"context"
|
|
"go.mongodb.org/mongo-driver/bson"
|
|
"go.mongodb.org/mongo-driver/mongo"
|
|
"go.mongodb.org/mongo-driver/mongo/options"
|
|
"time"
|
|
)
|
|
|
|
// This is an explicit derived-data index operation, never API startup behavior.
|
|
// Production execution requires its own reviewed migration/index authorization.
|
|
func (s *MongoStore) EnsureQueryReadModelIndexes(ctx context.Context) error {
|
|
if s.publishedWorkforceReads() {
|
|
_, err := s.db.Collection(workforcePublicationData).Indexes().CreateOne(ctx, mongo.IndexModel{Keys: bson.D{{Key: "computedAt", Value: 1}}, Options: options.Index().SetName("workforce_publication_retention_v1").SetExpireAfterSeconds(172800)})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
_, err = s.db.Collection(workforcePublicationRegistry).Indexes().CreateOne(ctx, mongo.IndexModel{Keys: bson.D{{Key: "updatedAt", Value: 1}}, Options: options.Index().SetName("workforce_publication_requests_retention_v1").SetExpireAfterSeconds(172800)})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
if s.queryReplicaFactStore {
|
|
if err := s.ensureWorkforceFactIndexes(ctx); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
_, err := s.queryData().Indexes().CreateOne(ctx, mongo.IndexModel{Keys: bson.D{{Key: "generation", Value: 1}, {Key: "organizationId", Value: 1}, {Key: "itemId", Value: 1}}, Options: options.Index().SetName("query_model_rows_v1")})
|
|
return err
|
|
}
|
|
func (s *MongoStore) cleanQueryModelOrphans(ctx context.Context) error {
|
|
var state struct {
|
|
Heads map[string]QueryModelHead `bson:"heads"`
|
|
}
|
|
if err := s.queryControl().FindOne(ctx, bson.M{"_id": "global"}).Decode(&state); err != nil {
|
|
return err
|
|
}
|
|
keep := []string{}
|
|
for _, h := range state.Heads {
|
|
keep = append(keep, h.Generation)
|
|
}
|
|
if s.queryReplicaFactStore && s.periodicQueryStore() {
|
|
cur, err := s.db.Collection(queryBuildJobsCollection).Find(ctx, bson.M{"schema": queryBuildJobSchema}, options.Find().SetProjection(bson.M{"production.generation": 1, "phase": 1, "retiredGenerations": 1, "snapshotRecoveries": 1}))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
var jobs []queryBuildJob
|
|
err = cur.All(ctx, &jobs)
|
|
cur.Close(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
for _, j := range jobs {
|
|
if j.Phase != "complete" {
|
|
for _, r := range j.SnapshotRecoveries {
|
|
keep = append(keep, r.Generation+"-production")
|
|
}
|
|
}
|
|
if j.Phase != "complete" && j.Production.Generation != "" {
|
|
keep = append(keep, j.Production.Generation)
|
|
}
|
|
for _, pin := range j.Retired {
|
|
if time.Now().Before(pin.Until) {
|
|
keep = append(keep, pin.Generation)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
filter := bson.M{"generation": bson.M{"$nin": keep}, "createdAt": bson.M{"$lt": time.Now().UTC().Add(-15 * time.Minute)}}
|
|
// Ordinary orphan chunks/rows must still be reclaimed when Typesense is
|
|
// offline. Retain only its collection intents until remote deletion succeeds.
|
|
if _, err := s.queryData().DeleteMany(ctx, bson.M{"$and": bson.A{filter, bson.M{"kind": bson.M{"$ne": "search_collection"}}}}); err != nil {
|
|
return err
|
|
}
|
|
// Only this database's registered, unreachable search generations are
|
|
// eligible; other deployments and current generations remain untouched.
|
|
cur, err := s.queryData().Find(ctx, bson.M{"$and": bson.A{filter, bson.M{"kind": "search_collection"}}}, options.Find().SetProjection(bson.M{"_id": 1, "collection": 1}))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer cur.Close(ctx)
|
|
for cur.Next(ctx) {
|
|
var row struct {
|
|
ID string `bson:"_id"`
|
|
Collection string `bson:"collection"`
|
|
}
|
|
if err = cur.Decode(&row); err != nil {
|
|
return err
|
|
}
|
|
if s.queryServing != nil && s.queryServing.search != nil {
|
|
if err = s.queryServing.search.Delete(ctx, row.Collection); err != nil {
|
|
return err
|
|
}
|
|
if _, err = s.queryData().DeleteOne(ctx, bson.M{"_id": row.ID}); err != nil {
|
|
return err
|
|
}
|
|
} else {
|
|
return nil
|
|
}
|
|
}
|
|
if err = cur.Err(); err != nil {
|
|
return err
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// Obsolete configured scopes cannot pin an unbounded history of generations.
|
|
// Only the current builder may change this derived manifest directory.
|
|
func (s *MongoStore) pruneQueryModelScopes(ctx context.Context, l queryModelLease, scopes map[string][]string) error {
|
|
var st struct {
|
|
Heads map[string]QueryModelHead `bson:"heads"`
|
|
}
|
|
if err := s.queryControl().FindOne(ctx, queryLeaseFilter(l)).Decode(&st); err != nil {
|
|
return err
|
|
}
|
|
keep := map[string]bool{}
|
|
for org, projects := range scopes {
|
|
keep[queryModelKey("production", org, projects)] = true
|
|
keep[queryModelKey("search", org, nil)] = true
|
|
keep[workforceFactKey(org)] = true
|
|
}
|
|
unset := bson.M{}
|
|
for key := range st.Heads {
|
|
if !keep[key] {
|
|
unset["heads."+key] = ""
|
|
}
|
|
}
|
|
if len(unset) == 0 {
|
|
return nil
|
|
}
|
|
_, err := s.queryControl().UpdateOne(ctx, queryLeaseFilter(l), bson.M{"$unset": unset})
|
|
return err
|
|
}
|