droboticflow/internal/store/query_source_changes.go

173 lines
5.3 KiB
Go

package store
import (
"context"
"crypto/sha256"
"droboticflow/internal/domain"
"encoding/json"
"fmt"
"go.mongodb.org/mongo-driver/mongo/options"
"time"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo"
)
type querySourceDigest struct {
Digest [32]byte
Organization string
}
var querySearchSources = map[string]bool{"humanItems": true, "annotationTasks": true, "captureRecords": true, "runs": true, "projects": true}
// Only names, source identities and references affect search. Saving a draft,
// renewing a lease or changing task status must not invalidate the name index.
func querySearchSourceProjection(collection string) bson.M {
p := bson.M{"_id": 1, "organizationId": 1, "projectId": 1, "id": 1, "poolSearch.text": 1}
fields := map[string][]string{
"humanItems": {"legacyTaskId", "runId", "subject.itemKey", "subject.artifactId"},
"annotationTasks": {"artifactId", "rightArtifactId", "sourceVideo"},
"captureRecords": {"captureId", "prefix"},
"projects": {"name"},
"runs": {"taskId", "projectName", "captureDirectories", "inputs", "artifacts.id", "artifacts.name", "artifacts.path", "nodeExecutions.artifacts.id", "nodeExecutions.artifacts.name", "nodeExecutions.artifacts.path"},
}
for _, f := range fields[collection] {
p[f] = 1
}
if collection == "runs" {
p["inputs"] = bson.M{"$cond": bson.A{poolSearchValidExpression("runs"), bson.M{}, "$inputs"}}
}
if collection == "humanItems" {
p["legacyPayload"] = bson.M{"$cond": bson.A{poolSearchValidExpression("humanItems"), bson.M{}, "$subject.payload"}}
}
return p
}
func querySourceIdentity(collection string, id any) string {
b, _ := json.Marshal(id)
return collection + ":" + string(b)
}
func querySourceHash(doc bson.M) querySourceDigest {
if payload, ok := doc["legacyPayload"]; ok {
names := []string{}
domain.AppendCaptureDirectoriesFromPayload(payload, &names)
doc["legacyPayload"] = names
}
b, _ := json.Marshal(doc)
org, _ := doc["organizationId"].(string)
return querySourceDigest{Digest: sha256.Sum256(b), Organization: org}
}
func (b *QueryReadBuilder) seedSourceDigests(ctx context.Context) error {
for collection := range querySearchSources {
cur, err := b.store.queryCollection(collection).Aggregate(ctx, mongo.Pipeline{{{Key: "$project", Value: querySearchSourceProjection(collection)}}})
if err != nil {
return err
}
for cur.Next(ctx) {
var doc bson.M
if err = cur.Decode(&doc); err != nil {
_ = cur.Close(ctx)
return err
}
b.sourceDigests[querySourceIdentity(collection, doc["_id"])] = querySourceHash(doc)
}
err = cur.Err()
_ = cur.Close(ctx)
if err != nil {
return err
}
}
return nil
}
func (b *QueryReadBuilder) searchSourceChanged(ctx context.Context, collection string, key bson.M) (bool, string, error) {
identity := querySourceIdentity(collection, key["_id"])
old, exists := b.sourceDigests[identity]
cur, err := b.store.queryCollection(collection).Aggregate(ctx, mongo.Pipeline{{{Key: "$match", Value: key}}, {{Key: "$project", Value: querySearchSourceProjection(collection)}}})
if err != nil {
return false, "", err
}
defer cur.Close(ctx)
if !cur.Next(ctx) {
if err = cur.Err(); err != nil {
return false, "", err
}
delete(b.sourceDigests, identity)
return true, old.Organization, nil
}
var doc bson.M
if err = cur.Decode(&doc); err != nil {
return false, "", err
}
next := querySourceHash(doc)
b.sourceDigests[identity] = next
if exists && old.Digest == next.Digest {
return false, next.Organization, nil
}
if exists && old.Organization != next.Organization {
return true, "", nil
}
return true, next.Organization, nil
}
func (b *QueryReadBuilder) observeQuerySourceEvent(ctx context.Context, l queryModelLease, collection, operation string, key bson.M) error {
if operation == "drop" || operation == "rename" || operation == "invalidate" {
b.mu.Lock()
b.all, b.allSearch = true, true
b.mu.Unlock()
return b.store.markQuerySearchDirty(ctx, l, "")
}
org := ""
searchChanged := false
if querySearchSources[collection] {
var err error
searchChanged, org, err = b.searchSourceChanged(ctx, collection, key)
if err != nil {
return err
}
} else if operation != "delete" && operation != "drop" && operation != "rename" {
var doc struct {
Org string `bson:"organizationId"`
}
if err := b.store.queryCollection(collection).FindOne(ctx, key, options.FindOne().SetProjection(bson.M{"organizationId": 1})).Decode(&doc); err != nil && err != mongo.ErrNoDocuments {
return err
}
org = doc.Org
}
if operation == "drop" || operation == "rename" || operation == "invalidate" {
org = ""
searchChanged = true
}
b.mu.Lock()
if org == "" {
b.all = true
} else {
b.dirty[org] = true
}
if searchChanged {
if org == "" {
b.allSearch = true
} else {
b.searchDirty[org] = true
}
}
b.mu.Unlock()
if searchChanged {
return b.store.markQuerySearchDirty(ctx, l, org)
}
return nil
}
func (b *QueryReadBuilder) recordObserved(ctx context.Context, l queryModelLease, at time.Time, changed bool) error {
update := bson.M{"$set": bson.M{"observedAt": at}}
if changed {
update["$inc"] = bson.M{"productionRevision": int64(1)}
}
r, err := b.store.queryControl().UpdateOne(ctx, queryLeaseFilter(l), update)
if err != nil {
return err
}
if r.MatchedCount != 1 {
return fmt.Errorf("query observer fenced")
}
return nil
}