forked from longtao.wu/droboticflow
173 lines
5.3 KiB
Go
173 lines
5.3 KiB
Go
package store
|
|
|
|
import (
|
|
"context"
|
|
"crypto/sha256"
|
|
"droboticflow/internal/domain"
|
|
"encoding/json"
|
|
"fmt"
|
|
"go.mongodb.org/mongo-driver/mongo/options"
|
|
"time"
|
|
|
|
"go.mongodb.org/mongo-driver/bson"
|
|
"go.mongodb.org/mongo-driver/mongo"
|
|
)
|
|
|
|
type querySourceDigest struct {
|
|
Digest [32]byte
|
|
Organization string
|
|
}
|
|
|
|
var querySearchSources = map[string]bool{"humanItems": true, "annotationTasks": true, "captureRecords": true, "runs": true, "projects": true}
|
|
|
|
// Only names, source identities and references affect search. Saving a draft,
|
|
// renewing a lease or changing task status must not invalidate the name index.
|
|
func querySearchSourceProjection(collection string) bson.M {
|
|
p := bson.M{"_id": 1, "organizationId": 1, "projectId": 1, "id": 1, "poolSearch.text": 1}
|
|
fields := map[string][]string{
|
|
"humanItems": {"legacyTaskId", "runId", "subject.itemKey", "subject.artifactId"},
|
|
"annotationTasks": {"artifactId", "rightArtifactId", "sourceVideo"},
|
|
"captureRecords": {"captureId", "prefix"},
|
|
"projects": {"name"},
|
|
"runs": {"taskId", "projectName", "captureDirectories", "inputs", "artifacts.id", "artifacts.name", "artifacts.path", "nodeExecutions.artifacts.id", "nodeExecutions.artifacts.name", "nodeExecutions.artifacts.path"},
|
|
}
|
|
for _, f := range fields[collection] {
|
|
p[f] = 1
|
|
}
|
|
if collection == "runs" {
|
|
p["inputs"] = bson.M{"$cond": bson.A{poolSearchValidExpression("runs"), bson.M{}, "$inputs"}}
|
|
}
|
|
if collection == "humanItems" {
|
|
p["legacyPayload"] = bson.M{"$cond": bson.A{poolSearchValidExpression("humanItems"), bson.M{}, "$subject.payload"}}
|
|
}
|
|
return p
|
|
}
|
|
func querySourceIdentity(collection string, id any) string {
|
|
b, _ := json.Marshal(id)
|
|
return collection + ":" + string(b)
|
|
}
|
|
func querySourceHash(doc bson.M) querySourceDigest {
|
|
if payload, ok := doc["legacyPayload"]; ok {
|
|
names := []string{}
|
|
domain.AppendCaptureDirectoriesFromPayload(payload, &names)
|
|
doc["legacyPayload"] = names
|
|
}
|
|
b, _ := json.Marshal(doc)
|
|
org, _ := doc["organizationId"].(string)
|
|
return querySourceDigest{Digest: sha256.Sum256(b), Organization: org}
|
|
}
|
|
func (b *QueryReadBuilder) seedSourceDigests(ctx context.Context) error {
|
|
for collection := range querySearchSources {
|
|
cur, err := b.store.db.Collection(collection).Aggregate(ctx, mongo.Pipeline{{{Key: "$project", Value: querySearchSourceProjection(collection)}}})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
for cur.Next(ctx) {
|
|
var doc bson.M
|
|
if err = cur.Decode(&doc); err != nil {
|
|
_ = cur.Close(ctx)
|
|
return err
|
|
}
|
|
b.sourceDigests[querySourceIdentity(collection, doc["_id"])] = querySourceHash(doc)
|
|
}
|
|
err = cur.Err()
|
|
_ = cur.Close(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
func (b *QueryReadBuilder) searchSourceChanged(ctx context.Context, collection string, key bson.M) (bool, string, error) {
|
|
identity := querySourceIdentity(collection, key["_id"])
|
|
old, exists := b.sourceDigests[identity]
|
|
cur, err := b.store.db.Collection(collection).Aggregate(ctx, mongo.Pipeline{{{Key: "$match", Value: key}}, {{Key: "$project", Value: querySearchSourceProjection(collection)}}})
|
|
if err != nil {
|
|
return false, "", err
|
|
}
|
|
defer cur.Close(ctx)
|
|
if !cur.Next(ctx) {
|
|
if err = cur.Err(); err != nil {
|
|
return false, "", err
|
|
}
|
|
delete(b.sourceDigests, identity)
|
|
return true, old.Organization, nil
|
|
}
|
|
var doc bson.M
|
|
if err = cur.Decode(&doc); err != nil {
|
|
return false, "", err
|
|
}
|
|
next := querySourceHash(doc)
|
|
b.sourceDigests[identity] = next
|
|
if exists && old.Digest == next.Digest {
|
|
return false, next.Organization, nil
|
|
}
|
|
if exists && old.Organization != next.Organization {
|
|
return true, "", nil
|
|
}
|
|
return true, next.Organization, nil
|
|
}
|
|
|
|
func (b *QueryReadBuilder) observeQuerySourceEvent(ctx context.Context, l queryModelLease, collection, operation string, key bson.M) error {
|
|
if operation == "drop" || operation == "rename" || operation == "invalidate" {
|
|
b.mu.Lock()
|
|
b.all, b.allSearch = true, true
|
|
b.mu.Unlock()
|
|
return b.store.markQuerySearchDirty(ctx, l, "")
|
|
}
|
|
org := ""
|
|
searchChanged := false
|
|
if querySearchSources[collection] {
|
|
var err error
|
|
searchChanged, org, err = b.searchSourceChanged(ctx, collection, key)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
} else if operation != "delete" && operation != "drop" && operation != "rename" {
|
|
var doc struct {
|
|
Org string `bson:"organizationId"`
|
|
}
|
|
if err := b.store.db.Collection(collection).FindOne(ctx, key, options.FindOne().SetProjection(bson.M{"organizationId": 1})).Decode(&doc); err != nil && err != mongo.ErrNoDocuments {
|
|
return err
|
|
}
|
|
org = doc.Org
|
|
}
|
|
if operation == "drop" || operation == "rename" || operation == "invalidate" {
|
|
org = ""
|
|
searchChanged = true
|
|
}
|
|
b.mu.Lock()
|
|
if org == "" {
|
|
b.all = true
|
|
} else {
|
|
b.dirty[org] = true
|
|
}
|
|
if searchChanged {
|
|
if org == "" {
|
|
b.allSearch = true
|
|
} else {
|
|
b.searchDirty[org] = true
|
|
}
|
|
}
|
|
b.mu.Unlock()
|
|
if searchChanged {
|
|
return b.store.markQuerySearchDirty(ctx, l, org)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (b *QueryReadBuilder) recordObserved(ctx context.Context, l queryModelLease, at time.Time, changed bool) error {
|
|
update := bson.M{"$set": bson.M{"observedAt": at}}
|
|
if changed {
|
|
update["$inc"] = bson.M{"productionRevision": int64(1)}
|
|
}
|
|
r, err := b.store.queryControl().UpdateOne(ctx, queryLeaseFilter(l), update)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if r.MatchedCount != 1 {
|
|
return fmt.Errorf("query observer fenced")
|
|
}
|
|
return nil
|
|
}
|