droboticflow/internal/store/query_pool_buckets.go

220 lines
6.8 KiB
Go

package store
import (
"context"
"droboticflow/internal/domain"
"droboticflow/internal/queryserving"
"errors"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
"go.mongodb.org/mongo-driver/mongo/readconcern"
"time"
)
const queryPoolBuckets = "queryPoolCountBuckets"
const queryPoolBucketSchema = "pool-count/v1"
// These are the inputs of the API's visibility predicate, not an authorization
// result. Identity-sensitive grant draining uses the legacy per-record reader.
var poolBucketFields = []string{"organizationId", "pluginId", "projectId", "batchId", "supplierId", "processingScope", "processingProfile", "workgroupId", "vendorId", "currentStageId", "status", "assignedUserId", "latestAuthorUserId", "acceptanceClaim", "reviewRound", "claimedAt", "inheritedRework"}
func poolBucketProjection() bson.M {
p := bson.M{"_id": 1}
for _, field := range poolBucketFields {
p[field] = 1
}
return p
}
func poolBucket(doc bson.M) (string, bson.M, error) {
projected := bson.M{}
for _, field := range poolBucketFields {
if value, ok := doc[field]; ok {
projected[field] = value
}
}
raw, err := bson.Marshal(projected)
if err != nil {
return "", nil, err
}
var typed domain.HumanItemRecord
if err = bson.Unmarshal(raw, &typed); err != nil {
return "", nil, err
}
out := bson.M{}
for _, field := range poolBucketFields {
value := doc[field]
switch field {
case "reviewRound":
value = int(supplierNumber(value))
case "inheritedRework":
value = value == true
case "acceptanceClaim":
if value == nil {
value = nil
}
default:
text, _ := value.(string)
value = text
}
out[field] = value
}
return queryserving.Key(queryPoolBucketSchema, out), out, nil
}
type poolBucketDelta struct {
doc bson.M
count int64
}
func addPoolBucketDelta(deltas map[string]*poolBucketDelta, doc bson.M, count int64) error {
if len(doc) == 0 {
return nil
}
key, row, err := poolBucket(doc)
if err != nil {
return err
}
v := deltas[key]
if v == nil {
v = &poolBucketDelta{doc: row}
deltas[key] = v
}
v.count += count
return nil
}
func (q *QueryReplica) writePoolBucketDeltas(ctx mongo.SessionContext, deltas map[string]*poolBucketDelta) error {
models := []mongo.WriteModel{}
for key, delta := range deltas {
if delta.count != 0 {
models = append(models, mongo.NewUpdateOneModel().SetFilter(bson.M{"_id": key}).SetUpdate(bson.M{"$setOnInsert": delta.doc, "$inc": bson.M{"count": delta.count}}).SetUpsert(true))
}
}
if len(models) == 0 {
return nil
}
_, err := q.Target.db.Collection(queryPoolBuckets).BulkWrite(ctx, models)
return err
}
func (q *QueryReplica) ensurePoolBucketIndexes(ctx context.Context) error {
_, err := q.Target.db.Collection(queryPoolBuckets).Indexes().CreateOne(ctx, mongo.IndexModel{Keys: bson.D{{Key: "organizationId", Value: 1}, {Key: "pluginId", Value: 1}, {Key: "currentStageId", Value: 1}, {Key: "projectId", Value: 1}}, Options: options.Index().SetName("pool_bucket_scope_v1")})
if err != nil {
return err
}
// Existing read stores without buckets retain the original reader until an
// explicit rebuild. A newly empty replica starts counting in its transactions.
n, err := q.Target.humanItems.CountDocuments(ctx, bson.M{}, options.Count().SetLimit(1))
if err != nil || n != 0 {
return err
}
_, err = q.control().UpdateOne(ctx, bson.M{"_id": "state"}, bson.M{"$setOnInsert": bson.M{"poolBucketSchema": queryPoolBucketSchema, "poolBucketEpoch": int64(0)}}, options.Update().SetUpsert(true))
return err
}
// VisitHumanPoolCountGroups never supplies an authorized total. Every group is
// checked using current server memberships/batches. Dirty authoritative rows
// subtract their old group and add their current row in this same request.
func (s *MongoStore) VisitHumanPoolCountGroups(ctx context.Context, org string, q domain.HumanItemQuery, visit func(domain.HumanItemRecord, int) error) (bool, error) {
target, err := s.poolReplica(ctx)
if err != nil {
return true, err
}
if target == nil {
return false, nil
}
if humanPoolFiltered(q) || q.BlindAnnotationActorID != "" || q.NeedsReworkRouting || q.InheritedUnassigned || q.WorkflowID != "" || q.RunID != "" || q.LegacyTaskID != "" || q.AnnotationView != "" || q.IDs != nil || q.CountIdentitySensitive {
return false, nil
}
b, err := s.countReplicaBarrier(ctx, org)
if err != nil {
return true, err
}
type weightedRow struct {
domain.HumanItemRecord `bson:",inline"`
Count int `bson:"count"`
}
rows := []weightedRow{}
session, e := target.db.Client().StartSession()
if e != nil {
return true, e
}
defer session.EndSession(ctx)
capacity := errors.New("count groups exceed request budget")
incompatible := errors.New("count bucket writer compatibility changed")
_, err = session.WithTransaction(ctx, func(sc mongo.SessionContext) (any, error) {
rows = nil
var state struct {
Schema string `bson:"poolBucketSchema"`
FactEpoch int64 `bson:"factEpoch"`
BucketEpoch int64 `bson:"poolBucketEpoch"`
}
if e := target.db.Collection(queryReplicaControl).FindOne(sc, bson.M{"_id": "state"}).Decode(&state); e != nil {
return nil, e
}
if state.Schema != queryPoolBucketSchema || state.FactEpoch != state.BucketEpoch {
return nil, incompatible
}
filter := humanItemMongoFilter(org, q)
filter["count"] = bson.M{"$gt": 0}
cur, e := target.db.Collection(queryPoolBuckets).Find(sc, filter, options.Find().SetMaxTime(8*time.Second).SetLimit(10001))
if e != nil {
return nil, e
}
e = cur.All(sc, &rows)
cur.Close(sc)
if e != nil {
return nil, e
}
if len(rows) > 10000 {
return nil, capacity
}
if b == nil || len(b.ids) == 0 {
return nil, nil
}
ids := make([]string, 0, len(b.ids))
for id := range b.ids {
ids = append(ids, id)
}
oldFilter := bson.M{"$and": bson.A{humanItemMongoFilter(org, q), bson.M{"id": bson.M{"$in": ids}}}}
old, e := target.humanItems.Find(sc, oldFilter, options.Find().SetProjection(poolBucketProjection()))
if e != nil {
return nil, e
}
defer old.Close(sc)
for old.Next(sc) {
var row domain.HumanItemRecord
if e = old.Decode(&row); e != nil {
return nil, e
}
rows = append(rows, weightedRow{HumanItemRecord: row, Count: -1})
}
return nil, old.Err()
}, options.Transaction().SetReadConcern(readconcern.Snapshot()))
if errors.Is(err, capacity) || errors.Is(err, incompatible) {
return false, nil
}
if err != nil {
return true, err
}
for _, row := range rows {
if e := ctx.Err(); e != nil {
return true, e
}
if e := visit(row.HumanItemRecord, row.Count); e != nil {
return true, e
}
}
if b == nil {
return true, ctx.Err()
}
err = s.visitReplicaPendingCounts(ctx, org, q, b, func(rows []domain.HumanItemRecord) error {
for _, row := range rows {
if e := visit(row, 1); e != nil {
return e
}
}
return nil
})
return true, err
}