forked from longtao.wu/droboticflow
220 lines
6.8 KiB
Go
220 lines
6.8 KiB
Go
package store
|
|
|
|
import (
|
|
"context"
|
|
"droboticflow/internal/domain"
|
|
"droboticflow/internal/queryserving"
|
|
"errors"
|
|
"go.mongodb.org/mongo-driver/bson"
|
|
"go.mongodb.org/mongo-driver/mongo"
|
|
"go.mongodb.org/mongo-driver/mongo/options"
|
|
"go.mongodb.org/mongo-driver/mongo/readconcern"
|
|
"time"
|
|
)
|
|
|
|
const queryPoolBuckets = "queryPoolCountBuckets"
|
|
const queryPoolBucketSchema = "pool-count/v1"
|
|
|
|
// These are the inputs of the API's visibility predicate, not an authorization
|
|
// result. Identity-sensitive grant draining uses the legacy per-record reader.
|
|
var poolBucketFields = []string{"organizationId", "pluginId", "projectId", "batchId", "supplierId", "processingScope", "processingProfile", "workgroupId", "vendorId", "currentStageId", "status", "assignedUserId", "latestAuthorUserId", "acceptanceClaim", "reviewRound", "claimedAt", "inheritedRework"}
|
|
|
|
func poolBucketProjection() bson.M {
|
|
p := bson.M{"_id": 1}
|
|
for _, field := range poolBucketFields {
|
|
p[field] = 1
|
|
}
|
|
return p
|
|
}
|
|
func poolBucket(doc bson.M) (string, bson.M, error) {
|
|
projected := bson.M{}
|
|
for _, field := range poolBucketFields {
|
|
if value, ok := doc[field]; ok {
|
|
projected[field] = value
|
|
}
|
|
}
|
|
raw, err := bson.Marshal(projected)
|
|
if err != nil {
|
|
return "", nil, err
|
|
}
|
|
var typed domain.HumanItemRecord
|
|
if err = bson.Unmarshal(raw, &typed); err != nil {
|
|
return "", nil, err
|
|
}
|
|
out := bson.M{}
|
|
for _, field := range poolBucketFields {
|
|
value := doc[field]
|
|
switch field {
|
|
case "reviewRound":
|
|
value = int(supplierNumber(value))
|
|
case "inheritedRework":
|
|
value = value == true
|
|
case "acceptanceClaim":
|
|
if value == nil {
|
|
value = nil
|
|
}
|
|
default:
|
|
text, _ := value.(string)
|
|
value = text
|
|
}
|
|
out[field] = value
|
|
}
|
|
return queryserving.Key(queryPoolBucketSchema, out), out, nil
|
|
}
|
|
|
|
type poolBucketDelta struct {
|
|
doc bson.M
|
|
count int64
|
|
}
|
|
|
|
func addPoolBucketDelta(deltas map[string]*poolBucketDelta, doc bson.M, count int64) error {
|
|
if len(doc) == 0 {
|
|
return nil
|
|
}
|
|
key, row, err := poolBucket(doc)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
v := deltas[key]
|
|
if v == nil {
|
|
v = &poolBucketDelta{doc: row}
|
|
deltas[key] = v
|
|
}
|
|
v.count += count
|
|
return nil
|
|
}
|
|
func (q *QueryReplica) writePoolBucketDeltas(ctx mongo.SessionContext, deltas map[string]*poolBucketDelta) error {
|
|
models := []mongo.WriteModel{}
|
|
for key, delta := range deltas {
|
|
if delta.count != 0 {
|
|
models = append(models, mongo.NewUpdateOneModel().SetFilter(bson.M{"_id": key}).SetUpdate(bson.M{"$setOnInsert": delta.doc, "$inc": bson.M{"count": delta.count}}).SetUpsert(true))
|
|
}
|
|
}
|
|
if len(models) == 0 {
|
|
return nil
|
|
}
|
|
_, err := q.Target.db.Collection(queryPoolBuckets).BulkWrite(ctx, models)
|
|
return err
|
|
}
|
|
func (q *QueryReplica) ensurePoolBucketIndexes(ctx context.Context) error {
|
|
_, err := q.Target.db.Collection(queryPoolBuckets).Indexes().CreateOne(ctx, mongo.IndexModel{Keys: bson.D{{Key: "organizationId", Value: 1}, {Key: "pluginId", Value: 1}, {Key: "currentStageId", Value: 1}, {Key: "projectId", Value: 1}}, Options: options.Index().SetName("pool_bucket_scope_v1")})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
// Existing read stores without buckets retain the original reader until an
|
|
// explicit rebuild. A newly empty replica starts counting in its transactions.
|
|
n, err := q.Target.humanItems.CountDocuments(ctx, bson.M{}, options.Count().SetLimit(1))
|
|
if err != nil || n != 0 {
|
|
return err
|
|
}
|
|
_, err = q.control().UpdateOne(ctx, bson.M{"_id": "state"}, bson.M{"$setOnInsert": bson.M{"poolBucketSchema": queryPoolBucketSchema, "poolBucketEpoch": int64(0)}}, options.Update().SetUpsert(true))
|
|
return err
|
|
}
|
|
|
|
// VisitHumanPoolCountGroups never supplies an authorized total. Every group is
|
|
// checked using current server memberships/batches. Dirty authoritative rows
|
|
// subtract their old group and add their current row in this same request.
|
|
func (s *MongoStore) VisitHumanPoolCountGroups(ctx context.Context, org string, q domain.HumanItemQuery, visit func(domain.HumanItemRecord, int) error) (bool, error) {
|
|
target, err := s.poolReplica(ctx)
|
|
if err != nil {
|
|
return true, err
|
|
}
|
|
if target == nil {
|
|
return false, nil
|
|
}
|
|
if humanPoolFiltered(q) || q.BlindAnnotationActorID != "" || q.NeedsReworkRouting || q.InheritedUnassigned || q.WorkflowID != "" || q.RunID != "" || q.LegacyTaskID != "" || q.AnnotationView != "" || q.IDs != nil || q.CountIdentitySensitive {
|
|
return false, nil
|
|
}
|
|
b, err := s.countReplicaBarrier(ctx, org)
|
|
if err != nil {
|
|
return true, err
|
|
}
|
|
type weightedRow struct {
|
|
domain.HumanItemRecord `bson:",inline"`
|
|
Count int `bson:"count"`
|
|
}
|
|
rows := []weightedRow{}
|
|
session, e := target.db.Client().StartSession()
|
|
if e != nil {
|
|
return true, e
|
|
}
|
|
defer session.EndSession(ctx)
|
|
capacity := errors.New("count groups exceed request budget")
|
|
incompatible := errors.New("count bucket writer compatibility changed")
|
|
_, err = session.WithTransaction(ctx, func(sc mongo.SessionContext) (any, error) {
|
|
rows = nil
|
|
var state struct {
|
|
Schema string `bson:"poolBucketSchema"`
|
|
FactEpoch int64 `bson:"factEpoch"`
|
|
BucketEpoch int64 `bson:"poolBucketEpoch"`
|
|
}
|
|
if e := target.db.Collection(queryReplicaControl).FindOne(sc, bson.M{"_id": "state"}).Decode(&state); e != nil {
|
|
return nil, e
|
|
}
|
|
if state.Schema != queryPoolBucketSchema || state.FactEpoch != state.BucketEpoch {
|
|
return nil, incompatible
|
|
}
|
|
filter := humanItemMongoFilter(org, q)
|
|
filter["count"] = bson.M{"$gt": 0}
|
|
cur, e := target.db.Collection(queryPoolBuckets).Find(sc, filter, options.Find().SetMaxTime(8*time.Second).SetLimit(10001))
|
|
if e != nil {
|
|
return nil, e
|
|
}
|
|
e = cur.All(sc, &rows)
|
|
cur.Close(sc)
|
|
if e != nil {
|
|
return nil, e
|
|
}
|
|
if len(rows) > 10000 {
|
|
return nil, capacity
|
|
}
|
|
if b == nil || len(b.ids) == 0 {
|
|
return nil, nil
|
|
}
|
|
ids := make([]string, 0, len(b.ids))
|
|
for id := range b.ids {
|
|
ids = append(ids, id)
|
|
}
|
|
oldFilter := bson.M{"$and": bson.A{humanItemMongoFilter(org, q), bson.M{"id": bson.M{"$in": ids}}}}
|
|
old, e := target.humanItems.Find(sc, oldFilter, options.Find().SetProjection(poolBucketProjection()))
|
|
if e != nil {
|
|
return nil, e
|
|
}
|
|
defer old.Close(sc)
|
|
for old.Next(sc) {
|
|
var row domain.HumanItemRecord
|
|
if e = old.Decode(&row); e != nil {
|
|
return nil, e
|
|
}
|
|
rows = append(rows, weightedRow{HumanItemRecord: row, Count: -1})
|
|
}
|
|
return nil, old.Err()
|
|
}, options.Transaction().SetReadConcern(readconcern.Snapshot()))
|
|
if errors.Is(err, capacity) || errors.Is(err, incompatible) {
|
|
return false, nil
|
|
}
|
|
if err != nil {
|
|
return true, err
|
|
}
|
|
for _, row := range rows {
|
|
if e := ctx.Err(); e != nil {
|
|
return true, e
|
|
}
|
|
if e := visit(row.HumanItemRecord, row.Count); e != nil {
|
|
return true, e
|
|
}
|
|
}
|
|
if b == nil {
|
|
return true, ctx.Err()
|
|
}
|
|
err = s.visitReplicaPendingCounts(ctx, org, q, b, func(rows []domain.HumanItemRecord) error {
|
|
for _, row := range rows {
|
|
if e := visit(row, 1); e != nil {
|
|
return e
|
|
}
|
|
}
|
|
return nil
|
|
})
|
|
return true, err
|
|
}
|