forked from longtao.wu/droboticflow
346 lines
11 KiB
Go
346 lines
11 KiB
Go
package store
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"time"
|
|
|
|
"droboticflow/internal/workforcestats"
|
|
"go.mongodb.org/mongo-driver/bson"
|
|
"go.mongodb.org/mongo-driver/mongo"
|
|
"go.mongodb.org/mongo-driver/mongo/options"
|
|
)
|
|
|
|
type queryBuildPrevious struct {
|
|
ID string `bson:"_id"`
|
|
Key string `bson:"key"`
|
|
Hash string `bson:"hash"`
|
|
From int64 `bson:"fromEpoch"`
|
|
To int64 `bson:"toEpoch"`
|
|
Retired *time.Time `bson:"retiredAt"`
|
|
Expire *time.Time `bson:"expireAt"`
|
|
}
|
|
|
|
func (s *MongoStore) queryBuildPlanPage(ctx context.Context, j queryBuildJob, n int) ([]workforcestats.Fact, error) {
|
|
op, stop := queryBuildOperation(ctx)
|
|
defer stop()
|
|
var p queryBuildPart
|
|
if err := s.db.Collection(queryBuildPartsCollection).FindOne(op, bson.M{"_id": queryBuildPlanPartKey(j, "facts", n), "job": j.Generation}).Decode(&p); err != nil {
|
|
return nil, err
|
|
}
|
|
var facts []workforcestats.Fact
|
|
if err := queryBuildJSONDecode(p, &facts); err != nil {
|
|
return nil, err
|
|
}
|
|
if len(facts) == 0 || len(facts) > queryBuildFactPageRows {
|
|
return nil, errors.New("invalid sealed fact page")
|
|
}
|
|
return facts, nil
|
|
}
|
|
|
|
// A batch checkpoint commits in the very same majority+journal transaction as
|
|
// its version writes. A lost acknowledgement can only replay this bounded batch.
|
|
func (s *MongoStore) queryBuildApply(ctx context.Context, l queryModelLease, j *queryBuildJob, r *queryBuildRound) error {
|
|
c := s.db.Collection(workforceFactCollection)
|
|
for j.NextPart < j.PlanParts {
|
|
if err := r.check(ctx); err != nil {
|
|
return err
|
|
}
|
|
facts, err := s.queryBuildPlanPage(ctx, *j, j.NextPart)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
for j.NextRow < len(facts) {
|
|
if err = r.check(ctx); err != nil {
|
|
return err
|
|
}
|
|
models := []mongo.WriteModel{}
|
|
bytes, rows, reused := 0, 0, 0
|
|
next := *j
|
|
// Keep point lookups bounded; the key index avoids reading large old rows.
|
|
keys := []string{}
|
|
for _, f := range facts[j.NextRow:] {
|
|
keys = append(keys, f.Key)
|
|
}
|
|
op, stop := queryBuildOperation(ctx)
|
|
cur, e := c.Find(op, bson.M{"organizationId": j.Organization, "key": bson.M{"$in": keys}}, options.Find().SetProjection(bson.M{"_id": 1, "key": 1, "hash": 1, "fromEpoch": 1, "toEpoch": 1, "retiredAt": 1, "expireAt": 1}).SetHint("workforce_fact_key_v1"))
|
|
previous := map[string][]queryBuildPrevious{}
|
|
if e == nil {
|
|
var xs []queryBuildPrevious
|
|
e = cur.All(op, &xs)
|
|
cur.Close(op)
|
|
for _, x := range xs {
|
|
previous[x.Key] = append(previous[x.Key], x)
|
|
}
|
|
}
|
|
stop()
|
|
if e != nil {
|
|
return e
|
|
}
|
|
for next.NextRow < len(facts) {
|
|
f := facts[next.NextRow]
|
|
doc, hash, size, e := queryBuildFactDocument(j.Organization, j.Epoch, f)
|
|
if e != nil {
|
|
return e
|
|
}
|
|
if rows > 0 && bytes+size+512 > queryBuildBatchBytes {
|
|
break
|
|
}
|
|
versions := previous[f.Key]
|
|
var current *queryBuildPrevious
|
|
var staged *queryBuildPrevious
|
|
for i := range versions {
|
|
x := &versions[i]
|
|
if x.From == j.Epoch && x.ID == doc["_id"] {
|
|
staged = x
|
|
}
|
|
if x.From <= j.OldEpoch && (x.To == 0 || x.To > j.OldEpoch) {
|
|
if current != nil {
|
|
return errors.New("duplicate committed workforce key")
|
|
}
|
|
current = x
|
|
}
|
|
}
|
|
unchanged := current != nil && current.Hash == hash
|
|
keepID := ""
|
|
if unchanged {
|
|
keepID = current.ID
|
|
if current.To != 0 || current.Retired != nil || current.Expire != nil {
|
|
models = append(models, mongo.NewUpdateOneModel().SetFilter(bson.M{"_id": current.ID}).SetUpdate(bson.M{"$unset": bson.M{"toEpoch": "", "retiredAt": "", "expireAt": ""}}))
|
|
bytes += 256
|
|
}
|
|
reused++
|
|
} else {
|
|
next.Changed = true
|
|
keepID = doc["_id"].(string)
|
|
if current != nil && current.To != j.Epoch {
|
|
models = append(models, mongo.NewUpdateOneModel().SetFilter(bson.M{"_id": current.ID}).SetUpdate(bson.M{"$set": bson.M{"toEpoch": j.Epoch, "retiredAt": time.Now().UTC()}}))
|
|
bytes += 256
|
|
}
|
|
if staged != nil && staged.Hash == hash && staged.To == 0 && staged.Retired == nil && staged.Expire == nil {
|
|
reused++
|
|
} else {
|
|
models = append(models, mongo.NewReplaceOneModel().SetFilter(bson.M{"_id": doc["_id"]}).SetReplacement(doc).SetUpsert(true))
|
|
bytes += size
|
|
}
|
|
}
|
|
for _, x := range versions {
|
|
if x.From > j.OldEpoch && x.ID != keepID {
|
|
models = append(models, mongo.NewDeleteOneModel().SetFilter(bson.M{"_id": x.ID, "organizationId": j.Organization, "fromEpoch": bson.M{"$gt": j.OldEpoch}}))
|
|
bytes += 256
|
|
}
|
|
}
|
|
next.NextRow++
|
|
rows++
|
|
}
|
|
if rows == 0 {
|
|
return errors.New("fact batch made no progress")
|
|
}
|
|
if next.NextRow == len(facts) {
|
|
next.NextPart++
|
|
next.NextRow = 0
|
|
}
|
|
next.ConfirmedRows += int64(rows - reused)
|
|
next.ReusedRows += int64(reused)
|
|
next.ConfirmedBytes += int64(bytes)
|
|
next.ProgressAt = time.Now().UTC()
|
|
next.ActiveWithoutProgress = 0
|
|
err = s.queryBuildPace(ctx, bytes)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
err = s.queryBuildTransaction(ctx, l, func(sc mongo.SessionContext) error {
|
|
if len(models) > 0 {
|
|
if _, e := c.BulkWrite(sc, models, options.BulkWrite().SetOrdered(true)); e != nil {
|
|
return e
|
|
}
|
|
}
|
|
return s.saveQueryBuildJob(sc, next)
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
*j = next
|
|
if j.NextRow == 0 {
|
|
break
|
|
}
|
|
}
|
|
}
|
|
j.Phase = "cleanup"
|
|
return s.queryBuildTransaction(ctx, l, func(sc mongo.SessionContext) error { return s.saveQueryBuildJob(sc, *j) })
|
|
}
|
|
|
|
// Merge the sorted sealed plan with one bounded page of old/future versions.
|
|
// Persist both cursors, so restart never re-scans all previously compared keys.
|
|
func (s *MongoStore) queryBuildCleanup(ctx context.Context, l queryModelLease, j *queryBuildJob, r *queryBuildRound) error {
|
|
c := s.db.Collection(workforceFactCollection)
|
|
for {
|
|
if err := r.check(ctx); err != nil {
|
|
return err
|
|
}
|
|
f := bson.M{"organizationId": j.Organization}
|
|
if j.CleanupFuture {
|
|
f["fromEpoch"] = bson.M{"$gt": j.OldEpoch}
|
|
} else {
|
|
f = workforceVersionFilter(j.Organization, j.OldEpoch)
|
|
}
|
|
if j.CleanupHasAfter {
|
|
f["key"] = bson.M{"$gt": j.CleanupAfter}
|
|
}
|
|
op, stop := queryBuildOperation(ctx)
|
|
cur, err := c.Find(op, f, options.Find().SetProjection(bson.M{"_id": 1, "key": 1, "fromEpoch": 1}).SetSort(bson.D{{Key: "key", Value: 1}}).SetHint("workforce_fact_key_v1").SetLimit(200))
|
|
var xs []queryBuildPrevious
|
|
if err == nil {
|
|
err = cur.All(op, &xs)
|
|
cur.Close(op)
|
|
}
|
|
stop()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if len(xs) == 0 {
|
|
if j.CleanupFuture {
|
|
j.Phase = "publish"
|
|
} else {
|
|
j.CleanupFuture = true
|
|
j.CleanupAfter = ""
|
|
j.CleanupHasAfter = false
|
|
j.CleanupPlanPart = 0
|
|
j.CleanupPlanRow = 0
|
|
}
|
|
if err = s.queryBuildTransaction(ctx, l, func(sc mongo.SessionContext) error { return s.saveQueryBuildJob(sc, *j) }); err != nil {
|
|
return err
|
|
}
|
|
if j.Phase == "publish" {
|
|
return nil
|
|
}
|
|
continue
|
|
}
|
|
next := *j
|
|
models := []mongo.WriteModel{}
|
|
var page []workforcestats.Fact
|
|
pageN := -1
|
|
for _, x := range xs {
|
|
if x.Key == "" {
|
|
return errors.New("invalid workforce fact identity")
|
|
}
|
|
present := false
|
|
for next.CleanupPlanPart < j.PlanParts {
|
|
if pageN != next.CleanupPlanPart {
|
|
page, err = s.queryBuildPlanPage(ctx, *j, next.CleanupPlanPart)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
pageN = next.CleanupPlanPart
|
|
}
|
|
if next.CleanupPlanRow >= len(page) {
|
|
next.CleanupPlanPart++
|
|
next.CleanupPlanRow = 0
|
|
continue
|
|
}
|
|
key := page[next.CleanupPlanRow].Key
|
|
if key < x.Key {
|
|
next.CleanupPlanRow++
|
|
continue
|
|
}
|
|
present = key == x.Key
|
|
break
|
|
}
|
|
if !present {
|
|
next.Changed = true
|
|
if j.CleanupFuture {
|
|
models = append(models, mongo.NewDeleteManyModel().SetFilter(bson.M{"organizationId": j.Organization, "key": x.Key, "fromEpoch": bson.M{"$gt": j.OldEpoch}}))
|
|
} else {
|
|
models = append(models, mongo.NewUpdateManyModel().SetFilter(bson.M{"$and": bson.A{workforceVersionFilter(j.Organization, j.OldEpoch), bson.M{"key": x.Key}}}).SetUpdate(bson.M{"$set": bson.M{"toEpoch": j.Epoch, "retiredAt": time.Now().UTC()}}))
|
|
}
|
|
}
|
|
next.CleanupAfter = x.Key
|
|
next.CleanupHasAfter = true
|
|
}
|
|
next.ProgressAt = time.Now().UTC()
|
|
next.ActiveWithoutProgress = 0
|
|
if err = s.queryBuildPace(ctx, len(models)*256); err != nil {
|
|
return err
|
|
}
|
|
if err = s.queryBuildTransaction(ctx, l, func(sc mongo.SessionContext) error {
|
|
if len(models) > 0 {
|
|
if _, e := c.BulkWrite(sc, models); e != nil {
|
|
return e
|
|
}
|
|
}
|
|
return s.saveQueryBuildJob(sc, next)
|
|
}); err != nil {
|
|
return err
|
|
}
|
|
*j = next
|
|
}
|
|
}
|
|
|
|
func (s *MongoStore) queryBuildPublish(ctx context.Context, l queryModelLease, j *queryBuildJob) error {
|
|
op, stop := queryBuildOperation(ctx)
|
|
defer stop()
|
|
epoch := j.Epoch
|
|
if !j.Changed && j.OldEpoch > 0 {
|
|
epoch = j.OldEpoch
|
|
}
|
|
count, err := s.db.Collection(workforceFactCollection).CountDocuments(op, workforceVersionFilter(j.Organization, epoch))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if count != int64(j.PlanRows) {
|
|
return fmt.Errorf("sealed workforce row count differs: got=%d want=%d", count, j.PlanRows)
|
|
}
|
|
next := *j
|
|
next.Phase = "complete"
|
|
next.ProgressAt = time.Now().UTC()
|
|
next.ActiveWithoutProgress = 0
|
|
return s.queryBuildTransaction(ctx, l, func(sc mongo.SessionContext) error {
|
|
var state struct {
|
|
Heads map[string]QueryModelHead `bson:"heads"`
|
|
}
|
|
if e := s.queryControl().FindOne(sc, queryLeaseFilter(l)).Decode(&state); e != nil {
|
|
return e
|
|
}
|
|
if state.Heads[workforceFactKey(j.Organization)].FactEpoch != j.OldEpoch {
|
|
return errors.New("query build published epoch changed")
|
|
}
|
|
// A payload can be reused far longer than the orphan grace. Pin its
|
|
// retirement in the same commit as the replacement heads, rather than
|
|
// interpreting the payload's original createdAt as retirement time.
|
|
now := time.Now().UTC()
|
|
oldProduction := state.Heads[queryModelKey("production", j.Organization, j.Projects)]
|
|
next.Retired = nil
|
|
for _, pin := range j.Retired {
|
|
if now.Before(pin.Until) && pin.Generation != j.Production.Generation {
|
|
next.Retired = append(next.Retired, pin)
|
|
}
|
|
}
|
|
if oldProduction.Generation != "" && oldProduction.Generation != j.Production.Generation {
|
|
until := now.Add(15 * time.Minute)
|
|
found := false
|
|
for i := range next.Retired {
|
|
if next.Retired[i].Generation == oldProduction.Generation {
|
|
next.Retired[i].Until = until
|
|
found = true
|
|
}
|
|
}
|
|
if !found {
|
|
next.Retired = append(next.Retired, queryBuildRetiredGeneration{Generation: oldProduction.Generation, Until: until})
|
|
}
|
|
}
|
|
wf := QueryModelHead{Generation: workforceFactSchema, Collection: workforceFactCollection, FactEpoch: epoch, Rows: j.PlanRows, AsOf: j.AsOf, SourceRevision: j.SourceRevision, GlobalRevision: j.GlobalRevision}
|
|
result, e := s.queryControl().UpdateOne(sc, queryLeaseFilter(l), bson.M{"$set": bson.M{"heads." + workforceFactKey(j.Organization): wf, "heads." + queryModelKey("production", j.Organization, j.Projects): j.Production}})
|
|
if e != nil {
|
|
return e
|
|
}
|
|
if result.MatchedCount != 1 {
|
|
return errors.New("query build publication fenced")
|
|
}
|
|
if e = s.saveQueryBuildJob(sc, next); e == nil {
|
|
*j = next
|
|
}
|
|
return e
|
|
})
|
|
}
|