droboticflow/internal/store/query_build_apply.go
2026-11-15 09:12:57 -08:00

346 lines
11 KiB
Go

package store
import (
"context"
"errors"
"fmt"
"time"
"droboticflow/internal/workforcestats"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
)
type queryBuildPrevious struct {
ID string `bson:"_id"`
Key string `bson:"key"`
Hash string `bson:"hash"`
From int64 `bson:"fromEpoch"`
To int64 `bson:"toEpoch"`
Retired *time.Time `bson:"retiredAt"`
Expire *time.Time `bson:"expireAt"`
}
func (s *MongoStore) queryBuildPlanPage(ctx context.Context, j queryBuildJob, n int) ([]workforcestats.Fact, error) {
op, stop := queryBuildOperation(ctx)
defer stop()
var p queryBuildPart
if err := s.db.Collection(queryBuildPartsCollection).FindOne(op, bson.M{"_id": queryBuildPlanPartKey(j, "facts", n), "job": j.Generation}).Decode(&p); err != nil {
return nil, err
}
var facts []workforcestats.Fact
if err := queryBuildJSONDecode(p, &facts); err != nil {
return nil, err
}
if len(facts) == 0 || len(facts) > queryBuildFactPageRows {
return nil, errors.New("invalid sealed fact page")
}
return facts, nil
}
// A batch checkpoint commits in the very same majority+journal transaction as
// its version writes. A lost acknowledgement can only replay this bounded batch.
func (s *MongoStore) queryBuildApply(ctx context.Context, l queryModelLease, j *queryBuildJob, r *queryBuildRound) error {
c := s.db.Collection(workforceFactCollection)
for j.NextPart < j.PlanParts {
if err := r.check(ctx); err != nil {
return err
}
facts, err := s.queryBuildPlanPage(ctx, *j, j.NextPart)
if err != nil {
return err
}
for j.NextRow < len(facts) {
if err = r.check(ctx); err != nil {
return err
}
models := []mongo.WriteModel{}
bytes, rows, reused := 0, 0, 0
next := *j
// Keep point lookups bounded; the key index avoids reading large old rows.
keys := []string{}
for _, f := range facts[j.NextRow:] {
keys = append(keys, f.Key)
}
op, stop := queryBuildOperation(ctx)
cur, e := c.Find(op, bson.M{"organizationId": j.Organization, "key": bson.M{"$in": keys}}, options.Find().SetProjection(bson.M{"_id": 1, "key": 1, "hash": 1, "fromEpoch": 1, "toEpoch": 1, "retiredAt": 1, "expireAt": 1}).SetHint("workforce_fact_key_v1"))
previous := map[string][]queryBuildPrevious{}
if e == nil {
var xs []queryBuildPrevious
e = cur.All(op, &xs)
cur.Close(op)
for _, x := range xs {
previous[x.Key] = append(previous[x.Key], x)
}
}
stop()
if e != nil {
return e
}
for next.NextRow < len(facts) {
f := facts[next.NextRow]
doc, hash, size, e := queryBuildFactDocument(j.Organization, j.Epoch, f)
if e != nil {
return e
}
if rows > 0 && bytes+size+512 > queryBuildBatchBytes {
break
}
versions := previous[f.Key]
var current *queryBuildPrevious
var staged *queryBuildPrevious
for i := range versions {
x := &versions[i]
if x.From == j.Epoch && x.ID == doc["_id"] {
staged = x
}
if x.From <= j.OldEpoch && (x.To == 0 || x.To > j.OldEpoch) {
if current != nil {
return errors.New("duplicate committed workforce key")
}
current = x
}
}
unchanged := current != nil && current.Hash == hash
keepID := ""
if unchanged {
keepID = current.ID
if current.To != 0 || current.Retired != nil || current.Expire != nil {
models = append(models, mongo.NewUpdateOneModel().SetFilter(bson.M{"_id": current.ID}).SetUpdate(bson.M{"$unset": bson.M{"toEpoch": "", "retiredAt": "", "expireAt": ""}}))
bytes += 256
}
reused++
} else {
next.Changed = true
keepID = doc["_id"].(string)
if current != nil && current.To != j.Epoch {
models = append(models, mongo.NewUpdateOneModel().SetFilter(bson.M{"_id": current.ID}).SetUpdate(bson.M{"$set": bson.M{"toEpoch": j.Epoch, "retiredAt": time.Now().UTC()}}))
bytes += 256
}
if staged != nil && staged.Hash == hash && staged.To == 0 && staged.Retired == nil && staged.Expire == nil {
reused++
} else {
models = append(models, mongo.NewReplaceOneModel().SetFilter(bson.M{"_id": doc["_id"]}).SetReplacement(doc).SetUpsert(true))
bytes += size
}
}
for _, x := range versions {
if x.From > j.OldEpoch && x.ID != keepID {
models = append(models, mongo.NewDeleteOneModel().SetFilter(bson.M{"_id": x.ID, "organizationId": j.Organization, "fromEpoch": bson.M{"$gt": j.OldEpoch}}))
bytes += 256
}
}
next.NextRow++
rows++
}
if rows == 0 {
return errors.New("fact batch made no progress")
}
if next.NextRow == len(facts) {
next.NextPart++
next.NextRow = 0
}
next.ConfirmedRows += int64(rows - reused)
next.ReusedRows += int64(reused)
next.ConfirmedBytes += int64(bytes)
next.ProgressAt = time.Now().UTC()
next.ActiveWithoutProgress = 0
err = s.queryBuildPace(ctx, bytes)
if err != nil {
return err
}
err = s.queryBuildTransaction(ctx, l, func(sc mongo.SessionContext) error {
if len(models) > 0 {
if _, e := c.BulkWrite(sc, models, options.BulkWrite().SetOrdered(true)); e != nil {
return e
}
}
return s.saveQueryBuildJob(sc, next)
})
if err != nil {
return err
}
*j = next
if j.NextRow == 0 {
break
}
}
}
j.Phase = "cleanup"
return s.queryBuildTransaction(ctx, l, func(sc mongo.SessionContext) error { return s.saveQueryBuildJob(sc, *j) })
}
// Merge the sorted sealed plan with one bounded page of old/future versions.
// Persist both cursors, so restart never re-scans all previously compared keys.
func (s *MongoStore) queryBuildCleanup(ctx context.Context, l queryModelLease, j *queryBuildJob, r *queryBuildRound) error {
c := s.db.Collection(workforceFactCollection)
for {
if err := r.check(ctx); err != nil {
return err
}
f := bson.M{"organizationId": j.Organization}
if j.CleanupFuture {
f["fromEpoch"] = bson.M{"$gt": j.OldEpoch}
} else {
f = workforceVersionFilter(j.Organization, j.OldEpoch)
}
if j.CleanupHasAfter {
f["key"] = bson.M{"$gt": j.CleanupAfter}
}
op, stop := queryBuildOperation(ctx)
cur, err := c.Find(op, f, options.Find().SetProjection(bson.M{"_id": 1, "key": 1, "fromEpoch": 1}).SetSort(bson.D{{Key: "key", Value: 1}}).SetHint("workforce_fact_key_v1").SetLimit(200))
var xs []queryBuildPrevious
if err == nil {
err = cur.All(op, &xs)
cur.Close(op)
}
stop()
if err != nil {
return err
}
if len(xs) == 0 {
if j.CleanupFuture {
j.Phase = "publish"
} else {
j.CleanupFuture = true
j.CleanupAfter = ""
j.CleanupHasAfter = false
j.CleanupPlanPart = 0
j.CleanupPlanRow = 0
}
if err = s.queryBuildTransaction(ctx, l, func(sc mongo.SessionContext) error { return s.saveQueryBuildJob(sc, *j) }); err != nil {
return err
}
if j.Phase == "publish" {
return nil
}
continue
}
next := *j
models := []mongo.WriteModel{}
var page []workforcestats.Fact
pageN := -1
for _, x := range xs {
if x.Key == "" {
return errors.New("invalid workforce fact identity")
}
present := false
for next.CleanupPlanPart < j.PlanParts {
if pageN != next.CleanupPlanPart {
page, err = s.queryBuildPlanPage(ctx, *j, next.CleanupPlanPart)
if err != nil {
return err
}
pageN = next.CleanupPlanPart
}
if next.CleanupPlanRow >= len(page) {
next.CleanupPlanPart++
next.CleanupPlanRow = 0
continue
}
key := page[next.CleanupPlanRow].Key
if key < x.Key {
next.CleanupPlanRow++
continue
}
present = key == x.Key
break
}
if !present {
next.Changed = true
if j.CleanupFuture {
models = append(models, mongo.NewDeleteManyModel().SetFilter(bson.M{"organizationId": j.Organization, "key": x.Key, "fromEpoch": bson.M{"$gt": j.OldEpoch}}))
} else {
models = append(models, mongo.NewUpdateManyModel().SetFilter(bson.M{"$and": bson.A{workforceVersionFilter(j.Organization, j.OldEpoch), bson.M{"key": x.Key}}}).SetUpdate(bson.M{"$set": bson.M{"toEpoch": j.Epoch, "retiredAt": time.Now().UTC()}}))
}
}
next.CleanupAfter = x.Key
next.CleanupHasAfter = true
}
next.ProgressAt = time.Now().UTC()
next.ActiveWithoutProgress = 0
if err = s.queryBuildPace(ctx, len(models)*256); err != nil {
return err
}
if err = s.queryBuildTransaction(ctx, l, func(sc mongo.SessionContext) error {
if len(models) > 0 {
if _, e := c.BulkWrite(sc, models); e != nil {
return e
}
}
return s.saveQueryBuildJob(sc, next)
}); err != nil {
return err
}
*j = next
}
}
func (s *MongoStore) queryBuildPublish(ctx context.Context, l queryModelLease, j *queryBuildJob) error {
op, stop := queryBuildOperation(ctx)
defer stop()
epoch := j.Epoch
if !j.Changed && j.OldEpoch > 0 {
epoch = j.OldEpoch
}
count, err := s.db.Collection(workforceFactCollection).CountDocuments(op, workforceVersionFilter(j.Organization, epoch))
if err != nil {
return err
}
if count != int64(j.PlanRows) {
return fmt.Errorf("sealed workforce row count differs: got=%d want=%d", count, j.PlanRows)
}
next := *j
next.Phase = "complete"
next.ProgressAt = time.Now().UTC()
next.ActiveWithoutProgress = 0
return s.queryBuildTransaction(ctx, l, func(sc mongo.SessionContext) error {
var state struct {
Heads map[string]QueryModelHead `bson:"heads"`
}
if e := s.queryControl().FindOne(sc, queryLeaseFilter(l)).Decode(&state); e != nil {
return e
}
if state.Heads[workforceFactKey(j.Organization)].FactEpoch != j.OldEpoch {
return errors.New("query build published epoch changed")
}
// A payload can be reused far longer than the orphan grace. Pin its
// retirement in the same commit as the replacement heads, rather than
// interpreting the payload's original createdAt as retirement time.
now := time.Now().UTC()
oldProduction := state.Heads[queryModelKey("production", j.Organization, j.Projects)]
next.Retired = nil
for _, pin := range j.Retired {
if now.Before(pin.Until) && pin.Generation != j.Production.Generation {
next.Retired = append(next.Retired, pin)
}
}
if oldProduction.Generation != "" && oldProduction.Generation != j.Production.Generation {
until := now.Add(15 * time.Minute)
found := false
for i := range next.Retired {
if next.Retired[i].Generation == oldProduction.Generation {
next.Retired[i].Until = until
found = true
}
}
if !found {
next.Retired = append(next.Retired, queryBuildRetiredGeneration{Generation: oldProduction.Generation, Until: until})
}
}
wf := QueryModelHead{Generation: workforceFactSchema, Collection: workforceFactCollection, FactEpoch: epoch, Rows: j.PlanRows, AsOf: j.AsOf, SourceRevision: j.SourceRevision, GlobalRevision: j.GlobalRevision}
result, e := s.queryControl().UpdateOne(sc, queryLeaseFilter(l), bson.M{"$set": bson.M{"heads." + workforceFactKey(j.Organization): wf, "heads." + queryModelKey("production", j.Organization, j.Projects): j.Production}})
if e != nil {
return e
}
if result.MatchedCount != 1 {
return errors.New("query build publication fenced")
}
if e = s.saveQueryBuildJob(sc, next); e == nil {
*j = next
}
return e
})
}