droboticflow/internal/store/query_build_capture.go
2026-11-15 08:11:09 -08:00

363 lines
10 KiB
Go

package store
import (
"bytes"
"compress/gzip"
"context"
"crypto/sha256"
"encoding/binary"
"encoding/hex"
"errors"
"fmt"
"io"
"reflect"
"sort"
"time"
"droboticflow/internal/queryserving"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/bson/primitive"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
)
type queryBuildReadKey struct{}
type queryBuildReadFunc func(context.Context, mongoCollection, bson.M, bson.M, any) error
type queryBuildCaptureState struct {
ID string `bson:"_id"`
Job string `bson:"job"`
Parts int `bson:"parts"`
Last bson.RawValue `bson:"last,omitempty"`
HasLast bool `bson:"hasLast"`
Complete bool `bson:"complete"`
Rows int `bson:"rows"`
Bytes int `bson:"bytes"`
}
type queryBuildPart struct {
ID string `bson:"_id"`
Job string `bson:"job"`
Digest string `bson:"digest"`
Data []byte `bson:"data"`
}
func queryBuildCanonical(v any) any {
switch d := v.(type) {
case bson.M:
out := map[string]any{}
for k, x := range d {
out[k] = queryBuildCanonical(x)
}
return out
case map[string]any:
out := map[string]any{}
for k, x := range d {
out[k] = queryBuildCanonical(x)
}
return out
case bson.A:
out := make([]any, len(d))
for i, x := range d {
out[i] = queryBuildCanonical(x)
}
return out
default:
return v
}
}
func queryBuildQueryKey(generation, name string, f, p bson.M) string {
// JSON map ordering is deterministic. Membership order is semantically
// irrelevant, so normalize caller-generated $in lists before hashing.
var normalize func(any) any
normalize = func(v any) any {
switch d := v.(type) {
case bson.M:
out := map[string]any{}
for k, x := range d {
if k == "$in" {
if ids, ok := x.([]string); ok {
cp := append([]string{}, ids...)
sort.Strings(cp)
out[k] = cp
continue
}
if a, ok := x.(bson.A); ok {
s := make([]string, 0, len(a))
all := true
for _, id := range a {
t, ok := id.(string)
if !ok {
all = false
break
}
s = append(s, t)
}
if all {
sort.Strings(s)
out[k] = s
continue
}
}
}
out[k] = normalize(x)
}
return out
case bson.A:
a := make([]any, len(d))
for i, x := range d {
a[i] = normalize(x)
}
return a
default:
return v
}
}
return queryserving.Key("query-build-input/v1", generation, name, normalize(f), queryBuildCanonical(p))
}
func queryBuildPack(v any) ([]byte, string, error) {
raw, err := bson.Marshal(v)
if err != nil {
return nil, "", err
}
var out bytes.Buffer
z := gzip.NewWriter(&out)
if _, err = z.Write(raw); err != nil {
return nil, "", err
}
if err = z.Close(); err != nil {
return nil, "", err
}
data := out.Bytes()
h := sha256.Sum256(data)
return data, hex.EncodeToString(h[:]), nil
}
func queryBuildUnpack(data []byte, digest string, out any) error {
h := sha256.Sum256(data)
if hex.EncodeToString(h[:]) != digest {
return errors.New("query build part digest mismatch")
}
z, err := gzip.NewReader(bytes.NewReader(data))
if err != nil {
return err
}
defer z.Close()
raw, err := io.ReadAll(io.LimitReader(z, (64<<20)+1))
if err != nil {
return err
}
if len(raw) > 64<<20 {
return errors.New("query build part exceeds decode budget")
}
return bson.Unmarshal(raw, out)
}
func (s *MongoStore) queryBuildSnapshot(ctx context.Context) (primitive.Timestamp, error) {
op, stop := queryBuildOperation(ctx)
defer stop()
var response struct {
At primitive.Timestamp `bson:"atClusterTime"`
Cursor struct {
At primitive.Timestamp `bson:"atClusterTime"`
} `bson:"cursor"`
}
err := s.db.RunCommand(op, bson.D{{Key: "find", Value: "queryReadModelControl"}, {Key: "filter", Value: bson.M{"_id": "global"}}, {Key: "projection", Value: bson.M{"_id": 1}}, {Key: "limit", Value: 1}, {Key: "singleBatch", Value: true}, {Key: "readConcern", Value: bson.M{"level": "snapshot"}}, {Key: "maxTimeMS", Value: 1500}}).Decode(&response)
if err != nil {
return primitive.Timestamp{}, err
}
if response.At.T == 0 {
response.At = response.Cursor.At
}
if response.At.T == 0 {
return primitive.Timestamp{}, errors.New("query build snapshot timestamp unavailable")
}
return response.At, nil
}
func (s *MongoStore) queryBuildReadPage(ctx context.Context, c mongoCollection, f, p bson.M, last bson.RawValue, hasLast bool, at primitive.Timestamp) ([]bson.Raw, error) {
filter := any(f)
if hasLast {
filter = bson.M{"$and": bson.A{f, bson.M{"$expr": bson.M{"$gt": bson.A{"$_id", bson.M{"$literal": last}}}}}}
}
projection := bson.M{}
for k, v := range p {
projection[k] = v
}
projection["_id"] = 1
cmd := bson.D{{Key: "find", Value: c.Name()}, {Key: "filter", Value: filter}, {Key: "projection", Value: projection}, {Key: "sort", Value: bson.D{{Key: "_id", Value: 1}}}, {Key: "hint", Value: "_id_"}, {Key: "limit", Value: 200}, {Key: "batchSize", Value: 200}, {Key: "singleBatch", Value: true}, {Key: "readConcern", Value: bson.M{"level": "snapshot", "atClusterTime": at}}, {Key: "maxTimeMS", Value: 1500}}
op, stop := queryBuildOperation(ctx)
defer stop()
var response struct {
Cursor struct {
Rows []bson.Raw `bson:"firstBatch"`
} `bson:"cursor"`
}
readCtx, readStop := context.WithTimeout(op, 5*time.Second)
defer readStop()
for attempt := 0; attempt < 3; attempt++ {
err := c.Database().RunCommand(readCtx, cmd).Decode(&response)
if err == nil {
return response.Cursor.Rows, nil
}
var command mongo.CommandError
transient := mongo.IsNetworkError(err) || mongo.IsTimeout(err)
if errors.As(err, &command) {
switch command.Code {
case 50, 91, 6, 7, 89, 11600, 11602, 10107:
transient = true
}
}
if !transient || attempt == 2 || readCtx.Err() != nil {
return nil, err
}
}
return response.Cursor.Rows, nil
}
func queryBuildAppendRows(ctx context.Context, rows []bson.Raw, target any, projection bson.M) error {
value := reflect.ValueOf(target).Elem()
budget, _ := ctx.Value(workforceBuildBudgetKey{}).(*workforceBuildBudget)
for _, raw := range rows {
if excluded, ok := projection["_id"]; ok && (excluded == 0 || excluded == false) {
elements, e := raw.Elements()
if e != nil {
return e
}
clean := make([]byte, 4, len(raw))
for _, element := range elements {
if element.Key() != "_id" {
clean = append(clean, element...)
}
}
clean = append(clean, 0)
binary.LittleEndian.PutUint32(clean, uint32(len(clean)))
raw = bson.Raw(clean)
}
if budget != nil {
budget.rows++
budget.bytes += len(raw)
if budget.rows > 250000 || budget.bytes > 64<<20 {
return errWorkforceBuildBudget
}
}
v := reflect.New(value.Type().Elem())
if err := bson.Unmarshal(raw, v.Interface()); err != nil {
return err
}
value.Set(reflect.Append(value, v.Elem()))
}
return nil
}
// This capture reads a native range once. Its payload and cursor commit in the
// same majority+journal transaction. Replaying a completed range reads only the
// immutable captured part, not the native business collections.
func (s *MongoStore) queryBuildCapture(ctx context.Context, l queryModelLease, j *queryBuildJob, round *queryBuildRound, c mongoCollection, f, p bson.M, target any) error {
if !s.queryReplicaFactStore {
return errors.New("query build capture requires independent read store")
}
check, checkStop := queryBuildOperation(ctx)
leaseErr := s.queryControl().FindOne(check, queryLeaseFilter(l), options.FindOne().SetProjection(bson.M{"_id": 1})).Err()
checkStop()
if leaseErr != nil {
return fmt.Errorf("query capture lease check: %w", leaseErr)
}
inputGeneration := j.InputGeneration
if inputGeneration == "" {
inputGeneration = j.Generation
}
key := queryBuildQueryKey(inputGeneration, c.Name(), f, p)
var st queryBuildCaptureState
check, checkStop = queryBuildOperation(ctx)
err := s.db.Collection(queryBuildJobsCollection).FindOne(check, bson.M{"_id": key}).Decode(&st)
checkStop()
if errors.Is(err, mongo.ErrNoDocuments) {
if inputGeneration != j.Generation {
return errors.New("verified input capture missing")
}
st = queryBuildCaptureState{ID: key, Job: inputGeneration}
} else if err != nil {
return err
}
for i := 0; i < st.Parts; i++ {
if err = ctx.Err(); err != nil {
return err
}
var part queryBuildPart
op, stop := queryBuildOperation(ctx)
err = s.db.Collection(queryBuildPartsCollection).FindOne(op, bson.M{"_id": fmt.Sprintf("%s/%09d", key, i)}).Decode(&part)
stop()
if err != nil {
return err
}
var payload struct {
Rows []bson.Raw `bson:"rows"`
}
if err = queryBuildUnpack(part.Data, part.Digest, &payload); err != nil {
return err
}
if err = queryBuildAppendRows(ctx, payload.Rows, target, p); err != nil {
return err
}
}
for !st.Complete {
if err = round.check(ctx); err != nil {
return err
}
rows, e := s.queryBuildReadPage(ctx, c, f, p, st.Last, st.HasLast, j.Snapshot)
if e != nil {
return e
}
next := st
if len(rows) == 0 {
next.Complete = true
} else {
id, e := rows[len(rows)-1].LookupErr("_id")
if e != nil {
return e
}
next.Last = bson.RawValue{Type: id.Type, Value: append([]byte(nil), id.Value...)}
next.HasLast = true
next.Parts++
next.Rows += len(rows)
}
var part queryBuildPart
if len(rows) > 0 {
data, hash, e := queryBuildPack(bson.M{"rows": rows})
if e != nil {
return e
}
part = queryBuildPart{ID: fmt.Sprintf("%s/%09d", key, st.Parts), Job: inputGeneration, Digest: hash, Data: data}
next.Bytes += len(data)
if e = s.queryBuildPace(ctx, len(data)); e != nil {
return e
}
}
if err = s.queryBuildTransaction(ctx, l, func(sc mongo.SessionContext) error {
if len(rows) > 0 {
if _, e := s.db.Collection(queryBuildPartsCollection).ReplaceOne(sc, bson.M{"_id": part.ID}, part, options.Replace().SetUpsert(true)); e != nil {
return e
}
}
_, e := s.db.Collection(queryBuildJobsCollection).ReplaceOne(sc, bson.M{"_id": key}, next, options.Replace().SetUpsert(true))
if e != nil {
return e
}
if j.ID != "" {
_, e = s.db.Collection(queryBuildJobsCollection).UpdateOne(sc, bson.M{"_id": j.ID, "generation": j.Generation}, bson.M{"$set": bson.M{"progressAt": time.Now().UTC(), "activeWithoutProgress": int64(0)}})
}
return e
}); err != nil {
return err
}
st = next
if err = queryBuildAppendRows(ctx, rows, target, p); err != nil {
return err
}
}
return nil
}