forked from longtao.wu/droboticflow
363 lines
10 KiB
Go
363 lines
10 KiB
Go
package store
|
|
|
|
import (
|
|
"bytes"
|
|
"compress/gzip"
|
|
"context"
|
|
"crypto/sha256"
|
|
"encoding/binary"
|
|
"encoding/hex"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"reflect"
|
|
"sort"
|
|
"time"
|
|
|
|
"droboticflow/internal/queryserving"
|
|
"go.mongodb.org/mongo-driver/bson"
|
|
"go.mongodb.org/mongo-driver/bson/primitive"
|
|
"go.mongodb.org/mongo-driver/mongo"
|
|
"go.mongodb.org/mongo-driver/mongo/options"
|
|
)
|
|
|
|
type queryBuildReadKey struct{}
|
|
type queryBuildReadFunc func(context.Context, mongoCollection, bson.M, bson.M, any) error
|
|
|
|
type queryBuildCaptureState struct {
|
|
ID string `bson:"_id"`
|
|
Job string `bson:"job"`
|
|
Parts int `bson:"parts"`
|
|
Last bson.RawValue `bson:"last,omitempty"`
|
|
HasLast bool `bson:"hasLast"`
|
|
Complete bool `bson:"complete"`
|
|
Rows int `bson:"rows"`
|
|
Bytes int `bson:"bytes"`
|
|
}
|
|
|
|
type queryBuildPart struct {
|
|
ID string `bson:"_id"`
|
|
Job string `bson:"job"`
|
|
Digest string `bson:"digest"`
|
|
Data []byte `bson:"data"`
|
|
}
|
|
|
|
func queryBuildCanonical(v any) any {
|
|
switch d := v.(type) {
|
|
case bson.M:
|
|
out := map[string]any{}
|
|
for k, x := range d {
|
|
out[k] = queryBuildCanonical(x)
|
|
}
|
|
return out
|
|
case map[string]any:
|
|
out := map[string]any{}
|
|
for k, x := range d {
|
|
out[k] = queryBuildCanonical(x)
|
|
}
|
|
return out
|
|
case bson.A:
|
|
out := make([]any, len(d))
|
|
for i, x := range d {
|
|
out[i] = queryBuildCanonical(x)
|
|
}
|
|
return out
|
|
default:
|
|
return v
|
|
}
|
|
}
|
|
|
|
func queryBuildQueryKey(generation, name string, f, p bson.M) string {
|
|
// JSON map ordering is deterministic. Membership order is semantically
|
|
// irrelevant, so normalize caller-generated $in lists before hashing.
|
|
var normalize func(any) any
|
|
normalize = func(v any) any {
|
|
switch d := v.(type) {
|
|
case bson.M:
|
|
out := map[string]any{}
|
|
for k, x := range d {
|
|
if k == "$in" {
|
|
if ids, ok := x.([]string); ok {
|
|
cp := append([]string{}, ids...)
|
|
sort.Strings(cp)
|
|
out[k] = cp
|
|
continue
|
|
}
|
|
if a, ok := x.(bson.A); ok {
|
|
s := make([]string, 0, len(a))
|
|
all := true
|
|
for _, id := range a {
|
|
t, ok := id.(string)
|
|
if !ok {
|
|
all = false
|
|
break
|
|
}
|
|
s = append(s, t)
|
|
}
|
|
if all {
|
|
sort.Strings(s)
|
|
out[k] = s
|
|
continue
|
|
}
|
|
}
|
|
}
|
|
out[k] = normalize(x)
|
|
}
|
|
return out
|
|
case bson.A:
|
|
a := make([]any, len(d))
|
|
for i, x := range d {
|
|
a[i] = normalize(x)
|
|
}
|
|
return a
|
|
default:
|
|
return v
|
|
}
|
|
}
|
|
return queryserving.Key("query-build-input/v1", generation, name, normalize(f), queryBuildCanonical(p))
|
|
}
|
|
|
|
func queryBuildPack(v any) ([]byte, string, error) {
|
|
raw, err := bson.Marshal(v)
|
|
if err != nil {
|
|
return nil, "", err
|
|
}
|
|
var out bytes.Buffer
|
|
z := gzip.NewWriter(&out)
|
|
if _, err = z.Write(raw); err != nil {
|
|
return nil, "", err
|
|
}
|
|
if err = z.Close(); err != nil {
|
|
return nil, "", err
|
|
}
|
|
data := out.Bytes()
|
|
h := sha256.Sum256(data)
|
|
return data, hex.EncodeToString(h[:]), nil
|
|
}
|
|
func queryBuildUnpack(data []byte, digest string, out any) error {
|
|
h := sha256.Sum256(data)
|
|
if hex.EncodeToString(h[:]) != digest {
|
|
return errors.New("query build part digest mismatch")
|
|
}
|
|
z, err := gzip.NewReader(bytes.NewReader(data))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer z.Close()
|
|
raw, err := io.ReadAll(io.LimitReader(z, (64<<20)+1))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if len(raw) > 64<<20 {
|
|
return errors.New("query build part exceeds decode budget")
|
|
}
|
|
return bson.Unmarshal(raw, out)
|
|
}
|
|
|
|
func (s *MongoStore) queryBuildSnapshot(ctx context.Context) (primitive.Timestamp, error) {
|
|
op, stop := queryBuildOperation(ctx)
|
|
defer stop()
|
|
var response struct {
|
|
At primitive.Timestamp `bson:"atClusterTime"`
|
|
Cursor struct {
|
|
At primitive.Timestamp `bson:"atClusterTime"`
|
|
} `bson:"cursor"`
|
|
}
|
|
err := s.db.RunCommand(op, bson.D{{Key: "find", Value: "queryReadModelControl"}, {Key: "filter", Value: bson.M{"_id": "global"}}, {Key: "projection", Value: bson.M{"_id": 1}}, {Key: "limit", Value: 1}, {Key: "singleBatch", Value: true}, {Key: "readConcern", Value: bson.M{"level": "snapshot"}}, {Key: "maxTimeMS", Value: 1500}}).Decode(&response)
|
|
if err != nil {
|
|
return primitive.Timestamp{}, err
|
|
}
|
|
if response.At.T == 0 {
|
|
response.At = response.Cursor.At
|
|
}
|
|
if response.At.T == 0 {
|
|
return primitive.Timestamp{}, errors.New("query build snapshot timestamp unavailable")
|
|
}
|
|
return response.At, nil
|
|
}
|
|
|
|
func (s *MongoStore) queryBuildReadPage(ctx context.Context, c mongoCollection, f, p bson.M, last bson.RawValue, hasLast bool, at primitive.Timestamp) ([]bson.Raw, error) {
|
|
filter := any(f)
|
|
if hasLast {
|
|
filter = bson.M{"$and": bson.A{f, bson.M{"$expr": bson.M{"$gt": bson.A{"$_id", bson.M{"$literal": last}}}}}}
|
|
}
|
|
projection := bson.M{}
|
|
for k, v := range p {
|
|
projection[k] = v
|
|
}
|
|
projection["_id"] = 1
|
|
cmd := bson.D{{Key: "find", Value: c.Name()}, {Key: "filter", Value: filter}, {Key: "projection", Value: projection}, {Key: "sort", Value: bson.D{{Key: "_id", Value: 1}}}, {Key: "hint", Value: "_id_"}, {Key: "limit", Value: 200}, {Key: "batchSize", Value: 200}, {Key: "singleBatch", Value: true}, {Key: "readConcern", Value: bson.M{"level": "snapshot", "atClusterTime": at}}, {Key: "maxTimeMS", Value: 1500}}
|
|
op, stop := queryBuildOperation(ctx)
|
|
defer stop()
|
|
var response struct {
|
|
Cursor struct {
|
|
Rows []bson.Raw `bson:"firstBatch"`
|
|
} `bson:"cursor"`
|
|
}
|
|
readCtx, readStop := context.WithTimeout(op, 5*time.Second)
|
|
defer readStop()
|
|
for attempt := 0; attempt < 3; attempt++ {
|
|
err := c.Database().RunCommand(readCtx, cmd).Decode(&response)
|
|
if err == nil {
|
|
return response.Cursor.Rows, nil
|
|
}
|
|
var command mongo.CommandError
|
|
transient := mongo.IsNetworkError(err) || mongo.IsTimeout(err)
|
|
if errors.As(err, &command) {
|
|
switch command.Code {
|
|
case 50, 91, 6, 7, 89, 11600, 11602, 10107:
|
|
transient = true
|
|
}
|
|
}
|
|
if !transient || attempt == 2 || readCtx.Err() != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
|
|
return response.Cursor.Rows, nil
|
|
}
|
|
|
|
func queryBuildAppendRows(ctx context.Context, rows []bson.Raw, target any, projection bson.M) error {
|
|
value := reflect.ValueOf(target).Elem()
|
|
budget, _ := ctx.Value(workforceBuildBudgetKey{}).(*workforceBuildBudget)
|
|
for _, raw := range rows {
|
|
if excluded, ok := projection["_id"]; ok && (excluded == 0 || excluded == false) {
|
|
elements, e := raw.Elements()
|
|
if e != nil {
|
|
return e
|
|
}
|
|
clean := make([]byte, 4, len(raw))
|
|
for _, element := range elements {
|
|
if element.Key() != "_id" {
|
|
clean = append(clean, element...)
|
|
}
|
|
}
|
|
clean = append(clean, 0)
|
|
binary.LittleEndian.PutUint32(clean, uint32(len(clean)))
|
|
raw = bson.Raw(clean)
|
|
}
|
|
if budget != nil {
|
|
budget.rows++
|
|
budget.bytes += len(raw)
|
|
if budget.rows > 250000 || budget.bytes > 64<<20 {
|
|
return errWorkforceBuildBudget
|
|
}
|
|
}
|
|
v := reflect.New(value.Type().Elem())
|
|
if err := bson.Unmarshal(raw, v.Interface()); err != nil {
|
|
return err
|
|
}
|
|
value.Set(reflect.Append(value, v.Elem()))
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// This capture reads a native range once. Its payload and cursor commit in the
|
|
// same majority+journal transaction. Replaying a completed range reads only the
|
|
// immutable captured part, not the native business collections.
|
|
func (s *MongoStore) queryBuildCapture(ctx context.Context, l queryModelLease, j *queryBuildJob, round *queryBuildRound, c mongoCollection, f, p bson.M, target any) error {
|
|
if !s.queryReplicaFactStore {
|
|
return errors.New("query build capture requires independent read store")
|
|
}
|
|
check, checkStop := queryBuildOperation(ctx)
|
|
leaseErr := s.queryControl().FindOne(check, queryLeaseFilter(l), options.FindOne().SetProjection(bson.M{"_id": 1})).Err()
|
|
checkStop()
|
|
if leaseErr != nil {
|
|
return fmt.Errorf("query capture lease check: %w", leaseErr)
|
|
}
|
|
inputGeneration := j.InputGeneration
|
|
if inputGeneration == "" {
|
|
inputGeneration = j.Generation
|
|
}
|
|
key := queryBuildQueryKey(inputGeneration, c.Name(), f, p)
|
|
var st queryBuildCaptureState
|
|
check, checkStop = queryBuildOperation(ctx)
|
|
err := s.db.Collection(queryBuildJobsCollection).FindOne(check, bson.M{"_id": key}).Decode(&st)
|
|
checkStop()
|
|
if errors.Is(err, mongo.ErrNoDocuments) {
|
|
if inputGeneration != j.Generation {
|
|
return errors.New("verified input capture missing")
|
|
}
|
|
st = queryBuildCaptureState{ID: key, Job: inputGeneration}
|
|
} else if err != nil {
|
|
return err
|
|
}
|
|
for i := 0; i < st.Parts; i++ {
|
|
if err = ctx.Err(); err != nil {
|
|
return err
|
|
}
|
|
var part queryBuildPart
|
|
op, stop := queryBuildOperation(ctx)
|
|
err = s.db.Collection(queryBuildPartsCollection).FindOne(op, bson.M{"_id": fmt.Sprintf("%s/%09d", key, i)}).Decode(&part)
|
|
stop()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
var payload struct {
|
|
Rows []bson.Raw `bson:"rows"`
|
|
}
|
|
if err = queryBuildUnpack(part.Data, part.Digest, &payload); err != nil {
|
|
return err
|
|
}
|
|
if err = queryBuildAppendRows(ctx, payload.Rows, target, p); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
for !st.Complete {
|
|
if err = round.check(ctx); err != nil {
|
|
return err
|
|
}
|
|
rows, e := s.queryBuildReadPage(ctx, c, f, p, st.Last, st.HasLast, j.Snapshot)
|
|
if e != nil {
|
|
return e
|
|
}
|
|
next := st
|
|
if len(rows) == 0 {
|
|
next.Complete = true
|
|
} else {
|
|
id, e := rows[len(rows)-1].LookupErr("_id")
|
|
if e != nil {
|
|
return e
|
|
}
|
|
next.Last = bson.RawValue{Type: id.Type, Value: append([]byte(nil), id.Value...)}
|
|
next.HasLast = true
|
|
next.Parts++
|
|
next.Rows += len(rows)
|
|
}
|
|
var part queryBuildPart
|
|
if len(rows) > 0 {
|
|
data, hash, e := queryBuildPack(bson.M{"rows": rows})
|
|
if e != nil {
|
|
return e
|
|
}
|
|
part = queryBuildPart{ID: fmt.Sprintf("%s/%09d", key, st.Parts), Job: inputGeneration, Digest: hash, Data: data}
|
|
next.Bytes += len(data)
|
|
if e = s.queryBuildPace(ctx, len(data)); e != nil {
|
|
return e
|
|
}
|
|
}
|
|
if err = s.queryBuildTransaction(ctx, l, func(sc mongo.SessionContext) error {
|
|
if len(rows) > 0 {
|
|
if _, e := s.db.Collection(queryBuildPartsCollection).ReplaceOne(sc, bson.M{"_id": part.ID}, part, options.Replace().SetUpsert(true)); e != nil {
|
|
return e
|
|
}
|
|
}
|
|
_, e := s.db.Collection(queryBuildJobsCollection).ReplaceOne(sc, bson.M{"_id": key}, next, options.Replace().SetUpsert(true))
|
|
if e != nil {
|
|
return e
|
|
}
|
|
if j.ID != "" {
|
|
_, e = s.db.Collection(queryBuildJobsCollection).UpdateOne(sc, bson.M{"_id": j.ID, "generation": j.Generation}, bson.M{"$set": bson.M{"progressAt": time.Now().UTC(), "activeWithoutProgress": int64(0)}})
|
|
}
|
|
return e
|
|
}); err != nil {
|
|
return err
|
|
}
|
|
st = next
|
|
if err = queryBuildAppendRows(ctx, rows, target, p); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|