droboticflow/internal/store/query_read_models.go
2026-11-13 22:31:44 -08:00

321 lines
12 KiB
Go

package store
import (
"bytes"
"compress/gzip"
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"sort"
"sync/atomic"
"time"
"droboticflow/internal/domain"
"droboticflow/internal/productionstats"
"droboticflow/internal/queryserving"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
)
const queryModelSchema = "query-model/v3"
const queryModelChunkBytes = 2 << 20
const queryModelBuildChunkBytes = 128 << 10
const queryModelMaxBytes = 64 << 20
type queryServingOptions struct {
cache queryserving.Cache
search *queryserving.Typesense
stats queryServingCounters
}
type queryServingCounters struct{ reportHits, responseHits, searchHits, typesenseReads, mongoSearchReads atomic.Uint64 }
// QueryServingStatistics exposes counters only, never names or query contents.
func (s *MongoStore) QueryServingStatistics() map[string]uint64 {
if s.queryReplicaReads != nil && s.queryReplicaReads.target != nil {
return s.queryReplicaReads.target.QueryServingStatistics()
}
if s.queryServing == nil {
return map[string]uint64{}
}
c := &s.queryServing.stats
out := map[string]uint64{"report_hits": c.reportHits.Load(), "response_hits": c.responseHits.Load(), "search_hits": c.searchHits.Load(), "typesense_reads": c.typesenseReads.Load(), "materialized_search_reads": c.mongoSearchReads.Load()}
if cache, ok := s.SharedQueryCache().(interface{ Statistics() map[string]uint64 }); ok {
for k, v := range cache.Statistics() {
out["cache_"+k] = v
}
}
return out
}
// ConfigureQueryServing is startup-only. Optional accelerators are never
// created by a business read or mutation. Nil dependencies preserve old reads.
func (s *MongoStore) ConfigureQueryServing(cache queryserving.Cache, search *queryserving.Typesense) {
s.queryServing = &queryServingOptions{cache: cache, search: search}
}
func (s *MongoStore) SharedQueryCache() queryserving.Cache {
if s.queryReplicaReads != nil {
if s.queryReplicaReads.target == nil {
return nil
}
return s.queryReplicaReads.target.SharedQueryCache()
}
if s.queryServing == nil {
return nil
}
return s.queryServing.cache
}
type QueryModelHead struct {
verifiedUntil time.Time // Request-local proof; never serialized or persisted.
ChunkBytes int `bson:"chunkBytes,omitempty" json:"chunkBytes,omitempty"`
FactEpoch int64 `bson:"factEpoch,omitempty" json:"factEpoch,omitempty"`
SourceRevision int64 `bson:"sourceRevision" json:"sourceRevision"`
Revision int64 `bson:"revision" json:"revision"`
GlobalRevision int64 `bson:"globalRevision" json:"globalRevision"`
Generation string `bson:"generation" json:"generation"`
AsOf time.Time `bson:"asOf" json:"asOf"`
Chunks int `bson:"chunks" json:"chunks"`
Bytes int `bson:"bytes" json:"bytes"`
Digest string `bson:"digest" json:"digest"`
Collection string `bson:"searchCollection,omitempty" json:"searchCollection,omitempty"`
Rows int `bson:"rows" json:"rows"`
}
type queryModelLease struct {
Owner string
Epoch int64
}
func queryModelKey(kind, org string, projects []string) string {
return queryserving.Key(queryModelSchema, kind, normalizedOrganizationID(org), normalizedIDs(projects))
}
func normalizedIDs(ids []string) []string {
out := append([]string{}, ids...)
sort.Strings(out)
return out
}
func (s *MongoStore) queryControl() mongoCollection {
return s.queryCollection("queryReadModelControl")
}
func (s *MongoStore) queryData() mongoCollection { return s.queryCollection("queryReadModelData") }
func (s *MongoStore) acquireQueryModelLease(ctx context.Context) (queryModelLease, bool, error) {
now := time.Now().UTC()
owner := domain.NewID("query-builder")
var st struct {
Owner string `bson:"owner"`
Epoch int64 `bson:"epoch"`
}
err := s.queryControl().FindOneAndUpdate(ctx, bson.M{"_id": "global", "$or": bson.A{bson.M{"owner": ""}, bson.M{"until": bson.M{"$lte": now}}, bson.M{"owner": bson.M{"$exists": false}}}}, bson.M{"$set": bson.M{"owner": owner, "until": now.Add(60 * time.Second)}, "$inc": bson.M{"epoch": int64(1)}}, options.FindOneAndUpdate().SetUpsert(true).SetReturnDocument(options.After)).Decode(&st)
if mongo.IsDuplicateKeyError(err) {
return queryModelLease{}, false, nil
}
return queryModelLease{Owner: st.Owner, Epoch: st.Epoch}, err == nil, err
}
func queryLeaseFilter(l queryModelLease) bson.M {
return bson.M{"_id": "global", "owner": l.Owner, "epoch": l.Epoch, "until": bson.M{"$gt": time.Now().UTC()}}
}
func (s *MongoStore) renewQueryModelLease(ctx context.Context, l queryModelLease) error {
r, err := s.queryControl().UpdateOne(ctx, queryLeaseFilter(l), bson.M{"$set": bson.M{"until": time.Now().UTC().Add(60 * time.Second)}})
if err != nil {
return err
}
if r.MatchedCount != 1 {
return errors.New("query builder lease lost")
}
return nil
}
func (s *MongoStore) readQueryModelHead(ctx context.Context, key string, maxAge time.Duration) (QueryModelHead, bool) {
var st struct {
Heads map[string]QueryModelHead `bson:"heads"`
ObservedAt time.Time `bson:"observedAt"`
SourceRevision int64 `bson:"productionRevision"`
}
short, cancel := context.WithTimeout(ctx, 200*time.Millisecond)
defer cancel()
if s.queryControl().FindOne(short, bson.M{"_id": "global"}, options.FindOne().SetProjection(bson.M{"heads." + key: 1, "observedAt": 1, "productionRevision": 1})).Decode(&st) != nil {
return QueryModelHead{}, false
}
h, ok := st.Heads[key]
age := time.Since(h.AsOf)
verifiedUnchanged := st.SourceRevision == h.SourceRevision && time.Since(st.ObservedAt) >= 0 && time.Since(st.ObservedAt) <= 5*time.Second
if s.periodicQueryStore() {
maxAge = queryPeriodicDisplayMaxAge
}
return h, ok && h.Generation != "" && age >= 0 && (age <= maxAge || verifiedUnchanged)
}
func (s *MongoStore) publishQueryModelHead(ctx context.Context, l queryModelLease, key string, h QueryModelHead) error {
r, err := s.queryControl().UpdateOne(ctx, queryLeaseFilter(l), bson.M{"$set": bson.M{"heads." + key: h}})
if err != nil {
return err
}
if r.MatchedCount != 1 {
return errors.New("query publication fenced")
}
return nil
}
func queryModelBlobKey(key, generation string, index int) string {
return fmt.Sprintf("%s:%s:%09d", key, generation, index)
}
func (s *MongoStore) stageQueryModel(ctx context.Context, key string, value any, asOf time.Time) (QueryModelHead, error) {
var data bytes.Buffer
zw := gzip.NewWriter(&data)
budget := &queryModelBudgetWriter{writer: zw}
if err := json.NewEncoder(budget).Encode(value); err != nil {
return QueryModelHead{}, err
}
if err := zw.Close(); err != nil {
return QueryModelHead{}, err
}
if data.Len() > queryModelMaxBytes {
return QueryModelHead{}, errors.New("query materialization exceeds budget")
}
h := QueryModelHead{Generation: domain.NewID("query-generation"), AsOf: asOf, Bytes: data.Len(), ChunkBytes: queryModelBuildChunkBytes, Chunks: (data.Len() + queryModelBuildChunkBytes - 1) / queryModelBuildChunkBytes}
sum := sha256.Sum256(data.Bytes())
h.Digest = hex.EncodeToString(sum[:])
for i := 0; i < h.Chunks; i++ {
b := data.Bytes()[i*h.ChunkBytes : min((i+1)*h.ChunkBytes, data.Len())]
if err := s.paceQueryBuild(ctx, len(b)); err != nil {
return QueryModelHead{}, err
}
_, err := s.queryData().InsertOne(ctx, bson.M{"_id": queryModelBlobKey(key, h.Generation, i), "generation": h.Generation, "data": b, "createdAt": time.Now().UTC()})
if err != nil {
return QueryModelHead{}, err
}
if cache := s.SharedQueryCache(); cache != nil {
cache.Put(ctx, queryModelBlobKey(key, h.Generation, i), b, 15*time.Minute)
}
}
return h, nil
}
func (s *MongoStore) loadQueryModel(ctx context.Context, key string, h QueryModelHead, out any) error {
chunkBytes := h.ChunkBytes
if chunkBytes == 0 {
chunkBytes = queryModelChunkBytes
} // Existing 2MiB manifests remain readable.
if chunkBytes < 64<<10 || chunkBytes > queryModelChunkBytes || h.Chunks < 1 || h.Bytes < 1 || h.Bytes > queryModelMaxBytes || h.Chunks != (h.Bytes+chunkBytes-1)/chunkBytes {
return errors.New("invalid query manifest")
}
var data bytes.Buffer
for i := 0; i < h.Chunks; i++ {
blobKey := queryModelBlobKey(key, h.Generation, i)
var b []byte
cache := s.SharedQueryCache()
if cache != nil {
b, _ = cache.Get(ctx, blobKey)
}
if b == nil {
var doc struct {
Data []byte `bson:"data"`
}
if err := s.queryData().FindOne(ctx, bson.M{"_id": blobKey}).Decode(&doc); err != nil {
return err
}
b = doc.Data
if cache != nil {
cache.Put(ctx, blobKey, b, 15*time.Minute)
}
}
if len(b) > chunkBytes {
return errors.New("invalid query chunk")
}
data.Write(b)
}
sum := sha256.Sum256(data.Bytes())
if data.Len() != h.Bytes || hex.EncodeToString(sum[:]) != h.Digest {
return errors.New("query materialization corrupt")
}
zr, err := gzip.NewReader(bytes.NewReader(data.Bytes()))
if err != nil {
return err
}
defer zr.Close()
b, err := io.ReadAll(io.LimitReader(zr, 128<<20+1))
if err != nil || len(b) > 128<<20 {
return errors.New("query decode exceeds budget")
}
return json.Unmarshal(b, out)
}
// LoadProductionReport serves a derived, scope-qualified report if available.
// Legacy reads remain the compatibility/failure path. No refresh is triggered.
func (s *MongoStore) LoadProductionReport(ctx context.Context, org string, projects []string) (productionstats.Report, string, error) {
if replica, err := s.reportReplica(ctx); err != nil {
return productionstats.Report{}, "", err
} else if replica != nil {
return replica.LoadProductionReport(ctx, org, projects)
}
if s.queryServing != nil {
key := queryModelKey("production", org, projects)
if head, ok := s.readQueryModelHead(ctx, key, dashboardStatsCacheTTL); ok {
var report productionstats.Report
if err := s.loadQueryModel(ctx, key, head, &report); err == nil {
s.queryServing.stats.reportHits.Add(1)
return report, head.AsOf.Format(time.RFC3339Nano), nil
}
}
}
if s.periodicQueryStore() {
return productionstats.Report{}, "", ErrQueryReplicaNotReady
}
snapshot, err := s.LoadProductionStatsCached(ctx, org, projects)
if err != nil {
return productionstats.Report{}, "", err
}
return productionstats.Build(snapshot), snapshot.AsOf, nil
}
func (s *MongoStore) PreparedProductionResponse(ctx context.Context, org string, projects []string, query string) ([]byte, bool) {
if replica, err := s.reportReplica(ctx); err != nil {
return nil, false
} else if replica != nil {
return replica.PreparedProductionResponse(ctx, org, projects, query)
}
if s.queryServing == nil {
return nil, false
}
head, ok := s.readQueryModelHead(ctx, queryModelKey("production", org, projects), dashboardStatsCacheTTL)
if !ok || s.SharedQueryCache() == nil {
return nil, false
}
b, hit := s.SharedQueryCache().Get(ctx, queryserving.Key("production-response/v1", s.db.Name(), org, normalizedIDs(projects), head.Generation, query))
if hit {
s.queryServing.stats.responseHits.Add(1)
}
return b, hit
}
func (s *MongoStore) RememberProductionResponse(ctx context.Context, org string, projects []string, query, asOf string, value any) {
if replica, err := s.reportReplica(ctx); err != nil {
return
} else if replica != nil {
replica.RememberProductionResponse(ctx, org, projects, query, asOf, value)
return
}
if s.queryServing == nil || s.SharedQueryCache() == nil {
return
}
head, ok := s.readQueryModelHead(ctx, queryModelKey("production", org, projects), dashboardStatsCacheTTL)
if !ok || head.AsOf.Format(time.RFC3339Nano) != asOf {
return
}
queryserving.PutJSON(ctx, s.SharedQueryCache(), queryserving.Key("production-response/v1", s.db.Name(), org, normalizedIDs(projects), head.Generation, query), value, 5*time.Minute)
}
type queryModelBudgetWriter struct {
writer io.Writer
bytes int
}
func (w *queryModelBudgetWriter) Write(p []byte) (int, error) {
if len(p) > (128<<20)-w.bytes {
return 0, errors.New("query decode exceeds budget")
}
n, e := w.writer.Write(p)
w.bytes += n
return n, e
}