321 lines
12 KiB
Go
321 lines
12 KiB
Go
package store
|
|
|
|
import (
|
|
"bytes"
|
|
"compress/gzip"
|
|
"context"
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"sort"
|
|
"sync/atomic"
|
|
"time"
|
|
|
|
"droboticflow/internal/domain"
|
|
"droboticflow/internal/productionstats"
|
|
"droboticflow/internal/queryserving"
|
|
"go.mongodb.org/mongo-driver/bson"
|
|
"go.mongodb.org/mongo-driver/mongo"
|
|
"go.mongodb.org/mongo-driver/mongo/options"
|
|
)
|
|
|
|
const queryModelSchema = "query-model/v3"
|
|
const queryModelChunkBytes = 2 << 20
|
|
const queryModelBuildChunkBytes = 128 << 10
|
|
const queryModelMaxBytes = 64 << 20
|
|
|
|
type queryServingOptions struct {
|
|
cache queryserving.Cache
|
|
search *queryserving.Typesense
|
|
stats queryServingCounters
|
|
}
|
|
|
|
type queryServingCounters struct{ reportHits, responseHits, searchHits, typesenseReads, mongoSearchReads atomic.Uint64 }
|
|
|
|
// QueryServingStatistics exposes counters only, never names or query contents.
|
|
func (s *MongoStore) QueryServingStatistics() map[string]uint64 {
|
|
if s.queryReplicaReads != nil && s.queryReplicaReads.target != nil {
|
|
return s.queryReplicaReads.target.QueryServingStatistics()
|
|
}
|
|
if s.queryServing == nil {
|
|
return map[string]uint64{}
|
|
}
|
|
c := &s.queryServing.stats
|
|
out := map[string]uint64{"report_hits": c.reportHits.Load(), "response_hits": c.responseHits.Load(), "search_hits": c.searchHits.Load(), "typesense_reads": c.typesenseReads.Load(), "materialized_search_reads": c.mongoSearchReads.Load()}
|
|
if cache, ok := s.SharedQueryCache().(interface{ Statistics() map[string]uint64 }); ok {
|
|
for k, v := range cache.Statistics() {
|
|
out["cache_"+k] = v
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
// ConfigureQueryServing is startup-only. Optional accelerators are never
|
|
// created by a business read or mutation. Nil dependencies preserve old reads.
|
|
func (s *MongoStore) ConfigureQueryServing(cache queryserving.Cache, search *queryserving.Typesense) {
|
|
s.queryServing = &queryServingOptions{cache: cache, search: search}
|
|
}
|
|
func (s *MongoStore) SharedQueryCache() queryserving.Cache {
|
|
if s.queryReplicaReads != nil {
|
|
if s.queryReplicaReads.target == nil {
|
|
return nil
|
|
}
|
|
return s.queryReplicaReads.target.SharedQueryCache()
|
|
}
|
|
if s.queryServing == nil {
|
|
return nil
|
|
}
|
|
return s.queryServing.cache
|
|
}
|
|
|
|
type QueryModelHead struct {
|
|
verifiedUntil time.Time // Request-local proof; never serialized or persisted.
|
|
ChunkBytes int `bson:"chunkBytes,omitempty" json:"chunkBytes,omitempty"`
|
|
FactEpoch int64 `bson:"factEpoch,omitempty" json:"factEpoch,omitempty"`
|
|
SourceRevision int64 `bson:"sourceRevision" json:"sourceRevision"`
|
|
Revision int64 `bson:"revision" json:"revision"`
|
|
GlobalRevision int64 `bson:"globalRevision" json:"globalRevision"`
|
|
Generation string `bson:"generation" json:"generation"`
|
|
AsOf time.Time `bson:"asOf" json:"asOf"`
|
|
Chunks int `bson:"chunks" json:"chunks"`
|
|
Bytes int `bson:"bytes" json:"bytes"`
|
|
Digest string `bson:"digest" json:"digest"`
|
|
Collection string `bson:"searchCollection,omitempty" json:"searchCollection,omitempty"`
|
|
Rows int `bson:"rows" json:"rows"`
|
|
}
|
|
type queryModelLease struct {
|
|
Owner string
|
|
Epoch int64
|
|
}
|
|
|
|
func queryModelKey(kind, org string, projects []string) string {
|
|
return queryserving.Key(queryModelSchema, kind, normalizedOrganizationID(org), normalizedIDs(projects))
|
|
}
|
|
func normalizedIDs(ids []string) []string {
|
|
out := append([]string{}, ids...)
|
|
sort.Strings(out)
|
|
return out
|
|
}
|
|
func (s *MongoStore) queryControl() mongoCollection {
|
|
return s.queryCollection("queryReadModelControl")
|
|
}
|
|
func (s *MongoStore) queryData() mongoCollection { return s.queryCollection("queryReadModelData") }
|
|
func (s *MongoStore) acquireQueryModelLease(ctx context.Context) (queryModelLease, bool, error) {
|
|
now := time.Now().UTC()
|
|
owner := domain.NewID("query-builder")
|
|
var st struct {
|
|
Owner string `bson:"owner"`
|
|
Epoch int64 `bson:"epoch"`
|
|
}
|
|
err := s.queryControl().FindOneAndUpdate(ctx, bson.M{"_id": "global", "$or": bson.A{bson.M{"owner": ""}, bson.M{"until": bson.M{"$lte": now}}, bson.M{"owner": bson.M{"$exists": false}}}}, bson.M{"$set": bson.M{"owner": owner, "until": now.Add(60 * time.Second)}, "$inc": bson.M{"epoch": int64(1)}}, options.FindOneAndUpdate().SetUpsert(true).SetReturnDocument(options.After)).Decode(&st)
|
|
if mongo.IsDuplicateKeyError(err) {
|
|
return queryModelLease{}, false, nil
|
|
}
|
|
return queryModelLease{Owner: st.Owner, Epoch: st.Epoch}, err == nil, err
|
|
}
|
|
func queryLeaseFilter(l queryModelLease) bson.M {
|
|
return bson.M{"_id": "global", "owner": l.Owner, "epoch": l.Epoch, "until": bson.M{"$gt": time.Now().UTC()}}
|
|
}
|
|
func (s *MongoStore) renewQueryModelLease(ctx context.Context, l queryModelLease) error {
|
|
r, err := s.queryControl().UpdateOne(ctx, queryLeaseFilter(l), bson.M{"$set": bson.M{"until": time.Now().UTC().Add(60 * time.Second)}})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if r.MatchedCount != 1 {
|
|
return errors.New("query builder lease lost")
|
|
}
|
|
return nil
|
|
}
|
|
func (s *MongoStore) readQueryModelHead(ctx context.Context, key string, maxAge time.Duration) (QueryModelHead, bool) {
|
|
var st struct {
|
|
Heads map[string]QueryModelHead `bson:"heads"`
|
|
ObservedAt time.Time `bson:"observedAt"`
|
|
SourceRevision int64 `bson:"productionRevision"`
|
|
}
|
|
short, cancel := context.WithTimeout(ctx, 200*time.Millisecond)
|
|
defer cancel()
|
|
if s.queryControl().FindOne(short, bson.M{"_id": "global"}, options.FindOne().SetProjection(bson.M{"heads." + key: 1, "observedAt": 1, "productionRevision": 1})).Decode(&st) != nil {
|
|
return QueryModelHead{}, false
|
|
}
|
|
h, ok := st.Heads[key]
|
|
age := time.Since(h.AsOf)
|
|
verifiedUnchanged := st.SourceRevision == h.SourceRevision && time.Since(st.ObservedAt) >= 0 && time.Since(st.ObservedAt) <= 5*time.Second
|
|
if s.periodicQueryStore() {
|
|
maxAge = queryPeriodicDisplayMaxAge
|
|
}
|
|
return h, ok && h.Generation != "" && age >= 0 && (age <= maxAge || verifiedUnchanged)
|
|
}
|
|
func (s *MongoStore) publishQueryModelHead(ctx context.Context, l queryModelLease, key string, h QueryModelHead) error {
|
|
r, err := s.queryControl().UpdateOne(ctx, queryLeaseFilter(l), bson.M{"$set": bson.M{"heads." + key: h}})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if r.MatchedCount != 1 {
|
|
return errors.New("query publication fenced")
|
|
}
|
|
return nil
|
|
}
|
|
func queryModelBlobKey(key, generation string, index int) string {
|
|
return fmt.Sprintf("%s:%s:%09d", key, generation, index)
|
|
}
|
|
func (s *MongoStore) stageQueryModel(ctx context.Context, key string, value any, asOf time.Time) (QueryModelHead, error) {
|
|
var data bytes.Buffer
|
|
zw := gzip.NewWriter(&data)
|
|
budget := &queryModelBudgetWriter{writer: zw}
|
|
if err := json.NewEncoder(budget).Encode(value); err != nil {
|
|
return QueryModelHead{}, err
|
|
}
|
|
if err := zw.Close(); err != nil {
|
|
return QueryModelHead{}, err
|
|
}
|
|
if data.Len() > queryModelMaxBytes {
|
|
return QueryModelHead{}, errors.New("query materialization exceeds budget")
|
|
}
|
|
h := QueryModelHead{Generation: domain.NewID("query-generation"), AsOf: asOf, Bytes: data.Len(), ChunkBytes: queryModelBuildChunkBytes, Chunks: (data.Len() + queryModelBuildChunkBytes - 1) / queryModelBuildChunkBytes}
|
|
sum := sha256.Sum256(data.Bytes())
|
|
h.Digest = hex.EncodeToString(sum[:])
|
|
for i := 0; i < h.Chunks; i++ {
|
|
b := data.Bytes()[i*h.ChunkBytes : min((i+1)*h.ChunkBytes, data.Len())]
|
|
if err := s.paceQueryBuild(ctx, len(b)); err != nil {
|
|
return QueryModelHead{}, err
|
|
}
|
|
_, err := s.queryData().InsertOne(ctx, bson.M{"_id": queryModelBlobKey(key, h.Generation, i), "generation": h.Generation, "data": b, "createdAt": time.Now().UTC()})
|
|
if err != nil {
|
|
return QueryModelHead{}, err
|
|
}
|
|
if cache := s.SharedQueryCache(); cache != nil {
|
|
cache.Put(ctx, queryModelBlobKey(key, h.Generation, i), b, 15*time.Minute)
|
|
}
|
|
}
|
|
return h, nil
|
|
}
|
|
func (s *MongoStore) loadQueryModel(ctx context.Context, key string, h QueryModelHead, out any) error {
|
|
chunkBytes := h.ChunkBytes
|
|
if chunkBytes == 0 {
|
|
chunkBytes = queryModelChunkBytes
|
|
} // Existing 2MiB manifests remain readable.
|
|
if chunkBytes < 64<<10 || chunkBytes > queryModelChunkBytes || h.Chunks < 1 || h.Bytes < 1 || h.Bytes > queryModelMaxBytes || h.Chunks != (h.Bytes+chunkBytes-1)/chunkBytes {
|
|
return errors.New("invalid query manifest")
|
|
}
|
|
var data bytes.Buffer
|
|
for i := 0; i < h.Chunks; i++ {
|
|
blobKey := queryModelBlobKey(key, h.Generation, i)
|
|
var b []byte
|
|
cache := s.SharedQueryCache()
|
|
if cache != nil {
|
|
b, _ = cache.Get(ctx, blobKey)
|
|
}
|
|
if b == nil {
|
|
var doc struct {
|
|
Data []byte `bson:"data"`
|
|
}
|
|
if err := s.queryData().FindOne(ctx, bson.M{"_id": blobKey}).Decode(&doc); err != nil {
|
|
return err
|
|
}
|
|
b = doc.Data
|
|
if cache != nil {
|
|
cache.Put(ctx, blobKey, b, 15*time.Minute)
|
|
}
|
|
}
|
|
if len(b) > chunkBytes {
|
|
return errors.New("invalid query chunk")
|
|
}
|
|
data.Write(b)
|
|
}
|
|
sum := sha256.Sum256(data.Bytes())
|
|
if data.Len() != h.Bytes || hex.EncodeToString(sum[:]) != h.Digest {
|
|
return errors.New("query materialization corrupt")
|
|
}
|
|
zr, err := gzip.NewReader(bytes.NewReader(data.Bytes()))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer zr.Close()
|
|
b, err := io.ReadAll(io.LimitReader(zr, 128<<20+1))
|
|
if err != nil || len(b) > 128<<20 {
|
|
return errors.New("query decode exceeds budget")
|
|
}
|
|
return json.Unmarshal(b, out)
|
|
}
|
|
|
|
// LoadProductionReport serves a derived, scope-qualified report if available.
|
|
// Legacy reads remain the compatibility/failure path. No refresh is triggered.
|
|
func (s *MongoStore) LoadProductionReport(ctx context.Context, org string, projects []string) (productionstats.Report, string, error) {
|
|
if replica, err := s.reportReplica(ctx); err != nil {
|
|
return productionstats.Report{}, "", err
|
|
} else if replica != nil {
|
|
return replica.LoadProductionReport(ctx, org, projects)
|
|
}
|
|
if s.queryServing != nil {
|
|
key := queryModelKey("production", org, projects)
|
|
if head, ok := s.readQueryModelHead(ctx, key, dashboardStatsCacheTTL); ok {
|
|
var report productionstats.Report
|
|
if err := s.loadQueryModel(ctx, key, head, &report); err == nil {
|
|
s.queryServing.stats.reportHits.Add(1)
|
|
return report, head.AsOf.Format(time.RFC3339Nano), nil
|
|
}
|
|
}
|
|
}
|
|
if s.periodicQueryStore() {
|
|
return productionstats.Report{}, "", ErrQueryReplicaNotReady
|
|
}
|
|
snapshot, err := s.LoadProductionStatsCached(ctx, org, projects)
|
|
if err != nil {
|
|
return productionstats.Report{}, "", err
|
|
}
|
|
return productionstats.Build(snapshot), snapshot.AsOf, nil
|
|
}
|
|
|
|
func (s *MongoStore) PreparedProductionResponse(ctx context.Context, org string, projects []string, query string) ([]byte, bool) {
|
|
if replica, err := s.reportReplica(ctx); err != nil {
|
|
return nil, false
|
|
} else if replica != nil {
|
|
return replica.PreparedProductionResponse(ctx, org, projects, query)
|
|
}
|
|
if s.queryServing == nil {
|
|
return nil, false
|
|
}
|
|
head, ok := s.readQueryModelHead(ctx, queryModelKey("production", org, projects), dashboardStatsCacheTTL)
|
|
if !ok || s.SharedQueryCache() == nil {
|
|
return nil, false
|
|
}
|
|
b, hit := s.SharedQueryCache().Get(ctx, queryserving.Key("production-response/v1", s.db.Name(), org, normalizedIDs(projects), head.Generation, query))
|
|
if hit {
|
|
s.queryServing.stats.responseHits.Add(1)
|
|
}
|
|
return b, hit
|
|
}
|
|
func (s *MongoStore) RememberProductionResponse(ctx context.Context, org string, projects []string, query, asOf string, value any) {
|
|
if replica, err := s.reportReplica(ctx); err != nil {
|
|
return
|
|
} else if replica != nil {
|
|
replica.RememberProductionResponse(ctx, org, projects, query, asOf, value)
|
|
return
|
|
}
|
|
if s.queryServing == nil || s.SharedQueryCache() == nil {
|
|
return
|
|
}
|
|
head, ok := s.readQueryModelHead(ctx, queryModelKey("production", org, projects), dashboardStatsCacheTTL)
|
|
if !ok || head.AsOf.Format(time.RFC3339Nano) != asOf {
|
|
return
|
|
}
|
|
queryserving.PutJSON(ctx, s.SharedQueryCache(), queryserving.Key("production-response/v1", s.db.Name(), org, normalizedIDs(projects), head.Generation, query), value, 5*time.Minute)
|
|
}
|
|
|
|
type queryModelBudgetWriter struct {
|
|
writer io.Writer
|
|
bytes int
|
|
}
|
|
|
|
func (w *queryModelBudgetWriter) Write(p []byte) (int, error) {
|
|
if len(p) > (128<<20)-w.bytes {
|
|
return 0, errors.New("query decode exceeds budget")
|
|
}
|
|
n, e := w.writer.Write(p)
|
|
w.bytes += n
|
|
return n, e
|
|
}
|