droboticflow/internal/store/query_read_builder.go

497 lines
15 KiB
Go

package store
import (
"context"
"errors"
"fmt"
"log"
"strings"
"sync"
"time"
"droboticflow/internal/outputsnapshot"
"droboticflow/internal/productionstats"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
)
var querySourceCollections = []string{"organizations", "projects", "runs", "captureRecords", "humanItems", "humanBatches", "annotationTasks", "annotationWorklogEvents", "humanEvents", "deliveryPackages", "annotationRevisions", "sliceRevisions", "annotationReviews", "users", "suppliers", "productionStatsScopes", "reviewOperations", "reworkBaselines", "reworkHandlings", "annotationIndexJobs"}
// QueryReadBuilder runs only in a dedicated, explicitly configured process.
// It uses the existing source loaders and publication protocol, never claims or
// mutates a business task. Restart starts a fresh stream BEFORE the full rebuild.
type QueryReadBuilder struct {
store *MongoStore
mu sync.Mutex
dirty map[string]bool
all bool
reader *outputsnapshot.Reader
searchDirty map[string]bool
allSearch bool
sourceDigests map[string]querySourceDigest
}
func NewQueryReadBuilder(s *MongoStore) *QueryReadBuilder {
r := outputsnapshot.NewStreamReader(s, s.StreamAnnotationOutput)
return &QueryReadBuilder{store: s, dirty: map[string]bool{}, all: true, reader: r, searchDirty: map[string]bool{}, allSearch: true, sourceDigests: map[string]querySourceDigest{}}
}
func (b *QueryReadBuilder) Close() { b.reader.Close() }
// WaitReady is for explicit prewarming and isolated acceptance, not HTTP reads.
func (b *QueryReadBuilder) WaitReady(ctx context.Context, org string, projects []string) error {
tick := time.NewTicker(100 * time.Millisecond)
defer tick.Stop()
for {
productionHead, production := b.store.readQueryModelHead(ctx, queryModelKey("production", org, projects), 5*time.Minute)
st, err := b.store.querySearchState(ctx)
production = production && productionHead.GlobalRevision == st.Global
h, exists := st.Heads[queryModelKey("search", org, nil)]
search := err == nil && exists && h.Generation != "" && h.Revision == st.Revisions[queryModelKey("search", org, nil)] && h.GlobalRevision == st.Global && time.Since(st.ObservedAt) >= 0 && time.Since(st.ObservedAt) < 5*time.Second
output := len(projects) == 0
if !output {
health, e := b.reader.Inspect(ctx, org, projects)
output = e == nil && health.Available
}
if production && search && output {
return nil
}
select {
case <-ctx.Done():
return ctx.Err()
case <-tick.C:
}
}
}
func (b *QueryReadBuilder) scopes(ctx context.Context) (map[string][]string, error) {
// Public ListOrganizations also bootstraps a default business organization;
// the read builder must enumerate without that write side effect.
cur, err := b.store.organizations.Find(ctx, bson.M{}, options.Find().SetProjection(bson.M{"id": 1, "_id": 0}))
if err != nil {
return nil, err
}
defer cur.Close(ctx)
var orgs []struct {
ID string `bson:"id"`
}
if err = cur.All(ctx, &orgs); err != nil {
return nil, err
}
out := map[string][]string{}
for _, org := range orgs {
scope, err := b.store.GetProductionStatsScope(ctx, org.ID)
if err != nil {
return nil, err
}
projects, err := b.store.ListProjectsByOrganization(ctx, org.ID)
if err != nil {
return nil, err
}
selected := map[string]bool{}
for _, id := range scope.ProjectIDs {
selected[id] = true
}
ids := []string{}
for _, p := range projects {
if selected[p.ID] {
ids = append(ids, p.ID)
}
}
out[org.ID] = ids
}
return out, nil
}
func (b *QueryReadBuilder) rebuild(ctx context.Context, l queryModelLease, org string, projects []string) error {
started := time.Now()
var source struct {
Revision int64 `bson:"productionRevision"`
Global int64 `bson:"searchGlobalRevision"`
}
if err := b.store.queryControl().FindOne(ctx, bson.M{"_id": "global"}, options.FindOne().SetProjection(bson.M{"productionRevision": 1, "searchGlobalRevision": 1})).Decode(&source); err != nil {
return err
}
buildCtx, finish := context.WithTimeout(ctx, 5*time.Minute)
defer finish()
snapshot, err := b.store.loadProductionStats(buildCtx, org, projects)
if err != nil {
return err
}
asOf, err := time.Parse(time.RFC3339Nano, snapshot.AsOf)
if err != nil {
return err
}
key := queryModelKey("production", org, projects)
head, err := b.store.stageQueryModel(ctx, key, productionstats.Build(snapshot), asOf)
if err != nil {
return err
}
head.SourceRevision = source.Revision
head.GlobalRevision = source.Global
if err = b.store.publishQueryModelHead(ctx, l, key, head); err != nil {
return err
}
if len(projects) > 0 {
if _, err = b.reader.Prewarm(ctx, org, projects); err != nil {
return err
}
}
log.Printf("query_model_build result=complete duration_ms=%d", time.Since(started).Milliseconds())
return nil
}
// RunOnce is for bounded initial prewarming and isolated tests, not a freshness
// substitute for Run. After the heartbeat expires search falls back to old reads.
func (b *QueryReadBuilder) RunOnce(parent context.Context) error {
ctx, cancel := context.WithTimeout(parent, 5*time.Minute)
defer cancel()
started := make(chan struct{})
done := make(chan error, 1)
go func() { done <- b.run(ctx, started); cancel() }()
select {
case <-started:
case err := <-done:
return err
case <-ctx.Done():
cancel()
<-done
return ctx.Err()
}
scopes, err := b.scopes(ctx)
if err == nil {
for org, ids := range scopes {
if err = b.WaitReady(ctx, org, ids); err != nil {
break
}
}
}
cancel()
runErr := <-done
if err != nil {
return err
}
if runErr != nil && !errors.Is(runErr, context.Canceled) {
return runErr
}
return nil
}
func (b *QueryReadBuilder) withLease(ctx context.Context, l queryModelLease, run func(context.Context) error) error {
ctx, cancel := context.WithCancel(ctx)
defer cancel()
done := make(chan struct{})
go func() {
defer close(done)
tick := time.NewTicker(10 * time.Second)
defer tick.Stop()
for {
select {
case <-ctx.Done():
return
case <-tick.C:
if b.store.renewQueryModelLease(ctx, l) != nil {
cancel()
return
}
}
}
}()
err := run(ctx)
cancel()
<-done
cleanup, stop := context.WithTimeout(context.Background(), time.Second)
defer stop()
_, _ = b.store.queryControl().UpdateOne(cleanup, queryLeaseFilter(l), bson.M{"$set": bson.M{"owner": "", "until": time.Now().UTC()}})
return err
}
func (b *QueryReadBuilder) Run(ctx context.Context) error { return b.run(ctx, nil) }
func (b *QueryReadBuilder) run(ctx context.Context, started chan<- struct{}) error {
b.mu.Lock()
b.all, b.allSearch = true, true
b.sourceDigests = map[string]querySourceDigest{}
b.mu.Unlock()
l, ok, err := b.store.acquireQueryModelLease(ctx)
if err != nil {
return err
}
if !ok {
return errors.New("query builder already active")
}
return b.withLease(ctx, l, func(ctx context.Context) error {
ctx, cancelWork := context.WithCancel(ctx)
defer cancelWork()
// A new observer has not covered changes during downtime. Old search
// generations must remain unavailable until a complete reconciliation.
result, e := b.store.queryControl().UpdateOne(ctx, queryLeaseFilter(l), bson.M{"$inc": bson.M{"searchGlobalRevision": int64(1), "productionRevision": int64(1)}, "$set": bson.M{"observedAt": time.Time{}}})
if e != nil {
return e
}
if result.MatchedCount != 1 {
return errors.New("query restart fenced")
}
// No fullDocument lookup: projecting only event identity avoids copying
// annotation bodies, credentials or full video payloads on each save.
pipeline := mongo.Pipeline{
{{Key: "$match", Value: bson.M{"$or": bson.A{
bson.M{"ns.coll": bson.M{"$in": querySourceCollections}},
bson.M{"ns.coll": "queryReadModelControl", "documentKey._id": "global", "operationType": "update", "updateDescription.updatedFields.watchBarrier": bson.M{"$exists": true}},
}}}},
{{Key: "$project", Value: bson.M{"_id": 1, "ns": 1, "documentKey": 1, "operationType": 1, "updateDescription.updatedFields.watchBarrier": 1}}},
}
stream, err := b.store.db.Watch(ctx, pipeline, options.ChangeStream().SetMaxAwaitTime(time.Second))
if err != nil {
return fmt.Errorf("query builder needs available Mongo change streams: %w", err)
}
defer func() {
closeCtx, closeCancel := context.WithTimeout(context.Background(), time.Second)
defer closeCancel()
_ = stream.Close(closeCtx)
}()
// Seed before consuming: events already captured by the open stream
// close the initial snapshot/change gap without retaining customer data.
seedCtx, seedCancel := context.WithTimeout(ctx, 2*time.Minute)
err = b.seedSourceDigests(seedCtx)
seedCancel()
if err != nil {
return err
}
watchCtx, stop := context.WithCancel(ctx)
defer stop()
watchDone := make(chan error, 1)
watchExited := make(chan struct{})
var watchFailure error
failWatch := func(e error) {
if watchCtx.Err() == nil {
watchFailure = e
}
watchDone <- e
cancelWork()
}
go func() {
defer close(watchExited)
lastHeartbeat, lastBarrier, observed := time.Time{}, time.Time{}, time.Time{}
pendingChange := false
for watchCtx.Err() == nil {
// An empty filtered batch is not proof that the stream reached
// the oplog end. Publish a private marker, then advance proof
// only when that exact immutable update delta is observed.
if time.Since(lastBarrier) >= time.Second {
now := time.Now().UTC()
result, e := b.store.queryControl().UpdateOne(watchCtx, queryLeaseFilter(l), bson.M{"$set": bson.M{"watchBarrier": now}})
if e != nil {
failWatch(e)
return
}
if result.MatchedCount != 1 {
failWatch(errors.New("query barrier fenced"))
return
}
lastBarrier = now
}
if stream.TryNext(watchCtx) {
var event queryWatchEvent
if e := stream.Decode(&event); e != nil {
failWatch(e)
return
}
if marker, ok := event.barrierTime(); ok {
if marker.After(observed) {
observed = marker
}
} else if event.NS.Collection != "queryReadModelControl" {
pendingChange = true
if e := b.observeQuerySourceEvent(watchCtx, l, event.NS.Collection, event.Operation, event.Key); e != nil {
failWatch(e)
return
}
}
} else if e := stream.Err(); e != nil {
failWatch(e)
return
}
if observed.IsZero() || time.Since(lastHeartbeat) < time.Second {
continue
}
if e := b.recordObserved(watchCtx, l, observed, pendingChange); e != nil {
failWatch(e)
return
}
pendingChange = false
lastHeartbeat = time.Now()
}
watchDone <- watchCtx.Err()
}()
defer func() {
stop()
<-watchExited
if watchFailure != nil {
cleanup, finish := context.WithTimeout(context.Background(), time.Second)
defer finish()
_, _ = b.store.queryControl().UpdateOne(cleanup, queryLeaseFilter(l), bson.M{"$set": bson.M{"observedAt": time.Time{}}})
}
}()
if started != nil {
close(started)
}
tick := time.NewTicker(time.Second)
defer tick.Stop()
pendingStats, pendingSearch := map[string]bool{}, map[string]bool{}
lastStats := map[string]time.Time{}
searchRetries := map[string]queryBuildRetry{}
statsRetries := map[string]queryBuildRetry{}
var scopes map[string][]string
refreshScopes := true
lastScopeRead := time.Time{}
lastCleanup := time.Now()
for {
b.mu.Lock()
all, allSearch := b.all, b.allSearch
for org := range b.dirty {
pendingStats[org] = true
}
for org := range b.searchDirty {
pendingSearch[org] = true
}
b.dirty, b.searchDirty = map[string]bool{}, map[string]bool{}
b.all, b.allSearch = false, false
b.mu.Unlock()
if refreshScopes || all || time.Since(lastScopeRead) > 30*time.Second {
var e error
scopes, e = b.scopes(ctx)
if e != nil {
return e
}
if e = b.store.pruneQueryModelScopes(ctx, l, scopes); e != nil {
return e
}
refreshScopes = false
lastScopeRead = time.Now()
}
for org := range scopes {
if all {
pendingStats[org] = true
}
if allSearch {
pendingSearch[org] = true
}
}
for org := range pendingSearch {
if _, exists := scopes[org]; !exists {
delete(pendingSearch, org)
continue
}
if time.Now().Before(searchRetries[org].Next) {
continue
}
if e := b.store.buildQuerySearchModel(ctx, l, org); e == nil {
delete(searchRetries, org)
delete(pendingSearch, org)
} else {
retry := searchRetries[org]
retry.fail(5*time.Second, time.Minute)
searchRetries[org] = retry
log.Print("query_search_build result=retry category=source_or_publication")
}
}
for org := range pendingStats {
ids, exists := scopes[org]
if !exists {
delete(pendingStats, org)
continue
}
if time.Since(lastStats[org]) < 30*time.Second {
continue
}
if time.Now().Before(statsRetries[org].Next) {
continue
}
if e := b.rebuild(ctx, l, org, ids); e == nil {
delete(statsRetries, org)
delete(pendingStats, org)
lastStats[org] = time.Now()
} else {
retry := statsRetries[org]
retry.fail(30*time.Second, 5*time.Minute)
statsRetries[org] = retry
log.Print("query_model_build result=retry category=source_or_publication")
}
}
if time.Since(lastCleanup) > time.Minute {
cleanup, finish := context.WithTimeout(ctx, 5*time.Second)
_ = b.store.cleanQueryModelOrphans(cleanup)
finish()
lastCleanup = time.Now()
}
// Prewarm due generations even when no browser is connected.
for org, ids := range scopes {
if len(ids) > 0 {
_, _ = b.reader.Prewarm(ctx, org, ids)
}
}
select {
case <-ctx.Done():
return ctx.Err()
case e := <-watchDone:
return e
case <-tick.C:
}
}
})
}
func (s *MongoStore) markQuerySearchDirty(ctx context.Context, l queryModelLease, org string) error {
if org != "" {
_, err := s.queryControl().UpdateOne(ctx, queryLeaseFilter(l), bson.M{"$inc": bson.M{"searchRevisions." + queryModelKey("search", org, nil): int64(1)}})
return err
}
_, err := s.queryControl().UpdateOne(ctx, queryLeaseFilter(l), bson.M{"$inc": bson.M{"searchGlobalRevision": int64(1)}})
return err
}
// The builder's source collection list excludes all derived collections so its
// own publications cannot recursively schedule rebuilds.
func querySourceCollection(name string) bool {
for _, c := range querySourceCollections {
if strings.EqualFold(c, name) {
return true
}
}
return false
}
type queryBuildRetry struct {
Next time.Time
Delay time.Duration
}
func (r *queryBuildRetry) fail(initial, maximum time.Duration) {
if r.Delay == 0 {
r.Delay = initial
} else {
r.Delay = min(r.Delay*2, maximum)
}
r.Next = time.Now().Add(r.Delay)
}
// The change-stream delta is the value written by that operation, unlike
// fullDocument updateLookup which can already contain a later marker.
type queryWatchEvent struct {
NS struct {
Collection string `bson:"coll"`
} `bson:"ns"`
Operation string `bson:"operationType"`
Key bson.M `bson:"documentKey"`
Update struct {
Fields struct {
Barrier time.Time `bson:"watchBarrier"`
} `bson:"updatedFields"`
} `bson:"updateDescription"`
}
func (e queryWatchEvent) barrierTime() (time.Time, bool) {
return e.Update.Fields.Barrier, e.NS.Collection == "queryReadModelControl" && e.Operation == "update" && e.Key["_id"] == "global" && !e.Update.Fields.Barrier.IsZero()
}