forked from longtao.wu/droboticflow
497 lines
15 KiB
Go
497 lines
15 KiB
Go
package store
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"log"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"droboticflow/internal/outputsnapshot"
|
|
"droboticflow/internal/productionstats"
|
|
"go.mongodb.org/mongo-driver/bson"
|
|
"go.mongodb.org/mongo-driver/mongo"
|
|
"go.mongodb.org/mongo-driver/mongo/options"
|
|
)
|
|
|
|
var querySourceCollections = []string{"organizations", "projects", "runs", "captureRecords", "humanItems", "humanBatches", "annotationTasks", "annotationWorklogEvents", "humanEvents", "deliveryPackages", "annotationRevisions", "sliceRevisions", "annotationReviews", "users", "suppliers", "productionStatsScopes", "reviewOperations", "reworkBaselines", "reworkHandlings", "annotationIndexJobs"}
|
|
|
|
// QueryReadBuilder runs only in a dedicated, explicitly configured process.
|
|
// It uses the existing source loaders and publication protocol, never claims or
|
|
// mutates a business task. Restart starts a fresh stream BEFORE the full rebuild.
|
|
type QueryReadBuilder struct {
|
|
store *MongoStore
|
|
mu sync.Mutex
|
|
dirty map[string]bool
|
|
all bool
|
|
reader *outputsnapshot.Reader
|
|
searchDirty map[string]bool
|
|
allSearch bool
|
|
sourceDigests map[string]querySourceDigest
|
|
}
|
|
|
|
func NewQueryReadBuilder(s *MongoStore) *QueryReadBuilder {
|
|
r := outputsnapshot.NewStreamReader(s, s.StreamAnnotationOutput)
|
|
return &QueryReadBuilder{store: s, dirty: map[string]bool{}, all: true, reader: r, searchDirty: map[string]bool{}, allSearch: true, sourceDigests: map[string]querySourceDigest{}}
|
|
}
|
|
func (b *QueryReadBuilder) Close() { b.reader.Close() }
|
|
|
|
// WaitReady is for explicit prewarming and isolated acceptance, not HTTP reads.
|
|
func (b *QueryReadBuilder) WaitReady(ctx context.Context, org string, projects []string) error {
|
|
tick := time.NewTicker(100 * time.Millisecond)
|
|
defer tick.Stop()
|
|
for {
|
|
productionHead, production := b.store.readQueryModelHead(ctx, queryModelKey("production", org, projects), 5*time.Minute)
|
|
st, err := b.store.querySearchState(ctx)
|
|
production = production && productionHead.GlobalRevision == st.Global
|
|
h, exists := st.Heads[queryModelKey("search", org, nil)]
|
|
search := err == nil && exists && h.Generation != "" && h.Revision == st.Revisions[queryModelKey("search", org, nil)] && h.GlobalRevision == st.Global && time.Since(st.ObservedAt) >= 0 && time.Since(st.ObservedAt) < 5*time.Second
|
|
output := len(projects) == 0
|
|
if !output {
|
|
health, e := b.reader.Inspect(ctx, org, projects)
|
|
output = e == nil && health.Available
|
|
}
|
|
if production && search && output {
|
|
return nil
|
|
}
|
|
select {
|
|
case <-ctx.Done():
|
|
return ctx.Err()
|
|
case <-tick.C:
|
|
}
|
|
}
|
|
}
|
|
|
|
func (b *QueryReadBuilder) scopes(ctx context.Context) (map[string][]string, error) {
|
|
// Public ListOrganizations also bootstraps a default business organization;
|
|
// the read builder must enumerate without that write side effect.
|
|
cur, err := b.store.organizations.Find(ctx, bson.M{}, options.Find().SetProjection(bson.M{"id": 1, "_id": 0}))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer cur.Close(ctx)
|
|
var orgs []struct {
|
|
ID string `bson:"id"`
|
|
}
|
|
if err = cur.All(ctx, &orgs); err != nil {
|
|
return nil, err
|
|
}
|
|
out := map[string][]string{}
|
|
for _, org := range orgs {
|
|
scope, err := b.store.GetProductionStatsScope(ctx, org.ID)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
projects, err := b.store.ListProjectsByOrganization(ctx, org.ID)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
selected := map[string]bool{}
|
|
for _, id := range scope.ProjectIDs {
|
|
selected[id] = true
|
|
}
|
|
ids := []string{}
|
|
for _, p := range projects {
|
|
if selected[p.ID] {
|
|
ids = append(ids, p.ID)
|
|
}
|
|
}
|
|
out[org.ID] = ids
|
|
}
|
|
return out, nil
|
|
}
|
|
func (b *QueryReadBuilder) rebuild(ctx context.Context, l queryModelLease, org string, projects []string) error {
|
|
started := time.Now()
|
|
var source struct {
|
|
Revision int64 `bson:"productionRevision"`
|
|
Global int64 `bson:"searchGlobalRevision"`
|
|
}
|
|
if err := b.store.queryControl().FindOne(ctx, bson.M{"_id": "global"}, options.FindOne().SetProjection(bson.M{"productionRevision": 1, "searchGlobalRevision": 1})).Decode(&source); err != nil {
|
|
return err
|
|
}
|
|
buildCtx, finish := context.WithTimeout(ctx, 5*time.Minute)
|
|
defer finish()
|
|
snapshot, err := b.store.loadProductionStats(buildCtx, org, projects)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
asOf, err := time.Parse(time.RFC3339Nano, snapshot.AsOf)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
key := queryModelKey("production", org, projects)
|
|
head, err := b.store.stageQueryModel(ctx, key, productionstats.Build(snapshot), asOf)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
head.SourceRevision = source.Revision
|
|
head.GlobalRevision = source.Global
|
|
if err = b.store.publishQueryModelHead(ctx, l, key, head); err != nil {
|
|
return err
|
|
}
|
|
if len(projects) > 0 {
|
|
if _, err = b.reader.Prewarm(ctx, org, projects); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
log.Printf("query_model_build result=complete duration_ms=%d", time.Since(started).Milliseconds())
|
|
return nil
|
|
}
|
|
|
|
// RunOnce is for bounded initial prewarming and isolated tests, not a freshness
|
|
// substitute for Run. After the heartbeat expires search falls back to old reads.
|
|
func (b *QueryReadBuilder) RunOnce(parent context.Context) error {
|
|
ctx, cancel := context.WithTimeout(parent, 5*time.Minute)
|
|
defer cancel()
|
|
started := make(chan struct{})
|
|
done := make(chan error, 1)
|
|
go func() { done <- b.run(ctx, started); cancel() }()
|
|
select {
|
|
case <-started:
|
|
case err := <-done:
|
|
return err
|
|
case <-ctx.Done():
|
|
cancel()
|
|
<-done
|
|
return ctx.Err()
|
|
}
|
|
scopes, err := b.scopes(ctx)
|
|
if err == nil {
|
|
for org, ids := range scopes {
|
|
if err = b.WaitReady(ctx, org, ids); err != nil {
|
|
break
|
|
}
|
|
}
|
|
}
|
|
cancel()
|
|
runErr := <-done
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if runErr != nil && !errors.Is(runErr, context.Canceled) {
|
|
return runErr
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (b *QueryReadBuilder) withLease(ctx context.Context, l queryModelLease, run func(context.Context) error) error {
|
|
ctx, cancel := context.WithCancel(ctx)
|
|
defer cancel()
|
|
done := make(chan struct{})
|
|
go func() {
|
|
defer close(done)
|
|
tick := time.NewTicker(10 * time.Second)
|
|
defer tick.Stop()
|
|
for {
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case <-tick.C:
|
|
if b.store.renewQueryModelLease(ctx, l) != nil {
|
|
cancel()
|
|
return
|
|
}
|
|
}
|
|
}
|
|
}()
|
|
err := run(ctx)
|
|
cancel()
|
|
<-done
|
|
cleanup, stop := context.WithTimeout(context.Background(), time.Second)
|
|
defer stop()
|
|
_, _ = b.store.queryControl().UpdateOne(cleanup, queryLeaseFilter(l), bson.M{"$set": bson.M{"owner": "", "until": time.Now().UTC()}})
|
|
return err
|
|
}
|
|
|
|
func (b *QueryReadBuilder) Run(ctx context.Context) error { return b.run(ctx, nil) }
|
|
|
|
func (b *QueryReadBuilder) run(ctx context.Context, started chan<- struct{}) error {
|
|
b.mu.Lock()
|
|
b.all, b.allSearch = true, true
|
|
b.sourceDigests = map[string]querySourceDigest{}
|
|
b.mu.Unlock()
|
|
l, ok, err := b.store.acquireQueryModelLease(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if !ok {
|
|
return errors.New("query builder already active")
|
|
}
|
|
return b.withLease(ctx, l, func(ctx context.Context) error {
|
|
ctx, cancelWork := context.WithCancel(ctx)
|
|
defer cancelWork()
|
|
// A new observer has not covered changes during downtime. Old search
|
|
// generations must remain unavailable until a complete reconciliation.
|
|
result, e := b.store.queryControl().UpdateOne(ctx, queryLeaseFilter(l), bson.M{"$inc": bson.M{"searchGlobalRevision": int64(1), "productionRevision": int64(1)}, "$set": bson.M{"observedAt": time.Time{}}})
|
|
if e != nil {
|
|
return e
|
|
}
|
|
if result.MatchedCount != 1 {
|
|
return errors.New("query restart fenced")
|
|
}
|
|
// No fullDocument lookup: projecting only event identity avoids copying
|
|
// annotation bodies, credentials or full video payloads on each save.
|
|
pipeline := mongo.Pipeline{
|
|
{{Key: "$match", Value: bson.M{"$or": bson.A{
|
|
bson.M{"ns.coll": bson.M{"$in": querySourceCollections}},
|
|
bson.M{"ns.coll": "queryReadModelControl", "documentKey._id": "global", "operationType": "update", "updateDescription.updatedFields.watchBarrier": bson.M{"$exists": true}},
|
|
}}}},
|
|
{{Key: "$project", Value: bson.M{"_id": 1, "ns": 1, "documentKey": 1, "operationType": 1, "updateDescription.updatedFields.watchBarrier": 1}}},
|
|
}
|
|
stream, err := b.store.db.Watch(ctx, pipeline, options.ChangeStream().SetMaxAwaitTime(time.Second))
|
|
if err != nil {
|
|
return fmt.Errorf("query builder needs available Mongo change streams: %w", err)
|
|
}
|
|
defer func() {
|
|
closeCtx, closeCancel := context.WithTimeout(context.Background(), time.Second)
|
|
defer closeCancel()
|
|
_ = stream.Close(closeCtx)
|
|
}()
|
|
// Seed before consuming: events already captured by the open stream
|
|
// close the initial snapshot/change gap without retaining customer data.
|
|
seedCtx, seedCancel := context.WithTimeout(ctx, 2*time.Minute)
|
|
err = b.seedSourceDigests(seedCtx)
|
|
seedCancel()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
watchCtx, stop := context.WithCancel(ctx)
|
|
defer stop()
|
|
watchDone := make(chan error, 1)
|
|
watchExited := make(chan struct{})
|
|
var watchFailure error
|
|
failWatch := func(e error) {
|
|
if watchCtx.Err() == nil {
|
|
watchFailure = e
|
|
}
|
|
watchDone <- e
|
|
cancelWork()
|
|
}
|
|
go func() {
|
|
defer close(watchExited)
|
|
lastHeartbeat, lastBarrier, observed := time.Time{}, time.Time{}, time.Time{}
|
|
pendingChange := false
|
|
for watchCtx.Err() == nil {
|
|
// An empty filtered batch is not proof that the stream reached
|
|
// the oplog end. Publish a private marker, then advance proof
|
|
// only when that exact immutable update delta is observed.
|
|
if time.Since(lastBarrier) >= time.Second {
|
|
now := time.Now().UTC()
|
|
result, e := b.store.queryControl().UpdateOne(watchCtx, queryLeaseFilter(l), bson.M{"$set": bson.M{"watchBarrier": now}})
|
|
if e != nil {
|
|
failWatch(e)
|
|
return
|
|
}
|
|
if result.MatchedCount != 1 {
|
|
failWatch(errors.New("query barrier fenced"))
|
|
return
|
|
}
|
|
lastBarrier = now
|
|
}
|
|
if stream.TryNext(watchCtx) {
|
|
var event queryWatchEvent
|
|
if e := stream.Decode(&event); e != nil {
|
|
failWatch(e)
|
|
return
|
|
}
|
|
if marker, ok := event.barrierTime(); ok {
|
|
if marker.After(observed) {
|
|
observed = marker
|
|
}
|
|
} else if event.NS.Collection != "queryReadModelControl" {
|
|
pendingChange = true
|
|
if e := b.observeQuerySourceEvent(watchCtx, l, event.NS.Collection, event.Operation, event.Key); e != nil {
|
|
failWatch(e)
|
|
return
|
|
}
|
|
}
|
|
} else if e := stream.Err(); e != nil {
|
|
failWatch(e)
|
|
return
|
|
}
|
|
if observed.IsZero() || time.Since(lastHeartbeat) < time.Second {
|
|
continue
|
|
}
|
|
if e := b.recordObserved(watchCtx, l, observed, pendingChange); e != nil {
|
|
failWatch(e)
|
|
return
|
|
}
|
|
pendingChange = false
|
|
lastHeartbeat = time.Now()
|
|
}
|
|
watchDone <- watchCtx.Err()
|
|
}()
|
|
defer func() {
|
|
stop()
|
|
<-watchExited
|
|
if watchFailure != nil {
|
|
cleanup, finish := context.WithTimeout(context.Background(), time.Second)
|
|
defer finish()
|
|
_, _ = b.store.queryControl().UpdateOne(cleanup, queryLeaseFilter(l), bson.M{"$set": bson.M{"observedAt": time.Time{}}})
|
|
}
|
|
}()
|
|
if started != nil {
|
|
close(started)
|
|
}
|
|
tick := time.NewTicker(time.Second)
|
|
defer tick.Stop()
|
|
pendingStats, pendingSearch := map[string]bool{}, map[string]bool{}
|
|
lastStats := map[string]time.Time{}
|
|
searchRetries := map[string]queryBuildRetry{}
|
|
statsRetries := map[string]queryBuildRetry{}
|
|
var scopes map[string][]string
|
|
refreshScopes := true
|
|
lastScopeRead := time.Time{}
|
|
lastCleanup := time.Now()
|
|
for {
|
|
b.mu.Lock()
|
|
all, allSearch := b.all, b.allSearch
|
|
for org := range b.dirty {
|
|
pendingStats[org] = true
|
|
}
|
|
for org := range b.searchDirty {
|
|
pendingSearch[org] = true
|
|
}
|
|
b.dirty, b.searchDirty = map[string]bool{}, map[string]bool{}
|
|
b.all, b.allSearch = false, false
|
|
b.mu.Unlock()
|
|
if refreshScopes || all || time.Since(lastScopeRead) > 30*time.Second {
|
|
var e error
|
|
scopes, e = b.scopes(ctx)
|
|
if e != nil {
|
|
return e
|
|
}
|
|
if e = b.store.pruneQueryModelScopes(ctx, l, scopes); e != nil {
|
|
return e
|
|
}
|
|
refreshScopes = false
|
|
lastScopeRead = time.Now()
|
|
}
|
|
for org := range scopes {
|
|
if all {
|
|
pendingStats[org] = true
|
|
}
|
|
if allSearch {
|
|
pendingSearch[org] = true
|
|
}
|
|
}
|
|
for org := range pendingSearch {
|
|
if _, exists := scopes[org]; !exists {
|
|
delete(pendingSearch, org)
|
|
continue
|
|
}
|
|
if time.Now().Before(searchRetries[org].Next) {
|
|
continue
|
|
}
|
|
if e := b.store.buildQuerySearchModel(ctx, l, org); e == nil {
|
|
delete(searchRetries, org)
|
|
delete(pendingSearch, org)
|
|
} else {
|
|
retry := searchRetries[org]
|
|
retry.fail(5*time.Second, time.Minute)
|
|
searchRetries[org] = retry
|
|
log.Print("query_search_build result=retry category=source_or_publication")
|
|
}
|
|
}
|
|
for org := range pendingStats {
|
|
ids, exists := scopes[org]
|
|
if !exists {
|
|
delete(pendingStats, org)
|
|
continue
|
|
}
|
|
if time.Since(lastStats[org]) < 30*time.Second {
|
|
continue
|
|
}
|
|
if time.Now().Before(statsRetries[org].Next) {
|
|
continue
|
|
}
|
|
if e := b.rebuild(ctx, l, org, ids); e == nil {
|
|
delete(statsRetries, org)
|
|
delete(pendingStats, org)
|
|
lastStats[org] = time.Now()
|
|
} else {
|
|
retry := statsRetries[org]
|
|
retry.fail(30*time.Second, 5*time.Minute)
|
|
statsRetries[org] = retry
|
|
log.Print("query_model_build result=retry category=source_or_publication")
|
|
}
|
|
}
|
|
if time.Since(lastCleanup) > time.Minute {
|
|
cleanup, finish := context.WithTimeout(ctx, 5*time.Second)
|
|
_ = b.store.cleanQueryModelOrphans(cleanup)
|
|
finish()
|
|
lastCleanup = time.Now()
|
|
}
|
|
// Prewarm due generations even when no browser is connected.
|
|
for org, ids := range scopes {
|
|
if len(ids) > 0 {
|
|
_, _ = b.reader.Prewarm(ctx, org, ids)
|
|
}
|
|
}
|
|
select {
|
|
case <-ctx.Done():
|
|
return ctx.Err()
|
|
case e := <-watchDone:
|
|
return e
|
|
case <-tick.C:
|
|
}
|
|
}
|
|
})
|
|
}
|
|
|
|
func (s *MongoStore) markQuerySearchDirty(ctx context.Context, l queryModelLease, org string) error {
|
|
if org != "" {
|
|
_, err := s.queryControl().UpdateOne(ctx, queryLeaseFilter(l), bson.M{"$inc": bson.M{"searchRevisions." + queryModelKey("search", org, nil): int64(1)}})
|
|
return err
|
|
}
|
|
_, err := s.queryControl().UpdateOne(ctx, queryLeaseFilter(l), bson.M{"$inc": bson.M{"searchGlobalRevision": int64(1)}})
|
|
return err
|
|
}
|
|
|
|
// The builder's source collection list excludes all derived collections so its
|
|
// own publications cannot recursively schedule rebuilds.
|
|
func querySourceCollection(name string) bool {
|
|
for _, c := range querySourceCollections {
|
|
if strings.EqualFold(c, name) {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
type queryBuildRetry struct {
|
|
Next time.Time
|
|
Delay time.Duration
|
|
}
|
|
|
|
func (r *queryBuildRetry) fail(initial, maximum time.Duration) {
|
|
if r.Delay == 0 {
|
|
r.Delay = initial
|
|
} else {
|
|
r.Delay = min(r.Delay*2, maximum)
|
|
}
|
|
r.Next = time.Now().Add(r.Delay)
|
|
}
|
|
|
|
// The change-stream delta is the value written by that operation, unlike
|
|
// fullDocument updateLookup which can already contain a later marker.
|
|
type queryWatchEvent struct {
|
|
NS struct {
|
|
Collection string `bson:"coll"`
|
|
} `bson:"ns"`
|
|
Operation string `bson:"operationType"`
|
|
Key bson.M `bson:"documentKey"`
|
|
Update struct {
|
|
Fields struct {
|
|
Barrier time.Time `bson:"watchBarrier"`
|
|
} `bson:"updatedFields"`
|
|
} `bson:"updateDescription"`
|
|
}
|
|
|
|
func (e queryWatchEvent) barrierTime() (time.Time, bool) {
|
|
return e.Update.Fields.Barrier, e.NS.Collection == "queryReadModelControl" && e.Operation == "update" && e.Key["_id"] == "global" && !e.Update.Fields.Barrier.IsZero()
|
|
}
|