forked from longtao.wu/droboticflow
298 lines
10 KiB
Go
298 lines
10 KiB
Go
package store
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"log"
|
|
"reflect"
|
|
"sort"
|
|
"time"
|
|
|
|
"droboticflow/internal/queryserving"
|
|
"go.mongodb.org/mongo-driver/bson"
|
|
"go.mongodb.org/mongo-driver/mongo"
|
|
"go.mongodb.org/mongo-driver/mongo/options"
|
|
)
|
|
|
|
// This lifecycle is exclusive to the independent, periodic read database.
|
|
// Authority/legacy callers keep their existing loader and publication behavior.
|
|
func (s *MongoStore) queryBuildStart(ctx context.Context, l queryModelLease, org string, projects []string) (queryBuildJob, error) {
|
|
org = normalizedOrganizationID(org)
|
|
projects = append([]string{}, projects...)
|
|
sort.Strings(projects)
|
|
old, exists, err := s.queryBuildJob(ctx, org)
|
|
if err != nil {
|
|
return old, err
|
|
}
|
|
recoverSnapshot := exists && recoverableQueryBuildSnapshot(old)
|
|
if exists && old.Phase != "complete" {
|
|
if !reflect.DeepEqual(old.Projects, projects) {
|
|
return old, errors.New("query build scope changed; retained job requires reconciliation")
|
|
}
|
|
if !recoverSnapshot {
|
|
if old.FailureClass != "" {
|
|
return old, fmt.Errorf("%w: phase=%s category=%s", ErrQueryBuildStopped, old.FailurePhase, old.FailureClass)
|
|
}
|
|
return old, nil
|
|
}
|
|
}
|
|
|
|
observedAt := time.Now().UTC()
|
|
at, err := s.queryBuildSnapshot(ctx)
|
|
if err != nil {
|
|
return old, err
|
|
}
|
|
var state struct {
|
|
Revision int64 `bson:"productionRevision"`
|
|
Global int64 `bson:"searchGlobalRevision"`
|
|
Observed time.Time `bson:"observedAt"`
|
|
Heads map[string]QueryModelHead `bson:"heads"`
|
|
}
|
|
rows, e := s.queryBuildReadPage(ctx, s.queryControl(), bson.M{"_id": "global"}, bson.M{"productionRevision": 1, "searchGlobalRevision": 1, "heads": 1, "observedAt": 1}, bson.RawValue{}, false, at)
|
|
if e != nil {
|
|
return old, e
|
|
}
|
|
if len(rows) != 1 {
|
|
return old, errors.New("query build control unavailable at snapshot")
|
|
}
|
|
if e = bson.Unmarshal(rows[0], &state); e != nil {
|
|
return old, e
|
|
}
|
|
head := state.Heads[workforceFactKey(org)]
|
|
j := queryBuildJob{ID: queryBuildJobKey(s.db.Name(), org), Schema: queryBuildJobSchema, Organization: org, Projects: projects, Generation: queryserving.Key(queryBuildJobSchema, org, at.T, at.I, time.Now().UnixNano()), Snapshot: at, AsOf: observedAt, Phase: "production_plan", OldEpoch: head.FactEpoch, Epoch: head.FactEpoch + 1, Cold: head.FactEpoch == 0, SourceRevision: state.Revision, GlobalRevision: state.Global, ProgressAt: time.Now().UTC()}
|
|
j.Retired = append([]queryBuildRetiredGeneration{}, old.Retired...)
|
|
j.InputGeneration = j.Generation
|
|
if recoverSnapshot {
|
|
j.SnapshotRecoveries = append(append([]queryBuildSnapshotRecovery{}, old.SnapshotRecoveries...), queryBuildSnapshotRecovery{Generation: old.Generation, Snapshot: old.Snapshot, AsOf: old.AsOf, Phase: old.Phase, Code: old.FailureCode, RecoveredAt: time.Now().UTC()})
|
|
}
|
|
|
|
if exists && old.Phase == "complete" && reflect.DeepEqual(old.Projects, projects) && old.SourceRevision == state.Revision && old.GlobalRevision == state.Global && !state.Observed.Before(old.AsOf) && time.Since(state.Observed) >= 0 && time.Since(state.Observed) < 15*time.Second {
|
|
j.AsOf = state.Observed
|
|
j.InputGeneration = old.InputGeneration
|
|
if j.InputGeneration == "" {
|
|
j.InputGeneration = old.Generation
|
|
}
|
|
// The caught-up stream proves that every business input is unchanged.
|
|
// Reconfirm the complete cohort without re-materializing or walking
|
|
// 140k unchanged facts and checkpointing thousands of no-op batches.
|
|
production, exists := state.Heads[queryModelKey("production", org, projects)]
|
|
committed := old.Epoch
|
|
if !old.Changed && old.OldEpoch > 0 {
|
|
committed = old.OldEpoch
|
|
}
|
|
if exists && production.Generation != "" && head.Rows == old.PlanRows && head.FactEpoch == committed && production.SourceRevision == old.SourceRevision {
|
|
j.Phase = "publish"
|
|
j.PlanRows = old.PlanRows
|
|
j.ReusedRows = int64(old.PlanRows)
|
|
j.Production = production
|
|
j.Production.AsOf = j.AsOf
|
|
}
|
|
}
|
|
err = s.queryBuildTransaction(ctx, l, func(sc mongo.SessionContext) error { return s.saveQueryBuildJob(sc, j) })
|
|
return j, err
|
|
}
|
|
|
|
func (s *MongoStore) queryBuildRunRound(ctx context.Context, l queryModelLease, org string, projects []string, r *queryBuildRound) (err error) {
|
|
j, err := s.queryBuildStart(ctx, l, org, projects)
|
|
if err != nil {
|
|
if ctx.Err() != nil {
|
|
return ctx.Err()
|
|
}
|
|
d := QueryReplicaDiagnostic(err)
|
|
log.Printf("query_model_build result=stopped stage=resume category=%s mongoCode=%d", d.Class, d.MongoCode)
|
|
return fmt.Errorf("%w: resume: %w", ErrQueryBuildStopped, err)
|
|
}
|
|
if r == nil {
|
|
r = newQueryBuildRound(j.Cold)
|
|
}
|
|
started := time.Now()
|
|
before := j.ProgressAt
|
|
defer func() {
|
|
if err == nil || ctx.Err() != nil {
|
|
return
|
|
}
|
|
// Maintenance/shutdown time is not counted as active lack of progress.
|
|
current, ok, e := s.queryBuildJob(ctx, org)
|
|
if e != nil || !ok {
|
|
return
|
|
}
|
|
if !current.ProgressAt.After(before) {
|
|
current.ActiveWithoutProgress += time.Since(started)
|
|
} else {
|
|
current.ActiveWithoutProgress = 0
|
|
}
|
|
if current.ActiveWithoutProgress >= queryBuildNoProgressLimit && (errors.Is(err, ErrQueryBuildYielded) || isTransientQueryBuildFailureClass(QueryReplicaDiagnostic(err).Class)) {
|
|
err = ErrQueryBuildNoProgress
|
|
}
|
|
if !errors.Is(err, ErrQueryBuildYielded) {
|
|
current.FailurePhase = current.Phase
|
|
diagnostic := QueryReplicaDiagnostic(err)
|
|
current.FailureClass = diagnostic.Class
|
|
current.FailureCode = diagnostic.MongoCode
|
|
if errors.Is(err, ErrQueryBuildNoProgress) {
|
|
current.FailureClass = "no_durable_progress"
|
|
}
|
|
}
|
|
if !errors.Is(err, ErrQueryBuildYielded) {
|
|
err = fmt.Errorf("%w: %w", ErrQueryBuildStopped, err)
|
|
}
|
|
if e = s.queryBuildTransaction(ctx, l, func(sc mongo.SessionContext) error { return s.saveQueryBuildJob(sc, current) }); e != nil {
|
|
d := QueryReplicaDiagnostic(e)
|
|
log.Printf("query_model_build result=stopped stage=checkpoint category=%s mongoCode=%d", d.Class, d.MongoCode)
|
|
err = fmt.Errorf("%w: query build could not persist outcome: %w", ErrQueryBuildStopped, e)
|
|
}
|
|
if e == nil && recoverableQueryBuildSnapshot(current) {
|
|
log.Print("query_model_build result=yielded stage=input_snapshot category=expired checkpoint=retained")
|
|
err = fmt.Errorf("%w: input snapshot expired; replacement generation scheduled", ErrQueryBuildYielded)
|
|
}
|
|
|
|
}()
|
|
for {
|
|
if err = r.check(ctx); err != nil {
|
|
return err
|
|
}
|
|
switch j.Phase {
|
|
case "production_plan":
|
|
err = s.queryBuildProductionPlan(ctx, l, &j, r)
|
|
case "workforce_plan":
|
|
err = s.queryBuildWorkforcePlan(ctx, l, &j, r)
|
|
case "apply":
|
|
err = s.queryBuildApply(ctx, l, &j, r)
|
|
case "cleanup":
|
|
err = s.queryBuildCleanup(ctx, l, &j, r)
|
|
case "publish":
|
|
err = s.queryBuildPublish(ctx, l, &j)
|
|
case "complete":
|
|
return nil
|
|
default:
|
|
return errors.New("unsupported query build phase")
|
|
}
|
|
if err != nil {
|
|
return queryBuildPhaseFailure(j.Phase, err)
|
|
}
|
|
}
|
|
}
|
|
|
|
// Explicitly bounded derived-data housekeeping; active jobs are never swept.
|
|
func (s *MongoStore) cleanQueryBuildParts(ctx context.Context, l queryModelLease) error {
|
|
var jobs []queryBuildJob
|
|
cur, err := s.db.Collection(queryBuildJobsCollection).Find(ctx, bson.M{"schema": queryBuildJobSchema}, options.Find().SetProjection(bson.M{"generation": 1, "inputGeneration": 1, "phase": 1, "snapshotRecoveries": 1}))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
err = cur.All(ctx, &jobs)
|
|
cur.Close(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
keep := []string{}
|
|
for _, j := range jobs {
|
|
keep = append(keep, j.Generation, j.InputGeneration)
|
|
if j.Phase != "complete" {
|
|
for _, v := range j.SnapshotRecoveries {
|
|
keep = append(keep, v.Generation)
|
|
}
|
|
}
|
|
}
|
|
for _, collection := range []string{queryBuildPartsCollection, queryBuildJobsCollection} {
|
|
f := bson.M{"job": bson.M{"$exists": true, "$nin": keep}}
|
|
cur, e := s.db.Collection(collection).Find(ctx, f, options.Find().SetProjection(bson.M{"_id": 1}).SetLimit(1000))
|
|
if e != nil {
|
|
return e
|
|
}
|
|
var rows []bson.M
|
|
e = cur.All(ctx, &rows)
|
|
cur.Close(ctx)
|
|
if e != nil {
|
|
return e
|
|
}
|
|
ids := bson.A{}
|
|
for _, row := range rows {
|
|
ids = append(ids, row["_id"])
|
|
}
|
|
if len(ids) > 0 {
|
|
if e = s.queryBuildTransaction(ctx, l, func(sc mongo.SessionContext) error {
|
|
_, e := s.db.Collection(collection).DeleteMany(sc, bson.M{"_id": bson.M{"$in": ids}, "job": bson.M{"$nin": keep}})
|
|
return e
|
|
}); e != nil {
|
|
return e
|
|
}
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
var transientQueryBuildFailureClasses = []string{
|
|
"source_timeout",
|
|
"target_timeout",
|
|
"timeout",
|
|
"deadline",
|
|
"network",
|
|
"transient",
|
|
}
|
|
|
|
func isTransientQueryBuildFailureClass(class string) bool {
|
|
for _, candidate := range transientQueryBuildFailureClasses {
|
|
if class == candidate {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// ResetTransientQueryBuildFailures clears transient failure fences for active
|
|
// query build jobs while preserving durable checkpoints and schema/content blocks.
|
|
func (s *MongoStore) ResetTransientQueryBuildFailures(ctx context.Context) (int64, error) {
|
|
if s == nil || s.db == nil {
|
|
return 0, errors.New("store unavailable")
|
|
}
|
|
if !s.queryReplicaFactStore || !s.periodicQueryStore() {
|
|
return 0, ErrQueryReplicaUnsupported
|
|
}
|
|
classes := bson.A{}
|
|
for _, c := range transientQueryBuildFailureClasses {
|
|
classes = append(classes, c)
|
|
}
|
|
filter := bson.M{
|
|
"schema": queryBuildJobSchema,
|
|
"phase": bson.M{"$ne": "complete"},
|
|
"failureClass": bson.M{"$in": classes},
|
|
}
|
|
update := bson.M{
|
|
"$unset": bson.M{
|
|
"failurePhase": "",
|
|
"failureClass": "",
|
|
"failureCode": "",
|
|
},
|
|
}
|
|
lease, ok, err := s.acquireQueryModelLease(ctx)
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
if !ok {
|
|
return 0, ErrQueryReplicaLease
|
|
}
|
|
defer func() {
|
|
cleanup, stop := context.WithTimeout(context.Background(), time.Second)
|
|
defer stop()
|
|
_, _ = s.queryControl().UpdateOne(cleanup, queryLeaseFilter(lease), bson.M{"$set": bson.M{"owner": "", "until": time.Now().UTC()}})
|
|
}()
|
|
var changed int64
|
|
err = s.queryBuildTransaction(ctx, lease, func(sc mongo.SessionContext) error {
|
|
result, e := s.db.Collection(queryBuildJobsCollection).UpdateMany(sc, filter, update)
|
|
if e == nil {
|
|
changed = result.ModifiedCount
|
|
}
|
|
return e
|
|
})
|
|
return changed, err
|
|
}
|
|
|
|
// Only unfinished input capture can lose Mongo history. Sealed plans, commit
|
|
// failures, lease loss and corruption remain fenced. At most three replacements
|
|
// are attempted before reporting genuine inability to seal an input generation.
|
|
func recoverableQueryBuildSnapshot(j queryBuildJob) bool {
|
|
return j.FailureClass == "mongo_command" && j.FailureCode == 239 && (j.Phase == "production_plan" || j.Phase == "workforce_plan") && len(j.SnapshotRecoveries) < 3
|
|
}
|