droboticflow/internal/store/query_build_runner.go

298 lines
10 KiB
Go

package store
import (
"context"
"errors"
"fmt"
"log"
"reflect"
"sort"
"time"
"droboticflow/internal/queryserving"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
)
// This lifecycle is exclusive to the independent, periodic read database.
// Authority/legacy callers keep their existing loader and publication behavior.
func (s *MongoStore) queryBuildStart(ctx context.Context, l queryModelLease, org string, projects []string) (queryBuildJob, error) {
org = normalizedOrganizationID(org)
projects = append([]string{}, projects...)
sort.Strings(projects)
old, exists, err := s.queryBuildJob(ctx, org)
if err != nil {
return old, err
}
recoverSnapshot := exists && recoverableQueryBuildSnapshot(old)
if exists && old.Phase != "complete" {
if !reflect.DeepEqual(old.Projects, projects) {
return old, errors.New("query build scope changed; retained job requires reconciliation")
}
if !recoverSnapshot {
if old.FailureClass != "" {
return old, fmt.Errorf("%w: phase=%s category=%s", ErrQueryBuildStopped, old.FailurePhase, old.FailureClass)
}
return old, nil
}
}
observedAt := time.Now().UTC()
at, err := s.queryBuildSnapshot(ctx)
if err != nil {
return old, err
}
var state struct {
Revision int64 `bson:"productionRevision"`
Global int64 `bson:"searchGlobalRevision"`
Observed time.Time `bson:"observedAt"`
Heads map[string]QueryModelHead `bson:"heads"`
}
rows, e := s.queryBuildReadPage(ctx, s.queryControl(), bson.M{"_id": "global"}, bson.M{"productionRevision": 1, "searchGlobalRevision": 1, "heads": 1, "observedAt": 1}, bson.RawValue{}, false, at)
if e != nil {
return old, e
}
if len(rows) != 1 {
return old, errors.New("query build control unavailable at snapshot")
}
if e = bson.Unmarshal(rows[0], &state); e != nil {
return old, e
}
head := state.Heads[workforceFactKey(org)]
j := queryBuildJob{ID: queryBuildJobKey(s.db.Name(), org), Schema: queryBuildJobSchema, Organization: org, Projects: projects, Generation: queryserving.Key(queryBuildJobSchema, org, at.T, at.I, time.Now().UnixNano()), Snapshot: at, AsOf: observedAt, Phase: "production_plan", OldEpoch: head.FactEpoch, Epoch: head.FactEpoch + 1, Cold: head.FactEpoch == 0, SourceRevision: state.Revision, GlobalRevision: state.Global, ProgressAt: time.Now().UTC()}
j.Retired = append([]queryBuildRetiredGeneration{}, old.Retired...)
j.InputGeneration = j.Generation
if recoverSnapshot {
j.SnapshotRecoveries = append(append([]queryBuildSnapshotRecovery{}, old.SnapshotRecoveries...), queryBuildSnapshotRecovery{Generation: old.Generation, Snapshot: old.Snapshot, AsOf: old.AsOf, Phase: old.Phase, Code: old.FailureCode, RecoveredAt: time.Now().UTC()})
}
if exists && old.Phase == "complete" && reflect.DeepEqual(old.Projects, projects) && old.SourceRevision == state.Revision && old.GlobalRevision == state.Global && !state.Observed.Before(old.AsOf) && time.Since(state.Observed) >= 0 && time.Since(state.Observed) < 15*time.Second {
j.AsOf = state.Observed
j.InputGeneration = old.InputGeneration
if j.InputGeneration == "" {
j.InputGeneration = old.Generation
}
// The caught-up stream proves that every business input is unchanged.
// Reconfirm the complete cohort without re-materializing or walking
// 140k unchanged facts and checkpointing thousands of no-op batches.
production, exists := state.Heads[queryModelKey("production", org, projects)]
committed := old.Epoch
if !old.Changed && old.OldEpoch > 0 {
committed = old.OldEpoch
}
if exists && production.Generation != "" && head.Rows == old.PlanRows && head.FactEpoch == committed && production.SourceRevision == old.SourceRevision {
j.Phase = "publish"
j.PlanRows = old.PlanRows
j.ReusedRows = int64(old.PlanRows)
j.Production = production
j.Production.AsOf = j.AsOf
}
}
err = s.queryBuildTransaction(ctx, l, func(sc mongo.SessionContext) error { return s.saveQueryBuildJob(sc, j) })
return j, err
}
func (s *MongoStore) queryBuildRunRound(ctx context.Context, l queryModelLease, org string, projects []string, r *queryBuildRound) (err error) {
j, err := s.queryBuildStart(ctx, l, org, projects)
if err != nil {
if ctx.Err() != nil {
return ctx.Err()
}
d := QueryReplicaDiagnostic(err)
log.Printf("query_model_build result=stopped stage=resume category=%s mongoCode=%d", d.Class, d.MongoCode)
return fmt.Errorf("%w: resume: %w", ErrQueryBuildStopped, err)
}
if r == nil {
r = newQueryBuildRound(j.Cold)
}
started := time.Now()
before := j.ProgressAt
defer func() {
if err == nil || ctx.Err() != nil {
return
}
// Maintenance/shutdown time is not counted as active lack of progress.
current, ok, e := s.queryBuildJob(ctx, org)
if e != nil || !ok {
return
}
if !current.ProgressAt.After(before) {
current.ActiveWithoutProgress += time.Since(started)
} else {
current.ActiveWithoutProgress = 0
}
if current.ActiveWithoutProgress >= queryBuildNoProgressLimit && (errors.Is(err, ErrQueryBuildYielded) || isTransientQueryBuildFailureClass(QueryReplicaDiagnostic(err).Class)) {
err = ErrQueryBuildNoProgress
}
if !errors.Is(err, ErrQueryBuildYielded) {
current.FailurePhase = current.Phase
diagnostic := QueryReplicaDiagnostic(err)
current.FailureClass = diagnostic.Class
current.FailureCode = diagnostic.MongoCode
if errors.Is(err, ErrQueryBuildNoProgress) {
current.FailureClass = "no_durable_progress"
}
}
if !errors.Is(err, ErrQueryBuildYielded) {
err = fmt.Errorf("%w: %w", ErrQueryBuildStopped, err)
}
if e = s.queryBuildTransaction(ctx, l, func(sc mongo.SessionContext) error { return s.saveQueryBuildJob(sc, current) }); e != nil {
d := QueryReplicaDiagnostic(e)
log.Printf("query_model_build result=stopped stage=checkpoint category=%s mongoCode=%d", d.Class, d.MongoCode)
err = fmt.Errorf("%w: query build could not persist outcome: %w", ErrQueryBuildStopped, e)
}
if e == nil && recoverableQueryBuildSnapshot(current) {
log.Print("query_model_build result=yielded stage=input_snapshot category=expired checkpoint=retained")
err = fmt.Errorf("%w: input snapshot expired; replacement generation scheduled", ErrQueryBuildYielded)
}
}()
for {
if err = r.check(ctx); err != nil {
return err
}
switch j.Phase {
case "production_plan":
err = s.queryBuildProductionPlan(ctx, l, &j, r)
case "workforce_plan":
err = s.queryBuildWorkforcePlan(ctx, l, &j, r)
case "apply":
err = s.queryBuildApply(ctx, l, &j, r)
case "cleanup":
err = s.queryBuildCleanup(ctx, l, &j, r)
case "publish":
err = s.queryBuildPublish(ctx, l, &j)
case "complete":
return nil
default:
return errors.New("unsupported query build phase")
}
if err != nil {
return queryBuildPhaseFailure(j.Phase, err)
}
}
}
// Explicitly bounded derived-data housekeeping; active jobs are never swept.
func (s *MongoStore) cleanQueryBuildParts(ctx context.Context, l queryModelLease) error {
var jobs []queryBuildJob
cur, err := s.db.Collection(queryBuildJobsCollection).Find(ctx, bson.M{"schema": queryBuildJobSchema}, options.Find().SetProjection(bson.M{"generation": 1, "inputGeneration": 1, "phase": 1, "snapshotRecoveries": 1}))
if err != nil {
return err
}
err = cur.All(ctx, &jobs)
cur.Close(ctx)
if err != nil {
return err
}
keep := []string{}
for _, j := range jobs {
keep = append(keep, j.Generation, j.InputGeneration)
if j.Phase != "complete" {
for _, v := range j.SnapshotRecoveries {
keep = append(keep, v.Generation)
}
}
}
for _, collection := range []string{queryBuildPartsCollection, queryBuildJobsCollection} {
f := bson.M{"job": bson.M{"$exists": true, "$nin": keep}}
cur, e := s.db.Collection(collection).Find(ctx, f, options.Find().SetProjection(bson.M{"_id": 1}).SetLimit(1000))
if e != nil {
return e
}
var rows []bson.M
e = cur.All(ctx, &rows)
cur.Close(ctx)
if e != nil {
return e
}
ids := bson.A{}
for _, row := range rows {
ids = append(ids, row["_id"])
}
if len(ids) > 0 {
if e = s.queryBuildTransaction(ctx, l, func(sc mongo.SessionContext) error {
_, e := s.db.Collection(collection).DeleteMany(sc, bson.M{"_id": bson.M{"$in": ids}, "job": bson.M{"$nin": keep}})
return e
}); e != nil {
return e
}
}
}
return nil
}
var transientQueryBuildFailureClasses = []string{
"source_timeout",
"target_timeout",
"timeout",
"deadline",
"network",
"transient",
}
func isTransientQueryBuildFailureClass(class string) bool {
for _, candidate := range transientQueryBuildFailureClasses {
if class == candidate {
return true
}
}
return false
}
// ResetTransientQueryBuildFailures clears transient failure fences for active
// query build jobs while preserving durable checkpoints and schema/content blocks.
func (s *MongoStore) ResetTransientQueryBuildFailures(ctx context.Context) (int64, error) {
if s == nil || s.db == nil {
return 0, errors.New("store unavailable")
}
if !s.queryReplicaFactStore || !s.periodicQueryStore() {
return 0, ErrQueryReplicaUnsupported
}
classes := bson.A{}
for _, c := range transientQueryBuildFailureClasses {
classes = append(classes, c)
}
filter := bson.M{
"schema": queryBuildJobSchema,
"phase": bson.M{"$ne": "complete"},
"failureClass": bson.M{"$in": classes},
}
update := bson.M{
"$unset": bson.M{
"failurePhase": "",
"failureClass": "",
"failureCode": "",
},
}
lease, ok, err := s.acquireQueryModelLease(ctx)
if err != nil {
return 0, err
}
if !ok {
return 0, ErrQueryReplicaLease
}
defer func() {
cleanup, stop := context.WithTimeout(context.Background(), time.Second)
defer stop()
_, _ = s.queryControl().UpdateOne(cleanup, queryLeaseFilter(lease), bson.M{"$set": bson.M{"owner": "", "until": time.Now().UTC()}})
}()
var changed int64
err = s.queryBuildTransaction(ctx, lease, func(sc mongo.SessionContext) error {
result, e := s.db.Collection(queryBuildJobsCollection).UpdateMany(sc, filter, update)
if e == nil {
changed = result.ModifiedCount
}
return e
})
return changed, err
}
// Only unfinished input capture can lose Mongo history. Sealed plans, commit
// failures, lease loss and corruption remain fenced. At most three replacements
// are attempted before reporting genuine inability to seal an input generation.
func recoverableQueryBuildSnapshot(j queryBuildJob) bool {
return j.FailureClass == "mongo_command" && j.FailureCode == 239 && (j.Phase == "production_plan" || j.Phase == "workforce_plan") && len(j.SnapshotRecoveries) < 3
}