droboticflow/internal/store/query_replica.go

481 lines
17 KiB
Go

package store
import (
"context"
"errors"
"log"
"sort"
"time"
"droboticflow/internal/domain"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
"go.mongodb.org/mongo-driver/mongo/readconcern"
"go.mongodb.org/mongo-driver/mongo/writeconcern"
)
const queryReplicaControl = "queryReplicaControl"
const queryReplicaSchema = "query-replica/v1"
var ErrQueryReplicaLease = errors.New("query replica lease lost")
var ErrQueryReplicaNotReady = errors.New("查询暂不可用,请稍后重试")
// QueryReplica copies only the explicit query-fact schema. The source does not
// require transactions or Change Streams. Checkpoints and target writes share
// a target transaction; source acknowledgements happen only after its commit.
// Tracking must cover every source writer before Bootstrap is started.
type QueryReplica struct {
Source, Target *MongoStore
owner string
epoch int64
BatchSize int
readBatch map[string]int64
emptyDrainAt map[string]time.Time
dirtyAfter map[string]any
reconcileNext int
safety *queryReplicaSafety
}
type queryReplicaCheckpoint struct {
ID string `bson:"_id"`
Last any `bson:"last,omitempty"`
HasLast bool `bson:"hasLast"`
Complete bool `bson:"complete"`
CheckedAt time.Time `bson:"checkedAt"`
}
func NewQueryReplica(source, target *MongoStore) (*QueryReplica, error) {
if source == nil || target == nil || source.db == nil || target.db == nil || source.db.Client() == target.db.Client() {
return nil, errors.New("separate source and query clients required")
}
target.queryReplicaFactStore = true
return &QueryReplica{Source: source, Target: target, owner: domain.NewID("query-projector"), BatchSize: 200, dirtyAfter: map[string]any{}}, nil
}
func (q *QueryReplica) control() *mongo.Collection {
return q.Target.db.Collection(queryReplicaControl)
}
func (q *QueryReplica) limit() int64 {
if q.BatchSize < 1 || q.BatchSize > 1000 {
return 200
}
return int64(q.BatchSize)
}
func replicaCollections() []string {
names := make([]string, 0, len(queryReplicaFields))
for n := range queryReplicaFields {
names = append(names, n)
}
sort.Strings(names)
return names
}
// VerifyTopology rejects two database names on the same Mongo deployment. A
// replica set is required only on the independent target, for fencing/CDC.
func (q *QueryReplica) VerifyTopology(ctx context.Context) error {
var a, b bson.M
if e := q.Source.db.RunCommand(ctx, bson.M{"hello": 1}).Decode(&a); e != nil {
return e
}
if e := q.Target.db.RunCommand(ctx, bson.M{"hello": 1}).Decode(&b); e != nil {
return e
}
if b["setName"] == nil {
return errors.New("query database must support transactions")
}
atop, btop := deletionDoc(a["topologyVersion"]), deletionDoc(b["topologyVersion"])
if atop["processId"] != nil && atop["processId"] == btop["processId"] {
return errors.New("query database shares source process")
}
if a["setName"] != nil && a["setName"] == b["setName"] {
return errors.New("query database shares source replica set")
}
return nil
}
// EnsureSourceIndexes is an explicit administrative operation, never startup
// behavior. Production execution requires a separately approved index plan.
func (q *QueryReplica) EnsureSourceIndexes(ctx context.Context) error {
for _, n := range replicaCollections() {
_, e := q.Source.db.Collection(n).Indexes().CreateOne(ctx, mongo.IndexModel{Keys: bson.D{{Key: querySyncField + ".dirty", Value: 1}, {Key: "_id", Value: 1}}, Options: options.Index().SetName("query_sync_dirty_v1").SetPartialFilterExpression(bson.M{querySyncField + ".dirty": true})})
if e != nil {
return e
}
_, e = q.Source.db.Collection(n).Indexes().CreateOne(ctx, mongo.IndexModel{Keys: bson.D{{Key: querySyncField + ".deleteClaim", Value: 1}}, Options: options.Index().SetName("query_sync_delete_claim_v1").SetSparse(true)})
if e != nil {
return e
}
}
_, e := q.Source.db.Collection(querySyncDeleteCollection).Indexes().CreateMany(ctx, []mongo.IndexModel{
{Keys: bson.D{{Key: "phase", Value: 1}, {Key: "leaseUntil", Value: 1}}, Options: options.Index().SetName("query_sync_delete_v1")},
{Keys: bson.D{{Key: "collection", Value: 1}, {Key: "phase", Value: 1}, {Key: "audit", Value: 1}}, Options: options.Index().SetName("query_sync_delete_pending_v1")},
{Keys: bson.D{{Key: "expireAt", Value: 1}}, Options: options.Index().SetName("query_sync_delete_retention_v1").SetExpireAfterSeconds(0)},
})
return e
}
func (q *QueryReplica) Acquire(ctx context.Context) error {
if e := q.bind(ctx); e != nil {
return e
}
now := time.Now().UTC()
_, e := q.control().UpdateOne(ctx, bson.M{"_id": "lease"}, bson.M{"$setOnInsert": bson.M{"owner": "", "epoch": int64(0), "until": time.Time{}}}, options.Update().SetUpsert(true))
if e != nil && !mongo.IsDuplicateKeyError(e) {
return e
}
var d struct {
Epoch int64 `bson:"epoch"`
}
e = q.control().FindOneAndUpdate(ctx, bson.M{"_id": "lease", "$or": bson.A{bson.M{"until": bson.M{"$lte": now}}, bson.M{"owner": q.owner}}}, bson.M{"$set": bson.M{"owner": q.owner, "until": now.Add(30 * time.Second)}, "$inc": bson.M{"epoch": 1}}, options.FindOneAndUpdate().SetReturnDocument(options.After)).Decode(&d)
if errors.Is(e, mongo.ErrNoDocuments) {
return ErrQueryReplicaLease
}
q.epoch = d.Epoch
return e
}
func (q *QueryReplica) transaction(ctx context.Context, fn func(mongo.SessionContext) error) error {
session, e := q.Target.db.Client().StartSession()
if e != nil {
return e
}
defer session.EndSession(ctx)
_, e = session.WithTransaction(ctx, func(sc mongo.SessionContext) (any, error) {
now := time.Now().UTC()
r, e := q.control().UpdateOne(sc, bson.M{"_id": "lease", "owner": q.owner, "epoch": q.epoch, "until": bson.M{"$gt": now}}, bson.M{"$set": bson.M{"until": now.Add(30 * time.Second)}})
if e != nil {
return nil, e
}
if r.MatchedCount != 1 {
return nil, ErrQueryReplicaLease
}
return nil, fn(sc)
}, options.Transaction().SetReadConcern(readconcern.Snapshot()).SetWriteConcern(writeconcern.New(writeconcern.WMajority(), writeconcern.J(true))))
return e
}
func (q *QueryReplica) checkpoint(ctx context.Context, n string) (queryReplicaCheckpoint, error) {
var cp queryReplicaCheckpoint
e := q.control().FindOne(ctx, bson.M{"_id": "bootstrap/" + n}).Decode(&cp)
if errors.Is(e, mongo.ErrNoDocuments) {
return queryReplicaCheckpoint{ID: "bootstrap/" + n}, nil
}
return cp, e
}
func (q *QueryReplica) apply(ctx mongo.SessionContext, n string, records []queryReplicaRecord) error {
if len(records) == 0 {
return nil
}
c := q.Target.db.Collection(n)
ids := make([]any, 0, len(records))
seen := map[string]bool{}
for _, r := range records {
key := querySourceIdentity(n, r.Document["_id"])
if seen[key] {
return ErrQueryReplicaUnsupported
}
seen[key] = true
ids = append(ids, r.Document["_id"])
}
projection := bson.M{"_queryReplica": 1}
if n == "humanItems" {
projection = poolBucketProjection()
projection["_queryReplica"] = 1
}
cur, e := c.Find(ctx, bson.M{"_id": bson.M{"$in": ids}}, options.Find().SetProjection(projection))
if e != nil {
return e
}
current := map[string]bson.M{}
oldPoolRows := map[string]bson.M{}
for cur.Next(ctx) {
var d bson.M
if e = cur.Decode(&d); e != nil {
cur.Close(ctx)
return e
}
current[querySourceIdentity(n, d["_id"])] = deletionDoc(d["_queryReplica"])
if n == "humanItems" {
oldPoolRows[querySourceIdentity(n, d["_id"])] = d
}
}
e = cur.Err()
cur.Close(ctx)
if e != nil {
return e
}
models := make([]mongo.WriteModel, 0, len(records))
deltas := map[string]*poolBucketDelta{}
for i, r := range records {
m := current[querySourceIdentity(n, r.Document["_id"])]
records[i].TargetSourceID = m["sourceId"]
records[i].TargetRevision = int64(supplierNumber(m["revision"]))
if bsonEqual(m["sourceId"], r.SourceID) && m["schema"] == queryReplicaSchema && (records[i].TargetRevision > r.Revision || (records[i].TargetRevision == r.Revision && m["digest"] == deletionDoc(r.Document["_queryReplica"])["digest"])) {
continue
}
records[i].TargetSourceID = r.SourceID
records[i].TargetRevision = r.Revision
if n == "humanItems" {
if err := addPoolBucketDelta(deltas, oldPoolRows[querySourceIdentity(n, r.Document["_id"])], -1); err != nil {
return err
}
if err := addPoolBucketDelta(deltas, r.Document, 1); err != nil {
return err
}
}
models = append(models, mongo.NewReplaceOneModel().SetFilter(bson.M{"_id": r.Document["_id"]}).SetReplacement(r.Document).SetUpsert(true))
}
if len(models) == 0 {
return nil
}
_, e = c.BulkWrite(ctx, models, options.BulkWrite().SetOrdered(true))
if e != nil {
return e
}
if e = q.writePoolBucketDeltas(ctx, deltas); e != nil {
return e
}
_, e = q.control().UpdateOne(ctx, bson.M{"_id": "state"}, bson.M{"$inc": bson.M{"factEpoch": int64(1), "poolBucketEpoch": int64(1), "collectionEpoch." + n: int64(1)}}, options.Update().SetUpsert(true))
return e
}
func bsonEqual(a, b any) bool {
x, e := bson.Marshal(bson.M{"v": a})
if e != nil {
return false
}
y, e := bson.Marshal(bson.M{"v": b})
return e == nil && string(x) == string(y)
}
// Retry source reads only: each attempt has the original deadline and a new
// durable reservation. Failed rows are discarded, never checkpointed or acked.
// Writes, fencing, unsafe health and unsupported records are not retried here.
func (q *QueryReplica) readLimit(n string) int64 {
limit := q.limit()
if adaptive := q.readBatch[n]; adaptive > 0 && adaptive < limit {
return adaptive
}
return limit
}
func retryableReplicaRead(ctx context.Context, err error) bool {
if ctx.Err() != nil || errors.Is(err, context.Canceled) {
return false
}
d := QueryReplicaDiagnostic(err)
if d.Phase != "source_find" && d.Phase != "source_cursor" {
return false
}
return errors.Is(err, context.DeadlineExceeded) || mongo.IsTimeout(err) || mongo.IsNetworkError(err)
}
func (q *QueryReplica) read(ctx context.Context, n string, filter bson.M) ([]queryReplicaRecord, any, error) {
limit := q.readLimit(n)
for attempt := 0; ; attempt++ {
rows, last, err := q.readAttempt(ctx, n, filter, limit)
if err == nil {
if q.readBatch == nil {
q.readBatch = map[string]int64{}
}
q.readBatch[n] = limit
return rows, last, nil
}
if attempt >= 3 || !retryableReplicaRead(ctx, err) {
return nil, nil, err
}
// The final bounded retry must fit one legacy document, even when
// earlier batch reductions still exceed the fixed wire deadline.
if attempt == 2 {
limit = 1
} else if limit > 1 {
limit /= 2
}
d := QueryReplicaDiagnostic(err)
log.Printf("projector source read retry collection=%s phase=%s class=%s mongoCode=%d attempt=%d batch=%d", d.Collection, d.Phase, d.Class, d.MongoCode, attempt+2, limit)
timer := time.NewTimer(time.Duration(100*(1<<attempt)) * time.Millisecond)
select {
case <-ctx.Done():
timer.Stop()
return nil, nil, ctx.Err()
case <-timer.C:
}
}
}
func (q *QueryReplica) readAttempt(ctx context.Context, n string, filter bson.M, requested int64) ([]queryReplicaRecord, any, error) {
parent := ctx
limit, e := q.sourcePermit(ctx, requested)
if e != nil {
return nil, nil, queryReplicaFailure(n, "source_permit", e)
}
readStarted := time.Now()
ctx, cancel := q.sourceContext(ctx)
defer cancel()
opts := options.Find().SetMaxTime(1500 * time.Millisecond).SetProjection(querySourceProjection(n)).SetSort(bson.D{{Key: "_id", Value: 1}}).SetLimit(limit).SetBatchSize(int32(limit))
if filter[querySyncField+".dirty"] == true {
opts.SetHint("query_sync_dirty_v1")
} else {
opts.SetHint("_id_")
}
cur, e := q.Source.db.Collection(n).Find(ctx, filter, opts)
if e != nil {
return nil, nil, queryReplicaFailure(n, "source_find", e)
}
defer cur.Close(ctx)
records := []queryReplicaRecord{}
var last any
for cur.Next(ctx) {
var d bson.M
if e = cur.Decode(&d); e != nil {
return nil, nil, queryReplicaFailure(n, "source_decode", e)
}
last = d["_id"]
r, e := queryExportRecord(n, d)
if e != nil {
return nil, nil, queryReplicaFailure(n, "source_export", e)
}
records = append(records, r)
}
if e := cur.Err(); e != nil {
return nil, nil, queryReplicaFailure(n, "source_cursor", e)
}
if e := q.paceReadRows(parent, len(records), readStarted); e != nil {
return nil, nil, queryReplicaFailure(n, "row_pacing", e)
}
return records, last, nil
}
func (q *QueryReplica) acknowledge(ctx context.Context, n string, records []queryReplicaRecord) error {
models := make([]mongo.WriteModel, 0, len(records))
for _, r := range records {
if bsonEqual(r.TargetSourceID, r.SourceID) && r.TargetRevision > r.Revision {
// An old writer removed/restored an earlier marker. Never acknowledge
// that regression as current: re-mark metadata above the durable sink rev.
models = append(models, mongo.NewUpdateOneModel().SetFilter(queryReplicaSourceCAS(r)).SetUpdate(bson.M{"$set": queryReplicaRepairMarker(r.TargetRevision + 1)}))
continue
}
if r.Revision > 0 && r.SourceDirty {
models = append(models, mongo.NewUpdateOneModel().SetFilter(bson.M{"_id": r.SourceID, querySyncField + ".revision": r.Revision, querySyncField + ".dirty": true}).SetUpdate(bson.M{"$set": bson.M{querySyncField + ".dirty": false}}))
}
}
if len(models) == 0 {
return nil
}
return queryReplicaFailure(n, "source_ack", q.sourceMetadataBulk(ctx, n, models))
}
// BootstrapStep is bounded and resumable. $expr uses BSON total ordering across
// mixed historical _id types; a plain $gt would silently skip other BSON types.
// This one-time pass is rate limited by the caller, never a per-request scan.
func (q *QueryReplica) BootstrapStep(ctx context.Context, n string) (bool, int, error) {
if queryReplicaFields[n] == "" {
return false, 0, ErrQueryReplicaUnsupported
}
cp, e := q.checkpoint(ctx, n)
if e != nil || cp.Complete {
return cp.Complete, 0, queryReplicaFailure(n, "checkpoint", e)
}
filter := bson.M{}
if cp.HasLast || cp.Last != nil {
filter["$expr"] = bson.M{"$gt": bson.A{"$_id", bson.M{"$literal": cp.Last}}}
}
rows, last, e := q.read(ctx, n, filter)
if e != nil {
return false, 0, e
}
done := len(rows) == 0
e = q.transaction(ctx, func(sc mongo.SessionContext) error {
if e := q.apply(sc, n, rows); e != nil {
return e
}
update := bson.M{"complete": done}
if len(rows) > 0 {
update["hasLast"] = true
update["last"] = last
}
_, e := q.control().UpdateOne(sc, bson.M{"_id": cp.ID}, bson.M{"$set": update}, options.Update().SetUpsert(true))
return e
})
if e != nil {
return false, 0, queryReplicaFailure(n, "target_apply", e)
}
return done, len(rows), q.acknowledge(ctx, n, rows)
}
// DrainStep never acknowledges a newer concurrent save. A crash after target
// commit simply replays the same version; a crash before commit retains dirty.
func (q *QueryReplica) DrainStep(ctx context.Context, n string) (int, error) {
if queryReplicaFields[n] == "" {
return 0, ErrQueryReplicaUnsupported
}
filter := bson.M{querySyncField + ".dirty": true}
if last := q.dirtyAfter[n]; last != nil {
filter["$expr"] = bson.M{"$gt": bson.A{"$_id", bson.M{"$literal": last}}}
}
rows, last, e := q.read(ctx, n, filter)
if e == nil {
if int64(len(rows)) < q.readLimit(n) {
delete(q.dirtyAfter, n)
} else {
q.dirtyAfter[n] = last
}
}
if e != nil {
return 0, e
}
// An empty poll has no target effect or source acknowledgement to fence.
// Keep lease fencing on actual changes and PublishReady, rather than writing
// the lease once per empty collection on every incremental pass.
if len(rows) == 0 {
return 0, nil
}
e = q.transaction(ctx, func(sc mongo.SessionContext) error { return q.apply(sc, n, rows) })
if e != nil {
return 0, queryReplicaFailure(n, "target_apply", e)
}
return len(rows), q.acknowledge(ctx, n, rows)
}
// EnableQuerySourceTracking is deliberately explicit and defaults off. Every
// writer role must be upgraded before a replica may be marked ready.
func (s *MongoStore) EnableQuerySourceTracking(enabled bool) { s.querySyncEnabled.Store(enabled) }
// RunOnce performs at most one batch per collection. Backfill is resumable and
// bounded; the process does not begin unbounded scans on an HTTP cache miss.
func (q *QueryReplica) RunOnce(ctx context.Context, bootstrap bool) (bool, error) {
if q.safety != nil {
return q.runGuardedOnce(ctx, bootstrap)
}
if _, e := q.sourcePermit(ctx, 0); e != nil {
return false, e
}
allBootstrapped := true
for _, n := range replicaCollections() {
cp, e := q.checkpoint(ctx, n)
if e != nil {
return false, e
}
if !cp.Complete {
if !bootstrap {
return false, ErrQueryReplicaNotReady
}
var complete bool
if complete, _, e = q.BootstrapStep(ctx, n); e != nil {
return false, e
}
allBootstrapped = allBootstrapped && complete
}
if _, e = q.DrainStep(ctx, n); e != nil {
return false, e
}
}
if allBootstrapped {
if _, e := q.ReconcileOneDue(ctx, 6*time.Hour); e != nil {
return false, e
}
}
if _, e := q.ReconcileDeleteStep(ctx); e != nil {
return false, e
}
if e := q.PublishReady(ctx); e == ErrQueryReplicaNotReady {
return false, nil
} else if e != nil {
return false, e
}
return true, nil
}