forked from longtao.wu/droboticflow
481 lines
17 KiB
Go
481 lines
17 KiB
Go
package store
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"log"
|
|
"sort"
|
|
"time"
|
|
|
|
"droboticflow/internal/domain"
|
|
"go.mongodb.org/mongo-driver/bson"
|
|
"go.mongodb.org/mongo-driver/mongo"
|
|
"go.mongodb.org/mongo-driver/mongo/options"
|
|
"go.mongodb.org/mongo-driver/mongo/readconcern"
|
|
"go.mongodb.org/mongo-driver/mongo/writeconcern"
|
|
)
|
|
|
|
const queryReplicaControl = "queryReplicaControl"
|
|
const queryReplicaSchema = "query-replica/v1"
|
|
|
|
var ErrQueryReplicaLease = errors.New("query replica lease lost")
|
|
var ErrQueryReplicaNotReady = errors.New("查询暂不可用,请稍后重试")
|
|
|
|
// QueryReplica copies only the explicit query-fact schema. The source does not
|
|
// require transactions or Change Streams. Checkpoints and target writes share
|
|
// a target transaction; source acknowledgements happen only after its commit.
|
|
// Tracking must cover every source writer before Bootstrap is started.
|
|
type QueryReplica struct {
|
|
Source, Target *MongoStore
|
|
owner string
|
|
epoch int64
|
|
BatchSize int
|
|
readBatch map[string]int64
|
|
emptyDrainAt map[string]time.Time
|
|
dirtyAfter map[string]any
|
|
reconcileNext int
|
|
safety *queryReplicaSafety
|
|
}
|
|
type queryReplicaCheckpoint struct {
|
|
ID string `bson:"_id"`
|
|
Last any `bson:"last,omitempty"`
|
|
HasLast bool `bson:"hasLast"`
|
|
Complete bool `bson:"complete"`
|
|
CheckedAt time.Time `bson:"checkedAt"`
|
|
}
|
|
|
|
func NewQueryReplica(source, target *MongoStore) (*QueryReplica, error) {
|
|
if source == nil || target == nil || source.db == nil || target.db == nil || source.db.Client() == target.db.Client() {
|
|
return nil, errors.New("separate source and query clients required")
|
|
}
|
|
target.queryReplicaFactStore = true
|
|
return &QueryReplica{Source: source, Target: target, owner: domain.NewID("query-projector"), BatchSize: 200, dirtyAfter: map[string]any{}}, nil
|
|
}
|
|
func (q *QueryReplica) control() *mongo.Collection {
|
|
return q.Target.db.Collection(queryReplicaControl)
|
|
}
|
|
func (q *QueryReplica) limit() int64 {
|
|
if q.BatchSize < 1 || q.BatchSize > 1000 {
|
|
return 200
|
|
}
|
|
return int64(q.BatchSize)
|
|
}
|
|
func replicaCollections() []string {
|
|
names := make([]string, 0, len(queryReplicaFields))
|
|
for n := range queryReplicaFields {
|
|
names = append(names, n)
|
|
}
|
|
sort.Strings(names)
|
|
return names
|
|
}
|
|
|
|
// VerifyTopology rejects two database names on the same Mongo deployment. A
|
|
// replica set is required only on the independent target, for fencing/CDC.
|
|
func (q *QueryReplica) VerifyTopology(ctx context.Context) error {
|
|
var a, b bson.M
|
|
if e := q.Source.db.RunCommand(ctx, bson.M{"hello": 1}).Decode(&a); e != nil {
|
|
return e
|
|
}
|
|
if e := q.Target.db.RunCommand(ctx, bson.M{"hello": 1}).Decode(&b); e != nil {
|
|
return e
|
|
}
|
|
if b["setName"] == nil {
|
|
return errors.New("query database must support transactions")
|
|
}
|
|
atop, btop := deletionDoc(a["topologyVersion"]), deletionDoc(b["topologyVersion"])
|
|
if atop["processId"] != nil && atop["processId"] == btop["processId"] {
|
|
return errors.New("query database shares source process")
|
|
}
|
|
if a["setName"] != nil && a["setName"] == b["setName"] {
|
|
return errors.New("query database shares source replica set")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// EnsureSourceIndexes is an explicit administrative operation, never startup
|
|
// behavior. Production execution requires a separately approved index plan.
|
|
func (q *QueryReplica) EnsureSourceIndexes(ctx context.Context) error {
|
|
for _, n := range replicaCollections() {
|
|
_, e := q.Source.db.Collection(n).Indexes().CreateOne(ctx, mongo.IndexModel{Keys: bson.D{{Key: querySyncField + ".dirty", Value: 1}, {Key: "_id", Value: 1}}, Options: options.Index().SetName("query_sync_dirty_v1").SetPartialFilterExpression(bson.M{querySyncField + ".dirty": true})})
|
|
if e != nil {
|
|
return e
|
|
}
|
|
_, e = q.Source.db.Collection(n).Indexes().CreateOne(ctx, mongo.IndexModel{Keys: bson.D{{Key: querySyncField + ".deleteClaim", Value: 1}}, Options: options.Index().SetName("query_sync_delete_claim_v1").SetSparse(true)})
|
|
if e != nil {
|
|
return e
|
|
}
|
|
}
|
|
_, e := q.Source.db.Collection(querySyncDeleteCollection).Indexes().CreateMany(ctx, []mongo.IndexModel{
|
|
{Keys: bson.D{{Key: "phase", Value: 1}, {Key: "leaseUntil", Value: 1}}, Options: options.Index().SetName("query_sync_delete_v1")},
|
|
{Keys: bson.D{{Key: "collection", Value: 1}, {Key: "phase", Value: 1}, {Key: "audit", Value: 1}}, Options: options.Index().SetName("query_sync_delete_pending_v1")},
|
|
{Keys: bson.D{{Key: "expireAt", Value: 1}}, Options: options.Index().SetName("query_sync_delete_retention_v1").SetExpireAfterSeconds(0)},
|
|
})
|
|
return e
|
|
}
|
|
func (q *QueryReplica) Acquire(ctx context.Context) error {
|
|
if e := q.bind(ctx); e != nil {
|
|
return e
|
|
}
|
|
now := time.Now().UTC()
|
|
_, e := q.control().UpdateOne(ctx, bson.M{"_id": "lease"}, bson.M{"$setOnInsert": bson.M{"owner": "", "epoch": int64(0), "until": time.Time{}}}, options.Update().SetUpsert(true))
|
|
if e != nil && !mongo.IsDuplicateKeyError(e) {
|
|
return e
|
|
}
|
|
var d struct {
|
|
Epoch int64 `bson:"epoch"`
|
|
}
|
|
e = q.control().FindOneAndUpdate(ctx, bson.M{"_id": "lease", "$or": bson.A{bson.M{"until": bson.M{"$lte": now}}, bson.M{"owner": q.owner}}}, bson.M{"$set": bson.M{"owner": q.owner, "until": now.Add(30 * time.Second)}, "$inc": bson.M{"epoch": 1}}, options.FindOneAndUpdate().SetReturnDocument(options.After)).Decode(&d)
|
|
if errors.Is(e, mongo.ErrNoDocuments) {
|
|
return ErrQueryReplicaLease
|
|
}
|
|
q.epoch = d.Epoch
|
|
return e
|
|
}
|
|
func (q *QueryReplica) transaction(ctx context.Context, fn func(mongo.SessionContext) error) error {
|
|
session, e := q.Target.db.Client().StartSession()
|
|
if e != nil {
|
|
return e
|
|
}
|
|
defer session.EndSession(ctx)
|
|
_, e = session.WithTransaction(ctx, func(sc mongo.SessionContext) (any, error) {
|
|
now := time.Now().UTC()
|
|
r, e := q.control().UpdateOne(sc, bson.M{"_id": "lease", "owner": q.owner, "epoch": q.epoch, "until": bson.M{"$gt": now}}, bson.M{"$set": bson.M{"until": now.Add(30 * time.Second)}})
|
|
if e != nil {
|
|
return nil, e
|
|
}
|
|
if r.MatchedCount != 1 {
|
|
return nil, ErrQueryReplicaLease
|
|
}
|
|
return nil, fn(sc)
|
|
}, options.Transaction().SetReadConcern(readconcern.Snapshot()).SetWriteConcern(writeconcern.New(writeconcern.WMajority(), writeconcern.J(true))))
|
|
return e
|
|
}
|
|
func (q *QueryReplica) checkpoint(ctx context.Context, n string) (queryReplicaCheckpoint, error) {
|
|
var cp queryReplicaCheckpoint
|
|
e := q.control().FindOne(ctx, bson.M{"_id": "bootstrap/" + n}).Decode(&cp)
|
|
if errors.Is(e, mongo.ErrNoDocuments) {
|
|
return queryReplicaCheckpoint{ID: "bootstrap/" + n}, nil
|
|
}
|
|
return cp, e
|
|
}
|
|
func (q *QueryReplica) apply(ctx mongo.SessionContext, n string, records []queryReplicaRecord) error {
|
|
if len(records) == 0 {
|
|
return nil
|
|
}
|
|
c := q.Target.db.Collection(n)
|
|
ids := make([]any, 0, len(records))
|
|
seen := map[string]bool{}
|
|
for _, r := range records {
|
|
key := querySourceIdentity(n, r.Document["_id"])
|
|
if seen[key] {
|
|
return ErrQueryReplicaUnsupported
|
|
}
|
|
seen[key] = true
|
|
ids = append(ids, r.Document["_id"])
|
|
}
|
|
projection := bson.M{"_queryReplica": 1}
|
|
if n == "humanItems" {
|
|
projection = poolBucketProjection()
|
|
projection["_queryReplica"] = 1
|
|
}
|
|
cur, e := c.Find(ctx, bson.M{"_id": bson.M{"$in": ids}}, options.Find().SetProjection(projection))
|
|
if e != nil {
|
|
return e
|
|
}
|
|
current := map[string]bson.M{}
|
|
oldPoolRows := map[string]bson.M{}
|
|
for cur.Next(ctx) {
|
|
var d bson.M
|
|
if e = cur.Decode(&d); e != nil {
|
|
cur.Close(ctx)
|
|
return e
|
|
}
|
|
current[querySourceIdentity(n, d["_id"])] = deletionDoc(d["_queryReplica"])
|
|
if n == "humanItems" {
|
|
oldPoolRows[querySourceIdentity(n, d["_id"])] = d
|
|
}
|
|
}
|
|
e = cur.Err()
|
|
cur.Close(ctx)
|
|
if e != nil {
|
|
return e
|
|
}
|
|
models := make([]mongo.WriteModel, 0, len(records))
|
|
deltas := map[string]*poolBucketDelta{}
|
|
for i, r := range records {
|
|
m := current[querySourceIdentity(n, r.Document["_id"])]
|
|
records[i].TargetSourceID = m["sourceId"]
|
|
records[i].TargetRevision = int64(supplierNumber(m["revision"]))
|
|
if bsonEqual(m["sourceId"], r.SourceID) && m["schema"] == queryReplicaSchema && (records[i].TargetRevision > r.Revision || (records[i].TargetRevision == r.Revision && m["digest"] == deletionDoc(r.Document["_queryReplica"])["digest"])) {
|
|
continue
|
|
}
|
|
records[i].TargetSourceID = r.SourceID
|
|
records[i].TargetRevision = r.Revision
|
|
if n == "humanItems" {
|
|
if err := addPoolBucketDelta(deltas, oldPoolRows[querySourceIdentity(n, r.Document["_id"])], -1); err != nil {
|
|
return err
|
|
}
|
|
if err := addPoolBucketDelta(deltas, r.Document, 1); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
models = append(models, mongo.NewReplaceOneModel().SetFilter(bson.M{"_id": r.Document["_id"]}).SetReplacement(r.Document).SetUpsert(true))
|
|
}
|
|
if len(models) == 0 {
|
|
return nil
|
|
}
|
|
_, e = c.BulkWrite(ctx, models, options.BulkWrite().SetOrdered(true))
|
|
if e != nil {
|
|
return e
|
|
}
|
|
if e = q.writePoolBucketDeltas(ctx, deltas); e != nil {
|
|
return e
|
|
}
|
|
_, e = q.control().UpdateOne(ctx, bson.M{"_id": "state"}, bson.M{"$inc": bson.M{"factEpoch": int64(1), "poolBucketEpoch": int64(1), "collectionEpoch." + n: int64(1)}}, options.Update().SetUpsert(true))
|
|
return e
|
|
}
|
|
|
|
func bsonEqual(a, b any) bool {
|
|
x, e := bson.Marshal(bson.M{"v": a})
|
|
if e != nil {
|
|
return false
|
|
}
|
|
y, e := bson.Marshal(bson.M{"v": b})
|
|
return e == nil && string(x) == string(y)
|
|
}
|
|
|
|
// Retry source reads only: each attempt has the original deadline and a new
|
|
// durable reservation. Failed rows are discarded, never checkpointed or acked.
|
|
// Writes, fencing, unsafe health and unsupported records are not retried here.
|
|
func (q *QueryReplica) readLimit(n string) int64 {
|
|
limit := q.limit()
|
|
if adaptive := q.readBatch[n]; adaptive > 0 && adaptive < limit {
|
|
return adaptive
|
|
}
|
|
return limit
|
|
}
|
|
func retryableReplicaRead(ctx context.Context, err error) bool {
|
|
if ctx.Err() != nil || errors.Is(err, context.Canceled) {
|
|
return false
|
|
}
|
|
d := QueryReplicaDiagnostic(err)
|
|
if d.Phase != "source_find" && d.Phase != "source_cursor" {
|
|
return false
|
|
}
|
|
return errors.Is(err, context.DeadlineExceeded) || mongo.IsTimeout(err) || mongo.IsNetworkError(err)
|
|
}
|
|
func (q *QueryReplica) read(ctx context.Context, n string, filter bson.M) ([]queryReplicaRecord, any, error) {
|
|
limit := q.readLimit(n)
|
|
for attempt := 0; ; attempt++ {
|
|
rows, last, err := q.readAttempt(ctx, n, filter, limit)
|
|
if err == nil {
|
|
if q.readBatch == nil {
|
|
q.readBatch = map[string]int64{}
|
|
}
|
|
q.readBatch[n] = limit
|
|
return rows, last, nil
|
|
}
|
|
if attempt >= 3 || !retryableReplicaRead(ctx, err) {
|
|
return nil, nil, err
|
|
}
|
|
// The final bounded retry must fit one legacy document, even when
|
|
// earlier batch reductions still exceed the fixed wire deadline.
|
|
if attempt == 2 {
|
|
limit = 1
|
|
} else if limit > 1 {
|
|
limit /= 2
|
|
}
|
|
d := QueryReplicaDiagnostic(err)
|
|
log.Printf("projector source read retry collection=%s phase=%s class=%s mongoCode=%d attempt=%d batch=%d", d.Collection, d.Phase, d.Class, d.MongoCode, attempt+2, limit)
|
|
timer := time.NewTimer(time.Duration(100*(1<<attempt)) * time.Millisecond)
|
|
select {
|
|
case <-ctx.Done():
|
|
timer.Stop()
|
|
return nil, nil, ctx.Err()
|
|
case <-timer.C:
|
|
}
|
|
}
|
|
}
|
|
func (q *QueryReplica) readAttempt(ctx context.Context, n string, filter bson.M, requested int64) ([]queryReplicaRecord, any, error) {
|
|
|
|
parent := ctx
|
|
limit, e := q.sourcePermit(ctx, requested)
|
|
if e != nil {
|
|
return nil, nil, queryReplicaFailure(n, "source_permit", e)
|
|
}
|
|
readStarted := time.Now()
|
|
ctx, cancel := q.sourceContext(ctx)
|
|
defer cancel()
|
|
opts := options.Find().SetMaxTime(1500 * time.Millisecond).SetProjection(querySourceProjection(n)).SetSort(bson.D{{Key: "_id", Value: 1}}).SetLimit(limit).SetBatchSize(int32(limit))
|
|
if filter[querySyncField+".dirty"] == true {
|
|
opts.SetHint("query_sync_dirty_v1")
|
|
} else {
|
|
opts.SetHint("_id_")
|
|
}
|
|
cur, e := q.Source.db.Collection(n).Find(ctx, filter, opts)
|
|
if e != nil {
|
|
return nil, nil, queryReplicaFailure(n, "source_find", e)
|
|
}
|
|
defer cur.Close(ctx)
|
|
records := []queryReplicaRecord{}
|
|
var last any
|
|
for cur.Next(ctx) {
|
|
var d bson.M
|
|
if e = cur.Decode(&d); e != nil {
|
|
return nil, nil, queryReplicaFailure(n, "source_decode", e)
|
|
}
|
|
last = d["_id"]
|
|
r, e := queryExportRecord(n, d)
|
|
if e != nil {
|
|
return nil, nil, queryReplicaFailure(n, "source_export", e)
|
|
}
|
|
records = append(records, r)
|
|
}
|
|
if e := cur.Err(); e != nil {
|
|
return nil, nil, queryReplicaFailure(n, "source_cursor", e)
|
|
}
|
|
if e := q.paceReadRows(parent, len(records), readStarted); e != nil {
|
|
return nil, nil, queryReplicaFailure(n, "row_pacing", e)
|
|
}
|
|
return records, last, nil
|
|
}
|
|
func (q *QueryReplica) acknowledge(ctx context.Context, n string, records []queryReplicaRecord) error {
|
|
models := make([]mongo.WriteModel, 0, len(records))
|
|
for _, r := range records {
|
|
if bsonEqual(r.TargetSourceID, r.SourceID) && r.TargetRevision > r.Revision {
|
|
// An old writer removed/restored an earlier marker. Never acknowledge
|
|
// that regression as current: re-mark metadata above the durable sink rev.
|
|
models = append(models, mongo.NewUpdateOneModel().SetFilter(queryReplicaSourceCAS(r)).SetUpdate(bson.M{"$set": queryReplicaRepairMarker(r.TargetRevision + 1)}))
|
|
continue
|
|
}
|
|
if r.Revision > 0 && r.SourceDirty {
|
|
models = append(models, mongo.NewUpdateOneModel().SetFilter(bson.M{"_id": r.SourceID, querySyncField + ".revision": r.Revision, querySyncField + ".dirty": true}).SetUpdate(bson.M{"$set": bson.M{querySyncField + ".dirty": false}}))
|
|
}
|
|
}
|
|
if len(models) == 0 {
|
|
return nil
|
|
}
|
|
return queryReplicaFailure(n, "source_ack", q.sourceMetadataBulk(ctx, n, models))
|
|
}
|
|
|
|
// BootstrapStep is bounded and resumable. $expr uses BSON total ordering across
|
|
// mixed historical _id types; a plain $gt would silently skip other BSON types.
|
|
// This one-time pass is rate limited by the caller, never a per-request scan.
|
|
func (q *QueryReplica) BootstrapStep(ctx context.Context, n string) (bool, int, error) {
|
|
if queryReplicaFields[n] == "" {
|
|
return false, 0, ErrQueryReplicaUnsupported
|
|
}
|
|
cp, e := q.checkpoint(ctx, n)
|
|
if e != nil || cp.Complete {
|
|
return cp.Complete, 0, queryReplicaFailure(n, "checkpoint", e)
|
|
}
|
|
filter := bson.M{}
|
|
if cp.HasLast || cp.Last != nil {
|
|
filter["$expr"] = bson.M{"$gt": bson.A{"$_id", bson.M{"$literal": cp.Last}}}
|
|
}
|
|
rows, last, e := q.read(ctx, n, filter)
|
|
if e != nil {
|
|
return false, 0, e
|
|
}
|
|
done := len(rows) == 0
|
|
e = q.transaction(ctx, func(sc mongo.SessionContext) error {
|
|
if e := q.apply(sc, n, rows); e != nil {
|
|
return e
|
|
}
|
|
update := bson.M{"complete": done}
|
|
if len(rows) > 0 {
|
|
update["hasLast"] = true
|
|
update["last"] = last
|
|
}
|
|
_, e := q.control().UpdateOne(sc, bson.M{"_id": cp.ID}, bson.M{"$set": update}, options.Update().SetUpsert(true))
|
|
return e
|
|
})
|
|
if e != nil {
|
|
return false, 0, queryReplicaFailure(n, "target_apply", e)
|
|
}
|
|
return done, len(rows), q.acknowledge(ctx, n, rows)
|
|
}
|
|
|
|
// DrainStep never acknowledges a newer concurrent save. A crash after target
|
|
// commit simply replays the same version; a crash before commit retains dirty.
|
|
func (q *QueryReplica) DrainStep(ctx context.Context, n string) (int, error) {
|
|
if queryReplicaFields[n] == "" {
|
|
return 0, ErrQueryReplicaUnsupported
|
|
}
|
|
filter := bson.M{querySyncField + ".dirty": true}
|
|
if last := q.dirtyAfter[n]; last != nil {
|
|
filter["$expr"] = bson.M{"$gt": bson.A{"$_id", bson.M{"$literal": last}}}
|
|
}
|
|
rows, last, e := q.read(ctx, n, filter)
|
|
if e == nil {
|
|
if int64(len(rows)) < q.readLimit(n) {
|
|
delete(q.dirtyAfter, n)
|
|
} else {
|
|
q.dirtyAfter[n] = last
|
|
}
|
|
}
|
|
if e != nil {
|
|
return 0, e
|
|
}
|
|
// An empty poll has no target effect or source acknowledgement to fence.
|
|
// Keep lease fencing on actual changes and PublishReady, rather than writing
|
|
// the lease once per empty collection on every incremental pass.
|
|
if len(rows) == 0 {
|
|
return 0, nil
|
|
}
|
|
e = q.transaction(ctx, func(sc mongo.SessionContext) error { return q.apply(sc, n, rows) })
|
|
if e != nil {
|
|
return 0, queryReplicaFailure(n, "target_apply", e)
|
|
}
|
|
return len(rows), q.acknowledge(ctx, n, rows)
|
|
}
|
|
|
|
// EnableQuerySourceTracking is deliberately explicit and defaults off. Every
|
|
// writer role must be upgraded before a replica may be marked ready.
|
|
func (s *MongoStore) EnableQuerySourceTracking(enabled bool) { s.querySyncEnabled.Store(enabled) }
|
|
|
|
// RunOnce performs at most one batch per collection. Backfill is resumable and
|
|
// bounded; the process does not begin unbounded scans on an HTTP cache miss.
|
|
func (q *QueryReplica) RunOnce(ctx context.Context, bootstrap bool) (bool, error) {
|
|
if q.safety != nil {
|
|
return q.runGuardedOnce(ctx, bootstrap)
|
|
}
|
|
if _, e := q.sourcePermit(ctx, 0); e != nil {
|
|
return false, e
|
|
}
|
|
allBootstrapped := true
|
|
for _, n := range replicaCollections() {
|
|
cp, e := q.checkpoint(ctx, n)
|
|
if e != nil {
|
|
return false, e
|
|
}
|
|
if !cp.Complete {
|
|
if !bootstrap {
|
|
return false, ErrQueryReplicaNotReady
|
|
}
|
|
var complete bool
|
|
if complete, _, e = q.BootstrapStep(ctx, n); e != nil {
|
|
return false, e
|
|
}
|
|
allBootstrapped = allBootstrapped && complete
|
|
}
|
|
if _, e = q.DrainStep(ctx, n); e != nil {
|
|
return false, e
|
|
}
|
|
}
|
|
if allBootstrapped {
|
|
if _, e := q.ReconcileOneDue(ctx, 6*time.Hour); e != nil {
|
|
return false, e
|
|
}
|
|
}
|
|
if _, e := q.ReconcileDeleteStep(ctx); e != nil {
|
|
return false, e
|
|
}
|
|
if e := q.PublishReady(ctx); e == ErrQueryReplicaNotReady {
|
|
return false, nil
|
|
} else if e != nil {
|
|
return false, e
|
|
}
|
|
return true, nil
|
|
}
|