From d4ecc071c84ed09e009857426bd7dc5cd94b1547 Mon Sep 17 00:00:00 2001 From: d-robotics Date: Sun, 15 Nov 2026 00:02:45 -0800 Subject: [PATCH] =?UTF-8?q?=E2=9A=A1=20Reduce=20shadow=20sync=20overhead?= =?UTF-8?q?=20and=20bound=20maintenance=20throughput?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/operations/query-replica.md | 13 +- internal/store/query_replica.go | 6 +- internal/store/query_replica_projection.go | 4 +- .../query_replica_projection_local_test.go | 151 ++++++++++++++++++ internal/store/query_replica_safety.go | 14 +- internal/store/query_replica_safety_test.go | 57 +++++++ internal/store/query_replica_schedule.go | 47 +++++- internal/store/query_replica_transport.go | 15 +- 8 files changed, 295 insertions(+), 12 deletions(-) diff --git a/docs/operations/query-replica.md b/docs/operations/query-replica.md index 21242c05..6ab0a95f 100644 --- a/docs/operations/query-replica.md +++ b/docs/operations/query-replica.md @@ -257,4 +257,15 @@ Mongo 角色不能限制更新到某个字段:源账号按22个源集合授予 本机合成数据测得:保留256KiB/秒和8命令/秒,2000条精简运行事实的读取与多数提交,100条/秒节奏约76条/秒净吞吐,200条/秒节奏约123条/秒净吞吐。该测试没有启用生产安全策略,也未包含22集合的完整调度、历史大小分布、源确认和完整后台统计,不能宣称生产已达到100或200条/秒。大响应夹具由100条缩至25条后成功,单次5秒限制未延长。生产同断点恢复及吞吐仍待批准候选发布后验证。 -当前生产限速和截止时间保持原授权值。拟提速档位100/150/200条/秒、1/2/4MiB/秒、16/24/32命令/秒,批次100/150/200,仅用于有限维护模式;这些上限代码与实际配置均未调整,待用户统一确认。累计仍不超过1500万预留(含额外50条诊断),固定截止仍为2026-11-15 14:38:42 UTC(06:38:42 PST),不自动延长。生产入口不切换,客户业务正文、标注、归属和版本保持不变。 +当前生产限速和截止时间保持原授权值。方案0c0b8fefedff已获用户批准,本地实现提速档位100/150/200条/秒、1/2/4MiB/秒、16/24/32命令/秒,批次100/150/200,仅用于有限维护模式;高档上限仅对有限维护模式生效,普通模式保持原上限;实际部署仍等待最终不可变计划确认。累计仍不超过1500万预留(含额外50条诊断),固定截止仍为2026-11-15 14:38:42 UTC(06:38:42 PST),不自动延长。生产入口不切换,客户业务正文、标注、归属和版本保持不变。 + + +### 完整调度重复读取修复(方案0c0b8fefedff,尚未部署) + +相同1200条合成数据、200条/秒、4MiB/秒、32命令/秒下,完整守护调度原净吞吐49.12条/秒,源命令320个、读取预留36532次;消除重复后为107.29条/秒、86个命令、11230次预留。测试仍包含22集合、事务、预算、源元数据确认和最后就绪检查。该值不是生产200条/秒承诺;后续扩大同一数据夹具并保留原始失败记录。 + +- 只有维护模式且历史尚未完成时,对已经复制完成且脏队列为空的集合合并10秒内的重复探测。未复制完的集合不跳过;重启丢弃这份进程内时间表,重新探测。历史完成后停止此合并,沿原周期完整增量处理。 +- 历史缺失已足以证明不能发布时,不再扫描部分集合的干净标记后才返回未就绪;删除协调仍执行。正式发布仍实时检查全部集合和删除意图,测试覆盖合并窗口内新增脏数据不能获得新就绪证书。 +- 已经clean的源记录不发送空确认更新;真正dirty仍按修订号CAS确认,目标版本领先时的旧标记修复路径优先保留。源业务正文不修改。 +- 维护模式的行节奏计入本批实际读取耗时,避免读取后再次等待相同时间;不积累空闲额度,字节和命令节流不变,普通模式保留原节奏。 +- 最终计划需单列初始化`--min-pass-interval=0s`,移除额外的5秒轮次等待;全局限速仍生效,追平后的`--interval=60s`不变。该配置尚未在线上执行。 diff --git a/internal/store/query_replica.go b/internal/store/query_replica.go index 1fb8ccf0..f19fdabe 100644 --- a/internal/store/query_replica.go +++ b/internal/store/query_replica.go @@ -31,6 +31,7 @@ type QueryReplica struct { epoch int64 BatchSize int readBatch map[string]int64 + emptyDrainAt map[string]time.Time dirtyAfter map[string]any reconcileNext int safety *queryReplicaSafety @@ -298,6 +299,7 @@ func (q *QueryReplica) readAttempt(ctx context.Context, n string, filter bson.M, if e != nil { return nil, nil, queryReplicaFailure(n, "source_permit", e) } + readStarted := time.Now() ctx, cancel := q.sourceContext(ctx) defer cancel() opts := options.Find().SetMaxTime(1500 * time.Millisecond).SetProjection(querySourceProjection(n)).SetSort(bson.D{{Key: "_id", Value: 1}}).SetLimit(limit).SetBatchSize(int32(limit)) @@ -328,7 +330,7 @@ func (q *QueryReplica) readAttempt(ctx context.Context, n string, filter bson.M, if e := cur.Err(); e != nil { return nil, nil, queryReplicaFailure(n, "source_cursor", e) } - if e := q.paceRows(parent, len(records)); e != nil { + if e := q.paceReadRows(parent, len(records), readStarted); e != nil { return nil, nil, queryReplicaFailure(n, "row_pacing", e) } return records, last, nil @@ -342,7 +344,7 @@ func (q *QueryReplica) acknowledge(ctx context.Context, n string, records []quer models = append(models, mongo.NewUpdateOneModel().SetFilter(queryReplicaSourceCAS(r)).SetUpdate(bson.M{"$set": queryReplicaRepairMarker(r.TargetRevision + 1)})) continue } - if r.Revision > 0 { + if r.Revision > 0 && r.SourceDirty { models = append(models, mongo.NewUpdateOneModel().SetFilter(bson.M{"_id": r.SourceID, querySyncField + ".revision": r.Revision, querySyncField + ".dirty": true}).SetUpdate(bson.M{"$set": bson.M{querySyncField + ".dirty": false}})) } } diff --git a/internal/store/query_replica_projection.go b/internal/store/query_replica_projection.go index 38e2d495..d01581bc 100644 --- a/internal/store/query_replica_projection.go +++ b/internal/store/query_replica_projection.go @@ -17,6 +17,7 @@ var ErrQueryReplicaUnsupported = errors.New("query replica record requires compa type queryReplicaRecord struct { SourceID any + SourceDirty bool Revision int64 TargetRevision int64 TargetSourceID any @@ -167,7 +168,8 @@ func queryExportRecord(collection string, d bson.M) (queryReplicaRecord, error) d["_id"] = logical } d["_queryReplica"] = bson.M{"sourceId": source, "revision": revision, "schema": queryReplicaSchema, "digest": queryReplicaDigest(collection, d)} - return queryReplicaRecord{SourceID: source, Revision: revision, Document: d}, nil + dirty, _ := meta["dirty"].(bool) + return queryReplicaRecord{SourceID: source, SourceDirty: dirty, Revision: revision, Document: d}, nil } // A semantic fact digest ignores edits that cannot change query results. In diff --git a/internal/store/query_replica_projection_local_test.go b/internal/store/query_replica_projection_local_test.go index 11680288..40f1e633 100644 --- a/internal/store/query_replica_projection_local_test.go +++ b/internal/store/query_replica_projection_local_test.go @@ -355,3 +355,154 @@ func TestQueryReplicaSyntheticReadCommitThroughputLocal(t *testing.T) { }) } } + +// Full guarded scheduler, including22 collections, durable reservations, +// majority+j commits, source metadata ACKs and real shared transport budgets. +// The last-collection shape exposes overhead hidden by one-collection tests. +func TestQueryReplicaMaintenanceStagesGuardedThroughputLocal(t *testing.T) { + for _, stage := range []struct{ rows, bytes, commands, batch int }{{100, 1 << 20, 16, 100}, {150, 2 << 20, 24, 150}, {200, 4 << 20, 32, 200}} { + t.Run(fmt.Sprint(stage.rows), func(t *testing.T) { + source, target, q := newLocalReplica(t) + const total = 6000 + docs := make([]any, 0, total) + for i := 0; i < total; i++ { + docs = append(docs, bson.M{"_id": fmt.Sprintf("physical-%06d", i), "id": fmt.Sprintf("logical-%06d", i), "organizationId": "org", "projectId": "project", "status": "completed", "captureDirectories": bson.A{fmt.Sprintf("capture-%06d/part0001", i)}, "inputs": bson.M{"source": bson.M{"inlineValue": "preserve-source-body", "ref": bson.M{"captureUnit": fmt.Sprintf("capture-%06d/part0001", i)}}}, "_querySync": bson.M{"revision": int64(1), "dirty": false}}) + } + if _, e := source.db.Collection("runs").InsertMany(t.Context(), docs); e != nil { + t.Fatal(e) + } + transport, e := NewQueryReplicaTransport(stage.bytes, stage.commands) + if e != nil { + t.Fatal(e) + } + var replies, commands atomic.Int64 + monitor := &event.CommandMonitor{Started: func(_ context.Context, _ *event.CommandStartedEvent) { commands.Add(1) }, Succeeded: func(_ context.Context, e *event.CommandSucceededEvent) { replies.Add(int64(len(e.Reply))) }} + client, e := mongo.Connect(t.Context(), options.Client().ApplyURI("mongodb://127.0.0.1:27119/?directConnection=true").SetMaxPoolSize(1).SetDialer(transport).SetRetryReads(false).SetTimeout(5*time.Second).SetCompressors([]string{}).SetMonitor(monitor)) + if e != nil { + t.Fatal(e) + } + defer client.Disconnect(context.Background()) + q.Source = NewMongoStore(client.Database(source.db.Name())) + q.BatchSize = stage.batch + until := time.Now().Add(time.Hour) + p := QueryReplicaSafetyPolicy{Mode: QueryReplicaMaintenanceInitialization, MaintenanceUntil: &until, RunID: "finite-maintenance-stages", MaxRecords: 100000, RecordsPerSecond: stage.rows, BytesPerSecond: stage.bytes, CommandsPerSecond: stage.commands, MaxBatchRows: stage.batch} + maintenanceHealthLocal(t, q, 99, false) + if e = q.ConfigureSafety(t.Context(), p, ""); e != nil { + t.Fatal(e) + } + started := time.Now() + passes := 0 + for { + maintenanceHealthLocal(t, q, 99, false) + ready, e := q.RunOnce(t.Context(), true) + passes++ + if e != nil { + t.Fatal(e) + } + if ready { + break + } + if passes > 2*total/stage.batch+len(replicaCollections()) { + t.Fatal("guarded scheduler stalled") + } + } + elapsed := time.Since(started) + n, e := target.db.Collection("runs").CountDocuments(t.Context(), bson.M{}) + if e != nil || n != total { + t.Fatal("incomplete or duplicate facts", n, e) + } + var safety queryReplicaSafetyState + if e = q.control().FindOne(t.Context(), bson.M{"_id": "safety"}).Decode(&safety); e != nil || safety.Paused || safety.Reserved < total || safety.Reserved > 100000 { + t.Fatal("budget or health not enforced", e) + } + for _, collection := range replicaCollections() { + cp, e := q.checkpoint(t.Context(), collection) + if e != nil || !cp.Complete { + t.Fatal("missing collection checkpoint", collection, e) + } + } + var original bson.M + if e = source.db.Collection("runs").FindOne(t.Context(), bson.M{"_id": "physical-000000"}).Decode(&original); e != nil || deletionDoc(deletionDoc(original["inputs"])["source"])["inlineValue"] != "preserve-source-body" { + t.Fatal("source body changed", e) + } + if stage.rows == 200 && float64(n)/elapsed.Seconds() < 100 { + t.Errorf("guarded net throughput below100/s: %.2f", float64(n)/elapsed.Seconds()) + } + t.Logf("full guarded22-collection scheduler: rowCap=%d bytesCap=%d commandsCap=%d batch=%d facts=%d seconds=%.3f netFactsPerSecond=%.2f reservations=%d commands=%d replyBytes=%d majority+j=true minPassInterval=0 CPUObservationOnly=true productionCapacityAccepted=false", stage.rows, stage.bytes, stage.commands, stage.batch, n, elapsed.Seconds(), float64(n)/elapsed.Seconds(), safety.Reserved, commands.Load(), replies.Load()) + }) + } +} + +func TestQueryReplicaMaintenanceEmptyPollNeverPublishesNewDirtyFactLocal(t *testing.T) { + source, target, q := newLocalReplica(t) + q.BatchSize = 100 + until := time.Now().Add(time.Hour) + p := QueryReplicaSafetyPolicy{Mode: QueryReplicaMaintenanceInitialization, MaintenanceUntil: &until, RunID: "finite-empty-poll", MaxRecords: 100000, RecordsPerSecond: 200, BytesPerSecond: 4 << 20, CommandsPerSecond: 32, MaxBatchRows: 200} + maintenanceHealthLocal(t, q, 99, false) + if e := q.ConfigureSafety(t.Context(), p, ""); e != nil { + t.Fatal(e) + } + if ready, e := q.RunOnce(t.Context(), true); e != nil || !ready { + t.Fatal("initial empty history failed", ready, e) + } + // Simulate one checkpoint needing recovery while all other histories are + // complete: recently empty collections may be delayed, never certified. + if _, e := q.control().UpdateOne(t.Context(), bson.M{"_id": "bootstrap/runs"}, bson.M{"$set": bson.M{"complete": false}}); e != nil { + t.Fatal(e) + } + var before bson.M + if e := q.control().FindOne(t.Context(), bson.M{"_id": "state"}).Decode(&before); e != nil { + t.Fatal(e) + } + if _, e := source.db.Collection("projects").InsertOne(t.Context(), bson.M{"_id": "new-project", "id": "new-project", "organizationId": "org", "name": "new-project", "_querySync": bson.M{"revision": int64(1), "dirty": true}}); e != nil { + t.Fatal(e) + } + if ready, e := q.RunOnce(t.Context(), true); e != nil || ready { + t.Fatal("empty-poll cache bypassed final live dirty check", ready, e) + } + var after bson.M + if e := q.control().FindOne(t.Context(), bson.M{"_id": "state"}).Decode(&after); e != nil || !bsonEqual(before["observedAt"], after["observedAt"]) { + t.Fatal("published a fresh certificate while dirty", e) + } + q.emptyDrainAt["projects"] = time.Now().Add(-11 * time.Second) + if ready, e := q.RunOnce(t.Context(), true); e != nil || !ready { + t.Fatal("new dirty data not drained after bounded poll interval", ready, e) + } + if n, e := target.db.Collection("projects").CountDocuments(t.Context(), bson.M{"_id": "new-project"}); e != nil || n != 1 { + t.Fatal("new fact missing", n, e) + } +} + +func TestQueryReplicaCleanBootstrapDoesNotAcknowledgeSourceLocal(t *testing.T) { + var updates atomic.Int32 + monitor := &event.CommandMonitor{Started: func(_ context.Context, e *event.CommandStartedEvent) { + if e.CommandName == "update" { + updates.Add(1) + } + }} + source, _, q := newLocalReplica(t, monitor) + q.BatchSize = 10 + if _, e := source.db.Collection("projects").InsertOne(t.Context(), bson.M{"_id": "project", "id": "project", "name": "keep", "_querySync": bson.M{"revision": int64(1), "dirty": false}}); e != nil { + t.Fatal(e) + } + if _, n, e := q.BootstrapStep(t.Context(), "projects"); e != nil || n != 1 { + t.Fatal(n, e) + } + if updates.Load() != 0 { + t.Fatalf("clean historical copy wrote%d unnecessary source acknowledgement commands", updates.Load()) + } + if _, e := source.db.Collection("projects").UpdateOne(t.Context(), bson.M{"_id": "project"}, bson.M{"$set": bson.M{"name": "new", "_querySync.revision": int64(2), "_querySync.dirty": true}}); e != nil { + t.Fatal(e) + } + updates.Store(0) + if n, e := q.DrainStep(t.Context(), "projects"); e != nil || n != 1 { + t.Fatal(n, e) + } + if updates.Load() != 1 { + t.Fatal("real dirty update not acknowledged", updates.Load()) + } + var d bson.M + if e := source.db.Collection("projects").FindOne(t.Context(), bson.M{"_id": "project"}).Decode(&d); e != nil || deletionDoc(d["_querySync"])["dirty"] != false || d["name"] != "new" { + t.Fatal("wrong source ACK or business change", e) + } +} diff --git a/internal/store/query_replica_safety.go b/internal/store/query_replica_safety.go index dd770234..ab65ce41 100644 --- a/internal/store/query_replica_safety.go +++ b/internal/store/query_replica_safety.go @@ -80,7 +80,13 @@ func (p QueryReplicaSafetyPolicy) validate() error { if p.maintenance() && (p.MaxRecords <= 0 || p.MaintenanceUntil == nil || !p.MaintenanceUntil.After(time.Now())) { return errors.New("maintenance initialization requires a finite read budget and an explicit future deadline") } - if p.BytesPerSecond < 1024 || p.BytesPerSecond > 256*1024 || p.CommandsPerSecond < 1 || p.CommandsPerSecond > 8 || p.MaxBatchRows < 1 || p.MaxBatchRows > 100 || p.RunID == "" || len(p.RunID) > 100 || p.MaxRecords < 0 || p.RecordsPerSecond < 1 || p.RecordsPerSecond > 50 || math.IsNaN(p.BaselineCPU) || math.IsInf(p.BaselineCPU, 0) || (!p.maintenance() && (p.BaselineCPU <= 0 || p.BaselineCPU > 90)) || (p.maintenance() && (p.BaselineCPU < 0 || p.BaselineCPU > 100)) { + maxBytes, maxCommands, maxBatch, maxRows := 256*1024, 8, 100, 50 + // Expanded ceilings are reserved for explicitly finite maintenance runs. + // Normal operation retains its existing limits. + if p.maintenance() { + maxBytes, maxCommands, maxBatch, maxRows = 4*1024*1024, 32, 200, 200 + } + if p.BytesPerSecond < 1024 || p.BytesPerSecond > maxBytes || p.CommandsPerSecond < 1 || p.CommandsPerSecond > maxCommands || p.MaxBatchRows < 1 || p.MaxBatchRows > maxBatch || p.RunID == "" || len(p.RunID) > 100 || p.MaxRecords < 0 || p.RecordsPerSecond < 1 || p.RecordsPerSecond > maxRows || math.IsNaN(p.BaselineCPU) || math.IsInf(p.BaselineCPU, 0) || (!p.maintenance() && (p.BaselineCPU <= 0 || p.BaselineCPU > 90)) || (p.maintenance() && (p.BaselineCPU < 0 || p.BaselineCPU > 100)) { return errors.New("invalid shadow safety policy") } return nil @@ -337,6 +343,12 @@ func (q *QueryReplica) paceRows(ctx context.Context, n int) error { } return q.safety.rows.wait(ctx, float64(n)) } +func (q *QueryReplica) paceReadRows(ctx context.Context, n int, started time.Time) error { + if q.safety != nil && q.safety.policy.maintenance() { + return q.safety.rows.waitFrom(ctx, float64(n), started) + } + return q.paceRows(ctx, n) +} func (q *QueryReplica) sourceContext(ctx context.Context) (context.Context, context.CancelFunc) { return context.WithTimeout(ctx, 5*time.Second) } diff --git a/internal/store/query_replica_safety_test.go b/internal/store/query_replica_safety_test.go index ae945906..f2bb3205 100644 --- a/internal/store/query_replica_safety_test.go +++ b/internal/store/query_replica_safety_test.go @@ -186,3 +186,60 @@ func TestQueryReplicaMaintenancePolicyKeepsHardSafety(t *testing.T) { t.Fatal("legacy strict CPU protection changed") } } + +func TestQueryReplicaExpandedCeilingsRequireFiniteMaintenance(t *testing.T) { + until := time.Now().Add(time.Hour) + for _, stage := range []struct{ rows, bytes, commands, batch int }{{100, 1 << 20, 16, 100}, {150, 2 << 20, 24, 150}, {200, 4 << 20, 32, 200}} { + p := QueryReplicaSafetyPolicy{Mode: QueryReplicaMaintenanceInitialization, MaintenanceUntil: &until, RunID: "same-run", MaxRecords: 14999950, RecordsPerSecond: stage.rows, BytesPerSecond: stage.bytes, CommandsPerSecond: stage.commands, MaxBatchRows: stage.batch} + if e := p.validate(); e != nil { + t.Fatal("approved finite maintenance stage rejected", e) + } + if _, e := NewQueryReplicaTransport(stage.bytes, stage.commands); e != nil { + t.Fatal(e) + } + ordinary := p + ordinary.Mode = "" + ordinary.BaselineCPU = 85 + if ordinary.validate() == nil { + t.Fatal("higher limits leaked into normal operation") + } + for _, change := range []func(*QueryReplicaSafetyPolicy){func(p *QueryReplicaSafetyPolicy) { p.MaintenanceUntil = nil }, func(p *QueryReplicaSafetyPolicy) { past := time.Now().Add(-time.Second); p.MaintenanceUntil = &past }, func(p *QueryReplicaSafetyPolicy) { p.MaxRecords = 0 }, func(p *QueryReplicaSafetyPolicy) { p.RecordsPerSecond = 201 }, func(p *QueryReplicaSafetyPolicy) { p.BytesPerSecond = 4<<20 + 1 }, func(p *QueryReplicaSafetyPolicy) { p.CommandsPerSecond = 33 }, func(p *QueryReplicaSafetyPolicy) { p.MaxBatchRows = 201 }} { + bad := p + change(&bad) + if bad.validate() == nil { + t.Fatal("unapproved or unbounded maintenance accepted") + } + } + } + for _, budget := range [][2]int{{4<<20 + 1, 32}, {4 << 20, 33}} { + if _, e := NewQueryReplicaTransport(budget[0], budget[1]); e == nil { + t.Fatal("transport exceeded approved upper bound") + } + } +} + +func TestQueryReplicaReadPacingCountsWorkWithoutIdleCredit(t *testing.T) { + p := newQueryReplicaPacer(100) + started := time.Now() + time.Sleep(60 * time.Millisecond) + if e := p.waitFrom(t.Context(), 10, started); e != nil { + t.Fatal(e) + } + if elapsed := time.Since(started); elapsed < 95*time.Millisecond { + t.Fatalf("read work double counted or rate exceeded: %s", elapsed) + } + p.mu.Lock() + deadline := p.next + p.mu.Unlock() + if !deadline.Equal(started.Add(100 * time.Millisecond)) { + t.Fatal("read work was charged twice") + } + time.Sleep(100 * time.Millisecond) + started = time.Now() + if e := p.waitFrom(t.Context(), 10, started); e != nil { + t.Fatal(e) + } + if time.Since(started) < 95*time.Millisecond { + t.Fatal("idle time accumulated unreviewed row credit") + } +} diff --git a/internal/store/query_replica_schedule.go b/internal/store/query_replica_schedule.go index 1dd92356..1603d6c1 100644 --- a/internal/store/query_replica_schedule.go +++ b/internal/store/query_replica_schedule.go @@ -53,6 +53,19 @@ func (q *QueryReplica) runGuardedOnce(ctx context.Context, bootstrap bool) (bool if e != nil && e != mongo.ErrNoDocuments { return false, queryReplicaFailure(queryReplicaControl, "schedule", e) } + maintenance := q.safety != nil && q.safety.policy.maintenance() + bootstrapping := false + if maintenance { + ids := bson.A{} + for _, name := range names { + ids = append(ids, "bootstrap/"+name) + } + complete, err := q.control().CountDocuments(ctx, bson.M{"_id": bson.M{"$in": ids}, "complete": true}) + if err != nil { + return false, queryReplicaFailure(queryReplicaControl, "checkpoint", err) + } + bootstrapping = complete < int64(len(names)) + } start := cursor.Next % len(names) if start < 0 { start = 0 @@ -60,10 +73,34 @@ func (q *QueryReplica) runGuardedOnce(ctx context.Context, bootstrap bool) (bool for offset := 0; offset < len(names); offset++ { index := (start + offset) % len(names) name := names[index] + cp, e := q.checkpoint(ctx, name) + if e != nil { + return false, queryReplicaFailure(name, "checkpoint", e) + } + // Initialization may revisit a large remaining collection very often. + // A completed collection with a recently empty dirty queue does not + // need another source read/target transaction on every such pass. + // Missing history is never skipped; restart discards this small cache. + if maintenance && bootstrapping && cp.Complete { + age := time.Since(q.emptyDrainAt[name]) + if age >= 0 && age < 10*time.Second { + continue + } + } n, e := q.DrainStep(ctx, name) if e != nil { return false, e } + if maintenance { + if q.emptyDrainAt == nil { + q.emptyDrainAt = map[string]time.Time{} + } + if n == 0 { + q.emptyDrainAt[name] = time.Now() + } else { + delete(q.emptyDrainAt, name) + } + } age, e := q.backlogAge(ctx, name, n) if e != nil { return false, queryReplicaFailure(name, "backlog", e) @@ -82,10 +119,6 @@ func (q *QueryReplica) runGuardedOnce(ctx context.Context, bootstrap bool) (bool } } } - cp, e := q.checkpoint(ctx, name) - if e != nil { - return false, queryReplicaFailure(name, "checkpoint", e) - } if !cp.Complete { if !bootstrap { return false, ErrQueryReplicaNotReady @@ -117,6 +150,12 @@ func (q *QueryReplica) runGuardedOnce(ctx context.Context, bootstrap bool) (bool if _, e := q.ReconcileDeleteStep(ctx); e != nil { return false, e } + // Do not probe clean markers for a prefix of the collections when missing + // history already proves that publication cannot succeed. Deletions above + // still run; the final full authority checks below remain mandatory. + if !complete { + return false, nil + } if e := q.PublishReady(ctx); e == ErrQueryReplicaNotReady { return false, nil } else if e != nil { diff --git a/internal/store/query_replica_transport.go b/internal/store/query_replica_transport.go index c82b7f9c..a44bab98 100644 --- a/internal/store/query_replica_transport.go +++ b/internal/store/query_replica_transport.go @@ -19,13 +19,22 @@ type queryReplicaPacer struct { func newQueryReplicaPacer(rate float64) *queryReplicaPacer { return &queryReplicaPacer{rate: rate} } func (p *queryReplicaPacer) wait(ctx context.Context, units float64) error { + return p.waitFrom(ctx, units, time.Now()) +} + +// Account for work already spent reading this batch, without banking idle +// credit. The shared next deadline still orders concurrent reservations. +func (p *queryReplicaPacer) waitFrom(ctx context.Context, units float64, started time.Time) error { if units <= 0 { return ctx.Err() } p.mu.Lock() now := time.Now() - if p.next.Before(now) { - p.next = now + if started.IsZero() || started.After(now) { + started = now + } + if p.next.Before(started) { + p.next = started } p.next = p.next.Add(time.Duration(units / p.rate * float64(time.Second))) until := p.next @@ -43,7 +52,7 @@ func (p *queryReplicaPacer) wait(ctx context.Context, units float64) error { type QueryReplicaTransport struct{ bytes, commands *queryReplicaPacer } func NewQueryReplicaTransport(bytesPerSecond, commandsPerSecond int) (*QueryReplicaTransport, error) { - if bytesPerSecond < 1024 || bytesPerSecond > 256*1024 || commandsPerSecond < 1 || commandsPerSecond > 8 { + if bytesPerSecond < 1024 || bytesPerSecond > 4*1024*1024 || commandsPerSecond < 1 || commandsPerSecond > 32 { return nil, errors.New("invalid source transport budget") } return &QueryReplicaTransport{newQueryReplicaPacer(float64(bytesPerSecond)), newQueryReplicaPacer(float64(commandsPerSecond))}, nil