🔧 Retry transient query replica reads with adaptive batches

This commit is contained in:
d-robotics 2026-11-14 23:32:47 -08:00
parent b64d950f64
commit 480844cfcc
6 changed files with 333 additions and 5 deletions

View File

@ -177,7 +177,7 @@ func main() {
}
opts := options.Client().ApplyURI(uri).SetMaxPoolSize(4).SetServerSelectionTimeout(5 * time.Second)
if !target && transport != nil {
opts.SetMaxPoolSize(1).SetDialer(transport).SetCompressors([]string{}).SetTimeout(5 * time.Second)
opts.SetMaxPoolSize(1).SetDialer(transport).SetCompressors([]string{}).SetRetryReads(false).SetTimeout(5 * time.Second)
}
if target {
opts.SetReadPreference(readpref.Primary()).SetReadConcern(readconcern.Majority())

View File

@ -246,3 +246,15 @@ Mongo 角色不能限制更新到某个字段:源账号按22个源集合授予
现有首轮授权仍限1000条读取预留、10条/秒、64KiB/秒、2命令/秒、批次25,不因 CPU 策略更改扩大。该预算包含空读和重复探测,不能完成约76.4万源记录的初始化;完整历史预算、构建器启动及故障对照须合并列明再确认。实现核验与集群实际部署分开记录,新增镜像按正式发布规则处理。
成品链路验收须区分:统计事实构建成功、特定权限/筛选成品发布完成、Valkey 命中和生产读取切换是不同状态。新筛选仅登记到查询库,由后台计算并发布;不得在缺失成品时前台扫描原库。缓存丢失读取查询库最后完整成品;查询库也不可用时明确暂不可用,不集中回源。初始化完成或 Pod 就绪均不代表已分流;最终降载只能在另行批准切读后以同条件证据确认。
## 有界读取重试与缩批(2026-11-15,本地候选)
影子运行在升至100条/批后,于 `runs/source_find` 再次出现固定5秒读取超时;停于累计37015次预留、13914条事实。该现场没有足够信息区分服务端等待和传输等待,不将本地复现替代生产根因证明。本地真实驱动故障注入已复现一次读取失败直接结束;大查询事实夹具在256KiB/秒下也能触发同类读取超时。相同翻页表达式的两万条合成索引对照为100条结果、100个键/文档扫描,未证明存在从头扫描,不据此修改索引。
候选仅对 `source_find/source_cursor` 的网络错误和超时最多尝试4次,每次从相同条件重新读取、批量减半,等待依次为100/200/400毫秒。同一进程内该集合沿用成功的小批量,其他集合保持原批量;重启后的初始批量仍受既有配置限制。失败批次不应用、不确认、不推进断点。每次尝试独立消耗持久读取预留;关闭守护客户端的驱动隐式读取重试,避免遗漏计数。每次源操作5秒和服务端1500毫秒、目标事务 `majority+journal` 保持不变。
取消、安全健康、租约、权限、不支持的历史格式以及源确认/目标写入错误不进入此重试路径。最多4次仍失败则沿原持久暂停机制退出,不跳过记录、不自动更换run或重置预算。脏批次及积压判断按该集合的有效批量计算,避免缩批后误判已排空。
本机合成数据测得:保留256KiB/秒和8命令/秒,2000条精简运行事实的读取与多数提交,100条/秒节奏约76条/秒净吞吐,200条/秒节奏约123条/秒净吞吐。该测试没有启用生产安全策略,也未包含22集合的完整调度、历史大小分布、源确认和完整后台统计,不能宣称生产已达到100或200条/秒。大响应夹具由100条缩至25条后成功,单次5秒限制未延长。生产同断点恢复及吞吐仍待批准候选发布后验证。
当前生产限速和截止时间保持原授权值。拟提速档位100/150/200条/秒、1/2/4MiB/秒、16/24/32命令/秒,批次100/150/200,仅用于有限维护模式;这些上限代码与实际配置均未调整,待用户统一确认。累计仍不超过1500万预留(含额外50条诊断),固定截止仍为2026-11-15 14:38:42 UTC(06:38:42 PST),不自动延长。生产入口不切换,客户业务正文、标注、归属和版本保持不变。

View File

@ -3,6 +3,7 @@ package store
import (
"context"
"errors"
"log"
"sort"
"time"
@ -29,6 +30,7 @@ type QueryReplica struct {
owner string
epoch int64
BatchSize int
readBatch map[string]int64
dirtyAfter map[string]any
reconcileNext int
safety *queryReplicaSafety
@ -240,9 +242,59 @@ func bsonEqual(a, b any) bool {
y, e := bson.Marshal(bson.M{"v": b})
return e == nil && string(x) == string(y)
}
// Retry source reads only: each attempt has the original deadline and a new
// durable reservation. Failed rows are discarded, never checkpointed or acked.
// Writes, fencing, unsafe health and unsupported records are not retried here.
func (q *QueryReplica) readLimit(n string) int64 {
limit := q.limit()
if adaptive := q.readBatch[n]; adaptive > 0 && adaptive < limit {
return adaptive
}
return limit
}
func retryableReplicaRead(ctx context.Context, err error) bool {
if ctx.Err() != nil || errors.Is(err, context.Canceled) {
return false
}
d := QueryReplicaDiagnostic(err)
if d.Phase != "source_find" && d.Phase != "source_cursor" {
return false
}
return errors.Is(err, context.DeadlineExceeded) || mongo.IsTimeout(err) || mongo.IsNetworkError(err)
}
func (q *QueryReplica) read(ctx context.Context, n string, filter bson.M) ([]queryReplicaRecord, any, error) {
limit := q.readLimit(n)
for attempt := 0; ; attempt++ {
rows, last, err := q.readAttempt(ctx, n, filter, limit)
if err == nil {
if q.readBatch == nil {
q.readBatch = map[string]int64{}
}
q.readBatch[n] = limit
return rows, last, nil
}
if attempt >= 3 || !retryableReplicaRead(ctx, err) {
return nil, nil, err
}
if limit > 1 {
limit /= 2
}
d := QueryReplicaDiagnostic(err)
log.Printf("projector source read retry collection=%s phase=%s class=%s mongoCode=%d attempt=%d batch=%d", d.Collection, d.Phase, d.Class, d.MongoCode, attempt+2, limit)
timer := time.NewTimer(time.Duration(100*(1<<attempt)) * time.Millisecond)
select {
case <-ctx.Done():
timer.Stop()
return nil, nil, ctx.Err()
case <-timer.C:
}
}
}
func (q *QueryReplica) readAttempt(ctx context.Context, n string, filter bson.M, requested int64) ([]queryReplicaRecord, any, error) {
parent := ctx
limit, e := q.sourcePermit(ctx, q.limit())
limit, e := q.sourcePermit(ctx, requested)
if e != nil {
return nil, nil, queryReplicaFailure(n, "source_permit", e)
}
@ -350,7 +402,7 @@ func (q *QueryReplica) DrainStep(ctx context.Context, n string) (int, error) {
}
rows, last, e := q.read(ctx, n, filter)
if e == nil {
if int64(len(rows)) < q.limit() {
if int64(len(rows)) < q.readLimit(n) {
delete(q.dirtyAfter, n)
} else {
q.dirtyAfter[n] = last

View File

@ -66,3 +66,31 @@ func TestQueryReplicaDiagnosticPreservesCauseWithoutLeaking(t *testing.T) {
t.Fatal("uncontrolled diagnostic labels")
}
}
func TestQueryReplicaReadRetryOnlyTransientReads(t *testing.T) {
cases := []struct {
phase string
cause error
retry bool
}{
{"source_find", context.DeadlineExceeded, true},
{"source_cursor", mongo.CommandError{Code: 50}, true},
{"source_find", mongo.CommandError{Code: 13}, false},
{"source_find", context.Canceled, false},
{"source_permit", ErrQueryReplicaPaused, false},
{"source_ack", context.DeadlineExceeded, false},
{"target_apply", context.DeadlineExceeded, false},
{"source_export", ErrQueryReplicaUnsupported, false},
{"checkpoint", ErrQueryReplicaLease, false},
}
for _, c := range cases {
if got := retryableReplicaRead(context.Background(), queryReplicaFailure("runs", c.phase, c.cause)); got != c.retry {
t.Errorf("%s %T retry=%v", c.phase, c.cause, got)
}
}
ctx, cancel := context.WithCancel(context.Background())
cancel()
if retryableReplicaRead(ctx, queryReplicaFailure("runs", "source_find", context.DeadlineExceeded)) {
t.Fatal("retried caller cancellation")
}
}

View File

@ -8,6 +8,7 @@ import (
"go.mongodb.org/mongo-driver/event"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
"net"
"reflect"
"strings"
"sync/atomic"
@ -119,3 +120,238 @@ func TestQueryReplicaRunReferenceProjectionCompatibilityLocal(t *testing.T) {
})
}
}
// Exercise the real driver/network seam: an interrupted source read must retry
// from the same filter; no partially received rows may advance the checkpoint.
func TestQueryReplicaReadRecoversTransientDeadlineLocal(t *testing.T) {
source := newQuerySyncSource(t)
for i := 0; i < 100; i++ {
if _, err := source.db.Collection("runs").InsertOne(t.Context(), bson.M{"_id": fmt.Sprintf("run-%03d", i), "id": fmt.Sprintf("run-%03d", i), "status": "completed"}); err != nil {
t.Fatal(err)
}
}
var armed atomic.Bool
var finds atomic.Int32
var limits []int64
dialer := &replicaFaultDialer{armed: &armed}
monitor := &event.CommandMonitor{Started: func(_ context.Context, e *event.CommandStartedEvent) {
if e.CommandName == "find" {
limits = append(limits, e.Command.Lookup("limit").AsInt64())
if finds.Add(1) == 1 {
armed.Store(true)
}
}
}}
client, err := mongo.Connect(t.Context(), options.Client().ApplyURI("mongodb://127.0.0.1:27119/?directConnection=true").SetDialer(dialer).SetMaxPoolSize(1).SetRetryReads(false).SetTimeout(5*time.Second).SetMonitor(monitor))
if err != nil {
t.Fatal(err)
}
defer client.Disconnect(context.Background())
q := &QueryReplica{Source: NewMongoStore(client.Database(source.db.Name())), BatchSize: 100}
rows, last, err := q.read(t.Context(), "runs", bson.M{})
if err != nil {
t.Fatalf("single transient source deadline paused copy: %v", err)
}
if len(rows) != 50 || last != "run-049" || finds.Load() != 2 || !reflect.DeepEqual(limits, []int64{100, 50}) {
t.Fatalf("recovery must restart same filter with smaller batch: rows=%d last=%v finds=%d limits=%v", len(rows), last, finds.Load(), limits)
}
}
type replicaFaultDialer struct{ armed *atomic.Bool }
func (d *replicaFaultDialer) DialContext(ctx context.Context, network, address string) (net.Conn, error) {
c, e := (&net.Dialer{}).DialContext(ctx, network, address)
if e != nil {
return nil, e
}
return &replicaFaultConn{Conn: c, armed: d.armed}, nil
}
type replicaFaultConn struct {
net.Conn
armed *atomic.Bool
}
func (c *replicaFaultConn) Read(b []byte) (int, error) {
if c.armed.CompareAndSwap(true, false) {
return 0, context.DeadlineExceeded
}
return c.Conn.Read(b)
}
func TestQueryReplicaLargeResponseShrinksWithinFixedDeadlineLocal(t *testing.T) {
source := newQuerySyncSource(t)
docs := []any{}
artifacts := bson.A{}
for i := 0; i < 100; i++ {
artifacts = append(artifacts, bson.M{"id": fmt.Sprintf("artifact-%03d", i), "name": strings.Repeat("synthetic-query-name", 10)})
}
for i := 0; i < 100; i++ {
docs = append(docs, bson.M{"_id": fmt.Sprintf("run-%03d", i), "id": fmt.Sprintf("run-%03d", i), "status": "completed", "artifacts": artifacts})
}
if _, e := source.db.Collection("runs").InsertMany(t.Context(), docs); e != nil {
t.Fatal(e)
}
budget, _ := NewQueryReplicaTransport(256*1024, 8)
var finds atomic.Int32
monitor := &event.CommandMonitor{Started: func(_ context.Context, e *event.CommandStartedEvent) {
if e.CommandName == "find" {
finds.Add(1)
}
}}
client, e := mongo.Connect(t.Context(), options.Client().ApplyURI("mongodb://127.0.0.1:27119/?directConnection=true").SetMaxPoolSize(1).SetDialer(budget).SetRetryReads(false).SetCompressors([]string{}).SetTimeout(5*time.Second).SetMonitor(monitor))
if e != nil {
t.Fatal(e)
}
defer client.Disconnect(context.Background())
q := &QueryReplica{Source: NewMongoStore(client.Database(source.db.Name())), BatchSize: 100}
started := time.Now()
rows, last, e := q.read(t.Context(), "runs", bson.M{})
if e != nil {
t.Fatal(e)
}
if (len(rows) != 25 && len(rows) != 50) || last != fmt.Sprintf("run-%03d", len(rows)-1) || finds.Load() < 2 || finds.Load() > 3 || q.readLimit("runs") != int64(len(rows)) {
t.Fatalf("large response recovery rows=%d last=%v attempts=%d batch=%d", len(rows), last, finds.Load(), q.readLimit("runs"))
}
for _, r := range rows {
if len(r.Document["artifacts"].(bson.A)) != 100 {
t.Fatal("shrinking rows truncated query facts")
}
}
t.Logf("fixed5s attempts,256KiB/s,100→adaptive: recoveredRows=%d attempts=%d elapsed=%s", len(rows), finds.Load(), time.Since(started))
}
func TestQueryReplicaRetryBudgetAndCheckpointSurviveRestartLocal(t *testing.T) {
source, target, q := newLocalReplica(t)
q.BatchSize = 25
for i := 0; i < 20; i++ {
if _, e := source.db.Collection("runs").InsertOne(t.Context(), bson.M{"_id": fmt.Sprintf("run-%03d", i), "id": fmt.Sprintf("run-%03d", i), "status": "completed", "inputs": bson.M{"source": bson.M{"inlineValue": "keep-annotation"}}}); e != nil {
t.Fatal(e)
}
}
seedReplicaSafetyHealth(t, q)
p := safetyPolicyForTest()
if e := q.ConfigureSafety(t.Context(), p, ""); e != nil {
t.Fatal(e)
}
var armed atomic.Bool
var finds atomic.Int32
monitor := &event.CommandMonitor{Started: func(_ context.Context, e *event.CommandStartedEvent) {
if e.CommandName == "find" {
finds.Add(1)
armed.Store(true)
}
}}
client, e := mongo.Connect(t.Context(), options.Client().ApplyURI("mongodb://127.0.0.1:27119/?directConnection=true").SetDialer(&replicaFaultDialer{armed: &armed}).SetMaxPoolSize(1).SetRetryReads(false).SetMonitor(monitor))
if e != nil {
t.Fatal(e)
}
defer client.Disconnect(context.Background())
q.Source = NewMongoStore(client.Database(source.db.Name()))
if _, _, e = q.BootstrapStep(t.Context(), "runs"); e == nil || finds.Load() != 4 {
t.Fatalf("retries not bounded: err=%v attempts=%d", e, finds.Load())
}
cp, e := q.checkpoint(t.Context(), "runs")
if e != nil || cp.HasLast || cp.Complete {
t.Fatal("failed read moved checkpoint", e)
}
if n, _ := target.db.Collection("runs").CountDocuments(t.Context(), bson.M{}); n != 0 {
t.Fatal("failed read applied rows")
}
var state queryReplicaSafetyState
if e = q.control().FindOne(t.Context(), bson.M{"_id": "safety"}).Decode(&state); e != nil || state.Reserved != 46 {
t.Fatalf("attempts lost durable reservations: reserved=%d err=%v", state.Reserved, e)
}
if e = q.PauseSafety(t.Context(), "operation_error"); e != nil {
t.Fatal(e)
}
if e = q.control().FindOne(t.Context(), bson.M{"_id": "safety"}).Decode(&state); e != nil {
t.Fatal(e)
}
successor, e := NewQueryReplica(source, target)
if e != nil {
t.Fatal(e)
}
successor.BatchSize = 25
if _, e = q.control().UpdateOne(t.Context(), bson.M{"_id": "lease"}, bson.M{"$set": bson.M{"until": time.Time{}}}); e != nil {
t.Fatal(e)
}
if e = successor.Acquire(t.Context()); e != nil {
t.Fatal(e)
}
seedReplicaSafetyHealth(t, successor)
if e = successor.ConfigureSafety(t.Context(), p, state.ResumeToken); e != nil {
t.Fatal(e)
}
if _, n, e := successor.BootstrapStep(t.Context(), "runs"); e != nil || n != 20 {
t.Fatal("restart failed to replay original checkpoint", n, e)
}
if e = q.control().FindOne(t.Context(), bson.M{"_id": "safety"}).Decode(&state); e != nil || state.Reserved != 71 {
t.Fatal("restart reset budget", state.Reserved, e)
}
var original bson.M
if e = source.db.Collection("runs").FindOne(t.Context(), bson.M{"_id": "run-000"}).Decode(&original); e != nil || deletionDoc(deletionDoc(original["inputs"])["source"])["inlineValue"] != "keep-annotation" {
t.Fatal("source body changed", e)
}
}
// This is a local transport/majority-commit ceiling, not a production guarded
// run: the approved production safety caps are deliberately left unchanged.
func TestQueryReplicaSyntheticReadCommitThroughputLocal(t *testing.T) {
for _, rate := range []int{100, 200} {
t.Run(fmt.Sprint(rate), func(t *testing.T) {
source, target, q := newLocalReplica(t)
const total = 2000
docs := make([]any, 0, total)
for i := 0; i < total; i++ {
docs = append(docs, bson.M{"_id": fmt.Sprintf("physical-%06d", i), "id": fmt.Sprintf("logical-%06d", i), "organizationId": "org", "projectId": "project", "status": "completed", "captureDirectories": bson.A{fmt.Sprintf("capture-%06d/part0001", i)}, "inputs": bson.M{"source": bson.M{"inlineValue": "keep-original-private", "ref": bson.M{"captureUnit": fmt.Sprintf("capture-%06d/part0001", i)}}}})
}
if _, e := source.db.Collection("runs").InsertMany(t.Context(), docs); e != nil {
t.Fatal(e)
}
transport, _ := NewQueryReplicaTransport(256*1024, 8)
var replyBytes atomic.Int64
client, e := mongo.Connect(t.Context(), options.Client().ApplyURI("mongodb://127.0.0.1:27119/?directConnection=true").SetMaxPoolSize(1).SetDialer(transport).SetRetryReads(false).SetTimeout(5*time.Second).SetCompressors([]string{}).SetMonitor(&event.CommandMonitor{Succeeded: func(_ context.Context, e *event.CommandSucceededEvent) {
if e.CommandName == "find" {
replyBytes.Add(int64(len(e.Reply)))
}
}}))
if e != nil {
t.Fatal(e)
}
defer client.Disconnect(context.Background())
q.Source = NewMongoStore(client.Database(source.db.Name()))
q.BatchSize = 100
pacing := newQueryReplicaPacer(float64(rate))
copied := 0
started := time.Now()
for {
done, n, e := q.BootstrapStep(t.Context(), "runs")
if e != nil {
t.Fatal(e)
}
copied += n
if e = pacing.wait(t.Context(), float64(n)); e != nil {
t.Fatal(e)
}
if done {
break
}
}
elapsed := time.Since(started)
n, e := target.db.Collection("runs").CountDocuments(t.Context(), bson.M{})
if e != nil || n != total || copied != total {
t.Fatal("lost or duplicated records", n, copied, e)
}
var persisted bson.M
if e = target.db.Collection("runs").FindOne(t.Context(), bson.M{"_id": "logical-001999"}).Decode(&persisted); e != nil || persisted["_queryReplica"] == nil {
t.Fatal("majority committed fact missing", e)
}
effective := float64(copied) / elapsed.Seconds()
if rate == 200 && effective < 100 {
t.Fatalf("synthetic ceiling failed100/s: %.1f", effective)
}
t.Logf("synthetic thin-query rows only, fixed5s,256KiB/s,8commands/s,majority+j: configured=%d copied=%d seconds=%.3f netRowsPerSecond=%.2f replyBytes=%d productionSafetyEnabled=false productionCapacityAccepted=false", rate, copied, elapsed.Seconds(), effective, replyBytes.Load())
})
}
}

View File

@ -16,7 +16,7 @@ func (q *QueryReplica) backlogAge(ctx context.Context, n string, rows int) (time
var since time.Time
e := q.transaction(ctx, func(sc mongo.SessionContext) error {
key := "backlog/" + n
if rows < int(q.limit()) {
if rows < int(q.readLimit(n)) {
_, e := q.control().DeleteOne(sc, bson.M{"_id": key})
return e
}
@ -74,7 +74,7 @@ func (q *QueryReplica) runGuardedOnce(ctx context.Context, bootstrap bool) (bool
if e != nil {
return false, e
}
if n < int(q.limit()) {
if n < int(q.readLimit(name)) {
if _, e = q.backlogAge(ctx, name, n); e != nil {
return false, e
}