diff --git a/cmd/query-replica/main.go b/cmd/query-replica/main.go index d1d6a2f6..a5f30975 100644 --- a/cmd/query-replica/main.go +++ b/cmd/query-replica/main.go @@ -177,7 +177,7 @@ func main() { } opts := options.Client().ApplyURI(uri).SetMaxPoolSize(4).SetServerSelectionTimeout(5 * time.Second) if !target && transport != nil { - opts.SetMaxPoolSize(1).SetDialer(transport).SetCompressors([]string{}).SetTimeout(5 * time.Second) + opts.SetMaxPoolSize(1).SetDialer(transport).SetCompressors([]string{}).SetRetryReads(false).SetTimeout(5 * time.Second) } if target { opts.SetReadPreference(readpref.Primary()).SetReadConcern(readconcern.Majority()) diff --git a/docs/operations/query-replica.md b/docs/operations/query-replica.md index c7af7868..21242c05 100644 --- a/docs/operations/query-replica.md +++ b/docs/operations/query-replica.md @@ -246,3 +246,15 @@ Mongo 角色不能限制更新到某个字段:源账号按22个源集合授予 现有首轮授权仍限1000条读取预留、10条/秒、64KiB/秒、2命令/秒、批次25,不因 CPU 策略更改扩大。该预算包含空读和重复探测,不能完成约76.4万源记录的初始化;完整历史预算、构建器启动及故障对照须合并列明再确认。实现核验与集群实际部署分开记录,新增镜像按正式发布规则处理。 成品链路验收须区分:统计事实构建成功、特定权限/筛选成品发布完成、Valkey 命中和生产读取切换是不同状态。新筛选仅登记到查询库,由后台计算并发布;不得在缺失成品时前台扫描原库。缓存丢失读取查询库最后完整成品;查询库也不可用时明确暂不可用,不集中回源。初始化完成或 Pod 就绪均不代表已分流;最终降载只能在另行批准切读后以同条件证据确认。 + +## 有界读取重试与缩批(2026-11-15,本地候选) + +影子运行在升至100条/批后,于 `runs/source_find` 再次出现固定5秒读取超时;停于累计37015次预留、13914条事实。该现场没有足够信息区分服务端等待和传输等待,不将本地复现替代生产根因证明。本地真实驱动故障注入已复现一次读取失败直接结束;大查询事实夹具在256KiB/秒下也能触发同类读取超时。相同翻页表达式的两万条合成索引对照为100条结果、100个键/文档扫描,未证明存在从头扫描,不据此修改索引。 + +候选仅对 `source_find/source_cursor` 的网络错误和超时最多尝试4次,每次从相同条件重新读取、批量减半,等待依次为100/200/400毫秒。同一进程内该集合沿用成功的小批量,其他集合保持原批量;重启后的初始批量仍受既有配置限制。失败批次不应用、不确认、不推进断点。每次尝试独立消耗持久读取预留;关闭守护客户端的驱动隐式读取重试,避免遗漏计数。每次源操作5秒和服务端1500毫秒、目标事务 `majority+journal` 保持不变。 + +取消、安全健康、租约、权限、不支持的历史格式以及源确认/目标写入错误不进入此重试路径。最多4次仍失败则沿原持久暂停机制退出,不跳过记录、不自动更换run或重置预算。脏批次及积压判断按该集合的有效批量计算,避免缩批后误判已排空。 + +本机合成数据测得:保留256KiB/秒和8命令/秒,2000条精简运行事实的读取与多数提交,100条/秒节奏约76条/秒净吞吐,200条/秒节奏约123条/秒净吞吐。该测试没有启用生产安全策略,也未包含22集合的完整调度、历史大小分布、源确认和完整后台统计,不能宣称生产已达到100或200条/秒。大响应夹具由100条缩至25条后成功,单次5秒限制未延长。生产同断点恢复及吞吐仍待批准候选发布后验证。 + +当前生产限速和截止时间保持原授权值。拟提速档位100/150/200条/秒、1/2/4MiB/秒、16/24/32命令/秒,批次100/150/200,仅用于有限维护模式;这些上限代码与实际配置均未调整,待用户统一确认。累计仍不超过1500万预留(含额外50条诊断),固定截止仍为2026-11-15 14:38:42 UTC(06:38:42 PST),不自动延长。生产入口不切换,客户业务正文、标注、归属和版本保持不变。 diff --git a/internal/store/query_replica.go b/internal/store/query_replica.go index b3418f12..1fb8ccf0 100644 --- a/internal/store/query_replica.go +++ b/internal/store/query_replica.go @@ -3,6 +3,7 @@ package store import ( "context" "errors" + "log" "sort" "time" @@ -29,6 +30,7 @@ type QueryReplica struct { owner string epoch int64 BatchSize int + readBatch map[string]int64 dirtyAfter map[string]any reconcileNext int safety *queryReplicaSafety @@ -240,9 +242,59 @@ func bsonEqual(a, b any) bool { y, e := bson.Marshal(bson.M{"v": b}) return e == nil && string(x) == string(y) } + +// Retry source reads only: each attempt has the original deadline and a new +// durable reservation. Failed rows are discarded, never checkpointed or acked. +// Writes, fencing, unsafe health and unsupported records are not retried here. +func (q *QueryReplica) readLimit(n string) int64 { + limit := q.limit() + if adaptive := q.readBatch[n]; adaptive > 0 && adaptive < limit { + return adaptive + } + return limit +} +func retryableReplicaRead(ctx context.Context, err error) bool { + if ctx.Err() != nil || errors.Is(err, context.Canceled) { + return false + } + d := QueryReplicaDiagnostic(err) + if d.Phase != "source_find" && d.Phase != "source_cursor" { + return false + } + return errors.Is(err, context.DeadlineExceeded) || mongo.IsTimeout(err) || mongo.IsNetworkError(err) +} func (q *QueryReplica) read(ctx context.Context, n string, filter bson.M) ([]queryReplicaRecord, any, error) { + limit := q.readLimit(n) + for attempt := 0; ; attempt++ { + rows, last, err := q.readAttempt(ctx, n, filter, limit) + if err == nil { + if q.readBatch == nil { + q.readBatch = map[string]int64{} + } + q.readBatch[n] = limit + return rows, last, nil + } + if attempt >= 3 || !retryableReplicaRead(ctx, err) { + return nil, nil, err + } + if limit > 1 { + limit /= 2 + } + d := QueryReplicaDiagnostic(err) + log.Printf("projector source read retry collection=%s phase=%s class=%s mongoCode=%d attempt=%d batch=%d", d.Collection, d.Phase, d.Class, d.MongoCode, attempt+2, limit) + timer := time.NewTimer(time.Duration(100*(1< 3 || q.readLimit("runs") != int64(len(rows)) { + t.Fatalf("large response recovery rows=%d last=%v attempts=%d batch=%d", len(rows), last, finds.Load(), q.readLimit("runs")) + } + for _, r := range rows { + if len(r.Document["artifacts"].(bson.A)) != 100 { + t.Fatal("shrinking rows truncated query facts") + } + } + t.Logf("fixed5s attempts,256KiB/s,100→adaptive: recoveredRows=%d attempts=%d elapsed=%s", len(rows), finds.Load(), time.Since(started)) +} + +func TestQueryReplicaRetryBudgetAndCheckpointSurviveRestartLocal(t *testing.T) { + source, target, q := newLocalReplica(t) + q.BatchSize = 25 + for i := 0; i < 20; i++ { + if _, e := source.db.Collection("runs").InsertOne(t.Context(), bson.M{"_id": fmt.Sprintf("run-%03d", i), "id": fmt.Sprintf("run-%03d", i), "status": "completed", "inputs": bson.M{"source": bson.M{"inlineValue": "keep-annotation"}}}); e != nil { + t.Fatal(e) + } + } + seedReplicaSafetyHealth(t, q) + p := safetyPolicyForTest() + if e := q.ConfigureSafety(t.Context(), p, ""); e != nil { + t.Fatal(e) + } + var armed atomic.Bool + var finds atomic.Int32 + monitor := &event.CommandMonitor{Started: func(_ context.Context, e *event.CommandStartedEvent) { + if e.CommandName == "find" { + finds.Add(1) + armed.Store(true) + } + }} + client, e := mongo.Connect(t.Context(), options.Client().ApplyURI("mongodb://127.0.0.1:27119/?directConnection=true").SetDialer(&replicaFaultDialer{armed: &armed}).SetMaxPoolSize(1).SetRetryReads(false).SetMonitor(monitor)) + if e != nil { + t.Fatal(e) + } + defer client.Disconnect(context.Background()) + q.Source = NewMongoStore(client.Database(source.db.Name())) + if _, _, e = q.BootstrapStep(t.Context(), "runs"); e == nil || finds.Load() != 4 { + t.Fatalf("retries not bounded: err=%v attempts=%d", e, finds.Load()) + } + cp, e := q.checkpoint(t.Context(), "runs") + if e != nil || cp.HasLast || cp.Complete { + t.Fatal("failed read moved checkpoint", e) + } + if n, _ := target.db.Collection("runs").CountDocuments(t.Context(), bson.M{}); n != 0 { + t.Fatal("failed read applied rows") + } + var state queryReplicaSafetyState + if e = q.control().FindOne(t.Context(), bson.M{"_id": "safety"}).Decode(&state); e != nil || state.Reserved != 46 { + t.Fatalf("attempts lost durable reservations: reserved=%d err=%v", state.Reserved, e) + } + if e = q.PauseSafety(t.Context(), "operation_error"); e != nil { + t.Fatal(e) + } + if e = q.control().FindOne(t.Context(), bson.M{"_id": "safety"}).Decode(&state); e != nil { + t.Fatal(e) + } + successor, e := NewQueryReplica(source, target) + if e != nil { + t.Fatal(e) + } + successor.BatchSize = 25 + if _, e = q.control().UpdateOne(t.Context(), bson.M{"_id": "lease"}, bson.M{"$set": bson.M{"until": time.Time{}}}); e != nil { + t.Fatal(e) + } + if e = successor.Acquire(t.Context()); e != nil { + t.Fatal(e) + } + seedReplicaSafetyHealth(t, successor) + if e = successor.ConfigureSafety(t.Context(), p, state.ResumeToken); e != nil { + t.Fatal(e) + } + if _, n, e := successor.BootstrapStep(t.Context(), "runs"); e != nil || n != 20 { + t.Fatal("restart failed to replay original checkpoint", n, e) + } + if e = q.control().FindOne(t.Context(), bson.M{"_id": "safety"}).Decode(&state); e != nil || state.Reserved != 71 { + t.Fatal("restart reset budget", state.Reserved, e) + } + var original bson.M + if e = source.db.Collection("runs").FindOne(t.Context(), bson.M{"_id": "run-000"}).Decode(&original); e != nil || deletionDoc(deletionDoc(original["inputs"])["source"])["inlineValue"] != "keep-annotation" { + t.Fatal("source body changed", e) + } +} + +// This is a local transport/majority-commit ceiling, not a production guarded +// run: the approved production safety caps are deliberately left unchanged. +func TestQueryReplicaSyntheticReadCommitThroughputLocal(t *testing.T) { + for _, rate := range []int{100, 200} { + t.Run(fmt.Sprint(rate), func(t *testing.T) { + source, target, q := newLocalReplica(t) + const total = 2000 + docs := make([]any, 0, total) + for i := 0; i < total; i++ { + docs = append(docs, bson.M{"_id": fmt.Sprintf("physical-%06d", i), "id": fmt.Sprintf("logical-%06d", i), "organizationId": "org", "projectId": "project", "status": "completed", "captureDirectories": bson.A{fmt.Sprintf("capture-%06d/part0001", i)}, "inputs": bson.M{"source": bson.M{"inlineValue": "keep-original-private", "ref": bson.M{"captureUnit": fmt.Sprintf("capture-%06d/part0001", i)}}}}) + } + if _, e := source.db.Collection("runs").InsertMany(t.Context(), docs); e != nil { + t.Fatal(e) + } + transport, _ := NewQueryReplicaTransport(256*1024, 8) + var replyBytes atomic.Int64 + client, e := mongo.Connect(t.Context(), options.Client().ApplyURI("mongodb://127.0.0.1:27119/?directConnection=true").SetMaxPoolSize(1).SetDialer(transport).SetRetryReads(false).SetTimeout(5*time.Second).SetCompressors([]string{}).SetMonitor(&event.CommandMonitor{Succeeded: func(_ context.Context, e *event.CommandSucceededEvent) { + if e.CommandName == "find" { + replyBytes.Add(int64(len(e.Reply))) + } + }})) + if e != nil { + t.Fatal(e) + } + defer client.Disconnect(context.Background()) + q.Source = NewMongoStore(client.Database(source.db.Name())) + q.BatchSize = 100 + pacing := newQueryReplicaPacer(float64(rate)) + copied := 0 + started := time.Now() + for { + done, n, e := q.BootstrapStep(t.Context(), "runs") + if e != nil { + t.Fatal(e) + } + copied += n + if e = pacing.wait(t.Context(), float64(n)); e != nil { + t.Fatal(e) + } + if done { + break + } + } + elapsed := time.Since(started) + n, e := target.db.Collection("runs").CountDocuments(t.Context(), bson.M{}) + if e != nil || n != total || copied != total { + t.Fatal("lost or duplicated records", n, copied, e) + } + var persisted bson.M + if e = target.db.Collection("runs").FindOne(t.Context(), bson.M{"_id": "logical-001999"}).Decode(&persisted); e != nil || persisted["_queryReplica"] == nil { + t.Fatal("majority committed fact missing", e) + } + effective := float64(copied) / elapsed.Seconds() + if rate == 200 && effective < 100 { + t.Fatalf("synthetic ceiling failed100/s: %.1f", effective) + } + t.Logf("synthetic thin-query rows only, fixed5s,256KiB/s,8commands/s,majority+j: configured=%d copied=%d seconds=%.3f netRowsPerSecond=%.2f replyBytes=%d productionSafetyEnabled=false productionCapacityAccepted=false", rate, copied, elapsed.Seconds(), effective, replyBytes.Load()) + }) + } +} diff --git a/internal/store/query_replica_schedule.go b/internal/store/query_replica_schedule.go index cc50db8e..1dd92356 100644 --- a/internal/store/query_replica_schedule.go +++ b/internal/store/query_replica_schedule.go @@ -16,7 +16,7 @@ func (q *QueryReplica) backlogAge(ctx context.Context, n string, rows int) (time var since time.Time e := q.transaction(ctx, func(sc mongo.SessionContext) error { key := "backlog/" + n - if rows < int(q.limit()) { + if rows < int(q.readLimit(n)) { _, e := q.control().DeleteOne(sc, bson.M{"_id": key}) return e } @@ -74,7 +74,7 @@ func (q *QueryReplica) runGuardedOnce(ctx context.Context, bootstrap bool) (bool if e != nil { return false, e } - if n < int(q.limit()) { + if n < int(q.readLimit(name)) { if _, e = q.backlogAge(ctx, name, n); e != nil { return false, e }