🩹 Recover oversized legacy replica reads within fixed deadlines

This commit is contained in:
d-robotics 2026-11-15 01:31:55 -08:00
parent d4ecc071c8
commit 7624987768
10 changed files with 245 additions and 9 deletions

View File

@ -177,7 +177,7 @@ func main() {
}
opts := options.Client().ApplyURI(uri).SetMaxPoolSize(4).SetServerSelectionTimeout(5 * time.Second)
if !target && transport != nil {
opts.SetMaxPoolSize(1).SetDialer(transport).SetCompressors([]string{}).SetRetryReads(false).SetTimeout(5 * time.Second)
opts.SetMaxPoolSize(1).SetDialer(transport).SetCompressors(sourceCompressors(*maintenance)).SetRetryReads(false).SetTimeout(5 * time.Second)
}
if target {
opts.SetReadPreference(readpref.Primary()).SetReadConcern(readconcern.Majority())
@ -285,7 +285,11 @@ func main() {
}
d := store.QueryReplicaDiagnostic(e)
log.Printf("projector operation failed collection=%s phase=%s class=%s mongoCode=%d", d.Collection, d.Phase, d.Class, d.MongoCode)
pause("operation_error")
short, stop := context.WithTimeout(context.Background(), 5*time.Second)
if q.PauseSafetyFailure(short, e) != nil {
log.Print("pause diagnostics persistence unavailable; startup fencing remains required")
}
stop()
log.Print("projector paused; checkpoint retained (no customer payload logged)")
if *safetyRun != "" {
<-ctx.Done()
@ -323,3 +327,12 @@ func main() {
<-ctx.Done()
}
}
// Source compression changes only the finite maintenance transport. The driver
// negotiates supported codecs and falls back to uncompressed messages.
func sourceCompressors(maintenance bool) []string {
if maintenance {
return []string{"snappy", "zstd"}
}
return []string{}
}

View File

@ -0,0 +1,15 @@
package main
import (
"reflect"
"testing"
)
func TestSourceCompressionOnlyFiniteMaintenance(t *testing.T) {
if got := sourceCompressors(false); len(got) != 0 {
t.Fatal("ordinary source transport changed", got)
}
if got := sourceCompressors(true); !reflect.DeepEqual(got, []string{"snappy", "zstd"}) {
t.Fatal("maintenance codecs", got)
}
}

View File

@ -269,3 +269,18 @@ Mongo 角色不能限制更新到某个字段:源账号按22个源集合授予
- 已经clean的源记录不发送空确认更新;真正dirty仍按修订号CAS确认,目标版本领先时的旧标记修复路径优先保留。源业务正文不修改。
- 维护模式的行节奏计入本批实际读取耗时,避免读取后再次等待相同时间;不积累空闲额度,字节和命令节流不变,普通模式保留原节奏。
- 最终计划需单列初始化`--min-pass-interval=0s`,移除额外的5秒轮次等待;全局限速仍生效,追平后的`--interval=60s`不变。该配置尚未在线上执行。
### humanItems 旧载荷传输恢复(3052cc600167 诊断后,候选未部署)
只读诊断的断点后150条记录:扫描150条,执行约78ms;兼容投影中的单份旧 `subject.payload` 达7,343,668字节。完整旧载荷暂用于兼容嵌套来源字段,提取后不写入查询事实。该证据指向大响应与限速/固定5秒期限冲突;原失败 Pod 日志未保留,不能把推断当作原异常分类已确认。诊断根文档尺寸总计因命令参数的 `$$` 展开无效,不用于容量判断,字段尺寸及执行计划有效。
候选仅调整影子读取和诊断:
- 仍最多4次、同一条件重读,批量先减半、最后一次固定1条;失败不应用、不确认、不推进断点,每次预留持久计入预算。取消、权限、租约和写入异常不扩大重试。
- 字节节流按已批准速率使用约8ms的有界读取块(1–64KiB),每次读取前仍预留字节,无空闲额度。避免高速档仍每1KiB调度一次计时器造成实际吞吐不足。
- 仅有限维护初始化协商 snappy/zstd,服务器不支持时走未压缩协议和上述回退;普通同步仍不请求压缩。保留全部兼容提取规则,源操作5秒、服务端1500ms和目标 majority+journal 不变。编码带来的源CPU成本仍需在维护小批观察。
- 持久暂停保存固定枚举的集合、阶段、异常类别、Mongo代码和时间,不存原始错误、查询、正文或连接;后续 operator_stop 不覆盖原始故障或重置断点/预算。
隔离合成样本:两个各7MiB的旧载荷,原固定批量重试失败;单条回退成功。维护压缩读取相同两条,完整解码约14MiB、实际传输696,286字节、约634ms,查询事实及顺序与未压缩结果一致。此为代表性样本,不是所有历史数据的压缩率或生产吞吐承诺。
3052cc600167 临时 Job 已完成并回收。诊断预留600次加之前50次,合计650;投影器累计98,659,合并累计99,309。后续投影器上限改为14,999,350,总预算仍1500万,固定截止仍2026-11-15T14:38:42Z,不退款、不重置、不延期。影子投影器及构建器保持0副本,生产入口不切换,候选上线另列最终不可变计划确认。

View File

@ -278,7 +278,11 @@ func (q *QueryReplica) read(ctx context.Context, n string, filter bson.M) ([]que
if attempt >= 3 || !retryableReplicaRead(ctx, err) {
return nil, nil, err
}
if limit > 1 {
// The final bounded retry must fit one legacy document, even when
// earlier batch reductions still exceed the fixed wire deadline.
if attempt == 2 {
limit = 1
} else if limit > 1 {
limit /= 2
}
d := QueryReplicaDiagnostic(err)

View File

@ -259,7 +259,7 @@ func TestQueryReplicaRetryBudgetAndCheckpointSurviveRestartLocal(t *testing.T) {
t.Fatal("failed read applied rows")
}
var state queryReplicaSafetyState
if e = q.control().FindOne(t.Context(), bson.M{"_id": "safety"}).Decode(&state); e != nil || state.Reserved != 46 {
if e = q.control().FindOne(t.Context(), bson.M{"_id": "safety"}).Decode(&state); e != nil || state.Reserved != 44 {
t.Fatalf("attempts lost durable reservations: reserved=%d err=%v", state.Reserved, e)
}
if e = q.PauseSafety(t.Context(), "operation_error"); e != nil {
@ -286,7 +286,7 @@ func TestQueryReplicaRetryBudgetAndCheckpointSurviveRestartLocal(t *testing.T) {
if _, n, e := successor.BootstrapStep(t.Context(), "runs"); e != nil || n != 20 {
t.Fatal("restart failed to replay original checkpoint", n, e)
}
if e = q.control().FindOne(t.Context(), bson.M{"_id": "safety"}).Decode(&state); e != nil || state.Reserved != 71 {
if e = q.control().FindOne(t.Context(), bson.M{"_id": "safety"}).Decode(&state); e != nil || state.Reserved != 69 {
t.Fatal("restart reset budget", state.Reserved, e)
}
var original bson.M
@ -506,3 +506,104 @@ func TestQueryReplicaCleanBootstrapDoesNotAcknowledgeSourceLocal(t *testing.T) {
t.Fatal("wrong source ACK or business change", e)
}
}
func TestQueryReplicaLargeLegacyHumanItemFallsBackToSingleLocal(t *testing.T) {
source := newQuerySyncSource(t)
payload := strings.Repeat("synthetic-legacy-annotation-", (7<<20)/len("synthetic-legacy-annotation-"))
for i := 0; i < 2; i++ {
if _, e := source.db.Collection("humanItems").InsertOne(t.Context(), bson.M{"_id": fmt.Sprintf("legacy-%d", i), "id": fmt.Sprintf("legacy-%d", i), "organizationId": "org", "version": int64(1), "subject": bson.M{"itemKey": "capture/part0001", "payload": bson.M{"captureDirectory": "capture/part0001", "annotationDraft": payload}}}); e != nil {
t.Fatal(e)
}
}
transport, _ := NewQueryReplicaTransport(2<<20, 24)
var finds atomic.Int32
client, e := mongo.Connect(t.Context(), options.Client().ApplyURI("mongodb://127.0.0.1:27119/?directConnection=true").SetMaxPoolSize(1).SetDialer(transport).SetRetryReads(false).SetCompressors([]string{}).SetTimeout(5*time.Second).SetMonitor(&event.CommandMonitor{Started: func(_ context.Context, e *event.CommandStartedEvent) {
if e.CommandName == "find" {
finds.Add(1)
}
}}))
if e != nil {
t.Fatal(e)
}
defer client.Disconnect(context.Background())
q := &QueryReplica{Source: NewMongoStore(client.Database(source.db.Name())), BatchSize: 150}
started := time.Now()
rows, last, e := q.read(t.Context(), "humanItems", bson.M{})
if e != nil {
t.Fatalf("valid7MiB legacy payload still blocks bounded4attempt copy: %v", e)
}
if len(rows) != 1 || last != "legacy-0" || finds.Load() != 4 || q.readLimit("humanItems") != 1 {
t.Fatalf("single-row fallback lost order/bounds: rows=%d last=%v finds=%d limit=%d", len(rows), last, finds.Load(), q.readLimit("humanItems"))
}
raw, _ := bson.Marshal(rows[0].Document)
if strings.Contains(string(raw), "synthetic-legacy-annotation") {
t.Fatal("legacy draft leaked into query fact")
}
t.Logf("7MiB legacy payload, fixed5s attempts,2MiB/s,4attempts: recovered1 elapsed=%s", time.Since(started))
}
func TestQueryReplicaCompressedLegacyHumanItemsEquivalentLocal(t *testing.T) {
source := newQuerySyncSource(t)
payload := strings.Repeat("synthetic-legacy-annotation-", (7<<20)/len("synthetic-legacy-annotation-"))
for i := 0; i < 2; i++ {
id := fmt.Sprintf("compressed-%d", i)
if _, e := source.db.Collection("humanItems").InsertOne(t.Context(), bson.M{"_id": id, "id": id, "organizationId": "org", "version": int64(1), "subject": bson.M{"itemKey": "capture/part0001", "payload": bson.M{"captureDirectory": "capture/part0001", "annotationDraft": payload}}}); e != nil {
t.Fatal(e)
}
}
plain := &QueryReplica{Source: source, BatchSize: 150}
want, lastWant, err := plain.read(t.Context(), "humanItems", bson.M{})
if err != nil {
t.Fatal(err)
}
transport, _ := NewQueryReplicaTransport(2<<20, 24)
wire := &legacyWireCountingDialer{inner: transport}
var replies atomic.Int64
client, err := mongo.Connect(t.Context(), options.Client().ApplyURI("mongodb://127.0.0.1:27119/?directConnection=true").SetMaxPoolSize(1).SetDialer(wire).SetRetryReads(false).SetCompressors([]string{"snappy", "zstd"}).SetTimeout(5*time.Second).SetMonitor(&event.CommandMonitor{Succeeded: func(_ context.Context, e *event.CommandSucceededEvent) {
if e.CommandName == "find" {
replies.Add(int64(len(e.Reply)))
}
}}))
if err != nil {
t.Fatal(err)
}
defer client.Disconnect(context.Background())
q := &QueryReplica{Source: NewMongoStore(client.Database(source.db.Name())), BatchSize: 150}
started := time.Now()
got, last, err := q.read(t.Context(), "humanItems", bson.M{})
elapsed := time.Since(started)
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(want, got) || !reflect.DeepEqual(lastWant, last) || len(got) != 2 || q.readLimit("humanItems") != 150 {
t.Fatal("compression changed facts/order or required retry")
}
if elapsed >= 5*time.Second || replies.Load() < 14<<20 || wire.bytes.Load() > 1<<20 {
t.Fatal("large legacy transfer not verified", elapsed, replies.Load())
}
t.Logf("two7MiB legacy rows equivalent, decodedBytes=%d wireBytes=%d elapsed=%s fixed5s, source unchanged", replies.Load(), wire.bytes.Load(), elapsed)
}
type legacyWireCountingDialer struct {
inner *QueryReplicaTransport
bytes atomic.Int64
}
func (d *legacyWireCountingDialer) DialContext(ctx context.Context, network, address string) (net.Conn, error) {
c, e := d.inner.DialContext(ctx, network, address)
if e != nil {
return nil, e
}
return &legacyWireCountingConn{Conn: c, bytes: &d.bytes}, nil
}
type legacyWireCountingConn struct {
net.Conn
bytes *atomic.Int64
}
func (c *legacyWireCountingConn) Read(p []byte) (int, error) {
n, e := c.Conn.Read(p)
c.bytes.Add(int64(n))
return n, e
}

View File

@ -319,6 +319,17 @@ func (q *QueryReplica) sourcePermit(ctx context.Context, want int64) (int64, err
return allowed, nil
}
func (q *QueryReplica) PauseSafety(ctx context.Context, reason string) error {
return q.pauseSafety(ctx, reason, nil)
}
// PauseSafetyFailure persists only the existing whitelisted diagnostic fields;
// never the raw error, customer payload, query or connection string.
func (q *QueryReplica) PauseSafetyFailure(ctx context.Context, cause error) error {
d := QueryReplicaDiagnostic(cause)
return q.pauseSafety(ctx, "operation_error", bson.M{"at": time.Now().UTC(), "collection": d.Collection, "phase": d.Phase, "class": d.Class, "mongoCode": d.MongoCode})
}
func (q *QueryReplica) pauseSafety(ctx context.Context, reason string, failure bson.M) error {
if q.safety == nil {
return nil
}
@ -329,7 +340,11 @@ func (q *QueryReplica) PauseSafety(ctx context.Context, reason string) error {
reason = "operation_error"
}
return q.transaction(ctx, func(sc mongo.SessionContext) error {
_, e := q.control().UpdateOne(sc, bson.M{"_id": "safety", "paused": false}, bson.M{"$set": bson.M{"paused": true, "reason": reason, "resumeToken": domain.NewID("shadow-resume")}})
update := bson.M{"paused": true, "reason": reason, "resumeToken": domain.NewID("shadow-resume")}
if failure != nil {
update["lastFailure"] = failure
}
_, e := q.control().UpdateOne(sc, bson.M{"_id": "safety", "paused": false}, bson.M{"$set": update})
if e != nil {
return e
}
@ -358,7 +373,7 @@ func (q *QueryReplica) sourceContext(ctx context.Context) (context.Context, cont
// remain mandatory. It is never emitted by the running projector's logs.
func (s *MongoStore) QueryReplicaSafetyStatus(ctx context.Context) (bson.M, error) {
var state bson.M
e := s.db.Collection(queryReplicaControl).FindOne(ctx, bson.M{"_id": "safety"}, options.FindOne().SetProjection(bson.M{"_id": 0, "policy": 1, "reserved": 1, "paused": 1, "reason": 1, "resumeToken": 1, "lastObservation": 1, "high": 1})).Decode(&state)
e := s.db.Collection(queryReplicaControl).FindOne(ctx, bson.M{"_id": "safety"}, options.FindOne().SetProjection(bson.M{"_id": 0, "policy": 1, "reserved": 1, "paused": 1, "reason": 1, "resumeToken": 1, "lastObservation": 1, "lastFailure": 1, "high": 1})).Decode(&state)
return state, e
}

View File

@ -7,6 +7,8 @@ import (
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/event"
"go.mongodb.org/mongo-driver/mongo/options"
"reflect"
"strings"
"testing"
"time"
)
@ -522,3 +524,39 @@ func TestQueryReplicaHealthReplayAndOutOfOrderLocal(t *testing.T) {
t.Fatal("replay duplicated sample or changed result", e, state)
}
}
func TestQueryReplicaDurableFailureSummarySurvivesOperatorStopLocal(t *testing.T) {
_, _, q := newLocalReplica(t)
seedReplicaSafetyHealth(t, q)
if e := q.ConfigureSafety(t.Context(), safetyPolicyForTest(), ""); e != nil {
t.Fatal(e)
}
const private = "mongodb://private-user:private-password@private-host/customer?signature=private-token"
cause := queryReplicaFailure("humanItems", "source_find", errors.Join(errors.New(private), context.DeadlineExceeded))
if e := q.PauseSafetyFailure(t.Context(), cause); e != nil {
t.Fatal(e)
}
before, e := q.Target.QueryReplicaSafetyStatus(t.Context())
if e != nil {
t.Fatal(e)
}
failure := deletionDoc(before["lastFailure"])
if failure["collection"] != "humanItems" || failure["phase"] != "source_find" || failure["class"] != "deadline" || before["paused"] != true {
t.Fatal("safe failure absent")
}
if e := q.PauseSafety(t.Context(), "operator_stop"); e != nil {
t.Fatal(e)
}
after, e := q.Target.QueryReplicaSafetyStatus(t.Context())
if e != nil {
t.Fatal(e)
}
b, _ := bson.Marshal(after)
// BSON map order is unspecified; compare fields rather than bytes.
if !reflect.DeepEqual(before, after) {
t.Fatal("operator stop replaced original cause, nonce or budget")
}
if strings.Contains(string(b), "private-") || strings.Contains(string(b), "signature") {
t.Fatal("raw error leaked")
}
}

View File

@ -243,3 +243,33 @@ func TestQueryReplicaReadPacingCountsWorkWithoutIdleCredit(t *testing.T) {
t.Fatal("idle time accumulated unreviewed row credit")
}
}
// Higher rates must not incur a timer/goroutine for every fixed 1KiB read.
func TestQueryReplicaTransportHighRateKeepsByteBudget(t *testing.T) {
a, b := net.Pipe()
defer a.Close()
defer b.Close()
budget, _ := NewQueryReplicaTransport(2<<20, 24)
c := &queryReplicaRateConn{Conn: a, budget: budget, closed: make(chan struct{}), wake: make(chan struct{})}
defer c.Close()
payload := make([]byte, 256<<10)
go func() { _, _ = b.Write(payload) }()
started := time.Now()
reads := 0
for n := 0; n < len(payload); {
count, err := c.Read(payload[n:])
if err != nil {
t.Fatal(err)
}
n += count
reads++
}
elapsed := time.Since(started)
if reads > 16 {
t.Fatal("excessive read scheduling", reads)
}
if elapsed < 125*time.Millisecond {
t.Fatal("byte budget bypassed", elapsed)
}
t.Logf("256KiB/2MiBps read calls=%d elapsed=%s", reads, elapsed)
}

View File

@ -151,8 +151,12 @@ func (c *queryReplicaRateConn) Read(p []byte) (int, error) {
if len(p) == 0 {
return 0, nil
}
if len(p) > 1024 {
p = p[:1024]
// Keep approximately 8ms pacing quanta at higher approved rates.
// Fixed 1KiB chunks otherwise spend more time scheduling timers than
// reading. Bytes remain reserved before every socket read.
chunk := max(1024, min(64*1024, int(c.budget.bytes.rate/128)))
if len(p) > chunk {
p = p[:chunk]
}
if e := c.wait(c.budget.bytes, len(p), true); e != nil {
return 0, e

View File

@ -16,6 +16,7 @@ ROOT = Path(__file__).resolve().parents[1]
PRIVATE = ROOT / 'local-data/query-replica-local'
COMPOSE = ROOT / 'scripts/local/query-replica.compose.yml'
CASES = ('cmd/query-replica/main_local_test.go',
'cmd/query-replica/source_transport_test.go',
'internal/store/query_replica_source_auth_local_test.go',
'internal/store/query_replica_safety_local_test.go',
'internal/store/query_replica_maintenance_flow_local_test.go',