🛡️ Pace query replica production backfills
Some checks are pending
web-ci / checks (push) Waiting to run
web-ci / pool-search-mongo (push) Waiting to run
web-ci / work-availability (push) Blocked by required conditions
web-ci / query-serving (push) Blocked by required conditions

This commit is contained in:
d-robotics 2026-11-14 02:21:01 -08:00
parent c0512c8bf6
commit 1a6e61909e
5 changed files with 81 additions and 4 deletions

View File

@ -23,7 +23,12 @@ func main() {
writers := flag.Bool("writers-covered", false, "attest all source writers use atomic query-source/v1 markers")
once := flag.Bool("once", false, "one bounded pass only")
interval := flag.Duration("interval", time.Second, "minimum delay between bounded passes")
batchSize := flag.Int("batch-size", 200, "maximum source records per collection batch (1-200)")
minimumPass := flag.Duration("min-pass-interval", 0, "optional minimum start-to-start pass interval, including backfill (0-1m)")
flag.Parse()
if !validPacing(*batchSize, *minimumPass) {
log.Fatal("invalid bounded projector pacing")
}
mode := os.Getenv("QUERY_REPLICA_REFRESH_MODE")
if mode != "" && mode != "continuous" && mode != "periodic" {
log.Fatal("invalid query refresh mode")
@ -66,6 +71,7 @@ func main() {
if e != nil {
log.Fatal("invalid projector configuration")
}
q.BatchSize = *batchSize
if q.VerifyTopology(ctx) != nil {
log.Fatal("source/query databases are not independently usable")
}
@ -79,6 +85,7 @@ func main() {
log.Fatal("projector lease unavailable")
}
for ctx.Err() == nil {
started := time.Now()
ready, e := q.RunOnce(ctx, *bootstrap)
if e != nil {
log.Print("projector paused; checkpoint retained (no customer payload logged)")
@ -91,10 +98,10 @@ func main() {
}
return
}
// Drain bounded backlog immediately, then idle without repeated source
// scans. Keep the original lease alive only in the independent query DB.
if ready || mode != "periodic" {
if e := q.WaitNextPass(ctx, *interval); e != nil {
// Optional pacing covers backfill too. Idle lease maintenance only
// touches the independent query DB, never scans authority for keepalive.
if delay := passDelay(mode, ready, time.Since(started), *interval, *minimumPass); delay > 0 {
if e := q.WaitNextPass(ctx, delay); e != nil {
if ctx.Err() == nil {
log.Print("projector lease unavailable; checkpoint retained")
os.Exit(1)

View File

@ -0,0 +1,25 @@
package main
import "time"
// The default preserves immediate periodic backlog draining. A production
// backfill can explicitly pace every pass without changing query timeouts,
// transaction durability, or the projector's renewable lease.
func validPacing(batch int, minimum time.Duration) bool {
return batch >= 1 && batch <= 200 && minimum >= 0 && minimum <= time.Minute
}
func passDelay(mode string, ready bool, elapsed, interval, minimum time.Duration) time.Duration {
var delay time.Duration
if ready || mode != "periodic" {
delay = interval
}
if remaining := minimum - elapsed; remaining > delay {
delay = remaining
}
// WaitNextPass has a 100ms lower bound and preserves lease heartbeats.
if delay > 0 && delay < 100*time.Millisecond {
return 100 * time.Millisecond
}
return delay
}

View File

@ -0,0 +1,42 @@
package main
import (
"testing"
"time"
)
func TestPassDelayPreservesDefaultsAndPacesBackfill(t *testing.T) {
for _, tc := range []struct {
name, mode string
ready bool
elapsed time.Duration
minimum time.Duration
want time.Duration
}{
{"default backlog", "periodic", false, time.Second, 0, 0},
{"default idle", "periodic", true, time.Second, 0, time.Minute},
{"legacy continuous", "continuous", false, time.Second, 0, time.Minute},
{"paced bootstrap", "periodic", false, time.Second, 5 * time.Second, 4 * time.Second},
{"slow pass already paid budget", "periodic", false, 6 * time.Second, 5 * time.Second, 0},
{"small remaining budget", "periodic", false, 4999 * time.Millisecond, 5 * time.Second, 100 * time.Millisecond},
{"idle cadence retained", "periodic", true, time.Second, 5 * time.Second, time.Minute},
} {
t.Run(tc.name, func(t *testing.T) {
if got := passDelay(tc.mode, tc.ready, tc.elapsed, time.Minute, tc.minimum); got != tc.want {
t.Fatalf("delay=%s want=%s", got, tc.want)
}
})
}
}
func TestPacingRejectsUnboundedOrInvalidConfiguration(t *testing.T) {
for _, tc := range []struct {
batch int
min time.Duration
want bool
}{{200, 0, true}, {1, time.Minute, true}, {50, 5 * time.Second, true}, {0, 0, false}, {201, 0, false}, {50, -1, false}, {50, time.Minute + 1, false}} {
if got := validPacing(tc.batch, tc.min); got != tc.want {
t.Fatalf("batch=%d minimum=%s valid=%v want=%v", tc.batch, tc.min, got, tc.want)
}
}
}

View File

@ -37,6 +37,8 @@
投影进程:`query-replica --writers-covered --bootstrap --interval=1s`。`--writers-covered` 是完成写者盘点后的操作声明,不会自动升级旧作业。每集合每次最多200条、连接池4,投影/确认可重放;查询库租约代次使旧进程不能继续发布。历史复制是一次性、可恢复的读取;每6小时对每集合启动一轮限速摘要对账,每次只处理一个批次。
原库高负载时,生产影子回填应显式设置 `--batch-size=50 --min-pass-interval=5s`。前者限制每集合批次,后者限制每轮起点间隔,覆盖周期模式尚未追平时的连续排空;一轮执行本身已超过5秒时不额外等待。默认200条、零额外间隔仍兼容既有行为。该配置不是字节/CPU硬限额,一轮仍可能读取每个集合的历史和脏批次,必须配合负载观测与外部停止控制。等待继续使用查询库租约续期,不延长任何查询超时、事务或写入确认预算。
新代码只保存白名单字段,维持原 `productionstats`、`workforcestats`、名称匹配和历史回退。保留原标注和正式意见;作业记录正文按选中的正式记录 ID 回原库读取。不能安全复制的 URL/名称不作为“成功迁移”忽略,须先处理兼容项,或保持该路径开关关闭。
## 原库索引与写入成本

View File

@ -1047,6 +1047,7 @@
"^TestQueryReplicaRejectsSigned"
],
"./cmd/query-cdc-probe": ["^TestClassification"],
"./cmd/query-replica": ["^TestPassDelay", "^TestPacing"],
"./cmd/query-lab-bridge": ["^TestIsolatedBridge"],
"./internal/testsupport": ["^TestQueryReplicaTarget"]
},