🛡️ Pace query replica production backfills
This commit is contained in:
parent
c0512c8bf6
commit
1a6e61909e
@ -23,7 +23,12 @@ func main() {
|
||||
writers := flag.Bool("writers-covered", false, "attest all source writers use atomic query-source/v1 markers")
|
||||
once := flag.Bool("once", false, "one bounded pass only")
|
||||
interval := flag.Duration("interval", time.Second, "minimum delay between bounded passes")
|
||||
batchSize := flag.Int("batch-size", 200, "maximum source records per collection batch (1-200)")
|
||||
minimumPass := flag.Duration("min-pass-interval", 0, "optional minimum start-to-start pass interval, including backfill (0-1m)")
|
||||
flag.Parse()
|
||||
if !validPacing(*batchSize, *minimumPass) {
|
||||
log.Fatal("invalid bounded projector pacing")
|
||||
}
|
||||
mode := os.Getenv("QUERY_REPLICA_REFRESH_MODE")
|
||||
if mode != "" && mode != "continuous" && mode != "periodic" {
|
||||
log.Fatal("invalid query refresh mode")
|
||||
@ -66,6 +71,7 @@ func main() {
|
||||
if e != nil {
|
||||
log.Fatal("invalid projector configuration")
|
||||
}
|
||||
q.BatchSize = *batchSize
|
||||
if q.VerifyTopology(ctx) != nil {
|
||||
log.Fatal("source/query databases are not independently usable")
|
||||
}
|
||||
@ -79,6 +85,7 @@ func main() {
|
||||
log.Fatal("projector lease unavailable")
|
||||
}
|
||||
for ctx.Err() == nil {
|
||||
started := time.Now()
|
||||
ready, e := q.RunOnce(ctx, *bootstrap)
|
||||
if e != nil {
|
||||
log.Print("projector paused; checkpoint retained (no customer payload logged)")
|
||||
@ -91,10 +98,10 @@ func main() {
|
||||
}
|
||||
return
|
||||
}
|
||||
// Drain bounded backlog immediately, then idle without repeated source
|
||||
// scans. Keep the original lease alive only in the independent query DB.
|
||||
if ready || mode != "periodic" {
|
||||
if e := q.WaitNextPass(ctx, *interval); e != nil {
|
||||
// Optional pacing covers backfill too. Idle lease maintenance only
|
||||
// touches the independent query DB, never scans authority for keepalive.
|
||||
if delay := passDelay(mode, ready, time.Since(started), *interval, *minimumPass); delay > 0 {
|
||||
if e := q.WaitNextPass(ctx, delay); e != nil {
|
||||
if ctx.Err() == nil {
|
||||
log.Print("projector lease unavailable; checkpoint retained")
|
||||
os.Exit(1)
|
||||
|
||||
25
cmd/query-replica/pacing.go
Normal file
25
cmd/query-replica/pacing.go
Normal file
@ -0,0 +1,25 @@
|
||||
package main
|
||||
|
||||
import "time"
|
||||
|
||||
// The default preserves immediate periodic backlog draining. A production
|
||||
// backfill can explicitly pace every pass without changing query timeouts,
|
||||
// transaction durability, or the projector's renewable lease.
|
||||
func validPacing(batch int, minimum time.Duration) bool {
|
||||
return batch >= 1 && batch <= 200 && minimum >= 0 && minimum <= time.Minute
|
||||
}
|
||||
|
||||
func passDelay(mode string, ready bool, elapsed, interval, minimum time.Duration) time.Duration {
|
||||
var delay time.Duration
|
||||
if ready || mode != "periodic" {
|
||||
delay = interval
|
||||
}
|
||||
if remaining := minimum - elapsed; remaining > delay {
|
||||
delay = remaining
|
||||
}
|
||||
// WaitNextPass has a 100ms lower bound and preserves lease heartbeats.
|
||||
if delay > 0 && delay < 100*time.Millisecond {
|
||||
return 100 * time.Millisecond
|
||||
}
|
||||
return delay
|
||||
}
|
||||
42
cmd/query-replica/pacing_test.go
Normal file
42
cmd/query-replica/pacing_test.go
Normal file
@ -0,0 +1,42 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestPassDelayPreservesDefaultsAndPacesBackfill(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name, mode string
|
||||
ready bool
|
||||
elapsed time.Duration
|
||||
minimum time.Duration
|
||||
want time.Duration
|
||||
}{
|
||||
{"default backlog", "periodic", false, time.Second, 0, 0},
|
||||
{"default idle", "periodic", true, time.Second, 0, time.Minute},
|
||||
{"legacy continuous", "continuous", false, time.Second, 0, time.Minute},
|
||||
{"paced bootstrap", "periodic", false, time.Second, 5 * time.Second, 4 * time.Second},
|
||||
{"slow pass already paid budget", "periodic", false, 6 * time.Second, 5 * time.Second, 0},
|
||||
{"small remaining budget", "periodic", false, 4999 * time.Millisecond, 5 * time.Second, 100 * time.Millisecond},
|
||||
{"idle cadence retained", "periodic", true, time.Second, 5 * time.Second, time.Minute},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
if got := passDelay(tc.mode, tc.ready, tc.elapsed, time.Minute, tc.minimum); got != tc.want {
|
||||
t.Fatalf("delay=%s want=%s", got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestPacingRejectsUnboundedOrInvalidConfiguration(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
batch int
|
||||
min time.Duration
|
||||
want bool
|
||||
}{{200, 0, true}, {1, time.Minute, true}, {50, 5 * time.Second, true}, {0, 0, false}, {201, 0, false}, {50, -1, false}, {50, time.Minute + 1, false}} {
|
||||
if got := validPacing(tc.batch, tc.min); got != tc.want {
|
||||
t.Fatalf("batch=%d minimum=%s valid=%v want=%v", tc.batch, tc.min, got, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -37,6 +37,8 @@
|
||||
|
||||
投影进程:`query-replica --writers-covered --bootstrap --interval=1s`。`--writers-covered` 是完成写者盘点后的操作声明,不会自动升级旧作业。每集合每次最多200条、连接池4,投影/确认可重放;查询库租约代次使旧进程不能继续发布。历史复制是一次性、可恢复的读取;每6小时对每集合启动一轮限速摘要对账,每次只处理一个批次。
|
||||
|
||||
原库高负载时,生产影子回填应显式设置 `--batch-size=50 --min-pass-interval=5s`。前者限制每集合批次,后者限制每轮起点间隔,覆盖周期模式尚未追平时的连续排空;一轮执行本身已超过5秒时不额外等待。默认200条、零额外间隔仍兼容既有行为。该配置不是字节/CPU硬限额,一轮仍可能读取每个集合的历史和脏批次,必须配合负载观测与外部停止控制。等待继续使用查询库租约续期,不延长任何查询超时、事务或写入确认预算。
|
||||
|
||||
新代码只保存白名单字段,维持原 `productionstats`、`workforcestats`、名称匹配和历史回退。保留原标注和正式意见;作业记录正文按选中的正式记录 ID 回原库读取。不能安全复制的 URL/名称不作为“成功迁移”忽略,须先处理兼容项,或保持该路径开关关闭。
|
||||
|
||||
## 原库索引与写入成本
|
||||
|
||||
@ -1047,6 +1047,7 @@
|
||||
"^TestQueryReplicaRejectsSigned"
|
||||
],
|
||||
"./cmd/query-cdc-probe": ["^TestClassification"],
|
||||
"./cmd/query-replica": ["^TestPassDelay", "^TestPacing"],
|
||||
"./cmd/query-lab-bridge": ["^TestIsolatedBridge"],
|
||||
"./internal/testsupport": ["^TestQueryReplicaTarget"]
|
||||
},
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user