🛡️ Add durable budgets and bounded recovery for shadow synchronization

This commit is contained in:
d-robotics 2026-11-14 07:41:18 -08:00
parent 1a6e61909e
commit f73665cfa0
20 changed files with 2377 additions and 35 deletions

View File

@ -19,6 +19,7 @@ import (
func main() {
once := flag.Bool("once", false, "build once; streaming freshness expires after exit")
ensure := flag.Bool("ensure-indexes", false, "explicitly create derived-data indexes before building")
shadowRun := flag.String("shadow-safety-run", "", "opt-in independently monitored shadow builder; no authority gates")
flag.Parse()
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
defer stop()
@ -43,25 +44,58 @@ func main() {
log.Fatal("builder accelerator configuration invalid")
}
defer closeServing()
workCtx := ctx
if *shadowRun != "" {
guarded, cancel := context.WithCancel(ctx)
defer cancel()
workCtx = guarded
if s.CheckQueryReplicaSafety(ctx, *shadowRun) != nil {
log.Print("shadow builder safety paused")
<-ctx.Done()
return
}
go func() {
ticker := time.NewTicker(5 * time.Second)
defer ticker.Stop()
for {
select {
case <-guarded.Done():
return
case <-ticker.C:
short, stop := context.WithTimeout(guarded, 2*time.Second)
err := s.CheckQueryReplicaSafety(short, *shadowRun)
stop()
if err != nil {
cancel()
return
}
}
}
}()
}
if *ensure {
if s.EnsureQueryReadModelIndexes(ctx) != nil {
if s.EnsureQueryReadModelIndexes(workCtx) != nil {
log.Fatal("derived index creation failed")
}
}
b := store.NewQueryReadBuilder(s)
defer b.Close()
if *once {
if b.RunOnce(ctx) != nil {
if b.RunOnce(workCtx) != nil {
log.Fatal("initial materialization incomplete")
}
return
}
for ctx.Err() == nil {
if b.Run(ctx) != nil && ctx.Err() == nil {
for workCtx.Err() == nil {
if b.Run(workCtx) != nil && workCtx.Err() == nil {
log.Print("query builder paused; retrying with full reconciliation")
}
select {
case <-ctx.Done():
case <-workCtx.Done():
if ctx.Err() == nil {
log.Print("shadow builder safety paused; explicit operator restart required")
<-ctx.Done()
}
return
case <-time.After(5 * time.Second):
}

View File

@ -4,12 +4,17 @@ package main
import (
"context"
"crypto/sha256"
"droboticflow/internal/store"
"encoding/json"
"errors"
"flag"
"fmt"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
"go.mongodb.org/mongo-driver/mongo/readconcern"
"go.mongodb.org/mongo-driver/mongo/readpref"
"io"
"log"
"os"
"os/signal"
@ -25,7 +30,104 @@ func main() {
interval := flag.Duration("interval", time.Second, "minimum delay between bounded passes")
batchSize := flag.Int("batch-size", 200, "maximum source records per collection batch (1-200)")
minimumPass := flag.Duration("min-pass-interval", 0, "optional minimum start-to-start pass interval, including backfill (0-1m)")
safetyRun := flag.String("safety-run", "", "explicit durable shadow run identity; requires global transport and health budgets")
maxRecords := flag.Int64("max-source-records", 1000, "durable total read reservations;0 explicitly permits full backfill")
rowsRate := flag.Int("source-records-per-second", 10, "global source row pacing")
bytesRate := flag.Int("source-bytes-per-second", 65536, "shared actual source transport read budget")
commandsRate := flag.Int("source-commands-per-second", 2, "shared source socket-write budget, includes control overhead")
baseline := flag.Float64("source-cpu-baseline", 0, "fixed reviewed4core source CPU baseline, never increased on resume")
resume := flag.String("resume-pause", "", "exact current pause nonce after operator review")
initializeTarget := flag.Bool("initialize-target", false, "verify independent topology and bind target only; no source scans or indexes")
status := flag.Bool("safety-status", false, "target-only private safety status")
sampleSource := flag.Bool("sample-source", false, "readonly server counters for trusted observer, no customer records")
observation := flag.String("record-observation", "", "trusted observer JSON file or stdin '-'; target-only, never connects to source")
flag.Parse()
if *sampleSource {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
if os.Getenv("MONGO_URI") == "" {
log.Fatal("source binding required")
}
c, e := mongo.Connect(ctx, options.Client().ApplyURI(os.Getenv("MONGO_URI")).SetMaxPoolSize(1))
if e != nil {
log.Fatal("source unavailable")
}
defer c.Disconnect(context.Background())
var info struct {
Uptime int64 `bson:"uptimeMillis"`
Extra struct {
User int64 `bson:"user_time_us"`
System int64 `bson:"system_time_us"`
} `bson:"extra_info"`
}
if c.Database("admin").RunCommand(ctx, map[string]any{"serverStatus": 1}).Decode(&info) != nil {
log.Fatal("source counters unavailable")
}
var hello struct {
Topology struct {
Process any `bson:"processId"`
} `bson:"topologyVersion"`
}
if c.Database("admin").RunCommand(ctx, map[string]any{"hello": 1}).Decode(&hello) != nil {
log.Fatal("source identity unavailable")
}
identity := sha256.Sum256([]byte(fmt.Sprint(hello.Topology.Process)))
_ = json.NewEncoder(os.Stdout).Encode(map[string]any{"uptimeMillis": info.Uptime, "cpuMicroseconds": info.Extra.User + info.Extra.System, "sourceProcess": fmt.Sprintf("%x", identity), "sampledAt": time.Now().UTC().Format(time.RFC3339Nano)})
return
}
if *observation != "" || *status {
var value store.QueryReplicaSafetyObservation
if !*status {
input := io.Reader(os.Stdin)
if *observation != "-" {
f, e := os.Open(*observation)
if e != nil {
log.Fatal("observation unavailable")
}
defer f.Close()
input = f
}
decoder := json.NewDecoder(io.LimitReader(input, 16384))
decoder.DisallowUnknownFields()
if decoder.Decode(&value) != nil {
log.Fatal("invalid observation")
}
}
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
uri, db := os.Getenv("QUERY_READ_MONGO_URI"), os.Getenv("QUERY_READ_MONGO_DB")
if uri == "" || db == "" || os.Getenv("QUERY_SERVING_NAMESPACE") == "" {
log.Fatal("explicit independent target required")
}
c, e := mongo.Connect(ctx, options.Client().ApplyURI(uri).SetMaxPoolSize(1))
if e != nil {
log.Fatal("target unavailable")
}
defer c.Disconnect(context.Background())
if *status {
value, e := store.NewMongoStore(c.Database(db)).QueryReplicaSafetyStatus(ctx)
if e != nil {
log.Fatal("safety state unavailable")
}
_ = json.NewEncoder(os.Stdout).Encode(value)
return
}
if store.NewMongoStore(c.Database(db)).RecordQueryReplicaObservation(ctx, value) != nil {
log.Fatal("observation rejected")
}
return
}
var transport *store.QueryReplicaTransport
if *safetyRun != "" {
var e error
transport, e = store.NewQueryReplicaTransport(*bytesRate, *commandsRate)
if e != nil || *ensure {
log.Fatal("invalid guarded source configuration")
}
} else if *resume != "" {
log.Fatal("resume requires durable safety")
}
if !validPacing(*batchSize, *minimumPass) {
log.Fatal("invalid bounded projector pacing")
}
@ -44,6 +146,9 @@ func main() {
*interval = store.QueryPeriodicSyncInterval
}
}
if *initializeTarget && *ensure {
log.Fatal("source index writes forbidden during target binding")
}
if !*writers || *interval < 100*time.Millisecond || *interval > 5*time.Minute || os.Getenv("QUERY_SERVING_NAMESPACE") == "" {
log.Fatal("writer coverage, namespace and bounded polling interval required")
}
@ -54,6 +159,9 @@ func main() {
log.Fatal("explicit source/query connection required")
}
opts := options.Client().ApplyURI(uri).SetMaxPoolSize(4).SetServerSelectionTimeout(5 * time.Second)
if !target && transport != nil {
opts.SetMaxPoolSize(1).SetDialer(transport).SetCompressors([]string{}).SetTimeout(5 * time.Second)
}
if target {
opts.SetReadPreference(readpref.Primary()).SetReadConcern(readconcern.Majority())
}
@ -78,21 +186,90 @@ func main() {
if *ensure && q.EnsureSourceIndexes(ctx) != nil {
log.Fatal("source index preparation failed")
}
if q.EnsureTargetIndexes(ctx) != nil {
log.Fatal("query index preparation failed")
for {
err := q.Acquire(ctx)
if err == nil {
break
}
if *safetyRun == "" || !errors.Is(err, store.ErrQueryReplicaLease) {
log.Fatal("projector lease unavailable")
}
select {
case <-ctx.Done():
return
case <-time.After(time.Second):
}
}
if q.Acquire(ctx) != nil {
log.Fatal("projector lease unavailable")
if *initializeTarget {
return
}
for ctx.Err() == nil {
if *safetyRun != "" {
if q.ConfigureSafety(ctx, store.QueryReplicaSafetyPolicy{BytesPerSecond: *bytesRate, CommandsPerSecond: *commandsRate, MaxBatchRows: *batchSize, RunID: *safetyRun, MaxRecords: *maxRecords, RecordsPerSecond: *rowsRate, BaselineCPU: *baseline}, *resume) != nil {
log.Print("shadow safety paused; inspect durable state and explicitly resume")
<-ctx.Done()
return
}
}
workCtx := ctx
if *safetyRun != "" {
guarded, cancel := context.WithCancel(ctx)
defer cancel()
workCtx = guarded
go func() {
tick := time.NewTicker(5 * time.Second)
defer tick.Stop()
for {
select {
case <-guarded.Done():
return
case <-tick.C:
short, stop := context.WithTimeout(guarded, 3*time.Second)
err := q.CheckSafety(short)
stop()
if err != nil {
cancel()
return
}
}
}
}()
}
pause := func(reason string) {
short, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
if q.PauseSafety(short, reason) != nil {
log.Print("pause persistence unavailable; startup fencing remains required")
}
}
defer func() {
if ctx.Err() != nil {
pause("operator_stop")
}
}()
if q.EnsureTargetIndexes(workCtx) != nil {
pause("target_error")
log.Print("query index preparation paused")
if *safetyRun == "" {
os.Exit(1)
}
<-ctx.Done()
return
}
for workCtx.Err() == nil {
started := time.Now()
ready, e := q.RunOnce(ctx, *bootstrap)
ready, e := q.RunOnce(workCtx, *bootstrap)
if e != nil {
pause("operation_error")
log.Print("projector paused; checkpoint retained (no customer payload logged)")
if *safetyRun != "" {
<-ctx.Done()
return
}
os.Exit(1)
}
if *once {
if !ready {
pause("incomplete_pilot")
log.Print("projection incomplete; resume required")
os.Exit(2)
}
@ -101,13 +278,22 @@ func main() {
// Optional pacing covers backfill too. Idle lease maintenance only
// touches the independent query DB, never scans authority for keepalive.
if delay := passDelay(mode, ready, time.Since(started), *interval, *minimumPass); delay > 0 {
if e := q.WaitNextPass(ctx, delay); e != nil {
if e := q.WaitNextPass(workCtx, delay); e != nil {
if ctx.Err() == nil {
pause("target_error")
log.Print("projector lease unavailable; checkpoint retained")
if *safetyRun != "" {
<-ctx.Done()
return
}
os.Exit(1)
}
return
}
}
}
if *safetyRun != "" && ctx.Err() == nil {
pause("operation_error")
<-ctx.Done()
}
}

View File

@ -0,0 +1,149 @@
package main
import (
"context"
"droboticflow/internal/domain"
"droboticflow/internal/store"
"encoding/json"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
"io"
"os"
"os/exec"
"path/filepath"
"testing"
"time"
)
func TestProjectorGuardedProcessRestartLocal(t *testing.T) {
if os.Getenv("QUERY_REPLICA_LOCAL_TEST") != "1" {
t.Skip("isolated source/query lab required")
}
ctx := t.Context()
sourceClient, e := mongo.Connect(ctx, options.Client().ApplyURI("mongodb://127.0.0.1:27119/?directConnection=true"))
if e != nil {
t.Fatal(e)
}
defer sourceClient.Disconnect(context.Background())
targetClient, e := mongo.Connect(ctx, options.Client().ApplyURI("mongodb://127.0.0.1:27129/?replicaSet=query-read-rs"))
if e != nil {
t.Fatal(e)
}
defer targetClient.Disconnect(context.Background())
source, target := sourceClient.Database(domain.NewID("query_sync_cmd")), targetClient.Database(domain.NewID("query_replica_cmd"))
t.Cleanup(func() { _ = source.Drop(context.Background()); _ = target.Drop(context.Background()) })
s, d := store.NewMongoStore(source), store.NewMongoStore(target)
q, e := store.NewQueryReplica(s, d)
if e != nil {
t.Fatal(e)
}
if e = q.EnsureSourceIndexes(ctx); e != nil {
t.Fatal(e)
}
if e = q.EnsureTargetIndexes(ctx); e != nil {
t.Fatal(e)
}
for _, id := range []string{"a", "b", "c"} {
_, e = source.Collection("annotationIndexJobs").InsertOne(ctx, bson.M{"_id": id, "id": id, "status": "completed", "organizationId": "org", "privateDraft": "not projected"})
if e != nil {
t.Fatal(e)
}
}
binary := filepath.Join(t.TempDir(), "projector")
if e = exec.Command("go", "build", "-o", binary, ".").Run(); e != nil {
t.Fatal("local projector build failed")
}
now := time.Now().UTC()
recent := []store.QueryReplicaSafetyObservation{}
for _, ago := range []time.Duration{60 * time.Second, 30 * time.Second, 0} {
recent = append(recent, store.QueryReplicaSafetyObservation{At: now.Add(-ago), CPU: 85, Ready: true, WritersCovered: true, HostFreeBytes: 120 << 30, TargetUsedFraction: .1})
}
raw, _ := bson.Marshal(recent[2])
var fields bson.M
_ = bson.Unmarshal(raw, &fields)
fields["recent"] = recent
_, e = target.Collection("queryReplicaControl").UpdateOne(ctx, bson.M{"_id": "safety-health"}, bson.M{"$set": fields}, options.Update().SetUpsert(true))
if e != nil {
t.Fatal(e)
}
env := []string{}
for _, entry := range os.Environ() {
if len(entry) >= 6 && (entry[:6] == "MONGO_" || len(entry) >= 6 && entry[:6] == "QUERY_") {
continue
}
env = append(env, entry)
}
env = append(env, "MONGO_URI=mongodb://127.0.0.1:27119/?directConnection=true", "MONGO_DB="+source.Name(), "QUERY_READ_MONGO_URI=mongodb://127.0.0.1:27129/?replicaSet=query-read-rs", "QUERY_READ_MONGO_DB="+target.Name(), "QUERY_SERVING_NAMESPACE=local:"+source.Name())
args := []string{"--writers-covered", "--bootstrap", "--batch-size=1", "--safety-run=process-test", "--max-source-records=2", "--source-records-per-second=50", "--source-bytes-per-second=262144", "--source-commands-per-second=8", "--source-cpu-baseline=85"}
start := func() (*exec.Cmd, <-chan error) {
t.Helper()
p := exec.Command(binary, args...)
p.Env = env
p.Stdout = io.Discard
p.Stderr = io.Discard
if e = p.Start(); e != nil {
t.Fatal(e)
}
done := make(chan error, 1)
go func() { done <- p.Wait() }()
t.Cleanup(func() {
_ = p.Process.Kill()
})
return p, done
}
child, done := start()
var state bson.M
deadline := time.Now().Add(15 * time.Second)
for time.Now().Before(deadline) {
_ = target.Collection("queryReplicaControl").FindOne(ctx, bson.M{"_id": "safety"}).Decode(&state)
if state["paused"] == true {
break
}
select {
case <-done:
t.Fatal("guarded process exited instead of parking")
default:
}
time.Sleep(50 * time.Millisecond)
}
if state["paused"] != true || state["reserved"] != int64(2) {
t.Fatal("process bypassed total budget", state)
}
_ = child.Process.Kill()
<-done
_, _ = target.Collection("queryReplicaControl").UpdateOne(ctx, bson.M{"_id": "lease"}, bson.M{"$set": bson.M{"until": time.Time{}}})
next, finished := start()
time.Sleep(600 * time.Millisecond)
select {
case <-finished:
t.Fatal("paused restart should park")
default:
}
var after bson.M
_ = target.Collection("queryReplicaControl").FindOne(ctx, bson.M{"_id": "safety"}).Decode(&after)
if after["reserved"] != state["reserved"] || after["resumeToken"] != state["resumeToken"] {
t.Fatal("process restart reset durable state")
}
_ = next.Process.Kill()
<-finished
// Operator status mode cannot connect to source, even with an invalid URI.
status := exec.Command(binary, "--safety-status")
status.Env = append(env, "MONGO_URI=mongodb://127.0.0.1:1/?directConnection=true")
out, e := status.Output()
if e != nil {
t.Fatal("target-only operator mode failed")
}
var v map[string]any
if json.Unmarshal(out, &v) != nil || v["paused"] != true {
t.Fatal("operator status invalid")
}
count, e := source.Collection("annotationIndexJobs").CountDocuments(ctx, bson.M{"privateDraft": "not projected"})
if e != nil || count != 3 {
t.Fatal("source payload changed")
}
var copied bson.M
if e = target.Collection("annotationIndexJobs").FindOne(ctx, bson.M{}).Decode(&copied); e != nil || copied["privateDraft"] != nil {
t.Fatal("projection boundary failed", e)
}
}

View File

@ -195,3 +195,37 @@ v5的广范围搜索及冷暖读取对照记录于 `final-products-pool-v5`;
最终集成报告:`local-data/test-results/verify-1794636683705497000/report.json`,源码未在运行中变化、无未映射文件。41项中39项通过,包含隔离副本/缓存/Mongo专项、源码校验不入队、标注/切片浏览器、搜索及面板、生产前端构建和静态启动。API整包原始检查仍报告已有失败,最终官方Go基线门禁确认68项均在70项既有基线中,无新增;store整包PASS,但默认跳过的外部环境用例标为未完整覆盖,不算通过。专项显式启用的本机副本/Valkey测试已执行通过,未扩大 known-failures。
`.4` 测后资源申请总量约6.475/7.8核(83%)、内存11,813Mi(42%),包含实验查询Pod;低实际使用率不能替代调度申请余量。后续继续扩核须重新核对该节点及生产峰值。当前仍是限时实验Pod,保留原自动结束机制和25Gi盘;后续恢复验证按已记录UID、配置及回退步骤重建,不将该单Pod直接当生产查询库。
## 影子同步全局预算与持久暂停(方案 8e8abb0cb64f)
本节替代上文仅按集合批次及轮次间隔控制的生产影子启动建议;旧命令保留兼容,但不得直接用于此次生产启动。当前仅完成本地实现与合成数据验证,未启动生产回填、未切换生产查询入口。
守护模式只作用于显式带 `--safety-run` 的独立投影器及带 `--shadow-safety-run` 的构建器,不进入保存、切片、送检、质检或最终验收准入。权威权限与版本判断不变。
| 项目 | 首轮候选值及行为 |
| --- | --- |
| 总读取预算 | `--max-source-records=1000`,读取前持久预留。空读、重复读及崩溃也可能消耗;不是已迁移条数。预算不能因重启、换 run ID 清零 |
| 全局节奏 | `--source-records-per-second=10 --source-bytes-per-second=65536 --source-commands-per-second=2 --batch-size=25`;共享源连接节流,包含握手/心跳开销,不按22个集合分别给额度 |
| 持久性 | 查询库事务 `majority+journal`,先提交投影/断点再 CAS 确认源标记,不降低写入确认,不延长超时 |
| 单次读取 | 源查询 `maxTimeMS=1500`,整个源操作5秒上限;大回复无法在预算内完成时暂停,不放开时限 |
| 源 CPU | 三个间隔30秒的有效样本,跨度至少60秒,CPU均≤90%、波动≤5个百分点才允许开始;基线固定,恢复不得抬高 |
| 停止 | 源 CPU≥95%一次,或比固定基线高3个百分点连续两个独立样本;受保护 Pod IO full avg10>1%连续两次或>5%一次;客户作业异常/延迟回退、健康缺失或超过45秒、写者变化、目标盘≥70%、宿主剩余<100GiB均暂停 |
| 调度 | 脏数据优先,正常每次历史批次前先排一个脏批次;积压超过5分钟时增加最多三个脏批次。按集合持久轮转,完整历史完成前不做全量审计,完成后首次审计延后6小时 |
| 恢复 | 私有控制表保存原因、预算、当前恢复令牌;进程异常重启转为暂停。仅显式提供当前 `--resume-pause`、相同 run ID、评审后的预算和健康窗口才能恢复。不会自行轮询恢复 |
源端修改集中在受控元数据接口,只允许同步标记及私有删除意图。过期删除声明的取消与清理按有界 ID 批次处理,不重放业务删除,不回写业务正文、版本、状态、归属或标注。若预算不足以确认所有待删除目标 ID,暂停并保留目标,不把尚未探测的记录当成已删除。
Mongo 角色不能限制更新到某个字段:源账号按22个源集合授予 find/update,私有删除意图集合 find/insert/update,另授 serverStatus;禁止 remove、DDL、跨库访问。字段白名单由应用强制,不能宣称源账号“只读”。构建器没有源凭证。
### 最终启动前逐项确认
1. 将本地改动核验合入并同步 `dev` 后,通过正式发布门禁,单独批准镜像上传及实际启动计划。绑定不可变镜像、二进制 SHA、两个命名空间 UID、五个写者 UID/镜像/提交/副本、影子控制器 UID、Mongo Pod UID、来源端点摘要和目标库;核对所有生产读取开关仍关闭。不要直接执行旧方案里的启动命令。
2. 核验源47个等价索引与最小权限角色,准备独立源 Secret。Secret 配额8、源网络出口、查询库4核8Gi/WiredTiger 2Gi/25Gi容量和 loop/底层 IO 双重限额保持现行已批准值。新增账号/Secret及私有同步元数据写入须列入实际启动授权。
3. 先执行一次 `query-replica --writers-covered --initialize-target`:核对独立拓扑并仅初始化目标绑定,不扫描历史或创建源索引。然后启动可信观察器,等待三个有效样本,避免未绑定目标造成观察器无法写入健康数据。
4. 接通可信客户作业健康输入 `--work-health`:JSON 必须含 UTC `at`、`coverage: "save-slice-submit-review-acceptance"`、布尔 `unexpectedErrors` 与 `latencyRegressed`。必须由已核实的指标提供者更新,≤45秒;不得用人工常量、readiness 或“日志没报错”替代。**实际生产提供者、延迟基线及故障演练尚待绑定,未完成不能启动。** 观察器来源 relay 目前仅适用于已审查的私网非TLS单端点;若实际连接启用TLS,必须先验证服务器名称处理,不得关闭TLS绕过。
5. 最终计划列明观察器 `--bindings-sha256`、`--binary-sha256` 和 `--stop-on-failure` 的授权范围。停止仅对两个影子控制器 UID/resourceVersion CAS 缩至0;不会修改 API、处理器、校验器、Worker、协调器。观察器缺失时进程5秒巡检健康,健康超过45秒会停止工作,不能无限继续扫描。
6. 仅投影器从0升至1,使用上表1000预留预算、10条/秒、64KiB/秒、2命令/秒、批次25及固定 CPU 基线。构建器保持0,生产查询开关保持关闭。影子目标数据不完整时不发布 ready。首轮停止后核对实际读取/预留/源确认量、源 CPU/读取字节、客户操作与共享盘 IO,对照同时间负载基线。
7. 首轮达标后再确认后续预算。25条/秒、128KiB/秒、5命令/秒、批次50,以及最高50条/秒、256KiB/秒、8命令/秒、批次100都只是后续上限,**本清单不授权自动升级**。完整复制与增量追平后才单列构建器启动、旧新结果及权限隔离对照,仍不切生产入口。
8. 回退:先停两个影子控制器并确认 Pod 退出,再停观察器;保留目标断点、暂停记录、最后完整结果和源标记。不删除源记录/标记,不清空查询库,不恢复业务旧快照。再次启动先读 `query-replica --safety-status`,重新确认健康、绑定和当前暂停令牌。生产入口从未切换,无需改客户作业配置。
此阶段只能证明新增同步负载受控及影子结果正确。生产入口未切换时不能宣称原 Mongo 总 CPU 已下降;降载验收仍需后续经授权切读后的同条件对照。

View File

@ -66,7 +66,12 @@ func (q *QueryReplica) WaitNextPass(ctx context.Context, interval time.Duration)
return nil
case <-heartbeat.C:
short, stop := context.WithTimeout(ctx, 3*time.Second)
err := q.transaction(short, func(_ mongo.SessionContext) error { return nil })
var err error
if q.safety != nil {
_, err = q.sourcePermit(short, 0)
} else {
err = q.transaction(short, func(_ mongo.SessionContext) error { return nil })
}
stop()
if err != nil {
return err

View File

@ -31,6 +31,7 @@ type QueryReplica struct {
BatchSize int
dirtyAfter map[string]any
reconcileNext int
safety *queryReplicaSafety
}
type queryReplicaCheckpoint struct {
ID string `bson:"_id"`
@ -143,7 +144,7 @@ func (q *QueryReplica) transaction(ctx context.Context, fn func(mongo.SessionCon
return nil, ErrQueryReplicaLease
}
return nil, fn(sc)
}, options.Transaction().SetReadConcern(readconcern.Snapshot()).SetWriteConcern(writeconcern.Majority()))
}, options.Transaction().SetReadConcern(readconcern.Snapshot()).SetWriteConcern(writeconcern.New(writeconcern.WMajority(), writeconcern.J(true))))
return e
}
func (q *QueryReplica) checkpoint(ctx context.Context, n string) (queryReplicaCheckpoint, error) {
@ -240,7 +241,14 @@ func bsonEqual(a, b any) bool {
return e == nil && string(x) == string(y)
}
func (q *QueryReplica) read(ctx context.Context, n string, filter bson.M) ([]queryReplicaRecord, any, error) {
opts := options.Find().SetProjection(querySourceProjection(n)).SetSort(bson.D{{Key: "_id", Value: 1}}).SetLimit(q.limit()).SetBatchSize(int32(q.limit()))
parent := ctx
limit, e := q.sourcePermit(ctx, q.limit())
if e != nil {
return nil, nil, e
}
ctx, cancel := q.sourceContext(ctx)
defer cancel()
opts := options.Find().SetMaxTime(1500 * time.Millisecond).SetProjection(querySourceProjection(n)).SetSort(bson.D{{Key: "_id", Value: 1}}).SetLimit(limit).SetBatchSize(int32(limit))
if filter[querySyncField+".dirty"] == true {
opts.SetHint("query_sync_dirty_v1")
} else {
@ -265,7 +273,13 @@ func (q *QueryReplica) read(ctx context.Context, n string, filter bson.M) ([]que
}
records = append(records, r)
}
return records, last, cur.Err()
if e := cur.Err(); e != nil {
return nil, nil, e
}
if e := q.paceRows(parent, len(records)); e != nil {
return nil, nil, e
}
return records, last, nil
}
func (q *QueryReplica) acknowledge(ctx context.Context, n string, records []queryReplicaRecord) error {
models := make([]mongo.WriteModel, 0, len(records))
@ -283,8 +297,7 @@ func (q *QueryReplica) acknowledge(ctx context.Context, n string, records []quer
if len(models) == 0 {
return nil
}
_, e := q.Source.db.Collection(n).BulkWrite(ctx, models, options.BulkWrite().SetOrdered(false))
return e
return q.sourceMetadataBulk(ctx, n, models)
}
// BootstrapStep is bounded and resumable. $expr uses BSON total ordering across
@ -306,7 +319,7 @@ func (q *QueryReplica) BootstrapStep(ctx context.Context, n string) (bool, int,
if e != nil {
return false, 0, e
}
done := int64(len(rows)) < q.limit()
done := len(rows) == 0
e = q.transaction(ctx, func(sc mongo.SessionContext) error {
if e := q.apply(sc, n, rows); e != nil {
return e
@ -366,6 +379,13 @@ func (s *MongoStore) EnableQuerySourceTracking(enabled bool) { s.querySyncEnable
// RunOnce performs at most one batch per collection. Backfill is resumable and
// bounded; the process does not begin unbounded scans on an HTTP cache miss.
func (q *QueryReplica) RunOnce(ctx context.Context, bootstrap bool) (bool, error) {
if q.safety != nil {
return q.runGuardedOnce(ctx, bootstrap)
}
if _, e := q.sourcePermit(ctx, 0); e != nil {
return false, e
}
allBootstrapped := true
for _, n := range replicaCollections() {
cp, e := q.checkpoint(ctx, n)
if e != nil {
@ -375,16 +395,20 @@ func (q *QueryReplica) RunOnce(ctx context.Context, bootstrap bool) (bool, error
if !bootstrap {
return false, ErrQueryReplicaNotReady
}
if _, _, e = q.BootstrapStep(ctx, n); e != nil {
var complete bool
if complete, _, e = q.BootstrapStep(ctx, n); e != nil {
return false, e
}
allBootstrapped = allBootstrapped && complete
}
if _, e = q.DrainStep(ctx, n); e != nil {
return false, e
}
}
if _, e := q.ReconcileOneDue(ctx, 6*time.Hour); e != nil {
return false, e
if allBootstrapped {
if _, e := q.ReconcileOneDue(ctx, 6*time.Hour); e != nil {
return false, e
}
}
if _, e := q.ReconcileDeleteStep(ctx); e != nil {
return false, e

View File

@ -13,8 +13,13 @@ import (
// It reads only source identities, never annotator payloads. Survivors retain
// their data; a cancelled deletion is not replayed or restored from snapshots.
func (q *QueryReplica) ReconcileDeleteStep(ctx context.Context) (bool, error) {
if _, e := q.sourcePermit(ctx, 1); e != nil {
return false, e
}
sourceCtx, sourceCancel := q.sourceContext(ctx)
defer sourceCancel()
var i queryDeleteIntent
e := q.Source.db.Collection(querySyncDeleteCollection).FindOne(ctx, bson.M{"$or": bson.A{bson.M{"phase": bson.M{"$in": bson.A{"complete", "cancelled", "reconcile"}}}, bson.M{"phase": bson.M{"$in": bson.A{"prepared", "deleting"}}, "leaseUntil": bson.M{"$lte": time.Now().UTC()}}}}, options.FindOne().SetSort(bson.D{{Key: "audit", Value: 1}, {Key: "createdAt", Value: 1}})).Decode(&i)
e := q.Source.db.Collection(querySyncDeleteCollection).FindOne(sourceCtx, bson.M{"$or": bson.A{bson.M{"phase": bson.M{"$in": bson.A{"complete", "cancelled", "reconcile"}}}, bson.M{"phase": bson.M{"$in": bson.A{"prepared", "deleting"}}, "leaseUntil": bson.M{"$lte": time.Now().UTC()}}}}, options.FindOne().SetMaxTime(1500*time.Millisecond).SetSort(bson.D{{Key: "audit", Value: 1}, {Key: "createdAt", Value: 1}})).Decode(&i)
if e == mongo.ErrNoDocuments {
return false, nil
}
@ -24,7 +29,7 @@ func (q *QueryReplica) ReconcileDeleteStep(ctx context.Context) (bool, error) {
if queryReplicaFields[i.Collection] == "" {
return false, ErrQueryReplicaUnsupported
}
if ok, e := q.Source.queryCollection(i.Collection).recoverDelete(ctx, i, time.Now().UTC()); e != nil || !ok {
if ok, e := q.recoverSourceClaims(ctx, i); e != nil || !ok {
return false, e
}
key := "delete/" + i.ID
@ -34,7 +39,7 @@ func (q *QueryReplica) ReconcileDeleteStep(ctx context.Context) (bool, error) {
return false, e
}
if cp.Complete {
_, e = q.Source.db.Collection(querySyncDeleteCollection).UpdateOne(ctx, bson.M{"_id": i.ID, "phase": "reconcile"}, bson.M{"$set": bson.M{"phase": "acknowledged", "expireAt": time.Now().UTC().Add(7 * 24 * time.Hour)}})
_, e = q.sourceIntentUpdate(ctx, bson.M{"_id": i.ID, "phase": "reconcile"}, bson.M{"$set": bson.M{"phase": "acknowledged", "expireAt": time.Now().UTC().Add(7 * 24 * time.Hour)}})
return true, e
}
filter := bson.M{}
@ -63,7 +68,19 @@ func (q *QueryReplica) ReconcileDeleteStep(ctx context.Context) (bool, error) {
}
present := []any{}
if len(ids) > 0 {
src, e := q.Source.db.Collection(i.Collection).Find(ctx, bson.M{"_id": bson.M{"$in": ids}}, options.Find().SetProjection(bson.M{"_id": 1}))
permitted, err := q.sourcePermit(ctx, int64(len(ids)))
if err != nil {
return false, err
}
if permitted < int64(len(ids)) {
if err = q.PauseSafety(ctx, "pilot_budget"); err != nil {
return false, err
}
return false, ErrQueryReplicaPaused
}
sourceCtx, sourceCancel := q.sourceContext(ctx)
defer sourceCancel()
src, e := q.Source.db.Collection(i.Collection).Find(sourceCtx, bson.M{"_id": bson.M{"$in": ids}}, options.Find().SetProjection(bson.M{"_id": 1}).SetLimit(int64(len(ids))).SetBatchSize(int32(len(ids))).SetMaxTime(1500*time.Millisecond))
if e != nil {
return false, e
}
@ -81,6 +98,9 @@ func (q *QueryReplica) ReconcileDeleteStep(ctx context.Context) (bool, error) {
return false, e
}
}
if e := q.paceRows(ctx, len(present)); e != nil {
return false, e
}
absent := []bson.M{}
for _, r := range rows {
id := deletionDoc(r["_queryReplica"])["sourceId"]

View File

@ -161,7 +161,12 @@ func (q *QueryReplica) PublishReady(ctx context.Context) error {
if !cp.Complete {
return ErrQueryReplicaNotReady
}
e = q.Source.db.Collection(n).FindOne(ctx, bson.M{querySyncField + ".dirty": true}, options.FindOne().SetProjection(bson.M{"_id": 1}).SetHint("query_sync_dirty_v1")).Err()
if _, e = q.sourcePermit(ctx, 1); e != nil {
return e
}
work, cancel := q.sourceContext(ctx)
e = q.Source.db.Collection(n).FindOne(work, bson.M{querySyncField + ".dirty": true}, options.FindOne().SetProjection(bson.M{"_id": 1}).SetHint("query_sync_dirty_v1").SetMaxTime(1500*time.Millisecond)).Err()
cancel()
if e == nil {
return ErrQueryReplicaNotReady
}
@ -169,7 +174,12 @@ func (q *QueryReplica) PublishReady(ctx context.Context) error {
return e
}
}
e := q.Source.db.Collection(querySyncDeleteCollection).FindOne(ctx, bson.M{"audit": bson.M{"$ne": true}, "phase": bson.M{"$in": queryDeletePendingPhases()}}, options.FindOne().SetProjection(bson.M{"_id": 1})).Err()
if _, e := q.sourcePermit(ctx, 1); e != nil {
return e
}
work, cancel := q.sourceContext(ctx)
defer cancel()
e := q.Source.db.Collection(querySyncDeleteCollection).FindOne(work, bson.M{"audit": bson.M{"$ne": true}, "phase": bson.M{"$in": queryDeletePendingPhases()}}, options.FindOne().SetProjection(bson.M{"_id": 1})).Err()
if e == nil {
return ErrQueryReplicaNotReady
}
@ -177,7 +187,11 @@ func (q *QueryReplica) PublishReady(ctx context.Context) error {
return e
}
return q.transaction(ctx, func(sc mongo.SessionContext) error {
_, e := q.control().UpdateOne(sc, bson.M{"_id": "state"}, bson.M{"$set": bson.M{"ready": true, "schema": queryReplicaSchema, "observedAt": time.Now().UTC(), "sourceDatabase": q.binding().SourceDatabase, "namespace": q.binding().Namespace}}, options.Update().SetUpsert(true))
_, e := q.control().UpdateOne(sc, bson.M{"_id": "state", "bootstrapAt": bson.M{"$exists": false}}, bson.M{"$set": bson.M{"bootstrapAt": time.Now().UTC()}})
if e != nil {
return e
}
_, e = q.control().UpdateOne(sc, bson.M{"_id": "state"}, bson.M{"$set": bson.M{"ready": true, "schema": queryReplicaSchema, "observedAt": time.Now().UTC(), "sourceDatabase": q.binding().SourceDatabase, "namespace": q.binding().Namespace}}, options.Update().SetUpsert(true))
return e
})
}

View File

@ -40,6 +40,17 @@ func (q *QueryReplica) ReconcileOneDue(ctx context.Context, interval time.Durati
if !boot.Complete {
continue
}
if q.safety != nil {
var first struct {
At time.Time `bson:"bootstrapAt"`
}
if e = q.control().FindOne(ctx, bson.M{"_id": "state"}).Decode(&first); e != nil && e != mongo.ErrNoDocuments {
return 0, e
}
if first.At.IsZero() || time.Since(first.At) < interval {
return 0, nil
}
}
key := "reconcile/" + name
var cp queryReplicaCheckpoint
e = q.control().FindOne(ctx, bson.M{"_id": key}).Decode(&cp)
@ -97,7 +108,7 @@ func (q *QueryReplica) ReconcileOneDue(ctx context.Context, interval time.Durati
}); e != nil {
return 0, e
}
if _, e = q.Source.db.Collection(name).BulkWrite(ctx, models, options.BulkWrite().SetOrdered(false)); e != nil {
if e = q.sourceMetadataBulk(ctx, name, models); e != nil {
return 0, e
}
}
@ -106,7 +117,7 @@ func (q *QueryReplica) ReconcileOneDue(ctx context.Context, interval time.Durati
// Reuse the durable identity sweeper for unreported physical deletions.
// Audits have lower priority and do not block unrelated report readiness.
i := queryDeleteIntent{ID: domain.NewID("query-audit"), Collection: name, Phase: "reconcile", CreatedAt: time.Now().UTC(), Audit: true}
if _, e = q.Source.db.Collection(querySyncDeleteCollection).InsertOne(ctx, i); e != nil {
if e = q.sourceAuditIntent(ctx, i); e != nil {
return 0, e
}
}

View File

@ -0,0 +1,331 @@
package store
import (
"context"
"crypto/sha256"
"droboticflow/internal/domain"
"encoding/hex"
"encoding/json"
"errors"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
"go.mongodb.org/mongo-driver/mongo/writeconcern"
"math"
"time"
)
var ErrQueryReplicaPaused = errors.New("shadow synchronization paused; explicit reviewed resume required")
// Safety is opt-in for the independent projector only. It never gates authority
// business operations. The policy and pause survive process and Pod replacement.
type QueryReplicaSafetyPolicy struct {
BytesPerSecond int `json:"bytesPerSecond" bson:"bytesPerSecond"`
CommandsPerSecond int `json:"commandsPerSecond" bson:"commandsPerSecond"`
MaxBatchRows int `json:"maxBatchRows" bson:"maxBatchRows"`
RunID string `json:"runId" bson:"runId"`
MaxRecords int64 `json:"maxRecords" bson:"maxRecords"`
RecordsPerSecond int `json:"recordsPerSecond" bson:"recordsPerSecond"`
BaselineCPU float64 `json:"baselineCPU" bson:"baselineCPU"`
}
type QueryReplicaSafetyObservation struct {
At time.Time `json:"at" bson:"at"`
CPU float64 `json:"cpu" bson:"cpu"`
Ready bool `json:"ready" bson:"ready"`
WritersCovered bool `json:"writersCovered" bson:"writersCovered"`
WorkErrors bool `json:"workErrors" bson:"workErrors"`
ProtectedIOFull float64 `json:"protectedIOFull" bson:"protectedIOFull"`
TargetUsedFraction float64 `json:"targetUsedFraction" bson:"targetUsedFraction"`
HostFreeBytes int64 `json:"hostFreeBytes" bson:"hostFreeBytes"`
}
type queryReplicaSafetyState struct {
Owner string `bson:"owner"`
Policy QueryReplicaSafetyPolicy `bson:"policy"`
Digest string `bson:"digest"`
Reserved int64 `bson:"reserved"`
Paused bool `bson:"paused"`
Reason string `bson:"reason"`
ResumeToken string `bson:"resumeToken"`
LastObservation time.Time `bson:"lastObservation"`
High int `bson:"high"`
HighIO int `bson:"highIO"`
}
type queryReplicaSafety struct {
policy QueryReplicaSafetyPolicy
rows *queryReplicaPacer
}
func safetyPolicyDigest(p QueryReplicaSafetyPolicy) string {
b, _ := json.Marshal(p)
h := sha256.Sum256(b)
return hex.EncodeToString(h[:])
}
func (p QueryReplicaSafetyPolicy) validate() error {
if p.BytesPerSecond < 1024 || p.BytesPerSecond > 256*1024 || p.CommandsPerSecond < 1 || p.CommandsPerSecond > 8 || p.MaxBatchRows < 1 || p.MaxBatchRows > 100 || p.RunID == "" || len(p.RunID) > 100 || p.MaxRecords < 0 || p.RecordsPerSecond < 1 || p.RecordsPerSecond > 50 || math.IsNaN(p.BaselineCPU) || math.IsInf(p.BaselineCPU, 0) || p.BaselineCPU <= 0 || p.BaselineCPU > 90 {
return errors.New("invalid shadow safety policy")
}
return nil
}
// RecordQueryReplicaObservation is for a trusted out-of-process observer. Only
// complete, fresh observations are accepted; callers never submit customer data.
func (s *MongoStore) RecordQueryReplicaObservation(ctx context.Context, o QueryReplicaSafetyObservation) error {
if o.At.IsZero() || time.Since(o.At) > 45*time.Second || o.At.After(time.Now().Add(time.Second)) || math.IsNaN(o.CPU) || math.IsInf(o.CPU, 0) || o.CPU < 0 || o.CPU > 100 || math.IsNaN(o.ProtectedIOFull) || math.IsInf(o.ProtectedIOFull, 0) || o.ProtectedIOFull < 0 || o.ProtectedIOFull > 100 || math.IsNaN(o.TargetUsedFraction) || math.IsInf(o.TargetUsedFraction, 0) || o.TargetUsedFraction < 0 || o.TargetUsedFraction > 1 || o.HostFreeBytes < 0 {
return errors.New("invalid shadow observation")
}
var hello bson.M
if e := s.db.RunCommand(ctx, bson.M{"hello": 1}).Decode(&hello); e != nil || hello["setName"] == nil {
return errors.New("independent query replica required")
}
var binding queryReplicaBinding
if e := s.db.Collection(queryReplicaControl).FindOne(ctx, bson.M{"_id": "source-binding"}).Decode(&binding); e != nil || binding.SourceDatabase == "" || binding.SourceDatabase == s.db.Name() || binding.Namespace != queryReplicaNamespace(binding.SourceDatabase) {
return errors.New("reviewed query binding required before observation writes")
}
c := s.db.Collection(queryReplicaControl, options.Collection().SetWriteConcern(writeconcern.New(writeconcern.WMajority(), writeconcern.J(true))))
// Monotonic CAS: an old observer cannot overwrite newer health evidence.
_, e := c.UpdateOne(ctx, bson.M{"_id": "safety-health", "$or": bson.A{bson.M{"at": bson.M{"$lt": o.At}}, bson.M{"at": bson.M{"$exists": false}}}}, bson.M{"$set": o, "$push": bson.M{"recent": bson.M{"$each": bson.A{o}, "$slice": -3}}}, options.Update().SetUpsert(true))
return e
}
func (q *QueryReplica) ConfigureSafety(ctx context.Context, p QueryReplicaSafetyPolicy, resumeToken string) error {
if e := p.validate(); e != nil {
return e
}
if q.limit() > int64(p.MaxBatchRows) {
return errors.New("batch exceeds reviewed safety policy")
}
digest := safetyPolicyDigest(p)
restarted := false
e := q.transaction(ctx, func(sc mongo.SessionContext) error {
var state queryReplicaSafetyState
e := q.control().FindOne(sc, bson.M{"_id": "safety"}).Decode(&state)
if e != nil && e != mongo.ErrNoDocuments {
return e
}
if e == nil {
if state.Paused {
if resumeToken == "" || state.ResumeToken != resumeToken || state.Policy.RunID != p.RunID || p.BaselineCPU > state.Policy.BaselineCPU || p.MaxRecords > 0 && p.MaxRecords <= state.Reserved {
return ErrQueryReplicaPaused
}
} else {
if state.Digest != digest || resumeToken != "" {
return errors.New("active shadow policy cannot be replaced")
}
if state.Owner == q.owner {
return nil
}
restarted = true
_, e = q.control().UpdateOne(sc, bson.M{"_id": "safety"}, bson.M{"$set": bson.M{"paused": true, "reason": "process_restarted", "resumeToken": domain.NewID("shadow-resume")}})
if e == nil {
_, e = q.control().UpdateOne(sc, bson.M{"_id": "state"}, bson.M{"$set": bson.M{"ready": false}}, options.Update().SetUpsert(true))
}
return e
}
} else if resumeToken != "" {
return errors.New("resume requires an existing pause")
}
denyStart := func() error {
if state.Paused {
return ErrQueryReplicaPaused
}
restarted = true
_, err := q.control().UpdateOne(sc, bson.M{"_id": "safety"}, bson.M{"$set": bson.M{"policy": p, "digest": digest, "owner": q.owner, "paused": true, "reason": "starting_health", "resumeToken": domain.NewID("shadow-resume"), "high": 0, "highIO": 0}, "$setOnInsert": bson.M{"reserved": int64(0)}}, options.Update().SetUpsert(true))
if err == nil {
_, err = q.control().UpdateOne(sc, bson.M{"_id": "state"}, bson.M{"$set": bson.M{"ready": false}}, options.Update().SetUpsert(true))
}
return err
}
var health struct {
Recent []QueryReplicaSafetyObservation `bson:"recent"`
}
if e = q.control().FindOne(sc, bson.M{"_id": "safety-health"}).Decode(&health); e != nil {
return denyStart()
}
if len(health.Recent) != 3 {
return denyStart()
}
low, high := 100., 0.
for i, o := range health.Recent {
if observationFailure(o, o.At) != "" || o.CPU > 90 {
return denyStart()
}
if i > 0 && !o.At.After(health.Recent[i-1].At) {
return denyStart()
}
low = math.Min(low, o.CPU)
high = math.Max(high, o.CPU)
}
if high-low > 5 || health.Recent[2].At.Sub(health.Recent[0].At) < 60*time.Second || observationFailure(health.Recent[2], time.Now()) != "" {
return denyStart()
}
_, e = q.control().UpdateOne(sc, bson.M{"_id": "safety"}, bson.M{"$set": bson.M{"policy": p, "digest": digest, "owner": q.owner, "paused": false, "reason": "", "resumeToken": "", "high": 0, "highIO": 0}, "$setOnInsert": bson.M{"reserved": int64(0)}}, options.Update().SetUpsert(true))
return e
})
if e != nil {
return e
}
if restarted {
return ErrQueryReplicaPaused
}
q.safety = &queryReplicaSafety{policy: p, rows: newQueryReplicaPacer(float64(p.RecordsPerSecond))}
_, e = q.sourcePermit(ctx, 0)
return e
}
func observationFailure(o QueryReplicaSafetyObservation, now time.Time) string {
switch {
case o.At.IsZero() || now.Sub(o.At) > 45*time.Second || o.At.After(now.Add(time.Second)):
return "health_missing_or_stale"
case !o.Ready || !o.WritersCovered || o.WorkErrors:
return "production_health"
case math.IsNaN(o.CPU) || math.IsInf(o.CPU, 0) || o.CPU < 0 || o.CPU > 100 || math.IsNaN(o.ProtectedIOFull) || math.IsInf(o.ProtectedIOFull, 0) || o.ProtectedIOFull < 0 || math.IsNaN(o.TargetUsedFraction) || math.IsInf(o.TargetUsedFraction, 0) || o.TargetUsedFraction < 0 || o.TargetUsedFraction > 1:
return "invalid_health"
case o.CPU >= 95:
return "source_cpu_absolute"
case o.ProtectedIOFull > 5:
return "protected_io"
case o.TargetUsedFraction >= 0.70 || o.HostFreeBytes < 100*1024*1024*1024:
return "storage_pressure"
}
return ""
}
// sourcePermit reserves a bounded maximum before a read. A crash consumes the
// reservation rather than silently resetting a pilot's total budget. Empty and
// repeated reads can consume it too; reserved is explicitly not progress.
func (q *QueryReplica) sourcePermit(ctx context.Context, want int64) (int64, error) {
if q.safety == nil {
return want, nil
}
allowed := want
paused := false
e := q.transaction(ctx, func(sc mongo.SessionContext) error {
var s queryReplicaSafetyState
if e := q.control().FindOne(sc, bson.M{"_id": "safety"}).Decode(&s); e != nil {
return e
}
if s.Paused {
paused = true
return nil
}
if s.Digest != safetyPolicyDigest(q.safety.policy) {
return errors.New("shadow policy changed")
}
var o QueryReplicaSafetyObservation
e := q.control().FindOne(sc, bson.M{"_id": "safety-health"}).Decode(&o)
if e != nil && e != mongo.ErrNoDocuments {
return e
}
reason := observationFailure(o, time.Now())
if reason == "" && !o.At.Equal(s.LastObservation) {
if o.CPU > s.Policy.BaselineCPU+3 {
s.High++
} else {
s.High = 0
}
if o.ProtectedIOFull > 1 {
s.HighIO++
} else {
s.HighIO = 0
}
s.LastObservation = o.At
if s.High >= 2 {
reason = "source_cpu_relative"
}
if s.HighIO >= 2 {
reason = "protected_io"
}
}
if reason == "" && want > 0 && s.Policy.MaxRecords > 0 {
left := s.Policy.MaxRecords - s.Reserved
if left <= 0 {
reason = "pilot_budget"
} else if allowed > left {
allowed = left
}
}
fields := bson.M{"lastObservation": s.LastObservation, "high": s.High, "highIO": s.HighIO}
if reason != "" {
paused = true
fields["paused"] = true
fields["reason"] = reason
fields["resumeToken"] = domain.NewID("shadow-resume")
if _, e = q.control().UpdateOne(sc, bson.M{"_id": "state"}, bson.M{"$set": bson.M{"ready": false}}, options.Update().SetUpsert(true)); e != nil {
return e
}
}
update := bson.M{"$set": fields}
if !paused && allowed > 0 {
update["$inc"] = bson.M{"reserved": allowed}
}
_, e = q.control().UpdateOne(sc, bson.M{"_id": "safety"}, update)
return e
})
if e != nil {
return 0, e
}
if paused {
return 0, ErrQueryReplicaPaused
}
return allowed, nil
}
func (q *QueryReplica) PauseSafety(ctx context.Context, reason string) error {
if q.safety == nil {
return nil
}
// Only fixed non-payload reasons reach durable diagnostics.
switch reason {
case "source_error", "target_error", "incomplete_pilot", "observer_error", "operator_stop", "pilot_budget":
default:
reason = "operation_error"
}
return q.transaction(ctx, func(sc mongo.SessionContext) error {
_, e := q.control().UpdateOne(sc, bson.M{"_id": "safety", "paused": false}, bson.M{"$set": bson.M{"paused": true, "reason": reason, "resumeToken": domain.NewID("shadow-resume")}})
if e != nil {
return e
}
_, e = q.control().UpdateOne(sc, bson.M{"_id": "state"}, bson.M{"$set": bson.M{"ready": false}}, options.Update().SetUpsert(true))
return e
})
}
func (q *QueryReplica) paceRows(ctx context.Context, n int) error {
if q.safety == nil {
return nil
}
return q.safety.rows.wait(ctx, float64(n))
}
func (q *QueryReplica) sourceContext(ctx context.Context) (context.Context, context.CancelFunc) {
return context.WithTimeout(ctx, 5*time.Second)
}
// QueryReplicaSafetyStatus contains no payloads or connection information.
// The nonce is an operator CAS token, not authorization: target credentials
// remain mandatory. It is never emitted by the running projector's logs.
func (s *MongoStore) QueryReplicaSafetyStatus(ctx context.Context) (bson.M, error) {
var state bson.M
e := s.db.Collection(queryReplicaControl).FindOne(ctx, bson.M{"_id": "safety"}, options.FindOne().SetProjection(bson.M{"_id": 0, "policy": 1, "reserved": 1, "paused": 1, "reason": 1, "resumeToken": 1, "lastObservation": 1, "high": 1})).Decode(&state)
return state, e
}
// CheckQueryReplicaSafety gates only the opt-in shadow builder, never API reads
// or any authority operation. A lost observer leaves the builder parked.
func (s *MongoStore) CheckQueryReplicaSafety(ctx context.Context, runID string) error {
var state queryReplicaSafetyState
if e := s.db.Collection(queryReplicaControl).FindOne(ctx, bson.M{"_id": "safety"}).Decode(&state); e != nil {
return e
}
if state.Paused || state.Policy.RunID != runID {
return ErrQueryReplicaPaused
}
var health QueryReplicaSafetyObservation
if e := s.db.Collection(queryReplicaControl).FindOne(ctx, bson.M{"_id": "safety-health"}).Decode(&health); e != nil {
return e
}
if observationFailure(health, time.Now()) != "" {
return ErrQueryReplicaPaused
}
return nil
}
func (q *QueryReplica) CheckSafety(ctx context.Context) error {
_, e := q.sourcePermit(ctx, 0)
return e
}

View File

@ -0,0 +1,427 @@
package store
import (
"context"
"droboticflow/internal/domain"
"errors"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/event"
"go.mongodb.org/mongo-driver/mongo/options"
"testing"
"time"
)
func seedReplicaSafetyHealth(t *testing.T, q *QueryReplica) {
t.Helper()
if q.BatchSize > 100 {
q.BatchSize = 25
}
now := time.Now().UTC()
recent := []QueryReplicaSafetyObservation{}
for _, ago := range []time.Duration{60 * time.Second, 30 * time.Second, 0} {
recent = append(recent, QueryReplicaSafetyObservation{At: now.Add(-ago), CPU: 85, Ready: true, WritersCovered: true, HostFreeBytes: 120 << 30, TargetUsedFraction: .1})
}
raw, e := bson.Marshal(recent[2])
if e != nil {
t.Fatal(e)
}
var fields bson.M
if e = bson.Unmarshal(raw, &fields); e != nil {
t.Fatal(e)
}
fields["recent"] = recent
if _, e = q.control().UpdateOne(t.Context(), bson.M{"_id": "safety-health"}, bson.M{"$set": fields}, options.Update().SetUpsert(true)); e != nil {
t.Fatal(e)
}
}
func safetyPolicyForTest() QueryReplicaSafetyPolicy {
return QueryReplicaSafetyPolicy{BytesPerSecond: 65536, CommandsPerSecond: 2, MaxBatchRows: 25, RunID: "synthetic-run", MaxRecords: 1000, RecordsPerSecond: 50, BaselineCPU: 85}
}
func TestQueryReplicaSafetyPilotBudgetAndRestartLocal(t *testing.T) {
source, target, q := newLocalReplica(t)
ctx := t.Context()
seedReplicaSafetyHealth(t, q)
p := safetyPolicyForTest()
p.MaxRecords = 4
q.BatchSize = 3
for _, n := range []string{"projects", "humanItems"} {
for i := 0; i < 3; i++ {
if _, e := source.db.Collection(n).InsertOne(ctx, bson.M{"_id": domain.NewID("source"), "id": domain.NewID("logical"), "organizationId": "o", "projectId": "p", "version": 7, "status": "claimed", "subject": bson.M{"payload": bson.M{"annotationDraft": "preserve"}}}); e != nil {
t.Fatal(e)
}
}
}
if e := q.ConfigureSafety(ctx, p, ""); e != nil {
t.Fatal(e)
}
if _, n, e := q.BootstrapStep(ctx, "projects"); e != nil || n != 3 {
t.Fatal(n, e)
}
complete, n, e := q.BootstrapStep(ctx, "humanItems")
if e != nil || n != 1 || complete {
t.Fatal("short budget falsely completed collection", complete, n, e)
}
if _, _, e = q.BootstrapStep(ctx, "humanItems"); !errors.Is(e, ErrQueryReplicaPaused) {
t.Fatal("pilot exceeded durable cap", e)
}
var state queryReplicaSafetyState
if e = q.control().FindOne(ctx, bson.M{"_id": "safety"}).Decode(&state); e != nil || state.Reserved != 4 || !state.Paused || state.ResumeToken == "" {
t.Fatal(state, e)
}
if _, e = q.control().UpdateOne(ctx, bson.M{"_id": "lease"}, bson.M{"$set": bson.M{"until": time.Time{}}}); e != nil {
t.Fatal(e)
}
successor, e := NewQueryReplica(source, target)
successor.BatchSize = q.BatchSize
if e != nil {
t.Fatal(e)
}
if e = successor.Acquire(ctx); e != nil {
t.Fatal(e)
}
p.MaxRecords = 20
if e = successor.ConfigureSafety(ctx, p, ""); !errors.Is(e, ErrQueryReplicaPaused) {
t.Fatal("restart bypassed pause", e)
}
changed := p
changed.RunID = "replacement-run"
if e = successor.ConfigureSafety(ctx, changed, state.ResumeToken); !errors.Is(e, ErrQueryReplicaPaused) {
t.Fatal("new run reset budget", e)
}
seedReplicaSafetyHealth(t, successor)
if e = successor.ConfigureSafety(ctx, p, state.ResumeToken); e != nil {
t.Fatal(e)
}
cp, e := successor.checkpoint(ctx, "humanItems")
if e != nil || !cp.HasLast || cp.Complete {
t.Fatal(cp, e)
}
if _, n, e = successor.BootstrapStep(ctx, "humanItems"); e != nil || n != 2 {
t.Fatal("resume lost remaining history", n, e)
}
var doc bson.M
if e = source.db.Collection("humanItems").FindOne(ctx, bson.M{}).Decode(&doc); e != nil || doc["version"] != int32(7) || deletionDoc(deletionDoc(doc["subject"])["payload"])["annotationDraft"] != "preserve" {
t.Fatal("source annotation altered", e)
}
}
func TestQueryReplicaSafetyRelativeCPUCountsDistinctObservationsLocal(t *testing.T) {
_, _, q := newLocalReplica(t)
seedReplicaSafetyHealth(t, q)
p := safetyPolicyForTest()
if e := q.ConfigureSafety(t.Context(), p, ""); e != nil {
t.Fatal(e)
}
now := time.Now()
for i := 1; i <= 2; i++ {
o := QueryReplicaSafetyObservation{At: now.Add(time.Duration(i) * time.Millisecond), CPU: 90, Ready: true, WritersCovered: true, HostFreeBytes: 120 << 30, TargetUsedFraction: .1}
if e := q.Target.RecordQueryReplicaObservation(t.Context(), o); e != nil {
t.Fatal(e)
}
_, e := q.sourcePermit(t.Context(), 0)
if i == 1 {
if e != nil {
t.Fatal(e)
}
for j := 0; j < 3; j++ {
if _, e = q.sourcePermit(t.Context(), 0); e != nil {
t.Fatal("same sample counted twice", e)
}
}
} else if !errors.Is(e, ErrQueryReplicaPaused) {
t.Fatal("CPU did not persist pause", e)
}
}
var state queryReplicaSafetyState
_ = q.control().FindOne(t.Context(), bson.M{"_id": "safety"}).Decode(&state)
if !state.Paused || state.Reason != "source_cpu_relative" {
t.Fatal(state)
}
seedReplicaSafetyHealth(t, q)
if _, e := q.sourcePermit(t.Context(), 0); !errors.Is(e, ErrQueryReplicaPaused) {
t.Fatal("cooldown silently resumed", e)
}
}
func TestQueryReplicaSafetyRestartWithoutPauseFailsClosedLocal(t *testing.T) {
source, target, q := newLocalReplica(t)
seedReplicaSafetyHealth(t, q)
p := safetyPolicyForTest()
if e := q.ConfigureSafety(t.Context(), p, ""); e != nil {
t.Fatal(e)
}
if _, e := q.sourcePermit(t.Context(), 7); e != nil {
t.Fatal(e)
}
_, _ = q.control().UpdateOne(t.Context(), bson.M{"_id": "lease"}, bson.M{"$set": bson.M{"until": time.Time{}}})
next, _ := NewQueryReplica(source, target)
next.BatchSize = q.BatchSize
if e := next.Acquire(t.Context()); e != nil {
t.Fatal(e)
}
if e := next.ConfigureSafety(t.Context(), p, ""); !errors.Is(e, ErrQueryReplicaPaused) {
t.Fatal("crash bypassed operator review", e)
}
var s queryReplicaSafetyState
_ = next.control().FindOne(t.Context(), bson.M{"_id": "safety"}).Decode(&s)
if !s.Paused || s.Reason != "process_restarted" || s.Reserved != 7 {
t.Fatal(s)
}
}
func TestQueryReplicaBoundedClaimRecoveryPreservesAnnotationsLocal(t *testing.T) {
var many int
monitor := &event.CommandMonitor{Started: func(_ context.Context, e *event.CommandStartedEvent) {
if e.CommandName == "delete" {
t.Error("projector attempted source delete")
}
if e.CommandName == "update" {
arr := e.Command.Lookup("updates").Array()
vs, _ := arr.Values()
for _, v := range vs {
u := v.Document()
if u.Lookup("multi").Type != 0 && u.Lookup("multi").Boolean() {
ids, _ := u.Lookup("q").Document().Lookup("_id").Document().Lookup("$in").Array().Values()
if len(ids) > 2 {
t.Error("unbounded cleanup")
}
many++
}
}
}
}}
source, _, q := newLocalReplica(t, monitor)
ctx := t.Context()
q.BatchSize = 2
i := queryDeleteIntent{ID: "expired-delete", Collection: "humanItems", Phase: "deleting", LeaseUntil: time.Now().Add(-time.Minute), CreatedAt: time.Now()}
if _, e := source.db.Collection(querySyncDeleteCollection).InsertOne(ctx, i); e != nil {
t.Fatal(e)
}
for n := 0; n < 7; n++ {
if _, e := source.db.Collection("humanItems").InsertOne(ctx, bson.M{"id": domain.NewID("item"), "version": 9, "status": "claimed", "annotation": "kept", "_querySync": bson.M{"deleteClaim": i.ID}}); e != nil {
t.Fatal(e)
}
}
for n := 0; n < 6; n++ {
if _, e := q.ReconcileDeleteStep(ctx); e != nil {
t.Fatal(e)
}
}
count, e := source.db.Collection("humanItems").CountDocuments(ctx, bson.M{"version": 9, "status": "claimed", "annotation": "kept", "_querySync.deleteClaim": bson.M{"$exists": false}})
if e != nil || count != 7 || many != 4 {
t.Fatal(count, many, e)
}
}
func TestQueryReplicaSafetyMajorityJournalAndNoEarlyAuditLocal(t *testing.T) {
var commits int
targetMonitor := &event.CommandMonitor{Started: func(_ context.Context, e *event.CommandStartedEvent) {
if e.CommandName == "commitTransaction" {
commits++
wc := e.Command.Lookup("writeConcern").Document()
if wc.Lookup("w").StringValue() != "majority" || !wc.Lookup("j").Boolean() {
t.Error("durability weakened")
}
}
}}
_, _, q := newLocalReplica(t, nil, targetMonitor)
seedReplicaSafetyHealth(t, q)
if e := q.ConfigureSafety(t.Context(), safetyPolicyForTest(), ""); e != nil {
t.Fatal(e)
}
if _, e := q.ReconcileOneDue(t.Context(), 6*time.Hour); e != nil {
t.Fatal(e)
}
if commits == 0 {
t.Fatal("no transactional durability evidence")
}
}
func TestQueryReplicaSafetyMissingHealthCannotRestartIntoRunningLocal(t *testing.T) {
_, _, q := newLocalReplica(t)
q.BatchSize = 25
p := safetyPolicyForTest()
if e := q.ConfigureSafety(t.Context(), p, ""); !errors.Is(e, ErrQueryReplicaPaused) {
t.Fatal(e)
}
seedReplicaSafetyHealth(t, q)
if e := q.ConfigureSafety(t.Context(), p, ""); !errors.Is(e, ErrQueryReplicaPaused) {
t.Fatal("missing health latch auto-resumed", e)
}
}
func TestQueryReplicaDeleteBudgetCannotDeleteUnprobedTargetLocal(t *testing.T) {
source, target, q := newLocalReplica(t)
q.BatchSize = 2
ctx := t.Context()
_, e := source.db.Collection(querySyncDeleteCollection).InsertOne(ctx, queryDeleteIntent{ID: "audit", Audit: true, Collection: "humanItems", Phase: "reconcile", CreatedAt: time.Now()})
if e != nil {
t.Fatal(e)
}
for _, id := range []string{"one", "two"} {
_, e = target.db.Collection("humanItems").InsertOne(ctx, bson.M{"_id": id, "_queryReplica": bson.M{"sourceId": id}})
if e != nil {
t.Fatal(e)
}
}
seedReplicaSafetyHealth(t, q)
p := safetyPolicyForTest()
p.MaxRecords = 4
if e = q.ConfigureSafety(ctx, p, ""); e != nil {
t.Fatal(e)
}
if _, e = q.ReconcileDeleteStep(ctx); !errors.Is(e, ErrQueryReplicaPaused) {
t.Fatal(e)
}
n, e := target.db.Collection("humanItems").CountDocuments(ctx, bson.M{})
if e != nil || n != 2 {
t.Fatal("unprobed target facts removed", n, e)
}
}
func TestQueryReplicaSafetyRejectsSourceObservationAndStopsBuilderLocal(t *testing.T) {
source, target, q := newLocalReplica(t)
seedReplicaSafetyHealth(t, q)
p := safetyPolicyForTest()
if e := q.ConfigureSafety(t.Context(), p, ""); e != nil {
t.Fatal(e)
}
now := time.Now().UTC()
o := QueryReplicaSafetyObservation{At: now, CPU: 85, Ready: true, WritersCovered: true, HostFreeBytes: 120 << 30, TargetUsedFraction: .1}
if e := source.RecordQueryReplicaObservation(t.Context(), o); e == nil {
t.Fatal("observer wrote into authority")
}
count, _ := source.db.Collection(queryReplicaControl).CountDocuments(t.Context(), bson.M{})
if count != 0 {
t.Fatal("observation mutated source")
}
if e := target.CheckQueryReplicaSafety(t.Context(), p.RunID); e != nil {
t.Fatal(e)
}
o.At = now.Add(-46 * time.Second)
if e := target.RecordQueryReplicaObservation(t.Context(), o); e == nil {
t.Fatal("stale health accepted")
}
if e := q.PauseSafety(t.Context(), "operator_stop"); e != nil {
t.Fatal(e)
}
if e := target.CheckQueryReplicaSafety(t.Context(), p.RunID); !errors.Is(e, ErrQueryReplicaPaused) {
t.Fatal("builder ignored durable pause", e)
}
}
func TestQueryReplicaSafetyTargetFailureRetainsDirtyAndCheckpointLocal(t *testing.T) {
source, target, q := newLocalReplica(t)
ctx := t.Context()
seedReplicaSafetyHealth(t, q)
if e := q.ConfigureSafety(ctx, safetyPolicyForTest(), ""); e != nil {
t.Fatal(e)
}
_, e := source.db.Collection("humanItems").InsertOne(ctx, bson.M{"_id": "source", "id": "item", "organizationId": "o", "projectId": "p", "version": 7, "status": "claimed", "annotation": "keep", "_querySync": bson.M{"revision": 1, "dirty": true}})
if e != nil {
t.Fatal(e)
}
// Failure injection is confined to this fixture's random target database.
if e = target.db.RunCommand(ctx, bson.D{{Key: "collMod", Value: "humanItems"}, {Key: "validator", Value: bson.M{"mustNeverExist": true}}}).Err(); e != nil {
t.Fatal(e)
}
if _, _, e = q.BootstrapStep(ctx, "humanItems"); e == nil {
t.Fatal("invalid target write unexpectedly committed")
}
cp, e := q.checkpoint(ctx, "humanItems")
if e != nil || cp.HasLast || cp.Complete {
t.Fatal("failed transaction advanced checkpoint", cp, e)
}
var row bson.M
if e = source.db.Collection("humanItems").FindOne(ctx, bson.M{"_id": "source"}).Decode(&row); e != nil || deletionDoc(row[querySyncField])["dirty"] != true || row["version"] != int32(7) || row["annotation"] != "keep" {
t.Fatal("failed sink damaged source", e)
}
if e = target.db.RunCommand(ctx, bson.D{{Key: "collMod", Value: "humanItems"}, {Key: "validator", Value: bson.M{}}}).Err(); e != nil {
t.Fatal(e)
}
if _, n, e := q.BootstrapStep(ctx, "humanItems"); e != nil || n != 1 {
t.Fatal("replay failed", n, e)
}
if n, e := q.DrainStep(ctx, "humanItems"); e != nil || n != 0 {
t.Fatal("dirty acknowledged before persistence or not replayable", n, e)
}
}
func TestQueryReplicaSafetyConcurrentSavePreservesNewRevisionLocal(t *testing.T) {
var source *MongoStore
armed := false
fired := false
mon := &event.CommandMonitor{Started: func(_ context.Context, e *event.CommandStartedEvent) {
if e.CommandName != "update" {
return
}
if !armed || fired {
return
}
name, _ := e.Command.Lookup("update").StringValueOK()
if name != "humanItems" {
return
}
fired = true
short, cancel := context.WithTimeout(context.Background(), 3*time.Second)
defer cancel()
_, err := source.db.Collection("humanItems").UpdateOne(short, bson.M{"_id": "source"}, bson.M{"$set": bson.M{"version": 8, "status": "submitted", "annotation": "new author edit", "_querySync.revision": 2, "_querySync.dirty": true}})
if err != nil {
t.Error(err)
}
}}
var q *QueryReplica
source, _, q = newLocalReplica(t, nil, mon)
ctx := t.Context()
seedReplicaSafetyHealth(t, q)
if e := q.ConfigureSafety(ctx, safetyPolicyForTest(), ""); e != nil {
t.Fatal(e)
}
_, e := source.db.Collection("humanItems").InsertOne(ctx, bson.M{"_id": "source", "id": "item", "organizationId": "o", "projectId": "p", "version": 7, "status": "claimed", "annotation": "old author edit", "_querySync": bson.M{"revision": 1, "dirty": true}})
if e != nil {
t.Fatal(e)
}
armed = true
if _, _, e = q.BootstrapStep(ctx, "humanItems"); e != nil {
t.Fatal(e)
}
var kept bson.M
_ = source.db.Collection("humanItems").FindOne(ctx, bson.M{"_id": "source"}).Decode(&kept)
if !fired || kept["version"] != int32(8) || kept["annotation"] != "new author edit" || deletionDoc(kept[querySyncField])["dirty"] != true {
t.Fatal("old copy acknowledged or overwrote concurrent save")
}
if n, e := q.DrainStep(ctx, "humanItems"); e != nil || n != 1 {
t.Fatal("new revision not recoverable", n, e)
}
}
func TestQueryReplicaSafetyBacklogPriorityAndRoundRobinLocal(t *testing.T) {
source, _, q := newLocalReplica(t)
q.BatchSize = 1
ctx := t.Context()
seedReplicaSafetyHealth(t, q)
p := safetyPolicyForTest()
p.MaxRecords = 0
if e := q.ConfigureSafety(ctx, p, ""); e != nil {
t.Fatal(e)
}
for _, id := range []string{"a", "b", "c", "d", "e"} {
_, e := source.db.Collection("annotationIndexJobs").InsertOne(ctx, bson.M{"_id": id, "id": id, "organizationId": "o", "status": "completed", "_querySync": bson.M{"revision": 1, "dirty": true}})
if e != nil {
t.Fatal(e)
}
}
_, e := q.control().InsertOne(ctx, bson.M{"_id": "backlog/annotationIndexJobs", "since": time.Now().Add(-6 * time.Minute)})
if e != nil {
t.Fatal(e)
}
if ready, e := q.RunOnce(ctx, true); e != nil || ready {
t.Fatal("partial history published", ready, e)
}
clean, e := source.db.Collection("annotationIndexJobs").CountDocuments(ctx, bson.M{"_querySync.dirty": false})
if e != nil || clean != 4 {
t.Fatal("old dirty backlog did not receive4:1 priority", clean, e)
}
var cursor struct {
Next int `bson:"next"`
}
if e = q.control().FindOne(ctx, bson.M{"_id": "schedule"}).Decode(&cursor); e != nil {
t.Fatal(e)
}
if cursor.Next != 0 {
t.Fatal("round-robin checkpoint not persisted")
}
}

View File

@ -0,0 +1,146 @@
package store
import (
"context"
"errors"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo"
"math"
"net"
"sync"
"testing"
"time"
)
func TestQueryReplicaMetadataWriterRejectsBusinessFields(t *testing.T) {
cases := []struct {
name string
model mongo.WriteModel
ok bool
}{
{"ack", mongo.NewUpdateOneModel().SetFilter(bson.M{"_id": "i", "_querySync.revision": 1}).SetUpdate(bson.M{"$set": bson.M{"_querySync.dirty": false}}), true},
{"repair", mongo.NewUpdateOneModel().SetFilter(bson.M{"_id": "i", "_querySync.revision": bson.M{"$exists": false}}).SetUpdate(bson.M{"$set": queryReplicaRepairMarker(2)}), true},
{"annotation", mongo.NewUpdateOneModel().SetFilter(bson.M{"_id": "i", "_querySync.revision": 1}).SetUpdate(bson.M{"$set": bson.M{"subject.payload.annotationDraft": "overwrite"}}), false},
{"version", mongo.NewUpdateOneModel().SetFilter(bson.M{"_id": "i", "_querySync.revision": 1}).SetUpdate(bson.M{"$inc": bson.M{"version": 1}}), false},
{"upsert", mongo.NewUpdateOneModel().SetFilter(bson.M{"_id": "i", "_querySync.revision": 1}).SetUpdate(bson.M{"$set": bson.M{"_querySync.dirty": false}}).SetUpsert(true), false},
{"missing-CAS", mongo.NewUpdateOneModel().SetFilter(bson.M{"_id": "i"}).SetUpdate(bson.M{"$set": bson.M{"_querySync.dirty": false}}), false},
{"delete", mongo.NewDeleteOneModel().SetFilter(bson.M{"_id": "i"}), false},
{"replace", mongo.NewReplaceOneModel().SetFilter(bson.M{"_id": "i"}).SetReplacement(bson.M{"_querySync": bson.M{"dirty": false}}), false},
{"pipeline", mongo.NewUpdateOneModel().SetFilter(bson.M{"_id": "i", "_querySync.revision": 1}).SetUpdate(bson.A{bson.M{"$set": bson.M{"_querySync.dirty": false}}}), false},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
e := validateReplicaMetadataWrite("humanItems", c.model)
if (e == nil) != c.ok {
t.Fatalf("allowed=%v error=%v", c.ok, e)
}
})
}
}
func TestQueryReplicaSafetyRejectsNonFiniteAndUnsafeHealth(t *testing.T) {
p := QueryReplicaSafetyPolicy{BytesPerSecond: 65536, CommandsPerSecond: 2, MaxBatchRows: 25, RunID: "r", RecordsPerSecond: 10, BaselineCPU: 85}
if p.validate() != nil {
t.Fatal("valid policy")
}
p.BaselineCPU = math.NaN()
if p.validate() == nil {
t.Fatal("NaN policy accepted")
}
now := time.Now()
good := QueryReplicaSafetyObservation{At: now, CPU: 85, Ready: true, WritersCovered: true, HostFreeBytes: 120 << 30, TargetUsedFraction: .1}
if observationFailure(good, now) != "" {
t.Fatal("healthy observation")
}
for _, change := range []func(*QueryReplicaSafetyObservation){func(o *QueryReplicaSafetyObservation) { o.CPU = 95 }, func(o *QueryReplicaSafetyObservation) { o.CPU = math.NaN() }, func(o *QueryReplicaSafetyObservation) { o.WorkErrors = true }, func(o *QueryReplicaSafetyObservation) { o.At = now.Add(-46 * time.Second) }, func(o *QueryReplicaSafetyObservation) { o.TargetUsedFraction = .7 }, func(o *QueryReplicaSafetyObservation) { o.HostFreeBytes = 99 << 30 }, func(o *QueryReplicaSafetyObservation) { o.ProtectedIOFull = 6 }} {
o := good
change(&o)
if observationFailure(o, now) == "" {
t.Fatal("unsafe observation accepted")
}
}
}
func TestQueryReplicaGlobalPacingAndCancellation(t *testing.T) {
p := newQueryReplicaPacer(100)
start := time.Now()
var wg sync.WaitGroup
for i := 0; i < 4; i++ {
wg.Add(1)
go func() {
defer wg.Done()
if e := p.wait(context.Background(), 5); e != nil {
t.Error(e)
}
}()
}
wg.Wait()
if time.Since(start) < 190*time.Millisecond {
t.Fatal("per-caller budgets bypassed shared rate")
}
ctx, cancel := context.WithCancel(context.Background())
cancel()
start = time.Now()
if e := p.wait(ctx, 100); !errors.Is(e, context.Canceled) || time.Since(start) > 50*time.Millisecond {
t.Fatal("uncancelable budget", e)
}
}
func TestQueryReplicaTransportReadDeadlineAndClose(t *testing.T) {
for _, closeIt := range []bool{false, true} {
t.Run(map[bool]string{false: "deadline", true: "close"}[closeIt], func(t *testing.T) {
a, b := net.Pipe()
defer b.Close()
budget, _ := NewQueryReplicaTransport(1024, 2)
c := &queryReplicaRateConn{Conn: a, budget: budget, closed: make(chan struct{}), wake: make(chan struct{}, 1)}
defer c.Close()
if !closeIt {
_ = c.SetReadDeadline(time.Now().Add(30 * time.Millisecond))
}
done := make(chan error, 1)
go func() { _, e := c.Read(make([]byte, 1024)); done <- e }()
if closeIt {
time.Sleep(10 * time.Millisecond)
_ = c.Close()
}
select {
case e := <-done:
if e == nil {
t.Fatal("unbounded read succeeded")
}
case <-time.After(300 * time.Millisecond):
t.Fatal("pacing ignores connection deadline/close")
}
})
}
}
func TestQueryReplicaTransportByteBudgetSharedAcrossConnections(t *testing.T) {
budget, _ := NewQueryReplicaTransport(20480, 8)
start := time.Now()
var wg sync.WaitGroup
for i := 0; i < 2; i++ {
a, b := net.Pipe()
c := &queryReplicaRateConn{Conn: a, budget: budget, closed: make(chan struct{}), wake: make(chan struct{}, 1)}
wg.Add(1)
go func() {
defer wg.Done()
defer c.Close()
defer b.Close()
go func() { _, _ = b.Write(make([]byte, 4096)) }()
got := 0
buf := make([]byte, 4096)
for got < 4096 {
n, e := c.Read(buf)
if e != nil {
t.Error(e)
return
}
if n > 1024 {
t.Error("unbounded transport burst")
}
got += n
}
}()
}
wg.Wait()
if time.Since(start) < 390*time.Millisecond {
t.Fatal("aggregate byte rate exceeded")
}
}

View File

@ -0,0 +1,126 @@
package store
import (
"context"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
"time"
)
// Preserve incremental progress during history copy. Full dirty batches reserve
// at least half the row opportunities (more than the30% floor); a backlog seen
// continuously for5min receives four batches for each history batch. This age
// is explicitly observation age, not an invented customer event timestamp.
func (q *QueryReplica) backlogAge(ctx context.Context, n string, rows int) (time.Duration, error) {
var since time.Time
e := q.transaction(ctx, func(sc mongo.SessionContext) error {
key := "backlog/" + n
if rows < int(q.limit()) {
_, e := q.control().DeleteOne(sc, bson.M{"_id": key})
return e
}
var d struct {
Since time.Time `bson:"since"`
}
e := q.control().FindOne(sc, bson.M{"_id": key}).Decode(&d)
if e != nil && e != mongo.ErrNoDocuments {
return e
}
if e == mongo.ErrNoDocuments {
d.Since = time.Now().UTC()
if _, e = q.control().InsertOne(sc, bson.M{"_id": key, "since": d.Since}); e != nil {
return e
}
}
since = d.Since
return nil
})
if e != nil || since.IsZero() {
return 0, e
}
return time.Since(since), nil
}
func (q *QueryReplica) runGuardedOnce(ctx context.Context, bootstrap bool) (bool, error) {
if _, e := q.sourcePermit(ctx, 0); e != nil {
return false, e
}
names := replicaCollections()
var cursor struct {
Next int `bson:"next"`
}
e := q.control().FindOne(ctx, bson.M{"_id": "schedule"}).Decode(&cursor)
if e != nil && e != mongo.ErrNoDocuments {
return false, e
}
start := cursor.Next % len(names)
if start < 0 {
start = 0
}
for offset := 0; offset < len(names); offset++ {
index := (start + offset) % len(names)
name := names[index]
n, e := q.DrainStep(ctx, name)
if e != nil {
return false, e
}
age, e := q.backlogAge(ctx, name, n)
if e != nil {
return false, e
}
if age > 5*time.Minute {
for extra := 0; extra < 3; extra++ {
n, e = q.DrainStep(ctx, name)
if e != nil {
return false, e
}
if n < int(q.limit()) {
if _, e = q.backlogAge(ctx, name, n); e != nil {
return false, e
}
break
}
}
}
cp, e := q.checkpoint(ctx, name)
if e != nil {
return false, e
}
if !cp.Complete {
if !bootstrap {
return false, ErrQueryReplicaNotReady
}
if _, _, e = q.BootstrapStep(ctx, name); e != nil {
return false, e
}
}
if e = q.transaction(ctx, func(sc mongo.SessionContext) error {
_, e := q.control().UpdateOne(sc, bson.M{"_id": "schedule"}, bson.M{"$set": bson.M{"next": (index + 1) % len(names)}}, options.Update().SetUpsert(true))
return e
}); e != nil {
return false, e
}
}
complete := true
for _, n := range names {
cp, e := q.checkpoint(ctx, n)
if e != nil {
return false, e
}
complete = complete && cp.Complete
}
if complete {
if _, e := q.ReconcileOneDue(ctx, 6*time.Hour); e != nil {
return false, e
}
}
if _, e := q.ReconcileDeleteStep(ctx); e != nil {
return false, e
}
if e := q.PublishReady(ctx); e == ErrQueryReplicaNotReady {
return false, nil
} else if e != nil {
return false, e
}
return true, nil
}

View File

@ -0,0 +1,165 @@
package store
import (
"context"
"errors"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
"strings"
"time"
)
// All projector source mutations pass this seam. Database grants are scoped to
// collections, not fields; neither raw replacements nor business fields belong
// in this writer. These checks remain on even when optional pacing is disabled.
func validateReplicaMetadataWrite(collection string, model mongo.WriteModel) error {
m, ok := model.(*mongo.UpdateOneModel)
if !ok || m.Upsert != nil && *m.Upsert {
return errors.New("source metadata write rejected")
}
var f, u bson.M
b, e := bson.Marshal(m.Filter)
if e != nil {
return e
}
if e = bson.Unmarshal(b, &f); e != nil {
return e
}
b, e = bson.Marshal(m.Update)
if e != nil {
return e
}
if e = bson.Unmarshal(b, &u); e != nil {
return e
}
if f["_id"] == nil {
return errors.New("source metadata identity required")
}
if id := deletionDoc(f["_id"]); len(id) > 0 {
for key := range id {
if strings.HasPrefix(key, "$") {
return errors.New("metadata identity selector rejected")
}
}
}
for op, raw := range u {
if op != "$set" && op != "$unset" {
return errors.New("source metadata operator rejected")
}
fields := deletionDoc(raw)
if len(fields) == 0 {
return errors.New("empty metadata update")
}
for path := range fields {
allowed := false
if collection == querySyncDeleteCollection {
allowed = op == "$set" && (path == "phase" || path == "expireAt")
} else if queryReplicaFields[collection] != "" {
switch path {
case "_querySync.dirty", "_querySync.revision", "_querySync.signature", "_querySync.mode", "_querySync.producer":
allowed = op == "$set"
}
}
if !allowed {
return errors.New("source business field write rejected")
}
}
}
if queryReplicaFields[collection] != "" && f[querySyncField+".revision"] == nil {
return errors.New("metadata revision CAS required")
}
return nil
}
func (q *QueryReplica) sourceMetadataBulk(ctx context.Context, n string, models []mongo.WriteModel) error {
if len(models) == 0 {
return nil
}
for _, m := range models {
if e := validateReplicaMetadataWrite(n, m); e != nil {
return e
}
}
if _, e := q.sourcePermit(ctx, 0); e != nil {
return e
}
ctx, cancel := q.sourceContext(ctx)
defer cancel()
_, e := q.Source.db.Collection(n).BulkWrite(ctx, models, options.BulkWrite().SetOrdered(false))
return e
}
func (q *QueryReplica) sourceIntentUpdate(ctx context.Context, filter, update bson.M) (*mongo.UpdateResult, error) {
m := mongo.NewUpdateOneModel().SetFilter(filter).SetUpdate(update)
if e := validateReplicaMetadataWrite(querySyncDeleteCollection, m); e != nil {
return nil, e
}
if _, e := q.sourcePermit(ctx, 0); e != nil {
return nil, e
}
ctx, cancel := q.sourceContext(ctx)
defer cancel()
return q.Source.db.Collection(querySyncDeleteCollection).UpdateOne(ctx, filter, update)
}
func (q *QueryReplica) sourceAuditIntent(ctx context.Context, i queryDeleteIntent) error {
if !i.Audit || i.Phase != "reconcile" || queryReplicaFields[i.Collection] == "" || i.ID == "" {
return errors.New("invalid source audit intent")
}
if _, e := q.sourcePermit(ctx, 0); e != nil {
return e
}
ctx, cancel := q.sourceContext(ctx)
defer cancel()
_, e := q.Source.db.Collection(querySyncDeleteCollection).InsertOne(ctx, i)
return e
}
// Cleanup is bounded even for a historical DeleteMany intent. We never replay
// the source delete. A cancelled old writer cannot claim additional rows.
func (q *QueryReplica) recoverSourceClaims(ctx context.Context, i queryDeleteIntent) (bool, error) {
if i.Phase == "prepared" || i.Phase == "deleting" {
now := time.Now().UTC()
if i.LeaseUntil.After(now) {
return false, nil
}
r, e := q.sourceIntentUpdate(ctx, bson.M{"_id": i.ID, "phase": i.Phase, "leaseUntil": bson.M{"$lte": now}}, bson.M{"$set": bson.M{"phase": "cancelled"}})
if e != nil || r.MatchedCount != 1 {
return false, e
}
} else if i.Phase != "complete" && i.Phase != "cancelled" && i.Phase != "reconcile" {
return false, nil
}
limit, e := q.sourcePermit(ctx, q.limit())
if e != nil {
return false, e
}
work, cancel := q.sourceContext(ctx)
defer cancel()
cur, e := q.Source.db.Collection(i.Collection).Find(work, bson.M{querySyncField + ".deleteClaim": i.ID}, options.Find().SetHint("query_sync_delete_claim_v1").SetProjection(bson.M{"_id": 1}).SetLimit(limit).SetBatchSize(int32(limit)).SetMaxTime(1500*time.Millisecond))
if e != nil {
return false, e
}
var rows []bson.M
if e = cur.All(work, &rows); e != nil {
return false, e
}
if len(rows) > 0 {
if e = q.paceRows(ctx, len(rows)); e != nil {
return false, e
}
if _, e = q.sourcePermit(ctx, 0); e != nil {
return false, e
}
ids := bson.A{}
for _, r := range rows {
ids = append(ids, r["_id"])
}
// This is the only UpdateMany: exact bounded identities and exact claim,
// with only its private claim field removed; no upsert or business delete.
work, cancel := q.sourceContext(ctx)
defer cancel()
_, e = q.Source.db.Collection(i.Collection).UpdateMany(work, bson.M{"_id": bson.M{"$in": ids}, querySyncField + ".deleteClaim": i.ID}, bson.M{"$unset": bson.M{querySyncField + ".deleteClaim": ""}})
return false, e
}
_, e = q.sourceIntentUpdate(ctx, bson.M{"_id": i.ID, "phase": bson.M{"$in": bson.A{"complete", "cancelled", "reconcile"}}}, bson.M{"$set": bson.M{"phase": "reconcile"}})
return e == nil, e
}

View File

@ -0,0 +1,118 @@
package store
import (
"context"
"droboticflow/internal/domain"
"encoding/json"
"errors"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
"os"
"os/exec"
"strings"
"testing"
"time"
)
// This fixture owns a separate ephemeral auth-enabled standalone Mongo. It
// never enables authorization or creates users on the shared source/query lab.
func TestQueryReplicaSourceAccountPermissionsLocal(t *testing.T) {
if os.Getenv("QUERY_REPLICA_LOCAL_TEST") != "1" {
t.Skip("isolated auth fixture required")
}
ctx := t.Context()
if host := os.Getenv("DOCKER_HOST"); host != "" && !strings.HasPrefix(host, "unix://") {
t.Fatal("remote Docker prohibited")
}
endpoint, e := exec.Command("docker", "context", "inspect", "--format", "{{.Endpoints.docker.Host}}").Output()
if e != nil || !strings.HasPrefix(strings.TrimSpace(string(endpoint)), "unix://") {
t.Fatal("local Docker socket required")
}
user, password := domain.NewID("root"), domain.NewID("secret")
name := strings.ToLower(domain.NewID("query-auth"))
cmd := exec.Command("docker", "run", "--rm", "-d", "--name", name, "--label", "codex.fixture=query-source-auth", "--cpus=1", "--memory=512m", "-p", "127.0.0.1::27017", "--env", "MONGO_INITDB_ROOT_USERNAME", "--env", "MONGO_INITDB_ROOT_PASSWORD", "mongo@sha256:5d7043a4ffe02b9ed1b6e0bab057546981af5ca0a79107e9c461e49bc44c0a7b", "--wiredTigerCacheSizeGB", "0.25", "--bind_ip_all")
cmd.Env = append(os.Environ(), "MONGO_INITDB_ROOT_USERNAME="+user, "MONGO_INITDB_ROOT_PASSWORD="+password)
out, e := cmd.Output()
if e != nil {
t.Fatal("owned authentication fixture startup failed")
}
container := strings.TrimSpace(string(out))
t.Cleanup(func() { _ = exec.Command("docker", "stop", "--time=5", container).Run() })
raw, e := exec.Command("docker", "inspect", "--format", "{{json .NetworkSettings.Ports}}", container).Output()
if e != nil {
t.Fatal("fixture binding unavailable")
}
var ports map[string][]struct{ HostIP, HostPort string }
if json.Unmarshal(raw, &ports) != nil || len(ports["27017/tcp"]) != 1 || ports["27017/tcp"][0].HostIP != "127.0.0.1" {
t.Fatal("fixture not loopback-only")
}
uri := "mongodb://127.0.0.1:" + ports["27017/tcp"][0].HostPort + "/?directConnection=true"
admin, e := mongo.Connect(ctx, options.Client().ApplyURI(uri).SetAuth(options.Credential{AuthSource: "admin", Username: user, Password: password}).SetServerSelectionTimeout(500*time.Millisecond))
if e != nil {
t.Fatal("fixture connection failed")
}
defer admin.Disconnect(context.Background())
ready := false
for n := 0; n < 80; n++ {
short, cancel := context.WithTimeout(ctx, 500*time.Millisecond)
e = admin.Ping(short, nil)
cancel()
if e == nil {
ready = true
break
}
time.Sleep(250 * time.Millisecond)
}
if !ready {
t.Fatal("authentication fixture readiness failed")
}
dbname := domain.NewID("query_sync_auth")
db := admin.Database(dbname)
privilege := bson.A{bson.M{"resource": bson.M{"cluster": true}, "actions": bson.A{"serverStatus"}}, bson.M{"resource": bson.M{"db": dbname, "collection": "humanItems"}, "actions": bson.A{"find", "update"}}, bson.M{"resource": bson.M{"db": dbname, "collection": querySyncDeleteCollection}, "actions": bson.A{"find", "insert", "update"}}}
if e = admin.Database("admin").RunCommand(ctx, bson.D{{Key: "createRole", Value: "metadata-sync"}, {Key: "privileges", Value: privilege}, {Key: "roles", Value: bson.A{}}}).Err(); e != nil {
t.Fatal("fixture role setup failed", e)
}
limitedPassword := domain.NewID("limited-secret")
if e = db.RunCommand(ctx, bson.D{{Key: "createUser", Value: "projector"}, {Key: "pwd", Value: limitedPassword}, {Key: "roles", Value: bson.A{bson.M{"role": "metadata-sync", "db": "admin"}}}}).Err(); e != nil {
t.Fatal("fixture user setup failed")
}
_, e = db.Collection("humanItems").InsertOne(ctx, bson.M{"_id": "i", "id": "i", "version": 7, "status": "claimed", "annotationDraft": "keep", "_querySync": bson.M{"revision": 1, "dirty": true}})
if e != nil {
t.Fatal(e)
}
client, e := mongo.Connect(ctx, options.Client().ApplyURI(uri).SetAuth(options.Credential{AuthSource: dbname, Username: "projector", Password: limitedPassword}))
if e != nil {
t.Fatal("fixture principal unavailable")
}
defer client.Disconnect(context.Background())
limited := client.Database(dbname)
q := &QueryReplica{Source: NewMongoStore(limited)}
if e = q.sourceMetadataBulk(ctx, "humanItems", []mongo.WriteModel{mongo.NewUpdateOneModel().SetFilter(bson.M{"_id": "i", "_querySync.revision": 1}).SetUpdate(bson.M{"$set": bson.M{"_querySync.dirty": false}})}); e != nil {
t.Fatal("allowed acknowledgement failed", e)
}
if e = q.sourceMetadataBulk(ctx, "humanItems", []mongo.WriteModel{mongo.NewUpdateOneModel().SetFilter(bson.M{"_id": "i", "_querySync.revision": 1}).SetUpdate(bson.M{"$set": bson.M{"annotationDraft": "overwrite"}})}); e == nil {
t.Fatal("application metadata boundary bypassed")
}
denied := func(label string, e error) {
t.Helper()
var ce mongo.CommandError
if !errors.As(e, &ce) || ce.Code != 13 {
t.Fatalf("%s was not denied by real authentication", label)
}
}
_, e = limited.Collection("humanItems").InsertOne(ctx, bson.M{"_id": "other"})
denied("insert", e)
_, e = limited.Collection("humanItems").DeleteOne(ctx, bson.M{"_id": "i"})
denied("delete", e)
denied("DDL", limited.RunCommand(ctx, bson.M{"drop": "humanItems"}).Err())
denied("other database", client.Database("foreign").Collection("humanItems").FindOne(ctx, bson.M{}).Err())
if e = client.Database("admin").RunCommand(ctx, bson.M{"serverStatus": 1}).Err(); e != nil {
t.Fatal("monitor privilege unavailable")
}
var kept bson.M
if e = db.Collection("humanItems").FindOne(ctx, bson.M{"_id": "i"}).Decode(&kept); e != nil || kept["annotationDraft"] != "keep" || kept["version"] != int32(7) || kept["status"] != "claimed" || deletionDoc(kept[querySyncField])["dirty"] != false {
t.Fatal("authority fields altered")
}
}

View File

@ -0,0 +1,158 @@
package store
import (
"context"
"errors"
"net"
"sync"
"time"
)
// A shared leaky bucket, with no accumulated idle credit. The source driver
// shares it across application and monitoring connections; TLS overhead is
// conservatively included. Waiting never occurs inside a target transaction.
type queryReplicaPacer struct {
mu sync.Mutex
next time.Time
rate float64
}
func newQueryReplicaPacer(rate float64) *queryReplicaPacer { return &queryReplicaPacer{rate: rate} }
func (p *queryReplicaPacer) wait(ctx context.Context, units float64) error {
if units <= 0 {
return ctx.Err()
}
p.mu.Lock()
now := time.Now()
if p.next.Before(now) {
p.next = now
}
p.next = p.next.Add(time.Duration(units / p.rate * float64(time.Second)))
until := p.next
p.mu.Unlock()
timer := time.NewTimer(time.Until(until))
defer timer.Stop()
select {
case <-ctx.Done():
return ctx.Err()
case <-timer.C:
return nil
}
}
type QueryReplicaTransport struct{ bytes, commands *queryReplicaPacer }
func NewQueryReplicaTransport(bytesPerSecond, commandsPerSecond int) (*QueryReplicaTransport, error) {
if bytesPerSecond < 1024 || bytesPerSecond > 256*1024 || commandsPerSecond < 1 || commandsPerSecond > 8 {
return nil, errors.New("invalid source transport budget")
}
return &QueryReplicaTransport{newQueryReplicaPacer(float64(bytesPerSecond)), newQueryReplicaPacer(float64(commandsPerSecond))}, nil
}
func (d *QueryReplicaTransport) DialContext(ctx context.Context, network, address string) (net.Conn, error) {
c, e := (&net.Dialer{}).DialContext(ctx, network, address)
if e != nil {
return nil, e
}
return &queryReplicaRateConn{Conn: c, budget: d, closed: make(chan struct{}), wake: make(chan struct{}, 1)}, nil
}
type queryReplicaRateConn struct {
net.Conn
budget *QueryReplicaTransport
once sync.Once
closed chan struct{}
mu sync.Mutex
readDeadline, writeDeadline time.Time
wake chan struct{}
}
func (c *queryReplicaRateConn) Close() error {
c.once.Do(func() { close(c.closed) })
return c.Conn.Close()
}
func (c *queryReplicaRateConn) SetDeadline(t time.Time) error {
c.mu.Lock()
c.readDeadline = t
c.writeDeadline = t
c.mu.Unlock()
c.notify()
return c.Conn.SetDeadline(t)
}
func (c *queryReplicaRateConn) SetReadDeadline(t time.Time) error {
c.mu.Lock()
c.readDeadline = t
c.mu.Unlock()
c.notify()
return c.Conn.SetReadDeadline(t)
}
func (c *queryReplicaRateConn) SetWriteDeadline(t time.Time) error {
c.mu.Lock()
c.writeDeadline = t
c.mu.Unlock()
c.notify()
return c.Conn.SetWriteDeadline(t)
}
func (c *queryReplicaRateConn) notify() {
c.mu.Lock()
old := c.wake
c.wake = make(chan struct{})
close(old)
c.mu.Unlock()
}
func (c *queryReplicaRateConn) wait(p *queryReplicaPacer, n int, read bool) error {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
done := make(chan error, 1)
go func() { done <- p.wait(ctx, float64(n)) }()
for {
c.mu.Lock()
wake := c.wake
deadline := c.writeDeadline
if read {
deadline = c.readDeadline
}
c.mu.Unlock()
var timer *time.Timer
var expired <-chan time.Time
if !deadline.IsZero() {
timer = time.NewTimer(time.Until(deadline))
expired = timer.C
}
select {
case e := <-done:
if timer != nil {
timer.Stop()
}
return e
case <-c.closed:
if timer != nil {
timer.Stop()
}
return net.ErrClosed
case <-expired:
return context.DeadlineExceeded
case <-wake:
if timer != nil {
timer.Stop()
}
}
}
}
func (c *queryReplicaRateConn) Read(p []byte) (int, error) {
if len(p) == 0 {
return 0, nil
}
if len(p) > 1024 {
p = p[:1024]
}
if e := c.wait(c.budget.bytes, len(p), true); e != nil {
return 0, e
}
return c.Conn.Read(p)
}
func (c *queryReplicaRateConn) Write(p []byte) (int, error) {
if e := c.wait(c.budget.commands, 1, false); e != nil {
return 0, e
}
return c.Conn.Write(p)
}

View File

@ -0,0 +1,296 @@
#!/usr/bin/env python3
"""Trusted operator-side observer. No kubeconfig or authority credentials enter builders.
Every run writes private health observations in the derived query database.
Only --stop-on-failure permits cluster configuration mutations: exact UID-tested scale-to-zero of the
TWO shadow controllers in the hash-bound plan. Never changes authority services.
Do not run against production before approving the final startup/stop scope.
"""
import argparse
import base64
import contextlib
import datetime as dt
import hashlib
import json
import os
from pathlib import Path
import socket
import subprocess
import threading
import time
import urllib.parse
import urllib.request
def check_bindings(b):
if b['sourceNamespace'] == b['shadowNamespace'] or len(b['writers']) != 5:
raise ValueError('invalid namespace or writer scope')
if {x['name'] for x in b['controllers']} != {'query-replica', 'query-read-builder'}:
raise ValueError('stop scope must contain only shadow controllers')
if b['sourceDatabase'] == b['targetDatabase'] or not 0 < b['baselineCPU'] <= 90:
raise ValueError('invalid source identity or baseline')
def cpu_percent(a, z):
if z['sourceProcess'] != a['sourceProcess']:
raise ValueError('source identity changed')
elapsed = z['uptimeMillis'] - a['uptimeMillis']
used = z['cpuMicroseconds'] - a['cpuMicroseconds']
if elapsed <= 0 or used < 0:
raise ValueError('nonmonotonic source counters')
return used / 1000 / elapsed / 4 * 100
def full_io_pressure(text):
for line in text.splitlines():
if line.startswith('full '):
return float(next(x.split('=', 1)[1] for x in line.split() if x.startswith('avg10=')))
raise ValueError('protected IO evidence missing')
def business_health(path, now=None):
feed = json.loads(Path(path).read_text())
at = dt.datetime.fromisoformat(feed['at'].replace('Z', '+00:00'))
age = ((now or dt.datetime.now(dt.timezone.utc)) - at).total_seconds()
if (not 0 <= age <= 45 or feed.get('coverage') != 'save-slice-submit-review-acceptance'
or type(feed.get('unexpectedErrors')) is not bool or type(feed.get('latencyRegressed')) is not bool):
raise ValueError('business-health evidence stale or incomplete')
return feed['unexpectedErrors'] or feed['latencyRegressed']
class Observer:
def __init__(self, args, bindings):
self.args, self.b = args, bindings
self.k = ['kubectl', '--kubeconfig', args.kubeconfig]
self.high = self.high_io = 0
def call(self, *args, data=None, timeout=15):
p = subprocess.run(self.k + list(args), input=data, capture_output=True,
text=True, timeout=timeout)
if p.returncode:
raise RuntimeError('bounded cluster operation unavailable')
return p.stdout
def get(self, ns, kind, name):
return json.loads(self.call('-n', ns, 'get', kind, name, '-o', 'json'))
def verify(self):
for key in ('source', 'shadow'):
n = self.get(self.b[key + 'Namespace'], 'namespace', self.b[key + 'Namespace'])
if n['metadata']['uid'] != self.b[key + 'NamespaceUID']:
raise ValueError('namespace identity changed')
for w in self.b['writers']:
d = self.get(self.b['sourceNamespace'], 'deployment', w['name'])
env = {x['name']: x.get('value') for x in d['spec']['template']['spec']['containers'][0].get('env', [])}
if (d['metadata']['uid'] != w['uid'] or env.get('DROBOTICFLOW_BUILD_COMMIT') != self.b['writerCommit']
or env.get('QUERY_SOURCE_TRACKING') != '1'
or d['spec']['template']['spec']['containers'][0]['image'] != w['image']
or d.get('status', {}).get('availableReplicas', 0) != w['replicas']):
raise ValueError('writer coverage or health changed')
for flag in ('QUERY_REPLICA_REPORTS', 'QUERY_REPLICA_COUNTS', 'QUERY_REPLICA_POOLS', 'QUERY_SERVING_READS', 'QUERY_PUBLISHED_RESULTS'):
if env.get(flag, '0') not in ('', '0', 'false'):
raise ValueError('production query entry changed')
def secret(self, name, key):
return base64.b64decode(self.get(self.b['shadowNamespace'], 'secret', name)['data'][key]).decode()
def target_environment(self, uri):
env = {k: v for k, v in os.environ.items() if not k.startswith(('MONGO_', 'QUERY_'))}
env.update(QUERY_READ_MONGO_URI=uri, QUERY_READ_MONGO_DB=self.b['targetDatabase'],
QUERY_SERVING_NAMESPACE=self.b['servingNamespace'])
return env
def run_binary(self, args, env, data=None):
if hashlib.sha256(Path(self.args.binary).read_bytes()).hexdigest() != self.args.binary_sha256:
raise ValueError('observer binary changed')
p = subprocess.run([self.args.binary] + args, input=data, env=env, capture_output=True,
text=True, timeout=10)
if p.returncode:
raise RuntimeError('bounded observation command unavailable')
return p.stdout
@contextlib.contextmanager
def source_relay(self, uri):
u = urllib.parse.urlsplit(uri)
if u.scheme != 'mongodb' or not u.hostname or ',' in u.netloc:
raise ValueError('single reviewed source endpoint required')
endpoint = u.hostname + ':' + str(u.port or 27017)
if hashlib.sha256(endpoint.encode()).hexdigest() != self.b['sourceEndpointSHA256']:
raise ValueError('source endpoint changed')
# No secret arguments or stderr: credentials remain in local child env.
pods = json.loads(self.call('-n', self.b['sourceNamespace'], 'get', 'pods', '-l', 'app=droboticflow-api', '-o', 'json'))['items']
pod = next(x['metadata']['name'] for x in pods if not x['metadata'].get('deletionTimestamp')
and x['status'].get('phase') == 'Running'
and all(c['ready'] for c in x['status'].get('containerStatuses', [])))
listener = socket.socket(); listener.bind(('127.0.0.1', 0)); listener.listen(8); listener.settimeout(.2)
stop = threading.Event(); children = []
relay = 'exec 3<>/dev/tcp/"$1"/"$2" || exit 1; cat <&3 & reader=$!; cat >&3; kill "$reader" 2>/dev/null || true; wait "$reader" 2>/dev/null || true'
def transfer(conn):
p = subprocess.Popen(self.k + ['-n', self.b['sourceNamespace'], 'exec', '-i', pod, '--', 'bash', '-c', relay,
'shadow-readonly-metrics', u.hostname, str(u.port or 27017)],
stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL)
children.append(p)
def upstream():
try:
while not stop.is_set():
chunk = conn.recv(65536)
if not chunk: break
p.stdin.write(chunk); p.stdin.flush()
except (OSError, ValueError): pass
finally:
with contextlib.suppress(OSError): p.stdin.close()
threading.Thread(target=upstream, daemon=True).start()
try:
while not stop.is_set():
chunk = p.stdout.read1(65536)
if not chunk: break
conn.sendall(chunk)
except (OSError, ValueError): pass
finally:
conn.close(); p.terminate()
def accept():
while not stop.is_set():
try: conn, _ = listener.accept()
except socket.timeout: continue
except OSError: return
threading.Thread(target=transfer, args=(conn,), daemon=True).start()
thread = threading.Thread(target=accept, daemon=True); thread.start()
params = urllib.parse.parse_qs(u.query); params['directConnection'] = ['true']
credential = u.netloc.rsplit('@', 1)[0] + '@' if '@' in u.netloc else ''
local = urllib.parse.urlunsplit(('mongodb', credential + '127.0.0.1:' + str(listener.getsockname()[1]), u.path,
urllib.parse.urlencode(params, doseq=True), ''))
# A TLS hostname is not rewritten/bypassed. This initial deployment uses
# its reviewed non-TLS private endpoint; other topologies need a new plan.
if params.get('tls', params.get('ssl', ['false']))[0].lower() == 'true':
listener.close(); stop.set(); raise ValueError('TLS relay needs separately verified server-name handling')
try: yield local
finally:
stop.set(); listener.close(); thread.join(timeout=1)
for child in children:
if child.poll() is None: child.terminate()
for child in children:
try: child.wait(timeout=2)
except subprocess.TimeoutExpired: child.kill()
@contextlib.contextmanager
def target_forward(self, uri):
with socket.socket() as listener:
listener.bind(('127.0.0.1', 0)); port = listener.getsockname()[1]
process = subprocess.Popen(self.k + ['-n', self.b['shadowNamespace'], 'port-forward', '--address=127.0.0.1',
'service/query-mongo', str(port) + ':27129'],
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
try:
for _ in range(40):
if process.poll() is not None: raise RuntimeError('query tunnel unavailable')
try:
with socket.create_connection(('127.0.0.1', port), timeout=.1): break
except OSError: time.sleep(.1)
else: raise RuntimeError('query tunnel deadline')
u = urllib.parse.urlsplit(uri)
credential = u.netloc.rsplit('@', 1)[0] + '@' if '@' in u.netloc else ''
yield urllib.parse.urlunsplit(('mongodb', credential + '127.0.0.1:' + str(port), u.path, u.query, ''))
finally:
process.terminate()
try: process.wait(timeout=2)
except subprocess.TimeoutExpired: process.kill()
def source_sample(self):
with self.source_relay(self.secret('query-shadow-source-connection', 'uri')) as uri:
env = self.target_environment(''); env.update(MONGO_URI=uri, MONGO_DB=self.b['sourceDatabase'])
return json.loads(self.run_binary(['--sample-source'], env))
def observation(self, cpu, sampled_at):
self.verify()
with urllib.request.urlopen(self.b['readyURL'], timeout=3) as response:
ready = response.status == 200 and json.load(response).get('ready') is True
ns = self.b['shadowNamespace']; pod = self.get(ns, 'pod', 'query-mongo-0')
if pod['metadata']['uid'] != self.b['mongoPodUID']:
raise ValueError('query Mongo rebuilt; recovery verification required')
fs = json.loads(self.call('get', '--raw', '/api/v1/nodes/10.189.16.4/proxy/stats/summary'))['node']['fs']
text = self.call('-n', ns, 'exec', 'query-mongo-0', '-c', 'mongo', '--', 'df', '-P', '-B1', '/lab/data')
fields = text.splitlines()[-1].split(); used = int(fields[2]) / int(fields[1])
protected = json.loads(self.call('-n', self.b['sourceNamespace'], 'get', 'pods', '--field-selector=spec.nodeName=10.189.16.4', '-o', 'json'))['items']
io = 0.
for p in protected:
if p['status'].get('phase') != 'Running': continue
uid = p['metadata']['uid'].replace('-', '_')
# Current production workloads are Burstable; refuse unknown cgroups.
path = '/hostcg/kubepods.slice/kubepods-burstable.slice/kubepods-burstable-pod' + uid + '.slice/io.pressure'
io = max(io, full_io_pressure(self.call('-n', ns, 'exec', 'query-mongo-0', '-c', 'mongo', '--', 'cat', path)))
# A trusted business-health feed is required; readiness/log absence is
# not evidence that saves/submits meet their latency/error baseline.
work_errors = business_health(self.args.work_health)
pods = json.loads(self.call('-n', self.b['sourceNamespace'], 'get', 'pods', '-l', 'app=droboticflow-api', '-o', 'json'))['items']
for p in pods:
if p['metadata'].get('deletionTimestamp'): continue
logs = self.call('-n', self.b['sourceNamespace'], 'logs', p['metadata']['name'], '--since=45s', '--tail=2000').lower()
work_errors |= any(s in logs for s in ('panic:', 'i/o timeout', 'write conflict retry exhausted', 'submit failed', 'save failed'))
return dict(at=sampled_at, cpu=cpu, ready=ready, writersCovered=True,
workErrors=work_errors, protectedIOFull=io, targetUsedFraction=used, hostFreeBytes=fs['availableBytes'])
def publish(self, observation):
with self.target_forward(self.secret('query-shadow-query-connections', 'builder-uri')) as uri:
self.run_binary(['--record-observation', '-'], self.target_environment(uri), json.dumps(observation))
def unsafe(self, o):
self.high = self.high + 1 if o['cpu'] > self.b['baselineCPU'] + 3 else 0
self.high_io = self.high_io + 1 if o['protectedIOFull'] > 1 else 0
return (not o['ready'] or not o['writersCovered'] or o['workErrors'] or o['cpu'] >= 95 or self.high >= 2
or self.high_io >= 2 or o['protectedIOFull'] > 5 or o['targetUsedFraction'] >= .7 or o['hostFreeBytes'] < (100 << 30))
def stop_shadow(self):
for c in self.b['controllers']:
d = self.get(self.b['shadowNamespace'], 'deployment', c['name'])
if d['metadata']['uid'] != c['uid']: raise ValueError('stop controller replaced')
if d['spec']['replicas'] == 0: continue
patch = [{'op': 'test', 'path': '/metadata/uid', 'value': c['uid']},
{'op': 'test', 'path': '/metadata/resourceVersion', 'value': d['metadata']['resourceVersion']},
{'op': 'replace', 'path': '/spec/replicas', 'value': 0}]
self.call('-n', self.b['shadowNamespace'], 'patch', 'deployment', c['name'], '--type=json', '-p', json.dumps(patch))
def run(self):
try: prior = self.source_sample()
except Exception:
if self.args.stop_on_failure: self.stop_shadow()
raise RuntimeError('initial source observation unavailable') from None
while True:
try:
for _ in range(3):
time.sleep(10)
with urllib.request.urlopen(self.b['readyURL'],timeout=3) as response:
if response.status!=200 or json.load(response).get('ready') is not True:
raise ValueError('production readiness unavailable')
current = self.source_sample(); cpu = cpu_percent(prior, current); prior = current
sampled_at=current['sampledAt'];age=(dt.datetime.now(dt.timezone.utc)-dt.datetime.fromisoformat(sampled_at.replace('Z','+00:00'))).total_seconds()
if not 0<=age<=45: raise ValueError('source observation stale')
o = self.observation(cpu,sampled_at); self.publish(o)
unsafe=self.unsafe(o)
print(json.dumps({'at':o['at'],'cpu':cpu,'ready':o['ready'],'unsafe':unsafe}),flush=True)
if unsafe:
if self.args.stop_on_failure: self.stop_shadow()
return
except Exception:
# No raw exception may print a URI, secret or customer log line.
with contextlib.suppress(Exception):
self.publish(dict(at=dt.datetime.now(dt.timezone.utc).isoformat(), cpu=0, ready=False, writersCovered=False,
workErrors=True, protectedIOFull=0, targetUsedFraction=0, hostFreeBytes=0))
if self.args.stop_on_failure: self.stop_shadow()
raise RuntimeError('observer paused; inspect private metadata and verify before resume') from None
def main():
p = argparse.ArgumentParser(description=__doc__)
p.add_argument('--kubeconfig', required=True); p.add_argument('--bindings', required=True)
p.add_argument('--bindings-sha256', required=True); p.add_argument('--binary', required=True)
p.add_argument('--binary-sha256', required=True); p.add_argument('--work-health', required=True); p.add_argument('--stop-on-failure', action='store_true')
args = p.parse_args(); raw = Path(args.bindings).read_bytes()
if hashlib.sha256(raw).hexdigest() != args.bindings_sha256: raise ValueError('bindings changed')
b = json.loads(raw); check_bindings(b); Observer(args, b).run()
if __name__ == '__main__':
try: main()
except KeyboardInterrupt: raise SystemExit(0)
except Exception: raise SystemExit('observer stopped; private checkpoint retained; no automatic resume')

View File

@ -0,0 +1,63 @@
import importlib.util
import json
from pathlib import Path
from types import SimpleNamespace
import unittest
import tempfile
import datetime as dt
spec = importlib.util.spec_from_file_location('shadow_observer', Path(__file__).with_name('query-shadow-observer.py'))
m = importlib.util.module_from_spec(spec); spec.loader.exec_module(m)
class ObserverTests(unittest.TestCase):
def bindings(self):
return dict(sourceNamespace='authority', shadowNamespace='shadow', sourceDatabase='source', targetDatabase='query',
baselineCPU=85, writers=[{} for _ in range(5)], controllers=[dict(name='query-replica', uid='p'), dict(name='query-read-builder', uid='b')])
def test_cpu_and_identity(self):
self.assertAlmostEqual(m.cpu_percent(dict(sourceProcess='s', uptimeMillis=1000, cpuMicroseconds=0),
dict(sourceProcess='s', uptimeMillis=2000, cpuMicroseconds=2000000)), 50)
with self.assertRaises(ValueError):
m.cpu_percent(dict(sourceProcess='s', uptimeMillis=2, cpuMicroseconds=2), dict(sourceProcess='x', uptimeMillis=3, cpuMicroseconds=3))
def test_stop_scope_and_uid_cas(self):
b = self.bindings(); m.check_bindings(b)
observer = m.Observer(SimpleNamespace(kubeconfig='unused', stop_on_failure=True), b); calls = []
observer.get = lambda ns, kind, name: dict(metadata=dict(uid='p' if name=='query-replica' else 'b', resourceVersion='7'), spec=dict(replicas=1))
observer.call = lambda *args, **kwargs: calls.append(args) or ''
observer.stop_shadow()
self.assertEqual(len(calls), 2)
for args in calls:
self.assertEqual(args[:4], ('-n', 'shadow', 'patch', 'deployment'))
payload = json.loads(args[-1]); self.assertEqual(payload[0]['op'], 'test'); self.assertEqual(payload[-1]['value'], 0)
observer.get = lambda *args: dict(metadata=dict(uid='replaced', resourceVersion='8'), spec=dict(replicas=1))
with self.assertRaises(ValueError): observer.stop_shadow()
self.assertEqual(len(calls), 2)
b['controllers'][0]['name'] = 'droboticflow-api'
with self.assertRaises(ValueError): m.check_bindings(b)
def test_distinct_samples_and_thresholds(self):
o = m.Observer(SimpleNamespace(kubeconfig='unused'), self.bindings())
good = dict(cpu=90, ready=True, writersCovered=True, workErrors=False, protectedIOFull=0, targetUsedFraction=.1, hostFreeBytes=120<<30)
self.assertFalse(o.unsafe(good)); self.assertTrue(o.unsafe(good))
good['cpu']=85; self.assertFalse(o.unsafe(good));good['targetUsedFraction']=.7;self.assertTrue(o.unsafe(good))
def test_business_health_missing_stale_and_real_errors(self):
now = dt.datetime.now(dt.timezone.utc)
with tempfile.TemporaryDirectory() as folder:
path = Path(folder) / 'health.json'
with self.assertRaises(FileNotFoundError): m.business_health(path, now)
feed = dict(at=now.isoformat(), coverage='save-slice-submit-review-acceptance', unexpectedErrors=False, latencyRegressed=False)
path.write_text(json.dumps(feed)); self.assertFalse(m.business_health(path, now))
feed['latencyRegressed'] = True
path.write_text(json.dumps(feed)); self.assertTrue(m.business_health(path, now))
feed['latencyRegressed'] = 'false'
path.write_text(json.dumps(feed))
with self.assertRaises(ValueError): m.business_health(path, now)
feed['latencyRegressed'] = False; feed['at'] = (now - dt.timedelta(seconds=46)).isoformat()
path.write_text(json.dumps(feed))
with self.assertRaises(ValueError): m.business_health(path, now)
feed['at'] = now.isoformat(); feed['coverage'] = 'readiness-only'
path.write_text(json.dumps(feed))
with self.assertRaises(ValueError): m.business_health(path, now)
def test_unknown_io_is_not_zero(self):
with self.assertRaises(ValueError): m.full_io_pressure('no evidence')
self.assertEqual(m.full_io_pressure('some avg10=2.0\nfull avg10=1.25 avg60=0'), 1.25)
if __name__ == '__main__': unittest.main()

View File

@ -15,7 +15,11 @@ from lib.release_checks import check_environment, source_fingerprint, write_log
ROOT = Path(__file__).resolve().parents[1]
PRIVATE = ROOT / 'local-data/query-replica-local'
COMPOSE = ROOT / 'scripts/local/query-replica.compose.yml'
CASES = ('internal/store/query_replica_local_test.go',
CASES = ('cmd/query-replica/main_local_test.go',
'internal/store/query_replica_source_auth_local_test.go',
'internal/store/query_replica_safety_local_test.go',
'internal/store/query_replica_safety_test.go',
'internal/store/query_replica_local_test.go',
'internal/store/query_sync_local_test.go',
'internal/store/query_replica_count_cache_test.go',
'internal/api/query_replica_local_test.go',
@ -91,6 +95,21 @@ def start(env):
required = {27119: 'source', 27129: 'query'}
if any(required[port] not in running for port in busy):
raise RuntimeError('lab ports are occupied by unowned services; use an owned native lab or free these ports')
if len(busy) == 2:
# Reuse isolated random databases, never recreate a shared fixture just
# because another authorized checkout owns its Compose metadata.
info = json.loads(subprocess.check_output(['docker', 'inspect', 'fanyu-query-replica-local-source-1',
'fanyu-query-replica-local-query-1'], cwd=ROOT, env=env, text=True))
resources = [{'name': x['Name'], 'cpu': x['HostConfig']['NanoCpus'],
'memory': x['HostConfig']['Memory']} for x in info]
if env.get('QUERY_REPLICA_LOCAL_QUERY_BUDGET') == 'query25':
query = next(x for x in resources if x['name'].endswith('query-1'))
if query['cpu'] != 1_000_000_000 or query['memory'] != 1024 ** 3:
raise RuntimeError('existing shared query fixture differs from1CPU/1Gi; do not recreate it')
env['QUERY_REPLICA_REUSED_RESOURCES'] = json.dumps(resources)
compose(env, 'exec', '-T', 'query', 'mongosh', '--quiet', '--port', '27129', '--eval',
"if (!db.hello().isWritablePrimary || db.hello().setName !== 'query-read-rs') quit(2)")
return
compose(env, 'up', '-d')
for _ in range(60):
try:
@ -135,7 +154,8 @@ def main():
else:
raise RuntimeError('select --native, --start or --auto explicitly')
if args.cache:
compose(env, 'up', '-d', 'valkey')
if 'valkey' not in compose(env, 'ps', '--status', 'running', '--services').splitlines():
compose(env, 'up', '-d', 'valkey')
env['QUERY_REPLICA_VALKEY_LOCAL_TEST'] = '1'
required = set()
for name in CASES:
@ -146,7 +166,7 @@ def main():
required = {'TestQueryReplicaScaleLocal'}
env['QUERY_REPLICA_SCALE_LOCAL'] = '1'
regex = '^(' + '|'.join(sorted(required)) + ')$'
argv = ['go', 'test', './internal/store', './internal/api', './cmd/query-cdc-probe',
argv = ['go', 'test', '-p=1', './internal/store', './internal/api', './cmd/query-cdc-probe', './cmd/query-replica',
'-run', regex, '-count=1', '-timeout=10m', '-v']
if args.race:
argv.append('-race')
@ -159,7 +179,7 @@ def main():
proof = write_log(PRIVATE / 'logs', argv, output, 'passed' if success else 'pending_or_failed')
proof.update(missing=missing, sourceChanged=changed, skipped=skipped,
environment='native-uncapped' if native else ('docker-source-4cpu-query-1cpu-1Gi' if args.bounded_query else 'docker-source-4cpu-query-4cpu'),
productionCapacityAccepted=False)
productionCapacityAccepted=False, reusedResources=json.loads(env.get('QUERY_REPLICA_REUSED_RESOURCES', '[]')))
print(json.dumps(proof, ensure_ascii=False))
return 0 if success else (code or 2)

View File

@ -1018,6 +1018,7 @@
{
"name": "independent-query-replica",
"paths": [
"^scripts/local/(query-shadow-observer|test_query_shadow_observer)\\.py$",
"^internal/store/query_(replica|sync|tracked_collection|collection_interface).*\\.go$",
"^internal/api/query_replica.*\\.go$",
"^internal/store/workforce_(report_cache|heap_diagnostic|facts|return_page|publication).*\\.go$",
@ -1052,6 +1053,20 @@
"./internal/testsupport": ["^TestQueryReplicaTarget"]
},
"task": [
{
"argv": [
"python3",
"-m",
"unittest",
"discover",
"-s",
"scripts/local",
"-p",
"test_query_shadow_observer.py",
"-v"
],
"tests": true
},
{
"argv": ["python3", "scripts/test_query_replica.py", "--auto", "--bounded-query", "--race", "--cache"],
"tests": true