diff --git a/cmd/query-read-builder/main.go b/cmd/query-read-builder/main.go index e5f37673..db49830a 100644 --- a/cmd/query-read-builder/main.go +++ b/cmd/query-read-builder/main.go @@ -19,6 +19,7 @@ import ( func main() { once := flag.Bool("once", false, "build once; streaming freshness expires after exit") ensure := flag.Bool("ensure-indexes", false, "explicitly create derived-data indexes before building") + shadowRun := flag.String("shadow-safety-run", "", "opt-in independently monitored shadow builder; no authority gates") flag.Parse() ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) defer stop() @@ -43,25 +44,58 @@ func main() { log.Fatal("builder accelerator configuration invalid") } defer closeServing() + workCtx := ctx + if *shadowRun != "" { + guarded, cancel := context.WithCancel(ctx) + defer cancel() + workCtx = guarded + if s.CheckQueryReplicaSafety(ctx, *shadowRun) != nil { + log.Print("shadow builder safety paused") + <-ctx.Done() + return + } + go func() { + ticker := time.NewTicker(5 * time.Second) + defer ticker.Stop() + for { + select { + case <-guarded.Done(): + return + case <-ticker.C: + short, stop := context.WithTimeout(guarded, 2*time.Second) + err := s.CheckQueryReplicaSafety(short, *shadowRun) + stop() + if err != nil { + cancel() + return + } + } + } + }() + } if *ensure { - if s.EnsureQueryReadModelIndexes(ctx) != nil { + if s.EnsureQueryReadModelIndexes(workCtx) != nil { log.Fatal("derived index creation failed") } } b := store.NewQueryReadBuilder(s) defer b.Close() if *once { - if b.RunOnce(ctx) != nil { + if b.RunOnce(workCtx) != nil { log.Fatal("initial materialization incomplete") } return } - for ctx.Err() == nil { - if b.Run(ctx) != nil && ctx.Err() == nil { + for workCtx.Err() == nil { + if b.Run(workCtx) != nil && workCtx.Err() == nil { log.Print("query builder paused; retrying with full reconciliation") } select { - case <-ctx.Done(): + case <-workCtx.Done(): + if ctx.Err() == nil { + log.Print("shadow builder safety paused; explicit operator restart required") + <-ctx.Done() + } return case <-time.After(5 * time.Second): } diff --git a/cmd/query-replica/main.go b/cmd/query-replica/main.go index ff09ec8a..aa34ee33 100644 --- a/cmd/query-replica/main.go +++ b/cmd/query-replica/main.go @@ -4,12 +4,17 @@ package main import ( "context" + "crypto/sha256" "droboticflow/internal/store" + "encoding/json" + "errors" "flag" + "fmt" "go.mongodb.org/mongo-driver/mongo" "go.mongodb.org/mongo-driver/mongo/options" "go.mongodb.org/mongo-driver/mongo/readconcern" "go.mongodb.org/mongo-driver/mongo/readpref" + "io" "log" "os" "os/signal" @@ -25,7 +30,104 @@ func main() { interval := flag.Duration("interval", time.Second, "minimum delay between bounded passes") batchSize := flag.Int("batch-size", 200, "maximum source records per collection batch (1-200)") minimumPass := flag.Duration("min-pass-interval", 0, "optional minimum start-to-start pass interval, including backfill (0-1m)") + safetyRun := flag.String("safety-run", "", "explicit durable shadow run identity; requires global transport and health budgets") + maxRecords := flag.Int64("max-source-records", 1000, "durable total read reservations;0 explicitly permits full backfill") + rowsRate := flag.Int("source-records-per-second", 10, "global source row pacing") + bytesRate := flag.Int("source-bytes-per-second", 65536, "shared actual source transport read budget") + commandsRate := flag.Int("source-commands-per-second", 2, "shared source socket-write budget, includes control overhead") + baseline := flag.Float64("source-cpu-baseline", 0, "fixed reviewed4core source CPU baseline, never increased on resume") + resume := flag.String("resume-pause", "", "exact current pause nonce after operator review") + initializeTarget := flag.Bool("initialize-target", false, "verify independent topology and bind target only; no source scans or indexes") + status := flag.Bool("safety-status", false, "target-only private safety status") + sampleSource := flag.Bool("sample-source", false, "readonly server counters for trusted observer, no customer records") + observation := flag.String("record-observation", "", "trusted observer JSON file or stdin '-'; target-only, never connects to source") flag.Parse() + if *sampleSource { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if os.Getenv("MONGO_URI") == "" { + log.Fatal("source binding required") + } + c, e := mongo.Connect(ctx, options.Client().ApplyURI(os.Getenv("MONGO_URI")).SetMaxPoolSize(1)) + if e != nil { + log.Fatal("source unavailable") + } + defer c.Disconnect(context.Background()) + var info struct { + Uptime int64 `bson:"uptimeMillis"` + Extra struct { + User int64 `bson:"user_time_us"` + System int64 `bson:"system_time_us"` + } `bson:"extra_info"` + } + if c.Database("admin").RunCommand(ctx, map[string]any{"serverStatus": 1}).Decode(&info) != nil { + log.Fatal("source counters unavailable") + } + var hello struct { + Topology struct { + Process any `bson:"processId"` + } `bson:"topologyVersion"` + } + if c.Database("admin").RunCommand(ctx, map[string]any{"hello": 1}).Decode(&hello) != nil { + log.Fatal("source identity unavailable") + } + identity := sha256.Sum256([]byte(fmt.Sprint(hello.Topology.Process))) + _ = json.NewEncoder(os.Stdout).Encode(map[string]any{"uptimeMillis": info.Uptime, "cpuMicroseconds": info.Extra.User + info.Extra.System, "sourceProcess": fmt.Sprintf("%x", identity), "sampledAt": time.Now().UTC().Format(time.RFC3339Nano)}) + return + } + if *observation != "" || *status { + var value store.QueryReplicaSafetyObservation + if !*status { + input := io.Reader(os.Stdin) + if *observation != "-" { + f, e := os.Open(*observation) + if e != nil { + log.Fatal("observation unavailable") + } + defer f.Close() + input = f + } + decoder := json.NewDecoder(io.LimitReader(input, 16384)) + decoder.DisallowUnknownFields() + if decoder.Decode(&value) != nil { + log.Fatal("invalid observation") + } + } + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + uri, db := os.Getenv("QUERY_READ_MONGO_URI"), os.Getenv("QUERY_READ_MONGO_DB") + if uri == "" || db == "" || os.Getenv("QUERY_SERVING_NAMESPACE") == "" { + log.Fatal("explicit independent target required") + } + c, e := mongo.Connect(ctx, options.Client().ApplyURI(uri).SetMaxPoolSize(1)) + if e != nil { + log.Fatal("target unavailable") + } + defer c.Disconnect(context.Background()) + if *status { + value, e := store.NewMongoStore(c.Database(db)).QueryReplicaSafetyStatus(ctx) + if e != nil { + log.Fatal("safety state unavailable") + } + _ = json.NewEncoder(os.Stdout).Encode(value) + return + } + if store.NewMongoStore(c.Database(db)).RecordQueryReplicaObservation(ctx, value) != nil { + log.Fatal("observation rejected") + } + return + } + var transport *store.QueryReplicaTransport + if *safetyRun != "" { + var e error + transport, e = store.NewQueryReplicaTransport(*bytesRate, *commandsRate) + if e != nil || *ensure { + log.Fatal("invalid guarded source configuration") + } + } else if *resume != "" { + log.Fatal("resume requires durable safety") + } + if !validPacing(*batchSize, *minimumPass) { log.Fatal("invalid bounded projector pacing") } @@ -44,6 +146,9 @@ func main() { *interval = store.QueryPeriodicSyncInterval } } + if *initializeTarget && *ensure { + log.Fatal("source index writes forbidden during target binding") + } if !*writers || *interval < 100*time.Millisecond || *interval > 5*time.Minute || os.Getenv("QUERY_SERVING_NAMESPACE") == "" { log.Fatal("writer coverage, namespace and bounded polling interval required") } @@ -54,6 +159,9 @@ func main() { log.Fatal("explicit source/query connection required") } opts := options.Client().ApplyURI(uri).SetMaxPoolSize(4).SetServerSelectionTimeout(5 * time.Second) + if !target && transport != nil { + opts.SetMaxPoolSize(1).SetDialer(transport).SetCompressors([]string{}).SetTimeout(5 * time.Second) + } if target { opts.SetReadPreference(readpref.Primary()).SetReadConcern(readconcern.Majority()) } @@ -78,21 +186,90 @@ func main() { if *ensure && q.EnsureSourceIndexes(ctx) != nil { log.Fatal("source index preparation failed") } - if q.EnsureTargetIndexes(ctx) != nil { - log.Fatal("query index preparation failed") + for { + err := q.Acquire(ctx) + if err == nil { + break + } + if *safetyRun == "" || !errors.Is(err, store.ErrQueryReplicaLease) { + log.Fatal("projector lease unavailable") + } + select { + case <-ctx.Done(): + return + case <-time.After(time.Second): + } } - if q.Acquire(ctx) != nil { - log.Fatal("projector lease unavailable") + if *initializeTarget { + return } - for ctx.Err() == nil { + if *safetyRun != "" { + if q.ConfigureSafety(ctx, store.QueryReplicaSafetyPolicy{BytesPerSecond: *bytesRate, CommandsPerSecond: *commandsRate, MaxBatchRows: *batchSize, RunID: *safetyRun, MaxRecords: *maxRecords, RecordsPerSecond: *rowsRate, BaselineCPU: *baseline}, *resume) != nil { + log.Print("shadow safety paused; inspect durable state and explicitly resume") + <-ctx.Done() + return + } + } + workCtx := ctx + if *safetyRun != "" { + guarded, cancel := context.WithCancel(ctx) + defer cancel() + workCtx = guarded + go func() { + tick := time.NewTicker(5 * time.Second) + defer tick.Stop() + for { + select { + case <-guarded.Done(): + return + case <-tick.C: + short, stop := context.WithTimeout(guarded, 3*time.Second) + err := q.CheckSafety(short) + stop() + if err != nil { + cancel() + return + } + } + } + }() + } + pause := func(reason string) { + short, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if q.PauseSafety(short, reason) != nil { + log.Print("pause persistence unavailable; startup fencing remains required") + } + } + defer func() { + if ctx.Err() != nil { + pause("operator_stop") + } + }() + if q.EnsureTargetIndexes(workCtx) != nil { + pause("target_error") + log.Print("query index preparation paused") + if *safetyRun == "" { + os.Exit(1) + } + <-ctx.Done() + return + } + for workCtx.Err() == nil { started := time.Now() - ready, e := q.RunOnce(ctx, *bootstrap) + ready, e := q.RunOnce(workCtx, *bootstrap) if e != nil { + pause("operation_error") log.Print("projector paused; checkpoint retained (no customer payload logged)") + if *safetyRun != "" { + <-ctx.Done() + return + } os.Exit(1) } if *once { if !ready { + pause("incomplete_pilot") log.Print("projection incomplete; resume required") os.Exit(2) } @@ -101,13 +278,22 @@ func main() { // Optional pacing covers backfill too. Idle lease maintenance only // touches the independent query DB, never scans authority for keepalive. if delay := passDelay(mode, ready, time.Since(started), *interval, *minimumPass); delay > 0 { - if e := q.WaitNextPass(ctx, delay); e != nil { + if e := q.WaitNextPass(workCtx, delay); e != nil { if ctx.Err() == nil { + pause("target_error") log.Print("projector lease unavailable; checkpoint retained") + if *safetyRun != "" { + <-ctx.Done() + return + } os.Exit(1) } return } } } + if *safetyRun != "" && ctx.Err() == nil { + pause("operation_error") + <-ctx.Done() + } } diff --git a/cmd/query-replica/main_local_test.go b/cmd/query-replica/main_local_test.go new file mode 100644 index 00000000..cce8bf82 --- /dev/null +++ b/cmd/query-replica/main_local_test.go @@ -0,0 +1,149 @@ +package main + +import ( + "context" + "droboticflow/internal/domain" + "droboticflow/internal/store" + "encoding/json" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + "io" + "os" + "os/exec" + "path/filepath" + "testing" + "time" +) + +func TestProjectorGuardedProcessRestartLocal(t *testing.T) { + if os.Getenv("QUERY_REPLICA_LOCAL_TEST") != "1" { + t.Skip("isolated source/query lab required") + } + ctx := t.Context() + sourceClient, e := mongo.Connect(ctx, options.Client().ApplyURI("mongodb://127.0.0.1:27119/?directConnection=true")) + if e != nil { + t.Fatal(e) + } + defer sourceClient.Disconnect(context.Background()) + targetClient, e := mongo.Connect(ctx, options.Client().ApplyURI("mongodb://127.0.0.1:27129/?replicaSet=query-read-rs")) + if e != nil { + t.Fatal(e) + } + defer targetClient.Disconnect(context.Background()) + source, target := sourceClient.Database(domain.NewID("query_sync_cmd")), targetClient.Database(domain.NewID("query_replica_cmd")) + t.Cleanup(func() { _ = source.Drop(context.Background()); _ = target.Drop(context.Background()) }) + s, d := store.NewMongoStore(source), store.NewMongoStore(target) + q, e := store.NewQueryReplica(s, d) + if e != nil { + t.Fatal(e) + } + if e = q.EnsureSourceIndexes(ctx); e != nil { + t.Fatal(e) + } + if e = q.EnsureTargetIndexes(ctx); e != nil { + t.Fatal(e) + } + for _, id := range []string{"a", "b", "c"} { + _, e = source.Collection("annotationIndexJobs").InsertOne(ctx, bson.M{"_id": id, "id": id, "status": "completed", "organizationId": "org", "privateDraft": "not projected"}) + if e != nil { + t.Fatal(e) + } + } + binary := filepath.Join(t.TempDir(), "projector") + if e = exec.Command("go", "build", "-o", binary, ".").Run(); e != nil { + t.Fatal("local projector build failed") + } + now := time.Now().UTC() + recent := []store.QueryReplicaSafetyObservation{} + for _, ago := range []time.Duration{60 * time.Second, 30 * time.Second, 0} { + recent = append(recent, store.QueryReplicaSafetyObservation{At: now.Add(-ago), CPU: 85, Ready: true, WritersCovered: true, HostFreeBytes: 120 << 30, TargetUsedFraction: .1}) + } + raw, _ := bson.Marshal(recent[2]) + var fields bson.M + _ = bson.Unmarshal(raw, &fields) + fields["recent"] = recent + _, e = target.Collection("queryReplicaControl").UpdateOne(ctx, bson.M{"_id": "safety-health"}, bson.M{"$set": fields}, options.Update().SetUpsert(true)) + if e != nil { + t.Fatal(e) + } + env := []string{} + for _, entry := range os.Environ() { + if len(entry) >= 6 && (entry[:6] == "MONGO_" || len(entry) >= 6 && entry[:6] == "QUERY_") { + continue + } + env = append(env, entry) + } + env = append(env, "MONGO_URI=mongodb://127.0.0.1:27119/?directConnection=true", "MONGO_DB="+source.Name(), "QUERY_READ_MONGO_URI=mongodb://127.0.0.1:27129/?replicaSet=query-read-rs", "QUERY_READ_MONGO_DB="+target.Name(), "QUERY_SERVING_NAMESPACE=local:"+source.Name()) + args := []string{"--writers-covered", "--bootstrap", "--batch-size=1", "--safety-run=process-test", "--max-source-records=2", "--source-records-per-second=50", "--source-bytes-per-second=262144", "--source-commands-per-second=8", "--source-cpu-baseline=85"} + start := func() (*exec.Cmd, <-chan error) { + t.Helper() + p := exec.Command(binary, args...) + p.Env = env + p.Stdout = io.Discard + p.Stderr = io.Discard + if e = p.Start(); e != nil { + t.Fatal(e) + } + done := make(chan error, 1) + go func() { done <- p.Wait() }() + t.Cleanup(func() { + _ = p.Process.Kill() + }) + return p, done + } + child, done := start() + var state bson.M + deadline := time.Now().Add(15 * time.Second) + for time.Now().Before(deadline) { + _ = target.Collection("queryReplicaControl").FindOne(ctx, bson.M{"_id": "safety"}).Decode(&state) + if state["paused"] == true { + break + } + select { + case <-done: + t.Fatal("guarded process exited instead of parking") + default: + } + time.Sleep(50 * time.Millisecond) + } + if state["paused"] != true || state["reserved"] != int64(2) { + t.Fatal("process bypassed total budget", state) + } + _ = child.Process.Kill() + <-done + _, _ = target.Collection("queryReplicaControl").UpdateOne(ctx, bson.M{"_id": "lease"}, bson.M{"$set": bson.M{"until": time.Time{}}}) + next, finished := start() + time.Sleep(600 * time.Millisecond) + select { + case <-finished: + t.Fatal("paused restart should park") + default: + } + var after bson.M + _ = target.Collection("queryReplicaControl").FindOne(ctx, bson.M{"_id": "safety"}).Decode(&after) + if after["reserved"] != state["reserved"] || after["resumeToken"] != state["resumeToken"] { + t.Fatal("process restart reset durable state") + } + _ = next.Process.Kill() + <-finished + // Operator status mode cannot connect to source, even with an invalid URI. + status := exec.Command(binary, "--safety-status") + status.Env = append(env, "MONGO_URI=mongodb://127.0.0.1:1/?directConnection=true") + out, e := status.Output() + if e != nil { + t.Fatal("target-only operator mode failed") + } + var v map[string]any + if json.Unmarshal(out, &v) != nil || v["paused"] != true { + t.Fatal("operator status invalid") + } + count, e := source.Collection("annotationIndexJobs").CountDocuments(ctx, bson.M{"privateDraft": "not projected"}) + if e != nil || count != 3 { + t.Fatal("source payload changed") + } + var copied bson.M + if e = target.Collection("annotationIndexJobs").FindOne(ctx, bson.M{}).Decode(&copied); e != nil || copied["privateDraft"] != nil { + t.Fatal("projection boundary failed", e) + } +} diff --git a/docs/operations/query-replica.md b/docs/operations/query-replica.md index b21d325b..8d3a60b1 100644 --- a/docs/operations/query-replica.md +++ b/docs/operations/query-replica.md @@ -195,3 +195,37 @@ v5的广范围搜索及冷暖读取对照记录于 `final-products-pool-v5`; 最终集成报告:`local-data/test-results/verify-1794636683705497000/report.json`,源码未在运行中变化、无未映射文件。41项中39项通过,包含隔离副本/缓存/Mongo专项、源码校验不入队、标注/切片浏览器、搜索及面板、生产前端构建和静态启动。API整包原始检查仍报告已有失败,最终官方Go基线门禁确认68项均在70项既有基线中,无新增;store整包PASS,但默认跳过的外部环境用例标为未完整覆盖,不算通过。专项显式启用的本机副本/Valkey测试已执行通过,未扩大 known-failures。 `.4` 测后资源申请总量约6.475/7.8核(83%)、内存11,813Mi(42%),包含实验查询Pod;低实际使用率不能替代调度申请余量。后续继续扩核须重新核对该节点及生产峰值。当前仍是限时实验Pod,保留原自动结束机制和25Gi盘;后续恢复验证按已记录UID、配置及回退步骤重建,不将该单Pod直接当生产查询库。 + +## 影子同步全局预算与持久暂停(方案 8e8abb0cb64f) + +本节替代上文仅按集合批次及轮次间隔控制的生产影子启动建议;旧命令保留兼容,但不得直接用于此次生产启动。当前仅完成本地实现与合成数据验证,未启动生产回填、未切换生产查询入口。 + +守护模式只作用于显式带 `--safety-run` 的独立投影器及带 `--shadow-safety-run` 的构建器,不进入保存、切片、送检、质检或最终验收准入。权威权限与版本判断不变。 + +| 项目 | 首轮候选值及行为 | +| --- | --- | +| 总读取预算 | `--max-source-records=1000`,读取前持久预留。空读、重复读及崩溃也可能消耗;不是已迁移条数。预算不能因重启、换 run ID 清零 | +| 全局节奏 | `--source-records-per-second=10 --source-bytes-per-second=65536 --source-commands-per-second=2 --batch-size=25`;共享源连接节流,包含握手/心跳开销,不按22个集合分别给额度 | +| 持久性 | 查询库事务 `majority+journal`,先提交投影/断点再 CAS 确认源标记,不降低写入确认,不延长超时 | +| 单次读取 | 源查询 `maxTimeMS=1500`,整个源操作5秒上限;大回复无法在预算内完成时暂停,不放开时限 | +| 源 CPU | 三个间隔30秒的有效样本,跨度至少60秒,CPU均≤90%、波动≤5个百分点才允许开始;基线固定,恢复不得抬高 | +| 停止 | 源 CPU≥95%一次,或比固定基线高3个百分点连续两个独立样本;受保护 Pod IO full avg10>1%连续两次或>5%一次;客户作业异常/延迟回退、健康缺失或超过45秒、写者变化、目标盘≥70%、宿主剩余<100GiB均暂停 | +| 调度 | 脏数据优先,正常每次历史批次前先排一个脏批次;积压超过5分钟时增加最多三个脏批次。按集合持久轮转,完整历史完成前不做全量审计,完成后首次审计延后6小时 | +| 恢复 | 私有控制表保存原因、预算、当前恢复令牌;进程异常重启转为暂停。仅显式提供当前 `--resume-pause`、相同 run ID、评审后的预算和健康窗口才能恢复。不会自行轮询恢复 | + +源端修改集中在受控元数据接口,只允许同步标记及私有删除意图。过期删除声明的取消与清理按有界 ID 批次处理,不重放业务删除,不回写业务正文、版本、状态、归属或标注。若预算不足以确认所有待删除目标 ID,暂停并保留目标,不把尚未探测的记录当成已删除。 + +Mongo 角色不能限制更新到某个字段:源账号按22个源集合授予 find/update,私有删除意图集合 find/insert/update,另授 serverStatus;禁止 remove、DDL、跨库访问。字段白名单由应用强制,不能宣称源账号“只读”。构建器没有源凭证。 + +### 最终启动前逐项确认 + +1. 将本地改动核验合入并同步 `dev` 后,通过正式发布门禁,单独批准镜像上传及实际启动计划。绑定不可变镜像、二进制 SHA、两个命名空间 UID、五个写者 UID/镜像/提交/副本、影子控制器 UID、Mongo Pod UID、来源端点摘要和目标库;核对所有生产读取开关仍关闭。不要直接执行旧方案里的启动命令。 +2. 核验源47个等价索引与最小权限角色,准备独立源 Secret。Secret 配额8、源网络出口、查询库4核8Gi/WiredTiger 2Gi/25Gi容量和 loop/底层 IO 双重限额保持现行已批准值。新增账号/Secret及私有同步元数据写入须列入实际启动授权。 +3. 先执行一次 `query-replica --writers-covered --initialize-target`:核对独立拓扑并仅初始化目标绑定,不扫描历史或创建源索引。然后启动可信观察器,等待三个有效样本,避免未绑定目标造成观察器无法写入健康数据。 +4. 接通可信客户作业健康输入 `--work-health`:JSON 必须含 UTC `at`、`coverage: "save-slice-submit-review-acceptance"`、布尔 `unexpectedErrors` 与 `latencyRegressed`。必须由已核实的指标提供者更新,≤45秒;不得用人工常量、readiness 或“日志没报错”替代。**实际生产提供者、延迟基线及故障演练尚待绑定,未完成不能启动。** 观察器来源 relay 目前仅适用于已审查的私网非TLS单端点;若实际连接启用TLS,必须先验证服务器名称处理,不得关闭TLS绕过。 +5. 最终计划列明观察器 `--bindings-sha256`、`--binary-sha256` 和 `--stop-on-failure` 的授权范围。停止仅对两个影子控制器 UID/resourceVersion CAS 缩至0;不会修改 API、处理器、校验器、Worker、协调器。观察器缺失时进程5秒巡检健康,健康超过45秒会停止工作,不能无限继续扫描。 +6. 仅投影器从0升至1,使用上表1000预留预算、10条/秒、64KiB/秒、2命令/秒、批次25及固定 CPU 基线。构建器保持0,生产查询开关保持关闭。影子目标数据不完整时不发布 ready。首轮停止后核对实际读取/预留/源确认量、源 CPU/读取字节、客户操作与共享盘 IO,对照同时间负载基线。 +7. 首轮达标后再确认后续预算。25条/秒、128KiB/秒、5命令/秒、批次50,以及最高50条/秒、256KiB/秒、8命令/秒、批次100都只是后续上限,**本清单不授权自动升级**。完整复制与增量追平后才单列构建器启动、旧新结果及权限隔离对照,仍不切生产入口。 +8. 回退:先停两个影子控制器并确认 Pod 退出,再停观察器;保留目标断点、暂停记录、最后完整结果和源标记。不删除源记录/标记,不清空查询库,不恢复业务旧快照。再次启动先读 `query-replica --safety-status`,重新确认健康、绑定和当前暂停令牌。生产入口从未切换,无需改客户作业配置。 + +此阶段只能证明新增同步负载受控及影子结果正确。生产入口未切换时不能宣称原 Mongo 总 CPU 已下降;降载验收仍需后续经授权切读后的同条件对照。 diff --git a/internal/store/query_periodic.go b/internal/store/query_periodic.go index 149d2940..eebdbabc 100644 --- a/internal/store/query_periodic.go +++ b/internal/store/query_periodic.go @@ -66,7 +66,12 @@ func (q *QueryReplica) WaitNextPass(ctx context.Context, interval time.Duration) return nil case <-heartbeat.C: short, stop := context.WithTimeout(ctx, 3*time.Second) - err := q.transaction(short, func(_ mongo.SessionContext) error { return nil }) + var err error + if q.safety != nil { + _, err = q.sourcePermit(short, 0) + } else { + err = q.transaction(short, func(_ mongo.SessionContext) error { return nil }) + } stop() if err != nil { return err diff --git a/internal/store/query_replica.go b/internal/store/query_replica.go index cdb0e768..437dc23a 100644 --- a/internal/store/query_replica.go +++ b/internal/store/query_replica.go @@ -31,6 +31,7 @@ type QueryReplica struct { BatchSize int dirtyAfter map[string]any reconcileNext int + safety *queryReplicaSafety } type queryReplicaCheckpoint struct { ID string `bson:"_id"` @@ -143,7 +144,7 @@ func (q *QueryReplica) transaction(ctx context.Context, fn func(mongo.SessionCon return nil, ErrQueryReplicaLease } return nil, fn(sc) - }, options.Transaction().SetReadConcern(readconcern.Snapshot()).SetWriteConcern(writeconcern.Majority())) + }, options.Transaction().SetReadConcern(readconcern.Snapshot()).SetWriteConcern(writeconcern.New(writeconcern.WMajority(), writeconcern.J(true)))) return e } func (q *QueryReplica) checkpoint(ctx context.Context, n string) (queryReplicaCheckpoint, error) { @@ -240,7 +241,14 @@ func bsonEqual(a, b any) bool { return e == nil && string(x) == string(y) } func (q *QueryReplica) read(ctx context.Context, n string, filter bson.M) ([]queryReplicaRecord, any, error) { - opts := options.Find().SetProjection(querySourceProjection(n)).SetSort(bson.D{{Key: "_id", Value: 1}}).SetLimit(q.limit()).SetBatchSize(int32(q.limit())) + parent := ctx + limit, e := q.sourcePermit(ctx, q.limit()) + if e != nil { + return nil, nil, e + } + ctx, cancel := q.sourceContext(ctx) + defer cancel() + opts := options.Find().SetMaxTime(1500 * time.Millisecond).SetProjection(querySourceProjection(n)).SetSort(bson.D{{Key: "_id", Value: 1}}).SetLimit(limit).SetBatchSize(int32(limit)) if filter[querySyncField+".dirty"] == true { opts.SetHint("query_sync_dirty_v1") } else { @@ -265,7 +273,13 @@ func (q *QueryReplica) read(ctx context.Context, n string, filter bson.M) ([]que } records = append(records, r) } - return records, last, cur.Err() + if e := cur.Err(); e != nil { + return nil, nil, e + } + if e := q.paceRows(parent, len(records)); e != nil { + return nil, nil, e + } + return records, last, nil } func (q *QueryReplica) acknowledge(ctx context.Context, n string, records []queryReplicaRecord) error { models := make([]mongo.WriteModel, 0, len(records)) @@ -283,8 +297,7 @@ func (q *QueryReplica) acknowledge(ctx context.Context, n string, records []quer if len(models) == 0 { return nil } - _, e := q.Source.db.Collection(n).BulkWrite(ctx, models, options.BulkWrite().SetOrdered(false)) - return e + return q.sourceMetadataBulk(ctx, n, models) } // BootstrapStep is bounded and resumable. $expr uses BSON total ordering across @@ -306,7 +319,7 @@ func (q *QueryReplica) BootstrapStep(ctx context.Context, n string) (bool, int, if e != nil { return false, 0, e } - done := int64(len(rows)) < q.limit() + done := len(rows) == 0 e = q.transaction(ctx, func(sc mongo.SessionContext) error { if e := q.apply(sc, n, rows); e != nil { return e @@ -366,6 +379,13 @@ func (s *MongoStore) EnableQuerySourceTracking(enabled bool) { s.querySyncEnable // RunOnce performs at most one batch per collection. Backfill is resumable and // bounded; the process does not begin unbounded scans on an HTTP cache miss. func (q *QueryReplica) RunOnce(ctx context.Context, bootstrap bool) (bool, error) { + if q.safety != nil { + return q.runGuardedOnce(ctx, bootstrap) + } + if _, e := q.sourcePermit(ctx, 0); e != nil { + return false, e + } + allBootstrapped := true for _, n := range replicaCollections() { cp, e := q.checkpoint(ctx, n) if e != nil { @@ -375,16 +395,20 @@ func (q *QueryReplica) RunOnce(ctx context.Context, bootstrap bool) (bool, error if !bootstrap { return false, ErrQueryReplicaNotReady } - if _, _, e = q.BootstrapStep(ctx, n); e != nil { + var complete bool + if complete, _, e = q.BootstrapStep(ctx, n); e != nil { return false, e } + allBootstrapped = allBootstrapped && complete } if _, e = q.DrainStep(ctx, n); e != nil { return false, e } } - if _, e := q.ReconcileOneDue(ctx, 6*time.Hour); e != nil { - return false, e + if allBootstrapped { + if _, e := q.ReconcileOneDue(ctx, 6*time.Hour); e != nil { + return false, e + } } if _, e := q.ReconcileDeleteStep(ctx); e != nil { return false, e diff --git a/internal/store/query_replica_delete.go b/internal/store/query_replica_delete.go index c06b8641..b0834f41 100644 --- a/internal/store/query_replica_delete.go +++ b/internal/store/query_replica_delete.go @@ -13,8 +13,13 @@ import ( // It reads only source identities, never annotator payloads. Survivors retain // their data; a cancelled deletion is not replayed or restored from snapshots. func (q *QueryReplica) ReconcileDeleteStep(ctx context.Context) (bool, error) { + if _, e := q.sourcePermit(ctx, 1); e != nil { + return false, e + } + sourceCtx, sourceCancel := q.sourceContext(ctx) + defer sourceCancel() var i queryDeleteIntent - e := q.Source.db.Collection(querySyncDeleteCollection).FindOne(ctx, bson.M{"$or": bson.A{bson.M{"phase": bson.M{"$in": bson.A{"complete", "cancelled", "reconcile"}}}, bson.M{"phase": bson.M{"$in": bson.A{"prepared", "deleting"}}, "leaseUntil": bson.M{"$lte": time.Now().UTC()}}}}, options.FindOne().SetSort(bson.D{{Key: "audit", Value: 1}, {Key: "createdAt", Value: 1}})).Decode(&i) + e := q.Source.db.Collection(querySyncDeleteCollection).FindOne(sourceCtx, bson.M{"$or": bson.A{bson.M{"phase": bson.M{"$in": bson.A{"complete", "cancelled", "reconcile"}}}, bson.M{"phase": bson.M{"$in": bson.A{"prepared", "deleting"}}, "leaseUntil": bson.M{"$lte": time.Now().UTC()}}}}, options.FindOne().SetMaxTime(1500*time.Millisecond).SetSort(bson.D{{Key: "audit", Value: 1}, {Key: "createdAt", Value: 1}})).Decode(&i) if e == mongo.ErrNoDocuments { return false, nil } @@ -24,7 +29,7 @@ func (q *QueryReplica) ReconcileDeleteStep(ctx context.Context) (bool, error) { if queryReplicaFields[i.Collection] == "" { return false, ErrQueryReplicaUnsupported } - if ok, e := q.Source.queryCollection(i.Collection).recoverDelete(ctx, i, time.Now().UTC()); e != nil || !ok { + if ok, e := q.recoverSourceClaims(ctx, i); e != nil || !ok { return false, e } key := "delete/" + i.ID @@ -34,7 +39,7 @@ func (q *QueryReplica) ReconcileDeleteStep(ctx context.Context) (bool, error) { return false, e } if cp.Complete { - _, e = q.Source.db.Collection(querySyncDeleteCollection).UpdateOne(ctx, bson.M{"_id": i.ID, "phase": "reconcile"}, bson.M{"$set": bson.M{"phase": "acknowledged", "expireAt": time.Now().UTC().Add(7 * 24 * time.Hour)}}) + _, e = q.sourceIntentUpdate(ctx, bson.M{"_id": i.ID, "phase": "reconcile"}, bson.M{"$set": bson.M{"phase": "acknowledged", "expireAt": time.Now().UTC().Add(7 * 24 * time.Hour)}}) return true, e } filter := bson.M{} @@ -63,7 +68,19 @@ func (q *QueryReplica) ReconcileDeleteStep(ctx context.Context) (bool, error) { } present := []any{} if len(ids) > 0 { - src, e := q.Source.db.Collection(i.Collection).Find(ctx, bson.M{"_id": bson.M{"$in": ids}}, options.Find().SetProjection(bson.M{"_id": 1})) + permitted, err := q.sourcePermit(ctx, int64(len(ids))) + if err != nil { + return false, err + } + if permitted < int64(len(ids)) { + if err = q.PauseSafety(ctx, "pilot_budget"); err != nil { + return false, err + } + return false, ErrQueryReplicaPaused + } + sourceCtx, sourceCancel := q.sourceContext(ctx) + defer sourceCancel() + src, e := q.Source.db.Collection(i.Collection).Find(sourceCtx, bson.M{"_id": bson.M{"$in": ids}}, options.Find().SetProjection(bson.M{"_id": 1}).SetLimit(int64(len(ids))).SetBatchSize(int32(len(ids))).SetMaxTime(1500*time.Millisecond)) if e != nil { return false, e } @@ -81,6 +98,9 @@ func (q *QueryReplica) ReconcileDeleteStep(ctx context.Context) (bool, error) { return false, e } } + if e := q.paceRows(ctx, len(present)); e != nil { + return false, e + } absent := []bson.M{} for _, r := range rows { id := deletionDoc(r["_queryReplica"])["sourceId"] diff --git a/internal/store/query_replica_reads.go b/internal/store/query_replica_reads.go index d8bf5b8c..07bec94d 100644 --- a/internal/store/query_replica_reads.go +++ b/internal/store/query_replica_reads.go @@ -161,7 +161,12 @@ func (q *QueryReplica) PublishReady(ctx context.Context) error { if !cp.Complete { return ErrQueryReplicaNotReady } - e = q.Source.db.Collection(n).FindOne(ctx, bson.M{querySyncField + ".dirty": true}, options.FindOne().SetProjection(bson.M{"_id": 1}).SetHint("query_sync_dirty_v1")).Err() + if _, e = q.sourcePermit(ctx, 1); e != nil { + return e + } + work, cancel := q.sourceContext(ctx) + e = q.Source.db.Collection(n).FindOne(work, bson.M{querySyncField + ".dirty": true}, options.FindOne().SetProjection(bson.M{"_id": 1}).SetHint("query_sync_dirty_v1").SetMaxTime(1500*time.Millisecond)).Err() + cancel() if e == nil { return ErrQueryReplicaNotReady } @@ -169,7 +174,12 @@ func (q *QueryReplica) PublishReady(ctx context.Context) error { return e } } - e := q.Source.db.Collection(querySyncDeleteCollection).FindOne(ctx, bson.M{"audit": bson.M{"$ne": true}, "phase": bson.M{"$in": queryDeletePendingPhases()}}, options.FindOne().SetProjection(bson.M{"_id": 1})).Err() + if _, e := q.sourcePermit(ctx, 1); e != nil { + return e + } + work, cancel := q.sourceContext(ctx) + defer cancel() + e := q.Source.db.Collection(querySyncDeleteCollection).FindOne(work, bson.M{"audit": bson.M{"$ne": true}, "phase": bson.M{"$in": queryDeletePendingPhases()}}, options.FindOne().SetProjection(bson.M{"_id": 1})).Err() if e == nil { return ErrQueryReplicaNotReady } @@ -177,7 +187,11 @@ func (q *QueryReplica) PublishReady(ctx context.Context) error { return e } return q.transaction(ctx, func(sc mongo.SessionContext) error { - _, e := q.control().UpdateOne(sc, bson.M{"_id": "state"}, bson.M{"$set": bson.M{"ready": true, "schema": queryReplicaSchema, "observedAt": time.Now().UTC(), "sourceDatabase": q.binding().SourceDatabase, "namespace": q.binding().Namespace}}, options.Update().SetUpsert(true)) + _, e := q.control().UpdateOne(sc, bson.M{"_id": "state", "bootstrapAt": bson.M{"$exists": false}}, bson.M{"$set": bson.M{"bootstrapAt": time.Now().UTC()}}) + if e != nil { + return e + } + _, e = q.control().UpdateOne(sc, bson.M{"_id": "state"}, bson.M{"$set": bson.M{"ready": true, "schema": queryReplicaSchema, "observedAt": time.Now().UTC(), "sourceDatabase": q.binding().SourceDatabase, "namespace": q.binding().Namespace}}, options.Update().SetUpsert(true)) return e }) } diff --git a/internal/store/query_replica_reconcile.go b/internal/store/query_replica_reconcile.go index b398d430..688dc47d 100644 --- a/internal/store/query_replica_reconcile.go +++ b/internal/store/query_replica_reconcile.go @@ -40,6 +40,17 @@ func (q *QueryReplica) ReconcileOneDue(ctx context.Context, interval time.Durati if !boot.Complete { continue } + if q.safety != nil { + var first struct { + At time.Time `bson:"bootstrapAt"` + } + if e = q.control().FindOne(ctx, bson.M{"_id": "state"}).Decode(&first); e != nil && e != mongo.ErrNoDocuments { + return 0, e + } + if first.At.IsZero() || time.Since(first.At) < interval { + return 0, nil + } + } key := "reconcile/" + name var cp queryReplicaCheckpoint e = q.control().FindOne(ctx, bson.M{"_id": key}).Decode(&cp) @@ -97,7 +108,7 @@ func (q *QueryReplica) ReconcileOneDue(ctx context.Context, interval time.Durati }); e != nil { return 0, e } - if _, e = q.Source.db.Collection(name).BulkWrite(ctx, models, options.BulkWrite().SetOrdered(false)); e != nil { + if e = q.sourceMetadataBulk(ctx, name, models); e != nil { return 0, e } } @@ -106,7 +117,7 @@ func (q *QueryReplica) ReconcileOneDue(ctx context.Context, interval time.Durati // Reuse the durable identity sweeper for unreported physical deletions. // Audits have lower priority and do not block unrelated report readiness. i := queryDeleteIntent{ID: domain.NewID("query-audit"), Collection: name, Phase: "reconcile", CreatedAt: time.Now().UTC(), Audit: true} - if _, e = q.Source.db.Collection(querySyncDeleteCollection).InsertOne(ctx, i); e != nil { + if e = q.sourceAuditIntent(ctx, i); e != nil { return 0, e } } diff --git a/internal/store/query_replica_safety.go b/internal/store/query_replica_safety.go new file mode 100644 index 00000000..e5d7fdfd --- /dev/null +++ b/internal/store/query_replica_safety.go @@ -0,0 +1,331 @@ +package store + +import ( + "context" + "crypto/sha256" + "droboticflow/internal/domain" + "encoding/hex" + "encoding/json" + "errors" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + "go.mongodb.org/mongo-driver/mongo/writeconcern" + "math" + "time" +) + +var ErrQueryReplicaPaused = errors.New("shadow synchronization paused; explicit reviewed resume required") + +// Safety is opt-in for the independent projector only. It never gates authority +// business operations. The policy and pause survive process and Pod replacement. +type QueryReplicaSafetyPolicy struct { + BytesPerSecond int `json:"bytesPerSecond" bson:"bytesPerSecond"` + CommandsPerSecond int `json:"commandsPerSecond" bson:"commandsPerSecond"` + MaxBatchRows int `json:"maxBatchRows" bson:"maxBatchRows"` + RunID string `json:"runId" bson:"runId"` + MaxRecords int64 `json:"maxRecords" bson:"maxRecords"` + RecordsPerSecond int `json:"recordsPerSecond" bson:"recordsPerSecond"` + BaselineCPU float64 `json:"baselineCPU" bson:"baselineCPU"` +} +type QueryReplicaSafetyObservation struct { + At time.Time `json:"at" bson:"at"` + CPU float64 `json:"cpu" bson:"cpu"` + Ready bool `json:"ready" bson:"ready"` + WritersCovered bool `json:"writersCovered" bson:"writersCovered"` + WorkErrors bool `json:"workErrors" bson:"workErrors"` + ProtectedIOFull float64 `json:"protectedIOFull" bson:"protectedIOFull"` + TargetUsedFraction float64 `json:"targetUsedFraction" bson:"targetUsedFraction"` + HostFreeBytes int64 `json:"hostFreeBytes" bson:"hostFreeBytes"` +} +type queryReplicaSafetyState struct { + Owner string `bson:"owner"` + Policy QueryReplicaSafetyPolicy `bson:"policy"` + Digest string `bson:"digest"` + Reserved int64 `bson:"reserved"` + Paused bool `bson:"paused"` + Reason string `bson:"reason"` + ResumeToken string `bson:"resumeToken"` + LastObservation time.Time `bson:"lastObservation"` + High int `bson:"high"` + HighIO int `bson:"highIO"` +} +type queryReplicaSafety struct { + policy QueryReplicaSafetyPolicy + rows *queryReplicaPacer +} + +func safetyPolicyDigest(p QueryReplicaSafetyPolicy) string { + b, _ := json.Marshal(p) + h := sha256.Sum256(b) + return hex.EncodeToString(h[:]) +} +func (p QueryReplicaSafetyPolicy) validate() error { + if p.BytesPerSecond < 1024 || p.BytesPerSecond > 256*1024 || p.CommandsPerSecond < 1 || p.CommandsPerSecond > 8 || p.MaxBatchRows < 1 || p.MaxBatchRows > 100 || p.RunID == "" || len(p.RunID) > 100 || p.MaxRecords < 0 || p.RecordsPerSecond < 1 || p.RecordsPerSecond > 50 || math.IsNaN(p.BaselineCPU) || math.IsInf(p.BaselineCPU, 0) || p.BaselineCPU <= 0 || p.BaselineCPU > 90 { + return errors.New("invalid shadow safety policy") + } + return nil +} + +// RecordQueryReplicaObservation is for a trusted out-of-process observer. Only +// complete, fresh observations are accepted; callers never submit customer data. +func (s *MongoStore) RecordQueryReplicaObservation(ctx context.Context, o QueryReplicaSafetyObservation) error { + if o.At.IsZero() || time.Since(o.At) > 45*time.Second || o.At.After(time.Now().Add(time.Second)) || math.IsNaN(o.CPU) || math.IsInf(o.CPU, 0) || o.CPU < 0 || o.CPU > 100 || math.IsNaN(o.ProtectedIOFull) || math.IsInf(o.ProtectedIOFull, 0) || o.ProtectedIOFull < 0 || o.ProtectedIOFull > 100 || math.IsNaN(o.TargetUsedFraction) || math.IsInf(o.TargetUsedFraction, 0) || o.TargetUsedFraction < 0 || o.TargetUsedFraction > 1 || o.HostFreeBytes < 0 { + return errors.New("invalid shadow observation") + } + var hello bson.M + if e := s.db.RunCommand(ctx, bson.M{"hello": 1}).Decode(&hello); e != nil || hello["setName"] == nil { + return errors.New("independent query replica required") + } + var binding queryReplicaBinding + if e := s.db.Collection(queryReplicaControl).FindOne(ctx, bson.M{"_id": "source-binding"}).Decode(&binding); e != nil || binding.SourceDatabase == "" || binding.SourceDatabase == s.db.Name() || binding.Namespace != queryReplicaNamespace(binding.SourceDatabase) { + return errors.New("reviewed query binding required before observation writes") + } + c := s.db.Collection(queryReplicaControl, options.Collection().SetWriteConcern(writeconcern.New(writeconcern.WMajority(), writeconcern.J(true)))) + // Monotonic CAS: an old observer cannot overwrite newer health evidence. + _, e := c.UpdateOne(ctx, bson.M{"_id": "safety-health", "$or": bson.A{bson.M{"at": bson.M{"$lt": o.At}}, bson.M{"at": bson.M{"$exists": false}}}}, bson.M{"$set": o, "$push": bson.M{"recent": bson.M{"$each": bson.A{o}, "$slice": -3}}}, options.Update().SetUpsert(true)) + return e +} +func (q *QueryReplica) ConfigureSafety(ctx context.Context, p QueryReplicaSafetyPolicy, resumeToken string) error { + if e := p.validate(); e != nil { + return e + } + if q.limit() > int64(p.MaxBatchRows) { + return errors.New("batch exceeds reviewed safety policy") + } + digest := safetyPolicyDigest(p) + restarted := false + e := q.transaction(ctx, func(sc mongo.SessionContext) error { + var state queryReplicaSafetyState + e := q.control().FindOne(sc, bson.M{"_id": "safety"}).Decode(&state) + if e != nil && e != mongo.ErrNoDocuments { + return e + } + if e == nil { + if state.Paused { + if resumeToken == "" || state.ResumeToken != resumeToken || state.Policy.RunID != p.RunID || p.BaselineCPU > state.Policy.BaselineCPU || p.MaxRecords > 0 && p.MaxRecords <= state.Reserved { + return ErrQueryReplicaPaused + } + } else { + if state.Digest != digest || resumeToken != "" { + return errors.New("active shadow policy cannot be replaced") + } + if state.Owner == q.owner { + return nil + } + restarted = true + _, e = q.control().UpdateOne(sc, bson.M{"_id": "safety"}, bson.M{"$set": bson.M{"paused": true, "reason": "process_restarted", "resumeToken": domain.NewID("shadow-resume")}}) + if e == nil { + _, e = q.control().UpdateOne(sc, bson.M{"_id": "state"}, bson.M{"$set": bson.M{"ready": false}}, options.Update().SetUpsert(true)) + } + return e + } + } else if resumeToken != "" { + return errors.New("resume requires an existing pause") + } + denyStart := func() error { + if state.Paused { + return ErrQueryReplicaPaused + } + restarted = true + _, err := q.control().UpdateOne(sc, bson.M{"_id": "safety"}, bson.M{"$set": bson.M{"policy": p, "digest": digest, "owner": q.owner, "paused": true, "reason": "starting_health", "resumeToken": domain.NewID("shadow-resume"), "high": 0, "highIO": 0}, "$setOnInsert": bson.M{"reserved": int64(0)}}, options.Update().SetUpsert(true)) + if err == nil { + _, err = q.control().UpdateOne(sc, bson.M{"_id": "state"}, bson.M{"$set": bson.M{"ready": false}}, options.Update().SetUpsert(true)) + } + return err + } + var health struct { + Recent []QueryReplicaSafetyObservation `bson:"recent"` + } + if e = q.control().FindOne(sc, bson.M{"_id": "safety-health"}).Decode(&health); e != nil { + return denyStart() + } + if len(health.Recent) != 3 { + return denyStart() + } + low, high := 100., 0. + for i, o := range health.Recent { + if observationFailure(o, o.At) != "" || o.CPU > 90 { + return denyStart() + } + if i > 0 && !o.At.After(health.Recent[i-1].At) { + return denyStart() + } + low = math.Min(low, o.CPU) + high = math.Max(high, o.CPU) + } + if high-low > 5 || health.Recent[2].At.Sub(health.Recent[0].At) < 60*time.Second || observationFailure(health.Recent[2], time.Now()) != "" { + return denyStart() + } + _, e = q.control().UpdateOne(sc, bson.M{"_id": "safety"}, bson.M{"$set": bson.M{"policy": p, "digest": digest, "owner": q.owner, "paused": false, "reason": "", "resumeToken": "", "high": 0, "highIO": 0}, "$setOnInsert": bson.M{"reserved": int64(0)}}, options.Update().SetUpsert(true)) + return e + }) + if e != nil { + return e + } + if restarted { + return ErrQueryReplicaPaused + } + q.safety = &queryReplicaSafety{policy: p, rows: newQueryReplicaPacer(float64(p.RecordsPerSecond))} + _, e = q.sourcePermit(ctx, 0) + return e +} +func observationFailure(o QueryReplicaSafetyObservation, now time.Time) string { + switch { + case o.At.IsZero() || now.Sub(o.At) > 45*time.Second || o.At.After(now.Add(time.Second)): + return "health_missing_or_stale" + case !o.Ready || !o.WritersCovered || o.WorkErrors: + return "production_health" + case math.IsNaN(o.CPU) || math.IsInf(o.CPU, 0) || o.CPU < 0 || o.CPU > 100 || math.IsNaN(o.ProtectedIOFull) || math.IsInf(o.ProtectedIOFull, 0) || o.ProtectedIOFull < 0 || math.IsNaN(o.TargetUsedFraction) || math.IsInf(o.TargetUsedFraction, 0) || o.TargetUsedFraction < 0 || o.TargetUsedFraction > 1: + return "invalid_health" + case o.CPU >= 95: + return "source_cpu_absolute" + case o.ProtectedIOFull > 5: + return "protected_io" + case o.TargetUsedFraction >= 0.70 || o.HostFreeBytes < 100*1024*1024*1024: + return "storage_pressure" + } + return "" +} + +// sourcePermit reserves a bounded maximum before a read. A crash consumes the +// reservation rather than silently resetting a pilot's total budget. Empty and +// repeated reads can consume it too; reserved is explicitly not progress. +func (q *QueryReplica) sourcePermit(ctx context.Context, want int64) (int64, error) { + if q.safety == nil { + return want, nil + } + allowed := want + paused := false + e := q.transaction(ctx, func(sc mongo.SessionContext) error { + var s queryReplicaSafetyState + if e := q.control().FindOne(sc, bson.M{"_id": "safety"}).Decode(&s); e != nil { + return e + } + if s.Paused { + paused = true + return nil + } + if s.Digest != safetyPolicyDigest(q.safety.policy) { + return errors.New("shadow policy changed") + } + var o QueryReplicaSafetyObservation + e := q.control().FindOne(sc, bson.M{"_id": "safety-health"}).Decode(&o) + if e != nil && e != mongo.ErrNoDocuments { + return e + } + reason := observationFailure(o, time.Now()) + if reason == "" && !o.At.Equal(s.LastObservation) { + if o.CPU > s.Policy.BaselineCPU+3 { + s.High++ + } else { + s.High = 0 + } + if o.ProtectedIOFull > 1 { + s.HighIO++ + } else { + s.HighIO = 0 + } + s.LastObservation = o.At + if s.High >= 2 { + reason = "source_cpu_relative" + } + if s.HighIO >= 2 { + reason = "protected_io" + } + } + if reason == "" && want > 0 && s.Policy.MaxRecords > 0 { + left := s.Policy.MaxRecords - s.Reserved + if left <= 0 { + reason = "pilot_budget" + } else if allowed > left { + allowed = left + } + } + fields := bson.M{"lastObservation": s.LastObservation, "high": s.High, "highIO": s.HighIO} + if reason != "" { + paused = true + fields["paused"] = true + fields["reason"] = reason + fields["resumeToken"] = domain.NewID("shadow-resume") + if _, e = q.control().UpdateOne(sc, bson.M{"_id": "state"}, bson.M{"$set": bson.M{"ready": false}}, options.Update().SetUpsert(true)); e != nil { + return e + } + } + update := bson.M{"$set": fields} + if !paused && allowed > 0 { + update["$inc"] = bson.M{"reserved": allowed} + } + _, e = q.control().UpdateOne(sc, bson.M{"_id": "safety"}, update) + return e + }) + if e != nil { + return 0, e + } + if paused { + return 0, ErrQueryReplicaPaused + } + return allowed, nil +} +func (q *QueryReplica) PauseSafety(ctx context.Context, reason string) error { + if q.safety == nil { + return nil + } + // Only fixed non-payload reasons reach durable diagnostics. + switch reason { + case "source_error", "target_error", "incomplete_pilot", "observer_error", "operator_stop", "pilot_budget": + default: + reason = "operation_error" + } + return q.transaction(ctx, func(sc mongo.SessionContext) error { + _, e := q.control().UpdateOne(sc, bson.M{"_id": "safety", "paused": false}, bson.M{"$set": bson.M{"paused": true, "reason": reason, "resumeToken": domain.NewID("shadow-resume")}}) + if e != nil { + return e + } + _, e = q.control().UpdateOne(sc, bson.M{"_id": "state"}, bson.M{"$set": bson.M{"ready": false}}, options.Update().SetUpsert(true)) + return e + }) +} +func (q *QueryReplica) paceRows(ctx context.Context, n int) error { + if q.safety == nil { + return nil + } + return q.safety.rows.wait(ctx, float64(n)) +} +func (q *QueryReplica) sourceContext(ctx context.Context) (context.Context, context.CancelFunc) { + return context.WithTimeout(ctx, 5*time.Second) +} + +// QueryReplicaSafetyStatus contains no payloads or connection information. +// The nonce is an operator CAS token, not authorization: target credentials +// remain mandatory. It is never emitted by the running projector's logs. +func (s *MongoStore) QueryReplicaSafetyStatus(ctx context.Context) (bson.M, error) { + var state bson.M + e := s.db.Collection(queryReplicaControl).FindOne(ctx, bson.M{"_id": "safety"}, options.FindOne().SetProjection(bson.M{"_id": 0, "policy": 1, "reserved": 1, "paused": 1, "reason": 1, "resumeToken": 1, "lastObservation": 1, "high": 1})).Decode(&state) + return state, e +} + +// CheckQueryReplicaSafety gates only the opt-in shadow builder, never API reads +// or any authority operation. A lost observer leaves the builder parked. +func (s *MongoStore) CheckQueryReplicaSafety(ctx context.Context, runID string) error { + var state queryReplicaSafetyState + if e := s.db.Collection(queryReplicaControl).FindOne(ctx, bson.M{"_id": "safety"}).Decode(&state); e != nil { + return e + } + if state.Paused || state.Policy.RunID != runID { + return ErrQueryReplicaPaused + } + var health QueryReplicaSafetyObservation + if e := s.db.Collection(queryReplicaControl).FindOne(ctx, bson.M{"_id": "safety-health"}).Decode(&health); e != nil { + return e + } + if observationFailure(health, time.Now()) != "" { + return ErrQueryReplicaPaused + } + return nil +} + +func (q *QueryReplica) CheckSafety(ctx context.Context) error { + _, e := q.sourcePermit(ctx, 0) + return e +} diff --git a/internal/store/query_replica_safety_local_test.go b/internal/store/query_replica_safety_local_test.go new file mode 100644 index 00000000..28b0a6b0 --- /dev/null +++ b/internal/store/query_replica_safety_local_test.go @@ -0,0 +1,427 @@ +package store + +import ( + "context" + "droboticflow/internal/domain" + "errors" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/event" + "go.mongodb.org/mongo-driver/mongo/options" + "testing" + "time" +) + +func seedReplicaSafetyHealth(t *testing.T, q *QueryReplica) { + t.Helper() + if q.BatchSize > 100 { + q.BatchSize = 25 + } + now := time.Now().UTC() + recent := []QueryReplicaSafetyObservation{} + for _, ago := range []time.Duration{60 * time.Second, 30 * time.Second, 0} { + recent = append(recent, QueryReplicaSafetyObservation{At: now.Add(-ago), CPU: 85, Ready: true, WritersCovered: true, HostFreeBytes: 120 << 30, TargetUsedFraction: .1}) + } + raw, e := bson.Marshal(recent[2]) + if e != nil { + t.Fatal(e) + } + var fields bson.M + if e = bson.Unmarshal(raw, &fields); e != nil { + t.Fatal(e) + } + fields["recent"] = recent + if _, e = q.control().UpdateOne(t.Context(), bson.M{"_id": "safety-health"}, bson.M{"$set": fields}, options.Update().SetUpsert(true)); e != nil { + t.Fatal(e) + } +} +func safetyPolicyForTest() QueryReplicaSafetyPolicy { + return QueryReplicaSafetyPolicy{BytesPerSecond: 65536, CommandsPerSecond: 2, MaxBatchRows: 25, RunID: "synthetic-run", MaxRecords: 1000, RecordsPerSecond: 50, BaselineCPU: 85} +} +func TestQueryReplicaSafetyPilotBudgetAndRestartLocal(t *testing.T) { + source, target, q := newLocalReplica(t) + ctx := t.Context() + seedReplicaSafetyHealth(t, q) + p := safetyPolicyForTest() + p.MaxRecords = 4 + q.BatchSize = 3 + for _, n := range []string{"projects", "humanItems"} { + for i := 0; i < 3; i++ { + if _, e := source.db.Collection(n).InsertOne(ctx, bson.M{"_id": domain.NewID("source"), "id": domain.NewID("logical"), "organizationId": "o", "projectId": "p", "version": 7, "status": "claimed", "subject": bson.M{"payload": bson.M{"annotationDraft": "preserve"}}}); e != nil { + t.Fatal(e) + } + } + } + if e := q.ConfigureSafety(ctx, p, ""); e != nil { + t.Fatal(e) + } + if _, n, e := q.BootstrapStep(ctx, "projects"); e != nil || n != 3 { + t.Fatal(n, e) + } + complete, n, e := q.BootstrapStep(ctx, "humanItems") + if e != nil || n != 1 || complete { + t.Fatal("short budget falsely completed collection", complete, n, e) + } + if _, _, e = q.BootstrapStep(ctx, "humanItems"); !errors.Is(e, ErrQueryReplicaPaused) { + t.Fatal("pilot exceeded durable cap", e) + } + var state queryReplicaSafetyState + if e = q.control().FindOne(ctx, bson.M{"_id": "safety"}).Decode(&state); e != nil || state.Reserved != 4 || !state.Paused || state.ResumeToken == "" { + t.Fatal(state, e) + } + if _, e = q.control().UpdateOne(ctx, bson.M{"_id": "lease"}, bson.M{"$set": bson.M{"until": time.Time{}}}); e != nil { + t.Fatal(e) + } + successor, e := NewQueryReplica(source, target) + successor.BatchSize = q.BatchSize + if e != nil { + t.Fatal(e) + } + if e = successor.Acquire(ctx); e != nil { + t.Fatal(e) + } + p.MaxRecords = 20 + if e = successor.ConfigureSafety(ctx, p, ""); !errors.Is(e, ErrQueryReplicaPaused) { + t.Fatal("restart bypassed pause", e) + } + changed := p + changed.RunID = "replacement-run" + if e = successor.ConfigureSafety(ctx, changed, state.ResumeToken); !errors.Is(e, ErrQueryReplicaPaused) { + t.Fatal("new run reset budget", e) + } + seedReplicaSafetyHealth(t, successor) + if e = successor.ConfigureSafety(ctx, p, state.ResumeToken); e != nil { + t.Fatal(e) + } + cp, e := successor.checkpoint(ctx, "humanItems") + if e != nil || !cp.HasLast || cp.Complete { + t.Fatal(cp, e) + } + if _, n, e = successor.BootstrapStep(ctx, "humanItems"); e != nil || n != 2 { + t.Fatal("resume lost remaining history", n, e) + } + var doc bson.M + if e = source.db.Collection("humanItems").FindOne(ctx, bson.M{}).Decode(&doc); e != nil || doc["version"] != int32(7) || deletionDoc(deletionDoc(doc["subject"])["payload"])["annotationDraft"] != "preserve" { + t.Fatal("source annotation altered", e) + } +} +func TestQueryReplicaSafetyRelativeCPUCountsDistinctObservationsLocal(t *testing.T) { + _, _, q := newLocalReplica(t) + seedReplicaSafetyHealth(t, q) + p := safetyPolicyForTest() + if e := q.ConfigureSafety(t.Context(), p, ""); e != nil { + t.Fatal(e) + } + now := time.Now() + for i := 1; i <= 2; i++ { + o := QueryReplicaSafetyObservation{At: now.Add(time.Duration(i) * time.Millisecond), CPU: 90, Ready: true, WritersCovered: true, HostFreeBytes: 120 << 30, TargetUsedFraction: .1} + if e := q.Target.RecordQueryReplicaObservation(t.Context(), o); e != nil { + t.Fatal(e) + } + _, e := q.sourcePermit(t.Context(), 0) + if i == 1 { + if e != nil { + t.Fatal(e) + } + for j := 0; j < 3; j++ { + if _, e = q.sourcePermit(t.Context(), 0); e != nil { + t.Fatal("same sample counted twice", e) + } + } + } else if !errors.Is(e, ErrQueryReplicaPaused) { + t.Fatal("CPU did not persist pause", e) + } + } + var state queryReplicaSafetyState + _ = q.control().FindOne(t.Context(), bson.M{"_id": "safety"}).Decode(&state) + if !state.Paused || state.Reason != "source_cpu_relative" { + t.Fatal(state) + } + seedReplicaSafetyHealth(t, q) + if _, e := q.sourcePermit(t.Context(), 0); !errors.Is(e, ErrQueryReplicaPaused) { + t.Fatal("cooldown silently resumed", e) + } +} +func TestQueryReplicaSafetyRestartWithoutPauseFailsClosedLocal(t *testing.T) { + source, target, q := newLocalReplica(t) + seedReplicaSafetyHealth(t, q) + p := safetyPolicyForTest() + if e := q.ConfigureSafety(t.Context(), p, ""); e != nil { + t.Fatal(e) + } + if _, e := q.sourcePermit(t.Context(), 7); e != nil { + t.Fatal(e) + } + _, _ = q.control().UpdateOne(t.Context(), bson.M{"_id": "lease"}, bson.M{"$set": bson.M{"until": time.Time{}}}) + next, _ := NewQueryReplica(source, target) + next.BatchSize = q.BatchSize + if e := next.Acquire(t.Context()); e != nil { + t.Fatal(e) + } + if e := next.ConfigureSafety(t.Context(), p, ""); !errors.Is(e, ErrQueryReplicaPaused) { + t.Fatal("crash bypassed operator review", e) + } + var s queryReplicaSafetyState + _ = next.control().FindOne(t.Context(), bson.M{"_id": "safety"}).Decode(&s) + if !s.Paused || s.Reason != "process_restarted" || s.Reserved != 7 { + t.Fatal(s) + } +} +func TestQueryReplicaBoundedClaimRecoveryPreservesAnnotationsLocal(t *testing.T) { + var many int + monitor := &event.CommandMonitor{Started: func(_ context.Context, e *event.CommandStartedEvent) { + if e.CommandName == "delete" { + t.Error("projector attempted source delete") + } + if e.CommandName == "update" { + arr := e.Command.Lookup("updates").Array() + vs, _ := arr.Values() + for _, v := range vs { + u := v.Document() + if u.Lookup("multi").Type != 0 && u.Lookup("multi").Boolean() { + ids, _ := u.Lookup("q").Document().Lookup("_id").Document().Lookup("$in").Array().Values() + if len(ids) > 2 { + t.Error("unbounded cleanup") + } + many++ + } + } + } + }} + source, _, q := newLocalReplica(t, monitor) + ctx := t.Context() + q.BatchSize = 2 + i := queryDeleteIntent{ID: "expired-delete", Collection: "humanItems", Phase: "deleting", LeaseUntil: time.Now().Add(-time.Minute), CreatedAt: time.Now()} + if _, e := source.db.Collection(querySyncDeleteCollection).InsertOne(ctx, i); e != nil { + t.Fatal(e) + } + for n := 0; n < 7; n++ { + if _, e := source.db.Collection("humanItems").InsertOne(ctx, bson.M{"id": domain.NewID("item"), "version": 9, "status": "claimed", "annotation": "kept", "_querySync": bson.M{"deleteClaim": i.ID}}); e != nil { + t.Fatal(e) + } + } + for n := 0; n < 6; n++ { + if _, e := q.ReconcileDeleteStep(ctx); e != nil { + t.Fatal(e) + } + } + count, e := source.db.Collection("humanItems").CountDocuments(ctx, bson.M{"version": 9, "status": "claimed", "annotation": "kept", "_querySync.deleteClaim": bson.M{"$exists": false}}) + if e != nil || count != 7 || many != 4 { + t.Fatal(count, many, e) + } +} +func TestQueryReplicaSafetyMajorityJournalAndNoEarlyAuditLocal(t *testing.T) { + var commits int + targetMonitor := &event.CommandMonitor{Started: func(_ context.Context, e *event.CommandStartedEvent) { + if e.CommandName == "commitTransaction" { + commits++ + wc := e.Command.Lookup("writeConcern").Document() + if wc.Lookup("w").StringValue() != "majority" || !wc.Lookup("j").Boolean() { + t.Error("durability weakened") + } + } + }} + _, _, q := newLocalReplica(t, nil, targetMonitor) + seedReplicaSafetyHealth(t, q) + if e := q.ConfigureSafety(t.Context(), safetyPolicyForTest(), ""); e != nil { + t.Fatal(e) + } + if _, e := q.ReconcileOneDue(t.Context(), 6*time.Hour); e != nil { + t.Fatal(e) + } + if commits == 0 { + t.Fatal("no transactional durability evidence") + } +} + +func TestQueryReplicaSafetyMissingHealthCannotRestartIntoRunningLocal(t *testing.T) { + _, _, q := newLocalReplica(t) + q.BatchSize = 25 + p := safetyPolicyForTest() + if e := q.ConfigureSafety(t.Context(), p, ""); !errors.Is(e, ErrQueryReplicaPaused) { + t.Fatal(e) + } + seedReplicaSafetyHealth(t, q) + if e := q.ConfigureSafety(t.Context(), p, ""); !errors.Is(e, ErrQueryReplicaPaused) { + t.Fatal("missing health latch auto-resumed", e) + } +} +func TestQueryReplicaDeleteBudgetCannotDeleteUnprobedTargetLocal(t *testing.T) { + source, target, q := newLocalReplica(t) + q.BatchSize = 2 + ctx := t.Context() + _, e := source.db.Collection(querySyncDeleteCollection).InsertOne(ctx, queryDeleteIntent{ID: "audit", Audit: true, Collection: "humanItems", Phase: "reconcile", CreatedAt: time.Now()}) + if e != nil { + t.Fatal(e) + } + for _, id := range []string{"one", "two"} { + _, e = target.db.Collection("humanItems").InsertOne(ctx, bson.M{"_id": id, "_queryReplica": bson.M{"sourceId": id}}) + if e != nil { + t.Fatal(e) + } + } + seedReplicaSafetyHealth(t, q) + p := safetyPolicyForTest() + p.MaxRecords = 4 + if e = q.ConfigureSafety(ctx, p, ""); e != nil { + t.Fatal(e) + } + if _, e = q.ReconcileDeleteStep(ctx); !errors.Is(e, ErrQueryReplicaPaused) { + t.Fatal(e) + } + n, e := target.db.Collection("humanItems").CountDocuments(ctx, bson.M{}) + if e != nil || n != 2 { + t.Fatal("unprobed target facts removed", n, e) + } +} + +func TestQueryReplicaSafetyRejectsSourceObservationAndStopsBuilderLocal(t *testing.T) { + source, target, q := newLocalReplica(t) + seedReplicaSafetyHealth(t, q) + p := safetyPolicyForTest() + if e := q.ConfigureSafety(t.Context(), p, ""); e != nil { + t.Fatal(e) + } + now := time.Now().UTC() + o := QueryReplicaSafetyObservation{At: now, CPU: 85, Ready: true, WritersCovered: true, HostFreeBytes: 120 << 30, TargetUsedFraction: .1} + if e := source.RecordQueryReplicaObservation(t.Context(), o); e == nil { + t.Fatal("observer wrote into authority") + } + count, _ := source.db.Collection(queryReplicaControl).CountDocuments(t.Context(), bson.M{}) + if count != 0 { + t.Fatal("observation mutated source") + } + if e := target.CheckQueryReplicaSafety(t.Context(), p.RunID); e != nil { + t.Fatal(e) + } + o.At = now.Add(-46 * time.Second) + if e := target.RecordQueryReplicaObservation(t.Context(), o); e == nil { + t.Fatal("stale health accepted") + } + if e := q.PauseSafety(t.Context(), "operator_stop"); e != nil { + t.Fatal(e) + } + if e := target.CheckQueryReplicaSafety(t.Context(), p.RunID); !errors.Is(e, ErrQueryReplicaPaused) { + t.Fatal("builder ignored durable pause", e) + } +} +func TestQueryReplicaSafetyTargetFailureRetainsDirtyAndCheckpointLocal(t *testing.T) { + source, target, q := newLocalReplica(t) + ctx := t.Context() + seedReplicaSafetyHealth(t, q) + if e := q.ConfigureSafety(ctx, safetyPolicyForTest(), ""); e != nil { + t.Fatal(e) + } + _, e := source.db.Collection("humanItems").InsertOne(ctx, bson.M{"_id": "source", "id": "item", "organizationId": "o", "projectId": "p", "version": 7, "status": "claimed", "annotation": "keep", "_querySync": bson.M{"revision": 1, "dirty": true}}) + if e != nil { + t.Fatal(e) + } + // Failure injection is confined to this fixture's random target database. + if e = target.db.RunCommand(ctx, bson.D{{Key: "collMod", Value: "humanItems"}, {Key: "validator", Value: bson.M{"mustNeverExist": true}}}).Err(); e != nil { + t.Fatal(e) + } + if _, _, e = q.BootstrapStep(ctx, "humanItems"); e == nil { + t.Fatal("invalid target write unexpectedly committed") + } + cp, e := q.checkpoint(ctx, "humanItems") + if e != nil || cp.HasLast || cp.Complete { + t.Fatal("failed transaction advanced checkpoint", cp, e) + } + var row bson.M + if e = source.db.Collection("humanItems").FindOne(ctx, bson.M{"_id": "source"}).Decode(&row); e != nil || deletionDoc(row[querySyncField])["dirty"] != true || row["version"] != int32(7) || row["annotation"] != "keep" { + t.Fatal("failed sink damaged source", e) + } + if e = target.db.RunCommand(ctx, bson.D{{Key: "collMod", Value: "humanItems"}, {Key: "validator", Value: bson.M{}}}).Err(); e != nil { + t.Fatal(e) + } + if _, n, e := q.BootstrapStep(ctx, "humanItems"); e != nil || n != 1 { + t.Fatal("replay failed", n, e) + } + if n, e := q.DrainStep(ctx, "humanItems"); e != nil || n != 0 { + t.Fatal("dirty acknowledged before persistence or not replayable", n, e) + } +} +func TestQueryReplicaSafetyConcurrentSavePreservesNewRevisionLocal(t *testing.T) { + var source *MongoStore + armed := false + fired := false + mon := &event.CommandMonitor{Started: func(_ context.Context, e *event.CommandStartedEvent) { + if e.CommandName != "update" { + return + } + if !armed || fired { + return + } + name, _ := e.Command.Lookup("update").StringValueOK() + if name != "humanItems" { + return + } + fired = true + short, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + _, err := source.db.Collection("humanItems").UpdateOne(short, bson.M{"_id": "source"}, bson.M{"$set": bson.M{"version": 8, "status": "submitted", "annotation": "new author edit", "_querySync.revision": 2, "_querySync.dirty": true}}) + if err != nil { + t.Error(err) + } + }} + var q *QueryReplica + source, _, q = newLocalReplica(t, nil, mon) + ctx := t.Context() + seedReplicaSafetyHealth(t, q) + if e := q.ConfigureSafety(ctx, safetyPolicyForTest(), ""); e != nil { + t.Fatal(e) + } + _, e := source.db.Collection("humanItems").InsertOne(ctx, bson.M{"_id": "source", "id": "item", "organizationId": "o", "projectId": "p", "version": 7, "status": "claimed", "annotation": "old author edit", "_querySync": bson.M{"revision": 1, "dirty": true}}) + if e != nil { + t.Fatal(e) + } + armed = true + if _, _, e = q.BootstrapStep(ctx, "humanItems"); e != nil { + t.Fatal(e) + } + var kept bson.M + _ = source.db.Collection("humanItems").FindOne(ctx, bson.M{"_id": "source"}).Decode(&kept) + if !fired || kept["version"] != int32(8) || kept["annotation"] != "new author edit" || deletionDoc(kept[querySyncField])["dirty"] != true { + t.Fatal("old copy acknowledged or overwrote concurrent save") + } + if n, e := q.DrainStep(ctx, "humanItems"); e != nil || n != 1 { + t.Fatal("new revision not recoverable", n, e) + } +} + +func TestQueryReplicaSafetyBacklogPriorityAndRoundRobinLocal(t *testing.T) { + source, _, q := newLocalReplica(t) + q.BatchSize = 1 + ctx := t.Context() + seedReplicaSafetyHealth(t, q) + p := safetyPolicyForTest() + p.MaxRecords = 0 + if e := q.ConfigureSafety(ctx, p, ""); e != nil { + t.Fatal(e) + } + for _, id := range []string{"a", "b", "c", "d", "e"} { + _, e := source.db.Collection("annotationIndexJobs").InsertOne(ctx, bson.M{"_id": id, "id": id, "organizationId": "o", "status": "completed", "_querySync": bson.M{"revision": 1, "dirty": true}}) + if e != nil { + t.Fatal(e) + } + } + _, e := q.control().InsertOne(ctx, bson.M{"_id": "backlog/annotationIndexJobs", "since": time.Now().Add(-6 * time.Minute)}) + if e != nil { + t.Fatal(e) + } + if ready, e := q.RunOnce(ctx, true); e != nil || ready { + t.Fatal("partial history published", ready, e) + } + clean, e := source.db.Collection("annotationIndexJobs").CountDocuments(ctx, bson.M{"_querySync.dirty": false}) + if e != nil || clean != 4 { + t.Fatal("old dirty backlog did not receive4:1 priority", clean, e) + } + var cursor struct { + Next int `bson:"next"` + } + if e = q.control().FindOne(ctx, bson.M{"_id": "schedule"}).Decode(&cursor); e != nil { + t.Fatal(e) + } + if cursor.Next != 0 { + t.Fatal("round-robin checkpoint not persisted") + } +} diff --git a/internal/store/query_replica_safety_test.go b/internal/store/query_replica_safety_test.go new file mode 100644 index 00000000..3a44008b --- /dev/null +++ b/internal/store/query_replica_safety_test.go @@ -0,0 +1,146 @@ +package store + +import ( + "context" + "errors" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "math" + "net" + "sync" + "testing" + "time" +) + +func TestQueryReplicaMetadataWriterRejectsBusinessFields(t *testing.T) { + cases := []struct { + name string + model mongo.WriteModel + ok bool + }{ + {"ack", mongo.NewUpdateOneModel().SetFilter(bson.M{"_id": "i", "_querySync.revision": 1}).SetUpdate(bson.M{"$set": bson.M{"_querySync.dirty": false}}), true}, + {"repair", mongo.NewUpdateOneModel().SetFilter(bson.M{"_id": "i", "_querySync.revision": bson.M{"$exists": false}}).SetUpdate(bson.M{"$set": queryReplicaRepairMarker(2)}), true}, + {"annotation", mongo.NewUpdateOneModel().SetFilter(bson.M{"_id": "i", "_querySync.revision": 1}).SetUpdate(bson.M{"$set": bson.M{"subject.payload.annotationDraft": "overwrite"}}), false}, + {"version", mongo.NewUpdateOneModel().SetFilter(bson.M{"_id": "i", "_querySync.revision": 1}).SetUpdate(bson.M{"$inc": bson.M{"version": 1}}), false}, + {"upsert", mongo.NewUpdateOneModel().SetFilter(bson.M{"_id": "i", "_querySync.revision": 1}).SetUpdate(bson.M{"$set": bson.M{"_querySync.dirty": false}}).SetUpsert(true), false}, + {"missing-CAS", mongo.NewUpdateOneModel().SetFilter(bson.M{"_id": "i"}).SetUpdate(bson.M{"$set": bson.M{"_querySync.dirty": false}}), false}, + {"delete", mongo.NewDeleteOneModel().SetFilter(bson.M{"_id": "i"}), false}, + {"replace", mongo.NewReplaceOneModel().SetFilter(bson.M{"_id": "i"}).SetReplacement(bson.M{"_querySync": bson.M{"dirty": false}}), false}, + {"pipeline", mongo.NewUpdateOneModel().SetFilter(bson.M{"_id": "i", "_querySync.revision": 1}).SetUpdate(bson.A{bson.M{"$set": bson.M{"_querySync.dirty": false}}}), false}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + e := validateReplicaMetadataWrite("humanItems", c.model) + if (e == nil) != c.ok { + t.Fatalf("allowed=%v error=%v", c.ok, e) + } + }) + } +} +func TestQueryReplicaSafetyRejectsNonFiniteAndUnsafeHealth(t *testing.T) { + p := QueryReplicaSafetyPolicy{BytesPerSecond: 65536, CommandsPerSecond: 2, MaxBatchRows: 25, RunID: "r", RecordsPerSecond: 10, BaselineCPU: 85} + if p.validate() != nil { + t.Fatal("valid policy") + } + p.BaselineCPU = math.NaN() + if p.validate() == nil { + t.Fatal("NaN policy accepted") + } + now := time.Now() + good := QueryReplicaSafetyObservation{At: now, CPU: 85, Ready: true, WritersCovered: true, HostFreeBytes: 120 << 30, TargetUsedFraction: .1} + if observationFailure(good, now) != "" { + t.Fatal("healthy observation") + } + for _, change := range []func(*QueryReplicaSafetyObservation){func(o *QueryReplicaSafetyObservation) { o.CPU = 95 }, func(o *QueryReplicaSafetyObservation) { o.CPU = math.NaN() }, func(o *QueryReplicaSafetyObservation) { o.WorkErrors = true }, func(o *QueryReplicaSafetyObservation) { o.At = now.Add(-46 * time.Second) }, func(o *QueryReplicaSafetyObservation) { o.TargetUsedFraction = .7 }, func(o *QueryReplicaSafetyObservation) { o.HostFreeBytes = 99 << 30 }, func(o *QueryReplicaSafetyObservation) { o.ProtectedIOFull = 6 }} { + o := good + change(&o) + if observationFailure(o, now) == "" { + t.Fatal("unsafe observation accepted") + } + } +} +func TestQueryReplicaGlobalPacingAndCancellation(t *testing.T) { + p := newQueryReplicaPacer(100) + start := time.Now() + var wg sync.WaitGroup + for i := 0; i < 4; i++ { + wg.Add(1) + go func() { + defer wg.Done() + if e := p.wait(context.Background(), 5); e != nil { + t.Error(e) + } + }() + } + wg.Wait() + if time.Since(start) < 190*time.Millisecond { + t.Fatal("per-caller budgets bypassed shared rate") + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + start = time.Now() + if e := p.wait(ctx, 100); !errors.Is(e, context.Canceled) || time.Since(start) > 50*time.Millisecond { + t.Fatal("uncancelable budget", e) + } +} +func TestQueryReplicaTransportReadDeadlineAndClose(t *testing.T) { + for _, closeIt := range []bool{false, true} { + t.Run(map[bool]string{false: "deadline", true: "close"}[closeIt], func(t *testing.T) { + a, b := net.Pipe() + defer b.Close() + budget, _ := NewQueryReplicaTransport(1024, 2) + c := &queryReplicaRateConn{Conn: a, budget: budget, closed: make(chan struct{}), wake: make(chan struct{}, 1)} + defer c.Close() + if !closeIt { + _ = c.SetReadDeadline(time.Now().Add(30 * time.Millisecond)) + } + done := make(chan error, 1) + go func() { _, e := c.Read(make([]byte, 1024)); done <- e }() + if closeIt { + time.Sleep(10 * time.Millisecond) + _ = c.Close() + } + select { + case e := <-done: + if e == nil { + t.Fatal("unbounded read succeeded") + } + case <-time.After(300 * time.Millisecond): + t.Fatal("pacing ignores connection deadline/close") + } + }) + } +} +func TestQueryReplicaTransportByteBudgetSharedAcrossConnections(t *testing.T) { + budget, _ := NewQueryReplicaTransport(20480, 8) + start := time.Now() + var wg sync.WaitGroup + for i := 0; i < 2; i++ { + a, b := net.Pipe() + c := &queryReplicaRateConn{Conn: a, budget: budget, closed: make(chan struct{}), wake: make(chan struct{}, 1)} + wg.Add(1) + go func() { + defer wg.Done() + defer c.Close() + defer b.Close() + go func() { _, _ = b.Write(make([]byte, 4096)) }() + got := 0 + buf := make([]byte, 4096) + for got < 4096 { + n, e := c.Read(buf) + if e != nil { + t.Error(e) + return + } + if n > 1024 { + t.Error("unbounded transport burst") + } + got += n + } + }() + } + wg.Wait() + if time.Since(start) < 390*time.Millisecond { + t.Fatal("aggregate byte rate exceeded") + } +} diff --git a/internal/store/query_replica_schedule.go b/internal/store/query_replica_schedule.go new file mode 100644 index 00000000..5ed0dbf3 --- /dev/null +++ b/internal/store/query_replica_schedule.go @@ -0,0 +1,126 @@ +package store + +import ( + "context" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + "time" +) + +// Preserve incremental progress during history copy. Full dirty batches reserve +// at least half the row opportunities (more than the30% floor); a backlog seen +// continuously for5min receives four batches for each history batch. This age +// is explicitly observation age, not an invented customer event timestamp. +func (q *QueryReplica) backlogAge(ctx context.Context, n string, rows int) (time.Duration, error) { + var since time.Time + e := q.transaction(ctx, func(sc mongo.SessionContext) error { + key := "backlog/" + n + if rows < int(q.limit()) { + _, e := q.control().DeleteOne(sc, bson.M{"_id": key}) + return e + } + var d struct { + Since time.Time `bson:"since"` + } + e := q.control().FindOne(sc, bson.M{"_id": key}).Decode(&d) + if e != nil && e != mongo.ErrNoDocuments { + return e + } + if e == mongo.ErrNoDocuments { + d.Since = time.Now().UTC() + if _, e = q.control().InsertOne(sc, bson.M{"_id": key, "since": d.Since}); e != nil { + return e + } + } + since = d.Since + return nil + }) + if e != nil || since.IsZero() { + return 0, e + } + return time.Since(since), nil +} +func (q *QueryReplica) runGuardedOnce(ctx context.Context, bootstrap bool) (bool, error) { + if _, e := q.sourcePermit(ctx, 0); e != nil { + return false, e + } + names := replicaCollections() + var cursor struct { + Next int `bson:"next"` + } + e := q.control().FindOne(ctx, bson.M{"_id": "schedule"}).Decode(&cursor) + if e != nil && e != mongo.ErrNoDocuments { + return false, e + } + start := cursor.Next % len(names) + if start < 0 { + start = 0 + } + for offset := 0; offset < len(names); offset++ { + index := (start + offset) % len(names) + name := names[index] + n, e := q.DrainStep(ctx, name) + if e != nil { + return false, e + } + age, e := q.backlogAge(ctx, name, n) + if e != nil { + return false, e + } + if age > 5*time.Minute { + for extra := 0; extra < 3; extra++ { + n, e = q.DrainStep(ctx, name) + if e != nil { + return false, e + } + if n < int(q.limit()) { + if _, e = q.backlogAge(ctx, name, n); e != nil { + return false, e + } + break + } + } + } + cp, e := q.checkpoint(ctx, name) + if e != nil { + return false, e + } + if !cp.Complete { + if !bootstrap { + return false, ErrQueryReplicaNotReady + } + if _, _, e = q.BootstrapStep(ctx, name); e != nil { + return false, e + } + } + if e = q.transaction(ctx, func(sc mongo.SessionContext) error { + _, e := q.control().UpdateOne(sc, bson.M{"_id": "schedule"}, bson.M{"$set": bson.M{"next": (index + 1) % len(names)}}, options.Update().SetUpsert(true)) + return e + }); e != nil { + return false, e + } + } + complete := true + for _, n := range names { + cp, e := q.checkpoint(ctx, n) + if e != nil { + return false, e + } + complete = complete && cp.Complete + } + if complete { + if _, e := q.ReconcileOneDue(ctx, 6*time.Hour); e != nil { + return false, e + } + } + if _, e := q.ReconcileDeleteStep(ctx); e != nil { + return false, e + } + if e := q.PublishReady(ctx); e == ErrQueryReplicaNotReady { + return false, nil + } else if e != nil { + return false, e + } + return true, nil +} diff --git a/internal/store/query_replica_source.go b/internal/store/query_replica_source.go new file mode 100644 index 00000000..6e511756 --- /dev/null +++ b/internal/store/query_replica_source.go @@ -0,0 +1,165 @@ +package store + +import ( + "context" + "errors" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + "strings" + "time" +) + +// All projector source mutations pass this seam. Database grants are scoped to +// collections, not fields; neither raw replacements nor business fields belong +// in this writer. These checks remain on even when optional pacing is disabled. +func validateReplicaMetadataWrite(collection string, model mongo.WriteModel) error { + m, ok := model.(*mongo.UpdateOneModel) + if !ok || m.Upsert != nil && *m.Upsert { + return errors.New("source metadata write rejected") + } + var f, u bson.M + b, e := bson.Marshal(m.Filter) + if e != nil { + return e + } + if e = bson.Unmarshal(b, &f); e != nil { + return e + } + b, e = bson.Marshal(m.Update) + if e != nil { + return e + } + if e = bson.Unmarshal(b, &u); e != nil { + return e + } + if f["_id"] == nil { + return errors.New("source metadata identity required") + } + if id := deletionDoc(f["_id"]); len(id) > 0 { + for key := range id { + if strings.HasPrefix(key, "$") { + return errors.New("metadata identity selector rejected") + } + } + } + for op, raw := range u { + if op != "$set" && op != "$unset" { + return errors.New("source metadata operator rejected") + } + fields := deletionDoc(raw) + if len(fields) == 0 { + return errors.New("empty metadata update") + } + for path := range fields { + allowed := false + if collection == querySyncDeleteCollection { + allowed = op == "$set" && (path == "phase" || path == "expireAt") + } else if queryReplicaFields[collection] != "" { + switch path { + case "_querySync.dirty", "_querySync.revision", "_querySync.signature", "_querySync.mode", "_querySync.producer": + allowed = op == "$set" + } + } + if !allowed { + return errors.New("source business field write rejected") + } + } + } + if queryReplicaFields[collection] != "" && f[querySyncField+".revision"] == nil { + return errors.New("metadata revision CAS required") + } + return nil +} +func (q *QueryReplica) sourceMetadataBulk(ctx context.Context, n string, models []mongo.WriteModel) error { + if len(models) == 0 { + return nil + } + for _, m := range models { + if e := validateReplicaMetadataWrite(n, m); e != nil { + return e + } + } + if _, e := q.sourcePermit(ctx, 0); e != nil { + return e + } + ctx, cancel := q.sourceContext(ctx) + defer cancel() + _, e := q.Source.db.Collection(n).BulkWrite(ctx, models, options.BulkWrite().SetOrdered(false)) + return e +} +func (q *QueryReplica) sourceIntentUpdate(ctx context.Context, filter, update bson.M) (*mongo.UpdateResult, error) { + m := mongo.NewUpdateOneModel().SetFilter(filter).SetUpdate(update) + if e := validateReplicaMetadataWrite(querySyncDeleteCollection, m); e != nil { + return nil, e + } + if _, e := q.sourcePermit(ctx, 0); e != nil { + return nil, e + } + ctx, cancel := q.sourceContext(ctx) + defer cancel() + return q.Source.db.Collection(querySyncDeleteCollection).UpdateOne(ctx, filter, update) +} +func (q *QueryReplica) sourceAuditIntent(ctx context.Context, i queryDeleteIntent) error { + if !i.Audit || i.Phase != "reconcile" || queryReplicaFields[i.Collection] == "" || i.ID == "" { + return errors.New("invalid source audit intent") + } + if _, e := q.sourcePermit(ctx, 0); e != nil { + return e + } + ctx, cancel := q.sourceContext(ctx) + defer cancel() + _, e := q.Source.db.Collection(querySyncDeleteCollection).InsertOne(ctx, i) + return e +} + +// Cleanup is bounded even for a historical DeleteMany intent. We never replay +// the source delete. A cancelled old writer cannot claim additional rows. +func (q *QueryReplica) recoverSourceClaims(ctx context.Context, i queryDeleteIntent) (bool, error) { + if i.Phase == "prepared" || i.Phase == "deleting" { + now := time.Now().UTC() + if i.LeaseUntil.After(now) { + return false, nil + } + r, e := q.sourceIntentUpdate(ctx, bson.M{"_id": i.ID, "phase": i.Phase, "leaseUntil": bson.M{"$lte": now}}, bson.M{"$set": bson.M{"phase": "cancelled"}}) + if e != nil || r.MatchedCount != 1 { + return false, e + } + } else if i.Phase != "complete" && i.Phase != "cancelled" && i.Phase != "reconcile" { + return false, nil + } + limit, e := q.sourcePermit(ctx, q.limit()) + if e != nil { + return false, e + } + work, cancel := q.sourceContext(ctx) + defer cancel() + cur, e := q.Source.db.Collection(i.Collection).Find(work, bson.M{querySyncField + ".deleteClaim": i.ID}, options.Find().SetHint("query_sync_delete_claim_v1").SetProjection(bson.M{"_id": 1}).SetLimit(limit).SetBatchSize(int32(limit)).SetMaxTime(1500*time.Millisecond)) + if e != nil { + return false, e + } + var rows []bson.M + if e = cur.All(work, &rows); e != nil { + return false, e + } + if len(rows) > 0 { + if e = q.paceRows(ctx, len(rows)); e != nil { + return false, e + } + if _, e = q.sourcePermit(ctx, 0); e != nil { + return false, e + } + ids := bson.A{} + for _, r := range rows { + ids = append(ids, r["_id"]) + } + // This is the only UpdateMany: exact bounded identities and exact claim, + // with only its private claim field removed; no upsert or business delete. + work, cancel := q.sourceContext(ctx) + defer cancel() + _, e = q.Source.db.Collection(i.Collection).UpdateMany(work, bson.M{"_id": bson.M{"$in": ids}, querySyncField + ".deleteClaim": i.ID}, bson.M{"$unset": bson.M{querySyncField + ".deleteClaim": ""}}) + return false, e + } + _, e = q.sourceIntentUpdate(ctx, bson.M{"_id": i.ID, "phase": bson.M{"$in": bson.A{"complete", "cancelled", "reconcile"}}}, bson.M{"$set": bson.M{"phase": "reconcile"}}) + return e == nil, e +} diff --git a/internal/store/query_replica_source_auth_local_test.go b/internal/store/query_replica_source_auth_local_test.go new file mode 100644 index 00000000..d70ec3ff --- /dev/null +++ b/internal/store/query_replica_source_auth_local_test.go @@ -0,0 +1,118 @@ +package store + +import ( + "context" + "droboticflow/internal/domain" + "encoding/json" + "errors" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + "os" + "os/exec" + "strings" + "testing" + "time" +) + +// This fixture owns a separate ephemeral auth-enabled standalone Mongo. It +// never enables authorization or creates users on the shared source/query lab. +func TestQueryReplicaSourceAccountPermissionsLocal(t *testing.T) { + if os.Getenv("QUERY_REPLICA_LOCAL_TEST") != "1" { + t.Skip("isolated auth fixture required") + } + ctx := t.Context() + if host := os.Getenv("DOCKER_HOST"); host != "" && !strings.HasPrefix(host, "unix://") { + t.Fatal("remote Docker prohibited") + } + endpoint, e := exec.Command("docker", "context", "inspect", "--format", "{{.Endpoints.docker.Host}}").Output() + if e != nil || !strings.HasPrefix(strings.TrimSpace(string(endpoint)), "unix://") { + t.Fatal("local Docker socket required") + } + user, password := domain.NewID("root"), domain.NewID("secret") + name := strings.ToLower(domain.NewID("query-auth")) + cmd := exec.Command("docker", "run", "--rm", "-d", "--name", name, "--label", "codex.fixture=query-source-auth", "--cpus=1", "--memory=512m", "-p", "127.0.0.1::27017", "--env", "MONGO_INITDB_ROOT_USERNAME", "--env", "MONGO_INITDB_ROOT_PASSWORD", "mongo@sha256:5d7043a4ffe02b9ed1b6e0bab057546981af5ca0a79107e9c461e49bc44c0a7b", "--wiredTigerCacheSizeGB", "0.25", "--bind_ip_all") + cmd.Env = append(os.Environ(), "MONGO_INITDB_ROOT_USERNAME="+user, "MONGO_INITDB_ROOT_PASSWORD="+password) + out, e := cmd.Output() + if e != nil { + t.Fatal("owned authentication fixture startup failed") + } + container := strings.TrimSpace(string(out)) + t.Cleanup(func() { _ = exec.Command("docker", "stop", "--time=5", container).Run() }) + raw, e := exec.Command("docker", "inspect", "--format", "{{json .NetworkSettings.Ports}}", container).Output() + if e != nil { + t.Fatal("fixture binding unavailable") + } + var ports map[string][]struct{ HostIP, HostPort string } + if json.Unmarshal(raw, &ports) != nil || len(ports["27017/tcp"]) != 1 || ports["27017/tcp"][0].HostIP != "127.0.0.1" { + t.Fatal("fixture not loopback-only") + } + uri := "mongodb://127.0.0.1:" + ports["27017/tcp"][0].HostPort + "/?directConnection=true" + admin, e := mongo.Connect(ctx, options.Client().ApplyURI(uri).SetAuth(options.Credential{AuthSource: "admin", Username: user, Password: password}).SetServerSelectionTimeout(500*time.Millisecond)) + if e != nil { + t.Fatal("fixture connection failed") + } + defer admin.Disconnect(context.Background()) + ready := false + for n := 0; n < 80; n++ { + short, cancel := context.WithTimeout(ctx, 500*time.Millisecond) + e = admin.Ping(short, nil) + cancel() + if e == nil { + ready = true + break + } + time.Sleep(250 * time.Millisecond) + } + if !ready { + t.Fatal("authentication fixture readiness failed") + } + dbname := domain.NewID("query_sync_auth") + db := admin.Database(dbname) + privilege := bson.A{bson.M{"resource": bson.M{"cluster": true}, "actions": bson.A{"serverStatus"}}, bson.M{"resource": bson.M{"db": dbname, "collection": "humanItems"}, "actions": bson.A{"find", "update"}}, bson.M{"resource": bson.M{"db": dbname, "collection": querySyncDeleteCollection}, "actions": bson.A{"find", "insert", "update"}}} + if e = admin.Database("admin").RunCommand(ctx, bson.D{{Key: "createRole", Value: "metadata-sync"}, {Key: "privileges", Value: privilege}, {Key: "roles", Value: bson.A{}}}).Err(); e != nil { + t.Fatal("fixture role setup failed", e) + } + limitedPassword := domain.NewID("limited-secret") + if e = db.RunCommand(ctx, bson.D{{Key: "createUser", Value: "projector"}, {Key: "pwd", Value: limitedPassword}, {Key: "roles", Value: bson.A{bson.M{"role": "metadata-sync", "db": "admin"}}}}).Err(); e != nil { + t.Fatal("fixture user setup failed") + } + _, e = db.Collection("humanItems").InsertOne(ctx, bson.M{"_id": "i", "id": "i", "version": 7, "status": "claimed", "annotationDraft": "keep", "_querySync": bson.M{"revision": 1, "dirty": true}}) + if e != nil { + t.Fatal(e) + } + client, e := mongo.Connect(ctx, options.Client().ApplyURI(uri).SetAuth(options.Credential{AuthSource: dbname, Username: "projector", Password: limitedPassword})) + if e != nil { + t.Fatal("fixture principal unavailable") + } + defer client.Disconnect(context.Background()) + limited := client.Database(dbname) + q := &QueryReplica{Source: NewMongoStore(limited)} + if e = q.sourceMetadataBulk(ctx, "humanItems", []mongo.WriteModel{mongo.NewUpdateOneModel().SetFilter(bson.M{"_id": "i", "_querySync.revision": 1}).SetUpdate(bson.M{"$set": bson.M{"_querySync.dirty": false}})}); e != nil { + t.Fatal("allowed acknowledgement failed", e) + } + if e = q.sourceMetadataBulk(ctx, "humanItems", []mongo.WriteModel{mongo.NewUpdateOneModel().SetFilter(bson.M{"_id": "i", "_querySync.revision": 1}).SetUpdate(bson.M{"$set": bson.M{"annotationDraft": "overwrite"}})}); e == nil { + t.Fatal("application metadata boundary bypassed") + } + denied := func(label string, e error) { + t.Helper() + var ce mongo.CommandError + if !errors.As(e, &ce) || ce.Code != 13 { + t.Fatalf("%s was not denied by real authentication", label) + } + } + _, e = limited.Collection("humanItems").InsertOne(ctx, bson.M{"_id": "other"}) + denied("insert", e) + _, e = limited.Collection("humanItems").DeleteOne(ctx, bson.M{"_id": "i"}) + denied("delete", e) + denied("DDL", limited.RunCommand(ctx, bson.M{"drop": "humanItems"}).Err()) + denied("other database", client.Database("foreign").Collection("humanItems").FindOne(ctx, bson.M{}).Err()) + if e = client.Database("admin").RunCommand(ctx, bson.M{"serverStatus": 1}).Err(); e != nil { + t.Fatal("monitor privilege unavailable") + } + var kept bson.M + if e = db.Collection("humanItems").FindOne(ctx, bson.M{"_id": "i"}).Decode(&kept); e != nil || kept["annotationDraft"] != "keep" || kept["version"] != int32(7) || kept["status"] != "claimed" || deletionDoc(kept[querySyncField])["dirty"] != false { + t.Fatal("authority fields altered") + } + +} diff --git a/internal/store/query_replica_transport.go b/internal/store/query_replica_transport.go new file mode 100644 index 00000000..c82b7f9c --- /dev/null +++ b/internal/store/query_replica_transport.go @@ -0,0 +1,158 @@ +package store + +import ( + "context" + "errors" + "net" + "sync" + "time" +) + +// A shared leaky bucket, with no accumulated idle credit. The source driver +// shares it across application and monitoring connections; TLS overhead is +// conservatively included. Waiting never occurs inside a target transaction. +type queryReplicaPacer struct { + mu sync.Mutex + next time.Time + rate float64 +} + +func newQueryReplicaPacer(rate float64) *queryReplicaPacer { return &queryReplicaPacer{rate: rate} } +func (p *queryReplicaPacer) wait(ctx context.Context, units float64) error { + if units <= 0 { + return ctx.Err() + } + p.mu.Lock() + now := time.Now() + if p.next.Before(now) { + p.next = now + } + p.next = p.next.Add(time.Duration(units / p.rate * float64(time.Second))) + until := p.next + p.mu.Unlock() + timer := time.NewTimer(time.Until(until)) + defer timer.Stop() + select { + case <-ctx.Done(): + return ctx.Err() + case <-timer.C: + return nil + } +} + +type QueryReplicaTransport struct{ bytes, commands *queryReplicaPacer } + +func NewQueryReplicaTransport(bytesPerSecond, commandsPerSecond int) (*QueryReplicaTransport, error) { + if bytesPerSecond < 1024 || bytesPerSecond > 256*1024 || commandsPerSecond < 1 || commandsPerSecond > 8 { + return nil, errors.New("invalid source transport budget") + } + return &QueryReplicaTransport{newQueryReplicaPacer(float64(bytesPerSecond)), newQueryReplicaPacer(float64(commandsPerSecond))}, nil +} +func (d *QueryReplicaTransport) DialContext(ctx context.Context, network, address string) (net.Conn, error) { + c, e := (&net.Dialer{}).DialContext(ctx, network, address) + if e != nil { + return nil, e + } + return &queryReplicaRateConn{Conn: c, budget: d, closed: make(chan struct{}), wake: make(chan struct{}, 1)}, nil +} + +type queryReplicaRateConn struct { + net.Conn + budget *QueryReplicaTransport + once sync.Once + closed chan struct{} + mu sync.Mutex + readDeadline, writeDeadline time.Time + wake chan struct{} +} + +func (c *queryReplicaRateConn) Close() error { + c.once.Do(func() { close(c.closed) }) + return c.Conn.Close() +} +func (c *queryReplicaRateConn) SetDeadline(t time.Time) error { + c.mu.Lock() + c.readDeadline = t + c.writeDeadline = t + c.mu.Unlock() + c.notify() + return c.Conn.SetDeadline(t) +} +func (c *queryReplicaRateConn) SetReadDeadline(t time.Time) error { + c.mu.Lock() + c.readDeadline = t + c.mu.Unlock() + c.notify() + return c.Conn.SetReadDeadline(t) +} +func (c *queryReplicaRateConn) SetWriteDeadline(t time.Time) error { + c.mu.Lock() + c.writeDeadline = t + c.mu.Unlock() + c.notify() + return c.Conn.SetWriteDeadline(t) +} +func (c *queryReplicaRateConn) notify() { + c.mu.Lock() + old := c.wake + c.wake = make(chan struct{}) + close(old) + c.mu.Unlock() +} +func (c *queryReplicaRateConn) wait(p *queryReplicaPacer, n int, read bool) error { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + done := make(chan error, 1) + go func() { done <- p.wait(ctx, float64(n)) }() + for { + c.mu.Lock() + wake := c.wake + deadline := c.writeDeadline + if read { + deadline = c.readDeadline + } + c.mu.Unlock() + var timer *time.Timer + var expired <-chan time.Time + if !deadline.IsZero() { + timer = time.NewTimer(time.Until(deadline)) + expired = timer.C + } + select { + case e := <-done: + if timer != nil { + timer.Stop() + } + return e + case <-c.closed: + if timer != nil { + timer.Stop() + } + return net.ErrClosed + case <-expired: + return context.DeadlineExceeded + case <-wake: + if timer != nil { + timer.Stop() + } + } + } +} +func (c *queryReplicaRateConn) Read(p []byte) (int, error) { + if len(p) == 0 { + return 0, nil + } + if len(p) > 1024 { + p = p[:1024] + } + if e := c.wait(c.budget.bytes, len(p), true); e != nil { + return 0, e + } + return c.Conn.Read(p) +} +func (c *queryReplicaRateConn) Write(p []byte) (int, error) { + if e := c.wait(c.budget.commands, 1, false); e != nil { + return 0, e + } + return c.Conn.Write(p) +} diff --git a/scripts/local/query-shadow-observer.py b/scripts/local/query-shadow-observer.py new file mode 100644 index 00000000..ba222111 --- /dev/null +++ b/scripts/local/query-shadow-observer.py @@ -0,0 +1,296 @@ +#!/usr/bin/env python3 +"""Trusted operator-side observer. No kubeconfig or authority credentials enter builders. + +Every run writes private health observations in the derived query database. +Only --stop-on-failure permits cluster configuration mutations: exact UID-tested scale-to-zero of the +TWO shadow controllers in the hash-bound plan. Never changes authority services. +Do not run against production before approving the final startup/stop scope. +""" +import argparse +import base64 +import contextlib +import datetime as dt +import hashlib +import json +import os +from pathlib import Path +import socket +import subprocess +import threading +import time +import urllib.parse +import urllib.request + + +def check_bindings(b): + if b['sourceNamespace'] == b['shadowNamespace'] or len(b['writers']) != 5: + raise ValueError('invalid namespace or writer scope') + if {x['name'] for x in b['controllers']} != {'query-replica', 'query-read-builder'}: + raise ValueError('stop scope must contain only shadow controllers') + if b['sourceDatabase'] == b['targetDatabase'] or not 0 < b['baselineCPU'] <= 90: + raise ValueError('invalid source identity or baseline') + + +def cpu_percent(a, z): + if z['sourceProcess'] != a['sourceProcess']: + raise ValueError('source identity changed') + elapsed = z['uptimeMillis'] - a['uptimeMillis'] + used = z['cpuMicroseconds'] - a['cpuMicroseconds'] + if elapsed <= 0 or used < 0: + raise ValueError('nonmonotonic source counters') + return used / 1000 / elapsed / 4 * 100 + + +def full_io_pressure(text): + for line in text.splitlines(): + if line.startswith('full '): + return float(next(x.split('=', 1)[1] for x in line.split() if x.startswith('avg10='))) + raise ValueError('protected IO evidence missing') + + +def business_health(path, now=None): + feed = json.loads(Path(path).read_text()) + at = dt.datetime.fromisoformat(feed['at'].replace('Z', '+00:00')) + age = ((now or dt.datetime.now(dt.timezone.utc)) - at).total_seconds() + if (not 0 <= age <= 45 or feed.get('coverage') != 'save-slice-submit-review-acceptance' + or type(feed.get('unexpectedErrors')) is not bool or type(feed.get('latencyRegressed')) is not bool): + raise ValueError('business-health evidence stale or incomplete') + return feed['unexpectedErrors'] or feed['latencyRegressed'] + + +class Observer: + def __init__(self, args, bindings): + self.args, self.b = args, bindings + self.k = ['kubectl', '--kubeconfig', args.kubeconfig] + self.high = self.high_io = 0 + + def call(self, *args, data=None, timeout=15): + p = subprocess.run(self.k + list(args), input=data, capture_output=True, + text=True, timeout=timeout) + if p.returncode: + raise RuntimeError('bounded cluster operation unavailable') + return p.stdout + + def get(self, ns, kind, name): + return json.loads(self.call('-n', ns, 'get', kind, name, '-o', 'json')) + + def verify(self): + for key in ('source', 'shadow'): + n = self.get(self.b[key + 'Namespace'], 'namespace', self.b[key + 'Namespace']) + if n['metadata']['uid'] != self.b[key + 'NamespaceUID']: + raise ValueError('namespace identity changed') + for w in self.b['writers']: + d = self.get(self.b['sourceNamespace'], 'deployment', w['name']) + env = {x['name']: x.get('value') for x in d['spec']['template']['spec']['containers'][0].get('env', [])} + if (d['metadata']['uid'] != w['uid'] or env.get('DROBOTICFLOW_BUILD_COMMIT') != self.b['writerCommit'] + or env.get('QUERY_SOURCE_TRACKING') != '1' + or d['spec']['template']['spec']['containers'][0]['image'] != w['image'] + or d.get('status', {}).get('availableReplicas', 0) != w['replicas']): + raise ValueError('writer coverage or health changed') + for flag in ('QUERY_REPLICA_REPORTS', 'QUERY_REPLICA_COUNTS', 'QUERY_REPLICA_POOLS', 'QUERY_SERVING_READS', 'QUERY_PUBLISHED_RESULTS'): + if env.get(flag, '0') not in ('', '0', 'false'): + raise ValueError('production query entry changed') + + def secret(self, name, key): + return base64.b64decode(self.get(self.b['shadowNamespace'], 'secret', name)['data'][key]).decode() + + def target_environment(self, uri): + env = {k: v for k, v in os.environ.items() if not k.startswith(('MONGO_', 'QUERY_'))} + env.update(QUERY_READ_MONGO_URI=uri, QUERY_READ_MONGO_DB=self.b['targetDatabase'], + QUERY_SERVING_NAMESPACE=self.b['servingNamespace']) + return env + + def run_binary(self, args, env, data=None): + if hashlib.sha256(Path(self.args.binary).read_bytes()).hexdigest() != self.args.binary_sha256: + raise ValueError('observer binary changed') + p = subprocess.run([self.args.binary] + args, input=data, env=env, capture_output=True, + text=True, timeout=10) + if p.returncode: + raise RuntimeError('bounded observation command unavailable') + return p.stdout + + @contextlib.contextmanager + def source_relay(self, uri): + u = urllib.parse.urlsplit(uri) + if u.scheme != 'mongodb' or not u.hostname or ',' in u.netloc: + raise ValueError('single reviewed source endpoint required') + endpoint = u.hostname + ':' + str(u.port or 27017) + if hashlib.sha256(endpoint.encode()).hexdigest() != self.b['sourceEndpointSHA256']: + raise ValueError('source endpoint changed') + # No secret arguments or stderr: credentials remain in local child env. + pods = json.loads(self.call('-n', self.b['sourceNamespace'], 'get', 'pods', '-l', 'app=droboticflow-api', '-o', 'json'))['items'] + pod = next(x['metadata']['name'] for x in pods if not x['metadata'].get('deletionTimestamp') + and x['status'].get('phase') == 'Running' + and all(c['ready'] for c in x['status'].get('containerStatuses', []))) + listener = socket.socket(); listener.bind(('127.0.0.1', 0)); listener.listen(8); listener.settimeout(.2) + stop = threading.Event(); children = [] + relay = 'exec 3<>/dev/tcp/"$1"/"$2" || exit 1; cat <&3 & reader=$!; cat >&3; kill "$reader" 2>/dev/null || true; wait "$reader" 2>/dev/null || true' + + def transfer(conn): + p = subprocess.Popen(self.k + ['-n', self.b['sourceNamespace'], 'exec', '-i', pod, '--', 'bash', '-c', relay, + 'shadow-readonly-metrics', u.hostname, str(u.port or 27017)], + stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL) + children.append(p) + def upstream(): + try: + while not stop.is_set(): + chunk = conn.recv(65536) + if not chunk: break + p.stdin.write(chunk); p.stdin.flush() + except (OSError, ValueError): pass + finally: + with contextlib.suppress(OSError): p.stdin.close() + threading.Thread(target=upstream, daemon=True).start() + try: + while not stop.is_set(): + chunk = p.stdout.read1(65536) + if not chunk: break + conn.sendall(chunk) + except (OSError, ValueError): pass + finally: + conn.close(); p.terminate() + def accept(): + while not stop.is_set(): + try: conn, _ = listener.accept() + except socket.timeout: continue + except OSError: return + threading.Thread(target=transfer, args=(conn,), daemon=True).start() + thread = threading.Thread(target=accept, daemon=True); thread.start() + params = urllib.parse.parse_qs(u.query); params['directConnection'] = ['true'] + credential = u.netloc.rsplit('@', 1)[0] + '@' if '@' in u.netloc else '' + local = urllib.parse.urlunsplit(('mongodb', credential + '127.0.0.1:' + str(listener.getsockname()[1]), u.path, + urllib.parse.urlencode(params, doseq=True), '')) + # A TLS hostname is not rewritten/bypassed. This initial deployment uses + # its reviewed non-TLS private endpoint; other topologies need a new plan. + if params.get('tls', params.get('ssl', ['false']))[0].lower() == 'true': + listener.close(); stop.set(); raise ValueError('TLS relay needs separately verified server-name handling') + try: yield local + finally: + stop.set(); listener.close(); thread.join(timeout=1) + for child in children: + if child.poll() is None: child.terminate() + for child in children: + try: child.wait(timeout=2) + except subprocess.TimeoutExpired: child.kill() + + @contextlib.contextmanager + def target_forward(self, uri): + with socket.socket() as listener: + listener.bind(('127.0.0.1', 0)); port = listener.getsockname()[1] + process = subprocess.Popen(self.k + ['-n', self.b['shadowNamespace'], 'port-forward', '--address=127.0.0.1', + 'service/query-mongo', str(port) + ':27129'], + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + try: + for _ in range(40): + if process.poll() is not None: raise RuntimeError('query tunnel unavailable') + try: + with socket.create_connection(('127.0.0.1', port), timeout=.1): break + except OSError: time.sleep(.1) + else: raise RuntimeError('query tunnel deadline') + u = urllib.parse.urlsplit(uri) + credential = u.netloc.rsplit('@', 1)[0] + '@' if '@' in u.netloc else '' + yield urllib.parse.urlunsplit(('mongodb', credential + '127.0.0.1:' + str(port), u.path, u.query, '')) + finally: + process.terminate() + try: process.wait(timeout=2) + except subprocess.TimeoutExpired: process.kill() + + def source_sample(self): + with self.source_relay(self.secret('query-shadow-source-connection', 'uri')) as uri: + env = self.target_environment(''); env.update(MONGO_URI=uri, MONGO_DB=self.b['sourceDatabase']) + return json.loads(self.run_binary(['--sample-source'], env)) + + def observation(self, cpu, sampled_at): + self.verify() + with urllib.request.urlopen(self.b['readyURL'], timeout=3) as response: + ready = response.status == 200 and json.load(response).get('ready') is True + ns = self.b['shadowNamespace']; pod = self.get(ns, 'pod', 'query-mongo-0') + if pod['metadata']['uid'] != self.b['mongoPodUID']: + raise ValueError('query Mongo rebuilt; recovery verification required') + fs = json.loads(self.call('get', '--raw', '/api/v1/nodes/10.189.16.4/proxy/stats/summary'))['node']['fs'] + text = self.call('-n', ns, 'exec', 'query-mongo-0', '-c', 'mongo', '--', 'df', '-P', '-B1', '/lab/data') + fields = text.splitlines()[-1].split(); used = int(fields[2]) / int(fields[1]) + protected = json.loads(self.call('-n', self.b['sourceNamespace'], 'get', 'pods', '--field-selector=spec.nodeName=10.189.16.4', '-o', 'json'))['items'] + io = 0. + for p in protected: + if p['status'].get('phase') != 'Running': continue + uid = p['metadata']['uid'].replace('-', '_') + # Current production workloads are Burstable; refuse unknown cgroups. + path = '/hostcg/kubepods.slice/kubepods-burstable.slice/kubepods-burstable-pod' + uid + '.slice/io.pressure' + io = max(io, full_io_pressure(self.call('-n', ns, 'exec', 'query-mongo-0', '-c', 'mongo', '--', 'cat', path))) + # A trusted business-health feed is required; readiness/log absence is + # not evidence that saves/submits meet their latency/error baseline. + work_errors = business_health(self.args.work_health) + pods = json.loads(self.call('-n', self.b['sourceNamespace'], 'get', 'pods', '-l', 'app=droboticflow-api', '-o', 'json'))['items'] + for p in pods: + if p['metadata'].get('deletionTimestamp'): continue + logs = self.call('-n', self.b['sourceNamespace'], 'logs', p['metadata']['name'], '--since=45s', '--tail=2000').lower() + work_errors |= any(s in logs for s in ('panic:', 'i/o timeout', 'write conflict retry exhausted', 'submit failed', 'save failed')) + return dict(at=sampled_at, cpu=cpu, ready=ready, writersCovered=True, + workErrors=work_errors, protectedIOFull=io, targetUsedFraction=used, hostFreeBytes=fs['availableBytes']) + + def publish(self, observation): + with self.target_forward(self.secret('query-shadow-query-connections', 'builder-uri')) as uri: + self.run_binary(['--record-observation', '-'], self.target_environment(uri), json.dumps(observation)) + + def unsafe(self, o): + self.high = self.high + 1 if o['cpu'] > self.b['baselineCPU'] + 3 else 0 + self.high_io = self.high_io + 1 if o['protectedIOFull'] > 1 else 0 + return (not o['ready'] or not o['writersCovered'] or o['workErrors'] or o['cpu'] >= 95 or self.high >= 2 + or self.high_io >= 2 or o['protectedIOFull'] > 5 or o['targetUsedFraction'] >= .7 or o['hostFreeBytes'] < (100 << 30)) + + def stop_shadow(self): + for c in self.b['controllers']: + d = self.get(self.b['shadowNamespace'], 'deployment', c['name']) + if d['metadata']['uid'] != c['uid']: raise ValueError('stop controller replaced') + if d['spec']['replicas'] == 0: continue + patch = [{'op': 'test', 'path': '/metadata/uid', 'value': c['uid']}, + {'op': 'test', 'path': '/metadata/resourceVersion', 'value': d['metadata']['resourceVersion']}, + {'op': 'replace', 'path': '/spec/replicas', 'value': 0}] + self.call('-n', self.b['shadowNamespace'], 'patch', 'deployment', c['name'], '--type=json', '-p', json.dumps(patch)) + + def run(self): + try: prior = self.source_sample() + except Exception: + if self.args.stop_on_failure: self.stop_shadow() + raise RuntimeError('initial source observation unavailable') from None + while True: + try: + for _ in range(3): + time.sleep(10) + with urllib.request.urlopen(self.b['readyURL'],timeout=3) as response: + if response.status!=200 or json.load(response).get('ready') is not True: + raise ValueError('production readiness unavailable') + current = self.source_sample(); cpu = cpu_percent(prior, current); prior = current + sampled_at=current['sampledAt'];age=(dt.datetime.now(dt.timezone.utc)-dt.datetime.fromisoformat(sampled_at.replace('Z','+00:00'))).total_seconds() + if not 0<=age<=45: raise ValueError('source observation stale') + o = self.observation(cpu,sampled_at); self.publish(o) + unsafe=self.unsafe(o) + print(json.dumps({'at':o['at'],'cpu':cpu,'ready':o['ready'],'unsafe':unsafe}),flush=True) + if unsafe: + if self.args.stop_on_failure: self.stop_shadow() + return + except Exception: + # No raw exception may print a URI, secret or customer log line. + with contextlib.suppress(Exception): + self.publish(dict(at=dt.datetime.now(dt.timezone.utc).isoformat(), cpu=0, ready=False, writersCovered=False, + workErrors=True, protectedIOFull=0, targetUsedFraction=0, hostFreeBytes=0)) + if self.args.stop_on_failure: self.stop_shadow() + raise RuntimeError('observer paused; inspect private metadata and verify before resume') from None + + + +def main(): + p = argparse.ArgumentParser(description=__doc__) + p.add_argument('--kubeconfig', required=True); p.add_argument('--bindings', required=True) + p.add_argument('--bindings-sha256', required=True); p.add_argument('--binary', required=True) + p.add_argument('--binary-sha256', required=True); p.add_argument('--work-health', required=True); p.add_argument('--stop-on-failure', action='store_true') + args = p.parse_args(); raw = Path(args.bindings).read_bytes() + if hashlib.sha256(raw).hexdigest() != args.bindings_sha256: raise ValueError('bindings changed') + b = json.loads(raw); check_bindings(b); Observer(args, b).run() + + +if __name__ == '__main__': + try: main() + except KeyboardInterrupt: raise SystemExit(0) + except Exception: raise SystemExit('observer stopped; private checkpoint retained; no automatic resume') diff --git a/scripts/local/test_query_shadow_observer.py b/scripts/local/test_query_shadow_observer.py new file mode 100644 index 00000000..db735d4b --- /dev/null +++ b/scripts/local/test_query_shadow_observer.py @@ -0,0 +1,63 @@ +import importlib.util +import json +from pathlib import Path +from types import SimpleNamespace +import unittest +import tempfile +import datetime as dt + +spec = importlib.util.spec_from_file_location('shadow_observer', Path(__file__).with_name('query-shadow-observer.py')) +m = importlib.util.module_from_spec(spec); spec.loader.exec_module(m) + +class ObserverTests(unittest.TestCase): + def bindings(self): + return dict(sourceNamespace='authority', shadowNamespace='shadow', sourceDatabase='source', targetDatabase='query', + baselineCPU=85, writers=[{} for _ in range(5)], controllers=[dict(name='query-replica', uid='p'), dict(name='query-read-builder', uid='b')]) + def test_cpu_and_identity(self): + self.assertAlmostEqual(m.cpu_percent(dict(sourceProcess='s', uptimeMillis=1000, cpuMicroseconds=0), + dict(sourceProcess='s', uptimeMillis=2000, cpuMicroseconds=2000000)), 50) + with self.assertRaises(ValueError): + m.cpu_percent(dict(sourceProcess='s', uptimeMillis=2, cpuMicroseconds=2), dict(sourceProcess='x', uptimeMillis=3, cpuMicroseconds=3)) + def test_stop_scope_and_uid_cas(self): + b = self.bindings(); m.check_bindings(b) + observer = m.Observer(SimpleNamespace(kubeconfig='unused', stop_on_failure=True), b); calls = [] + observer.get = lambda ns, kind, name: dict(metadata=dict(uid='p' if name=='query-replica' else 'b', resourceVersion='7'), spec=dict(replicas=1)) + observer.call = lambda *args, **kwargs: calls.append(args) or '' + observer.stop_shadow() + self.assertEqual(len(calls), 2) + for args in calls: + self.assertEqual(args[:4], ('-n', 'shadow', 'patch', 'deployment')) + payload = json.loads(args[-1]); self.assertEqual(payload[0]['op'], 'test'); self.assertEqual(payload[-1]['value'], 0) + observer.get = lambda *args: dict(metadata=dict(uid='replaced', resourceVersion='8'), spec=dict(replicas=1)) + with self.assertRaises(ValueError): observer.stop_shadow() + self.assertEqual(len(calls), 2) + b['controllers'][0]['name'] = 'droboticflow-api' + with self.assertRaises(ValueError): m.check_bindings(b) + def test_distinct_samples_and_thresholds(self): + o = m.Observer(SimpleNamespace(kubeconfig='unused'), self.bindings()) + good = dict(cpu=90, ready=True, writersCovered=True, workErrors=False, protectedIOFull=0, targetUsedFraction=.1, hostFreeBytes=120<<30) + self.assertFalse(o.unsafe(good)); self.assertTrue(o.unsafe(good)) + good['cpu']=85; self.assertFalse(o.unsafe(good));good['targetUsedFraction']=.7;self.assertTrue(o.unsafe(good)) + def test_business_health_missing_stale_and_real_errors(self): + now = dt.datetime.now(dt.timezone.utc) + with tempfile.TemporaryDirectory() as folder: + path = Path(folder) / 'health.json' + with self.assertRaises(FileNotFoundError): m.business_health(path, now) + feed = dict(at=now.isoformat(), coverage='save-slice-submit-review-acceptance', unexpectedErrors=False, latencyRegressed=False) + path.write_text(json.dumps(feed)); self.assertFalse(m.business_health(path, now)) + feed['latencyRegressed'] = True + path.write_text(json.dumps(feed)); self.assertTrue(m.business_health(path, now)) + feed['latencyRegressed'] = 'false' + path.write_text(json.dumps(feed)) + with self.assertRaises(ValueError): m.business_health(path, now) + feed['latencyRegressed'] = False; feed['at'] = (now - dt.timedelta(seconds=46)).isoformat() + path.write_text(json.dumps(feed)) + with self.assertRaises(ValueError): m.business_health(path, now) + feed['at'] = now.isoformat(); feed['coverage'] = 'readiness-only' + path.write_text(json.dumps(feed)) + with self.assertRaises(ValueError): m.business_health(path, now) + def test_unknown_io_is_not_zero(self): + with self.assertRaises(ValueError): m.full_io_pressure('no evidence') + self.assertEqual(m.full_io_pressure('some avg10=2.0\nfull avg10=1.25 avg60=0'), 1.25) + +if __name__ == '__main__': unittest.main() diff --git a/scripts/test_query_replica.py b/scripts/test_query_replica.py index 51a6d6f2..85754976 100644 --- a/scripts/test_query_replica.py +++ b/scripts/test_query_replica.py @@ -15,7 +15,11 @@ from lib.release_checks import check_environment, source_fingerprint, write_log ROOT = Path(__file__).resolve().parents[1] PRIVATE = ROOT / 'local-data/query-replica-local' COMPOSE = ROOT / 'scripts/local/query-replica.compose.yml' -CASES = ('internal/store/query_replica_local_test.go', +CASES = ('cmd/query-replica/main_local_test.go', + 'internal/store/query_replica_source_auth_local_test.go', + 'internal/store/query_replica_safety_local_test.go', + 'internal/store/query_replica_safety_test.go', + 'internal/store/query_replica_local_test.go', 'internal/store/query_sync_local_test.go', 'internal/store/query_replica_count_cache_test.go', 'internal/api/query_replica_local_test.go', @@ -91,6 +95,21 @@ def start(env): required = {27119: 'source', 27129: 'query'} if any(required[port] not in running for port in busy): raise RuntimeError('lab ports are occupied by unowned services; use an owned native lab or free these ports') + if len(busy) == 2: + # Reuse isolated random databases, never recreate a shared fixture just + # because another authorized checkout owns its Compose metadata. + info = json.loads(subprocess.check_output(['docker', 'inspect', 'fanyu-query-replica-local-source-1', + 'fanyu-query-replica-local-query-1'], cwd=ROOT, env=env, text=True)) + resources = [{'name': x['Name'], 'cpu': x['HostConfig']['NanoCpus'], + 'memory': x['HostConfig']['Memory']} for x in info] + if env.get('QUERY_REPLICA_LOCAL_QUERY_BUDGET') == 'query25': + query = next(x for x in resources if x['name'].endswith('query-1')) + if query['cpu'] != 1_000_000_000 or query['memory'] != 1024 ** 3: + raise RuntimeError('existing shared query fixture differs from1CPU/1Gi; do not recreate it') + env['QUERY_REPLICA_REUSED_RESOURCES'] = json.dumps(resources) + compose(env, 'exec', '-T', 'query', 'mongosh', '--quiet', '--port', '27129', '--eval', + "if (!db.hello().isWritablePrimary || db.hello().setName !== 'query-read-rs') quit(2)") + return compose(env, 'up', '-d') for _ in range(60): try: @@ -135,7 +154,8 @@ def main(): else: raise RuntimeError('select --native, --start or --auto explicitly') if args.cache: - compose(env, 'up', '-d', 'valkey') + if 'valkey' not in compose(env, 'ps', '--status', 'running', '--services').splitlines(): + compose(env, 'up', '-d', 'valkey') env['QUERY_REPLICA_VALKEY_LOCAL_TEST'] = '1' required = set() for name in CASES: @@ -146,7 +166,7 @@ def main(): required = {'TestQueryReplicaScaleLocal'} env['QUERY_REPLICA_SCALE_LOCAL'] = '1' regex = '^(' + '|'.join(sorted(required)) + ')$' - argv = ['go', 'test', './internal/store', './internal/api', './cmd/query-cdc-probe', + argv = ['go', 'test', '-p=1', './internal/store', './internal/api', './cmd/query-cdc-probe', './cmd/query-replica', '-run', regex, '-count=1', '-timeout=10m', '-v'] if args.race: argv.append('-race') @@ -159,7 +179,7 @@ def main(): proof = write_log(PRIVATE / 'logs', argv, output, 'passed' if success else 'pending_or_failed') proof.update(missing=missing, sourceChanged=changed, skipped=skipped, environment='native-uncapped' if native else ('docker-source-4cpu-query-1cpu-1Gi' if args.bounded_query else 'docker-source-4cpu-query-4cpu'), - productionCapacityAccepted=False) + productionCapacityAccepted=False, reusedResources=json.loads(env.get('QUERY_REPLICA_REUSED_RESOURCES', '[]'))) print(json.dumps(proof, ensure_ascii=False)) return 0 if success else (code or 2) diff --git a/scripts/verification-scopes.json b/scripts/verification-scopes.json index 9ebf31b4..5a5c3645 100644 --- a/scripts/verification-scopes.json +++ b/scripts/verification-scopes.json @@ -1018,6 +1018,7 @@ { "name": "independent-query-replica", "paths": [ + "^scripts/local/(query-shadow-observer|test_query_shadow_observer)\\.py$", "^internal/store/query_(replica|sync|tracked_collection|collection_interface).*\\.go$", "^internal/api/query_replica.*\\.go$", "^internal/store/workforce_(report_cache|heap_diagnostic|facts|return_page|publication).*\\.go$", @@ -1052,6 +1053,20 @@ "./internal/testsupport": ["^TestQueryReplicaTarget"] }, "task": [ + { + "argv": [ + "python3", + "-m", + "unittest", + "discover", + "-s", + "scripts/local", + "-p", + "test_query_shadow_observer.py", + "-v" + ], + "tests": true + }, { "argv": ["python3", "scripts/test_query_replica.py", "--auto", "--bounded-query", "--race", "--cache"], "tests": true