🧪 Validate isolated query replica routing and workload safety

This commit is contained in:
Codex 2026-11-12 23:02:02 -08:00
parent fbaa8546c0
commit e38c3be922
20 changed files with 1590 additions and 10 deletions

View File

@ -0,0 +1,105 @@
// query-lab-bridge is an isolated-fixture transport, never a production service.
package main
import (
"context"
"errors"
"flag"
"io"
"log"
"net"
"net/netip"
"os"
"os/signal"
"sync"
"syscall"
"time"
)
func allowedFlow(listen, upstream, node, actualUID, expectedUID string) bool {
if node != "10.189.16.4" || actualUID == "" || actualUID != expectedUID {
return false
}
l, le := netip.ParseAddrPort(listen)
u, ue := netip.ParseAddrPort(upstream)
if le != nil || ue != nil {
return false
}
private := netip.MustParsePrefix("10.189.16.0/24")
return l == netip.MustParseAddrPort("127.0.0.1:27229") && private.Contains(u.Addr()) && u.Port() == 27130 ||
private.Contains(l.Addr()) && l.Port() == 27130 && u == netip.MustParseAddrPort("127.0.0.1:27129")
}
// Do not embed TCPConn: promoted ReadFrom/WriteTo would bypass the deadlines
// when io.Copy selects its optimized interfaces.
type boundedConn struct{ conn *net.TCPConn }
func (c boundedConn) Read(p []byte) (int, error) {
_ = c.conn.SetReadDeadline(time.Now().Add(time.Minute))
return c.conn.Read(p)
}
func (c boundedConn) Write(p []byte) (int, error) {
_ = c.conn.SetWriteDeadline(time.Now().Add(time.Minute))
return c.conn.Write(p)
}
func main() {
listen := flag.String("listen", "", "fixed isolated listen address")
upstream := flag.String("upstream", "", "fixed isolated query address")
uid := flag.String("pod-uid", "", "preflight Pod identity")
flag.Parse()
if !allowedFlow(*listen, *upstream, os.Getenv("NODE_NAME"), os.Getenv("POD_UID"), *uid) {
log.Fatal("isolated transport identity or address rejected")
}
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGTERM, os.Interrupt)
defer stop()
listener, err := net.Listen("tcp4", *listen)
if err != nil {
log.Fatal(err)
}
defer listener.Close()
go func() { <-ctx.Done(); _ = listener.Close() }()
var workers sync.WaitGroup
defer workers.Wait()
slots := make(chan struct{}, 128)
for ctx.Err() == nil {
conn, err := listener.Accept()
if err != nil {
if ctx.Err() != nil || errors.Is(err, net.ErrClosed) {
return
}
log.Fatal("isolated accept failed")
}
select {
case slots <- struct{}{}:
default:
_ = conn.Close()
continue
}
workers.Add(1)
go func(conn net.Conn) {
defer workers.Done()
defer func() { <-slots }()
defer conn.Close()
dialer := net.Dialer{Timeout: 3 * time.Second}
remote, err := dialer.DialContext(ctx, "tcp4", *upstream)
if err != nil {
return
}
defer remote.Close()
a, b := conn.(*net.TCPConn), remote.(*net.TCPConn)
for _, c := range []*net.TCPConn{a, b} {
_ = c.SetReadBuffer(64 << 10)
_ = c.SetWriteBuffer(64 << 10)
}
localCtx, cancel := context.WithCancel(ctx)
defer cancel()
go func() { <-localCtx.Done(); _ = a.Close(); _ = b.Close() }()
done := make(chan struct{})
go func() { _, _ = io.Copy(boundedConn{b}, boundedConn{a}); _ = b.CloseWrite(); close(done) }()
_, _ = io.Copy(boundedConn{a}, boundedConn{b})
_ = a.CloseWrite()
<-done
}(conn)
}
}

View File

@ -0,0 +1,22 @@
package main
import "testing"
func TestIsolatedBridgeRejectsOtherNodesIdentitiesAndDestinations(t *testing.T) {
for _, tc := range []struct {
listen, upstream, node, uid, expected string
allowed bool
}{
{"127.0.0.1:27229", "10.189.16.46:27130", "10.189.16.4", "owned", "owned", true},
{"10.189.16.46:27130", "127.0.0.1:27129", "10.189.16.4", "owned", "owned", true},
{"0.0.0.0:27229", "10.189.16.46:27130", "10.189.16.4", "owned", "owned", false},
{"127.0.0.1:27229", "8.8.8.8:27130", "10.189.16.4", "owned", "owned", false},
{"127.0.0.1:27229", "10.189.16.46:27017", "10.189.16.4", "owned", "owned", false},
{"127.0.0.1:27229", "10.189.16.46:27130", "10.189.16.5", "owned", "owned", false},
{"127.0.0.1:27229", "10.189.16.46:27130", "10.189.16.4", "replaced", "owned", false},
} {
if got := allowedFlow(tc.listen, tc.upstream, tc.node, tc.uid, tc.expected); got != tc.allowed {
t.Fatalf("unexpected bridge admission: %+v", tc)
}
}
}

View File

@ -0,0 +1,97 @@
# 25Gi独立查询库:合成增量与HTTP对照
状态:隔离验证中,未准备生产切读。未修改生产Mongo、客户记录、业务部署、资源或带宽;没有购买磁盘。
## 本批范围
工作树 `mongo-cpu`,分支 `codex/mongo-cpu`,任务基线 `fbaa8546`。开始时合入了当时 `origin/dev` 的两项前端提交,保留现有后端修复;后续新鲜度检查确认包含远端主线及线上构建。
- 复用`.4`节点25Gi受限查询Mongo,接入持久标记投影器和查询库Change Streams聚合器。
- 使用真正的HTTP路由、会话Cookie与源库鉴权,对照任务计数、生产总览、个人看板和作业记录。验证程序不启动业务巡检/调度器。
- 新增 `QUERY_REPLICA_COUNTS=1`,配合报告模式,仅迁计数。`QUERY_REPLICA_POOLS`仍关闭,普通列表及实际作业留在权威库;旧池开关启用列表和计数的行为保持兼容。
- 夹具约2.6万任务、20项目、3供应商、500会话,包含大标注正文、切片版本、不可变提交及退回意见。规模对照固定数据和资源,前后都执行相同接口;计量包含新路径的增量轮询、确认与对账开销。
## 验证中修正的缺口
1. 人员快照缓存键缺少数据库/客户端身份,同进程对照可能串用源库与查询库的快照。已增加隔离维度;修复前后实际回归分别失败、通过。
2. 投影已提交新事件并重启恢复后,人员快照仍可能沿用45秒旧缓存。查询库缓存现以事务提交的事实代次失效;源库既有TTL不变。故障用例已先复现,再修复通过。
3. 空增量轮询对每个集合仍开启更新租约的事务。现无变化集合只读,实际变更和就绪发布仍验证租约/代次;旧投影器不能发布。真实Mongo回归确认空扫描不发目标写命令。
4. 计数与列表原先共用一个开关,无法分步上线。已拆出计数开关并验证旧开关兼容、查询服务不可用时普通列表不被计数模式阻断。
没有为减少读取而移除服务端权限、版本或领取约束,也没有让查询库回写业务正文。
## 环境与计量边界
| 组件 | 固定隔离预算 |
| --- | --- |
| 合成源Mongo、HTTP/投影/聚合/客户端 | 同一临时Pod,上限4CPU/4Gi,申请1CPU/2Gi;Mongo WT1Gi;Go最多2核、软内存限额1536MiB |
| 合成源数据 | 有界2Gi tmpfs;节点未启用交换盘;不是生产存储耐久性测试 |
| 查询Mongo | 原25Gi EXT4承载文件,1CPU/1Gi,WT0.25Gi,独立RS;数据盘继续`noexec` |
| 查询盘IO | loop和底层盘双层2MiB/s读、1MiB/s写,64/32 IOPS;配置保持不变 |
| 网络 | 无Service、无公网入口;默认拒绝,仅放行源试验Pod到查询Pod的27130端口 |
源Mongo CPU使用进程`utime+stime`,另记录整个试验Pod的CPU、内存及IO,不能把共享4核说成Mongo独占4核。记录源Mongo扫描文档/索引、逻辑及物理网络字节、全部源连接的命令及BSON响应字节、API分路径延迟/错误,以及查询库资源。BSON响应字节不是压缩后的公网流量。
投影器源/目标连接池各4,聚合器独立连接,API实际配置入口创建20连接的majority查询连接;核心鉴权及写入保持源库。共享Valkey和Typesense本轮未启用。
只在新建随机合成数据库内写入;结束清理各自测试数据库。对全部任务的标注、状态和版本计算前后摘要,防止把数据丢失误算为加速。目标原有`query25_synthetic`耐久标记只读核对。
监控现有API健康、生产Pod UID/重启数、节点压力/剩余空间、既有Pod独立IO PSI、物理盘等待及节点CPU。持续异常停止自有合成负载,绝不重启或改配生产Pod。短健康探针不等于客户完整作业验收。
## 为什么弃用管理隧道作为容量依据
早期本机API经`kubectl port-forward`查询`.4`库。81并发读同一合成文档,1KiB响应p95约100ms;约90KiB未压缩响应有63次15秒超时。相同大响应使用zstd后p95约85ms,无错误。Mongo本身多数命令执行很快,客户端出现连接/传输等待。
完整接口即使预热和压缩仍有秒级延迟,因此不把这些结果归咎于原Mongo或当作集群容量结论。全部失败证据保留,未扩大到生产。后续将源库与驱动放到同一隔离网段,管理链路只接收小型日志和结果。
首次桥接程序放在已禁止执行的数据盘上,启动被阻止。恢复仅将2.3MiB桥接程序放到有界`/tmp`内存卷,复用既有Pod和网络策略;没有取消数据盘`noexec`或改变IO限额。
## 已完成的小样本
同网段100任务、81个真实会话、两轮请求:四条接口业务JSON一致,无HTTP错误,任务标注/状态/版本摘要一致,确实命中后台生产报表。
| 指标 | 原路径 | 查询库路径 |
| --- | ---: | ---: |
| 整体p95 | 708.8ms | 637.8ms |
| 生产总览p95 | 3.0ms | 3.6ms |
| 人员看板p95 | 209.5ms | 381.3ms |
| 任务计数p95 | 724.6ms | 683.3ms |
| 作业记录p95 | 210.0ms | 384.9ms |
| 源Mongo CPU秒 | 2.91 | 2.42 |
| 源扫描文档 | 912,234 | 806,993 |
小样本不是性能达标结论;新路径的人员看板和作业记录反而较慢,须继续看代表性规模与后台成本,不能只展示整体p95下降。
## 大规模回填的首轮停止与修正
`cluster-scale-81-1`在回填约198.7秒处因租约过期停止,未进入HTTP对照。测试当时为了加快初始化采用1000条批次,和正式投影器默认200条不同。隔离库日志显示一次majority提交等待约41.5秒,超过30秒租约;此前也出现5~24秒写确认等待。IO采样触及32写IOPS及约1MiB/s预算,不能据此说需要给API加核。
没有延长租约、放宽IO限额或忽略错误。后续夹具改用正式默认200条;遇到租约错误时最多三次尝试重新取得自有/过期租约,沿用已提交检查点,另一活跃持有者仍会阻止接管。首次失败与新一轮证据分别保留,不清空源标注或把失败列为通过。
首轮监控39个样本,既有API健康最大约48ms、既有Pod独立IO PSI均为0、节点CPU最高12%、物理盘平均等待最高约2.43ms。只证明该窗口未观察到现有服务恶化,不代表完整作业或长期容量已验收。
第二轮`cluster-scale-81-2`采用200条批次,10次回填循环约26.4秒,随后保护监控因连续三次既有Pod IO等待超限而停止并按UID删除合成源库Pod。受影响的是`.4`节点一个生产API Pod,其IO `full avg10`依次约0.9、6.75、3.7;同期物理盘忙碌约2~3%、平均等待约0.4~0.6ms,API健康探针正常。不能把低磁盘忙碌当作隔离已达标,也不能据此确认共盘写回就是唯一原因。
停测后该API Pod IO等待回到0,且其`io.max`为空,未被试验意外设置限额。尚需区分共享文件系统写回/元数据依赖与同期业务自身IO。当前不再对该盘做大规模写入,不放宽保护阈值。这次对照未完成,不形成源Mongo大规模减负或API速度达标结论。
临时源库Pod已停止;桥接程序已按PID/可执行文件身份停止;两条新增网络策略按UID删除;试验配额CAS恢复原1Pod/1CPU/1Gi。原25Gi库、数据、标记、资源与IO限额保留,生产Pod未修改。部分合成查询库数据保留供诊断,不拿源库已消失的数据宣称可恢复生产。
## 证据与仍待完成
原始文件位于本工作树忽略目录 `local-data/diagnostics/mongo-cpu/query25-http/`:
- `smoke-*`:本机/管理隧道的各次失败及后续对照;不覆盖为通过。
- `transport-probe.jsonl`及对应诊断源码:大小响应与压缩的传输对照。
- `cluster-fixture/`:原计划、部分创建记录、恢复计划、精确Pod/策略身份和二进制SHA256。
- `cluster-smoke-1/`:同网段小样本的环境/源码摘要、资源采样、HTTP结果和退出状态。
- `cluster-scale-*`:代表性规模后续结果;按原始返回状态判断,不以Pod就绪宣称验收。
仍须完成26,218任务对照、81/200/500用户及长稳态/冷启动/缓存丢失/积压恢复;真实验收聚合包、复杂项目/权限分布的代表性也需单列确认。当前合成批次数20,不能代替真实批次数量的容量证明。查询/缓存优化不得靠减少返回任务或过滤授权记录达标。
下一步先只读核对保护触发窗口的API自身IO与挂载/文件系统关系,制定可证实不影响业务的写入隔离/低速回填方案,再恢复代表性规模对照;不以延长租约、增加API副本、删除安全判断或采购磁盘代替定位。
本批任务级检查中,浏览器看板、生产统计Mongo专项、查询服务三组件、节点保护和离线夹具检查通过;完整查询副本runner因本机只有源库、没有其要求的本地查询副本而未启动,保留失败记录,不另起默认8Gi容器挤占共享开发环境。新增count-only权限撤销/自审的真实Mongo用例尚待执行。测试目标连接白名单的单元检查及前端全量格式检查通过。任务检查不是全绿,不能作为发布证据。
停止保护同时加固:先落盘原始停止原因,再按UID清理自有Pod;即使远端已消失或删除失败也继续终止本机测试子进程,并独立记录清理错误。远程临时桥接的本机端口转发已关闭。
普通列表与搜索迁移、生产47个源索引/写者覆盖和历史回填、生产资源配置及发布均未执行。达标后才准备相应生产计划,不以本地或隔离结果宣称线上容量已验收。

View File

@ -27,7 +27,8 @@
| `QUERY_SOURCE_TRACKING=1` | API、切片处理器、校验器、Worker、协调器及写维护工具;启用同文档标记 |
| `QUERY_READ_MONGO_URI` / `QUERY_READ_MONGO_DB` | 独立查询库;不得与权威部署相同 |
| `QUERY_REPLICA_REPORTS=1` | API:计数/生产与人员看板/作业统计的目标读取;默认关闭 |
| `QUERY_REPLICA_POOLS=1` | API:任务池候选、计数及管理员标注池;报告模式可用后另启用 |
| `QUERY_REPLICA_COUNTS=1` | API:仅迁移带计数标记的任务池数量请求,普通列表和核心作业继续读权威库;需要报告模式 |
| `QUERY_REPLICA_POOLS=1` | API:任务池候选、计数及管理员标注池;保留原开关行为,计数验收后再单独启用 |
| `QUERY_SERVING_READS=1` / `QUERY_SERVING_NAMESPACE` | 查询库派生结果、不可变快照及缓存,namespace 必须隔离环境 |
| `QUERY_VALKEY_URL` | 可选共享缓存。失联只回到查询库 |
| `QUERY_TYPESENSE_URL` / `QUERY_TYPESENSE_API_KEY` | 可选搜索候选,后续独立验收;本批不要求启用 |

View File

@ -0,0 +1,704 @@
package api
import (
"context"
"crypto/sha256"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"net/http/httptest"
"os"
"os/exec"
"reflect"
"runtime"
"sort"
"strconv"
"strings"
"sync"
"testing"
"time"
"droboticflow/internal/domain"
"droboticflow/internal/productionstats"
"droboticflow/internal/store"
"droboticflow/internal/testsupport"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/event"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
"go.mongodb.org/mongo-driver/mongo/readconcern"
"go.mongodb.org/mongo-driver/mongo/readpref"
)
// This harness has no production URI input. Both DBs are newly named synthetic
// fixtures, and the target must carry the independently verified node4 marker.
// HTTP uses the actual session, ACL and count/report handlers. Business recovery
// schedulers are deliberately not duplicated in the two benchmark handlers.
func TestQueryReplicaIsolatedHTTPComparisonLocal(t *testing.T) {
if os.Getenv("QUERY_REPLICA_ISOLATED_E2E") != "1" {
t.Skip("explicit node4 synthetic comparison only")
}
if os.Getenv("QUERY_REPLICA_CLUSTER_TEST") != "1" {
t.Fatal("cluster synthetic target must be explicitly selected")
}
t.Setenv("ANNOTATION_SOURCE_VALIDATION_MODE", "advisory")
t.Setenv("QUERY_SERVING_NAMESPACE", "isolated-http-comparison")
ctx := t.Context()
n := isolatedComparisonInt(t, "QUERY_COMPARE_TASKS", 26218, 20, 30000)
waves := isolatedComparisonInt(t, "QUERY_COMPARE_WAVES", 2, 1, 100)
maxActors := isolatedComparisonInt(t, "QUERY_COMPARE_MAX_ACTORS", 81, 81, 500)
minPool := isolatedComparisonInt(t, "QUERY_COMPARE_MIN_POOL", 0, 0, 100)
compression := os.Getenv("QUERY_COMPARE_WIRE_COMPRESSION")
if compression != "" && compression != "zstd" {
t.Fatal("unsupported comparison compression")
}
var mu sync.Mutex
var phase string
inflight := map[int64]string{}
metrics := map[string]*isolatedCommands{}
var targetStatsMu sync.Mutex
targetCommands := map[string]int64{}
targetMaxMillis := map[string]float64{}
targetMonitor := &event.CommandMonitor{Succeeded: func(_ context.Context, e *event.CommandSucceededEvent) {
targetStatsMu.Lock()
defer targetStatsMu.Unlock()
targetCommands[e.CommandName]++
ms := float64(e.DurationNanos) / 1e6
targetMaxMillis[e.CommandName] = max(targetMaxMillis[e.CommandName], ms)
}}
monitor := &event.CommandMonitor{
Started: func(_ context.Context, e *event.CommandStartedEvent) {
mu.Lock()
defer mu.Unlock()
if phase != "" {
m := metrics[phase]
m.Commands[e.CommandName]++
m.RequestBytes += int64(len(e.Command))
inflight[e.RequestID] = phase
}
},
Succeeded: func(_ context.Context, e *event.CommandSucceededEvent) {
mu.Lock()
defer mu.Unlock()
if name := inflight[e.RequestID]; name != "" {
metrics[name].ReplyBytes += int64(len(e.Reply))
delete(inflight, e.RequestID)
}
},
Failed: func(_ context.Context, e *event.CommandFailedEvent) {
mu.Lock()
delete(inflight, e.RequestID)
mu.Unlock()
},
}
connect := func(uri string, monitor *event.CommandMonitor) (*mongo.Database, *store.MongoStore) {
opts := options.Client().ApplyURI(uri).SetMonitor(monitor).SetServerSelectionTimeout(5 * time.Second).SetMinPoolSize(uint64(minPool))
if compression != "" {
opts.SetCompressors([]string{compression})
}
c, err := mongo.Connect(ctx, opts)
if err != nil {
t.Fatal(err)
}
db := c.Database(domain.NewID("query_http_synthetic"))
t.Cleanup(func() {
clean, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
if err := db.Drop(clean); err != nil {
t.Errorf("synthetic cleanup %s: %v", db.Name(), err)
}
_ = c.Disconnect(clean)
})
return db, store.NewMongoStore(db)
}
sdb, old := connect("mongodb://127.0.0.1:27119/?directConnection=true", monitor)
db, _ := connect(testsupport.QueryReplicaTargetURI(t), targetMonitor)
var marker bson.M
if err := db.Client().Database("query25_synthetic").Collection("marker").FindOne(ctx, bson.M{"_id": "query25-confirmed"}).Decode(&marker); err != nil || marker["value"] != "preserve-query25-v1" {
t.Fatalf("owned 25Gi target identity not confirmed: %v", err)
}
newReader := store.NewMongoStore(sdb)
newReader.EnableQuerySourceTracking(true)
if err := old.EnsureIndexes(ctx); err != nil {
t.Fatal(err)
}
fixture := seedIsolatedHTTPComparison(t, sdb, old, n)
annotationDigest := isolatedAnnotationDigest(t, sdb)
peer := func(uri, name string, size uint64, monitor *event.CommandMonitor, majority bool) *store.MongoStore {
opts := options.Client().ApplyURI(uri).SetMaxPoolSize(size).SetMonitor(monitor).SetServerSelectionTimeout(5 * time.Second)
if compression != "" {
opts.SetCompressors([]string{compression})
}
if majority {
opts.SetReadPreference(readpref.Primary()).SetReadConcern(readconcern.Majority())
}
client, err := mongo.Connect(ctx, opts)
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() {
stop, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
_ = client.Disconnect(stop)
})
return store.NewMongoStore(client.Database(name))
}
workerSource := peer("mongodb://127.0.0.1:27119/?directConnection=true", sdb.Name(), 4, monitor, false)
workerTarget := peer(testsupport.QueryReplicaTargetURI(t), db.Name(), 4, targetMonitor, true)
q, err := store.NewQueryReplica(workerSource, workerTarget)
if err != nil {
t.Fatal(err)
}
for _, fn := range []func(context.Context) error{q.VerifyTopology, q.EnsureSourceIndexes, q.EnsureTargetIndexes, q.Acquire} {
if err := fn(ctx); err != nil {
t.Fatal(err)
}
}
start := time.Now()
q.BatchSize = isolatedComparisonInt(t, "QUERY_COMPARE_BOOTSTRAP_BATCH", 200, 20, 200)
bootstrapRestarts := 0
t.Logf("bootstrap starting source=%s target=%s batch=%d", sdb.Name(), db.Name(), q.BatchSize)
for i := 0; ; i++ {
ready, err := q.RunOnce(ctx, true)
if err != nil {
if errors.Is(err, store.ErrQueryReplicaLease) && bootstrapRestarts < 3 {
// Reacquire only our own/expired lease, never another live owner.
// Durable checkpoints, not a fresh database, resume slow commits.
if err := q.Acquire(ctx); err != nil {
t.Fatal("bootstrap lease recovery", err)
}
bootstrapRestarts++
t.Logf("bootstrap lease recovered count=%d", bootstrapRestarts)
continue
}
t.Fatal("bootstrap", err)
}
if ready {
break
}
if i > n/q.BatchSize+100 {
t.Fatal("bootstrap did not converge")
}
if i%10 == 0 {
t.Logf("bootstrap pass=%d seconds=%.1f", i, time.Since(start).Seconds())
}
}
t.Logf("bootstrap tasks=%d seconds=%.3f source=%s target=%s", n, time.Since(start).Seconds(), sdb.Name(), db.Name())
q.BatchSize = 200
builderTarget := peer(testsupport.QueryReplicaTargetURI(t), db.Name(), 100, targetMonitor, true)
if err := builderTarget.VerifyQueryReplicaTarget(ctx); err != nil {
t.Fatal(err)
}
builderTarget.ConfigureQueryServing(nil, nil)
if err := builderTarget.EnsureQueryReadModelIndexes(ctx); err != nil {
t.Fatal(err)
}
workerCtx, stopWorkers := context.WithCancel(ctx)
defer stopWorkers()
builder := store.NewQueryReadBuilder(builderTarget)
workerErrors := make(chan error, 2)
var workers sync.WaitGroup
workers.Add(2)
go func() {
defer workers.Done()
if err := builder.Run(workerCtx); err != nil && workerCtx.Err() == nil {
workerErrors <- err
}
}()
// The measured new phase includes every poll, reconcile and acknowledgement.
projectorEnabled := make(chan struct{})
go func() {
defer workers.Done()
select {
case <-projectorEnabled:
case <-workerCtx.Done():
return
}
tick := time.NewTicker(time.Second)
defer tick.Stop()
for workerCtx.Err() == nil {
if _, err := q.RunOnce(workerCtx, false); err != nil && workerCtx.Err() == nil {
workerErrors <- err
return
}
select {
case <-workerCtx.Done():
case <-tick.C:
}
}
}()
defer func() { stopWorkers(); workers.Wait(); builder.Close() }()
readyCtx, cancel := context.WithTimeout(ctx, 4*time.Minute)
err = builder.WaitReady(readyCtx, fixture.org, fixture.projects)
cancel()
if err != nil {
t.Fatal("query materialization", err)
}
// The projector is intentionally paused for the authority measurement. A
// long initial build must not leave that fixture's watermark expired before
// the first paired HTTP read, even though its source has had no changes.
if err := q.Acquire(ctx); err != nil {
t.Fatal(err)
}
if err := q.PublishReady(ctx); err != nil {
t.Fatal(err)
}
// Use the actual API configuration entry point (20-connection, majority
// target client), separate from projector and builder connection pools.
t.Setenv("QUERY_REPLICA_REPORTS", "1")
t.Setenv("QUERY_REPLICA_POOLS", "0")
t.Setenv("QUERY_REPLICA_COUNTS", "1")
t.Setenv("QUERY_SOURCE_TRACKING", "1")
uri := testsupport.QueryReplicaTargetURI(t)
if compression != "" {
uri += "&compressors=" + compression
}
t.Setenv("QUERY_READ_MONGO_URI", uri)
t.Setenv("QUERY_READ_MONGO_DB", db.Name())
t.Setenv("QUERY_SERVING_READS", "1")
t.Setenv("QUERY_VALKEY_URL", "")
t.Setenv("QUERY_TYPESENSE_URL", "")
closeReplica, err := newReader.ConfigureQueryReplicaFromEnvironment(ctx)
if err != nil {
t.Fatal(err)
}
t.Cleanup(closeReplica)
serve := func(s store.Store) *httptest.Server {
h := &server{store: s, requireSessionAuth: true, authLimiter: newRateLimiter(20, time.Minute), externalWriteLimiter: newRateLimiter(120, time.Minute), dashboardResponses: dashboardResponseCache{now: time.Now}}
ts := httptest.NewServer(h.routes())
t.Cleanup(ts.Close)
return ts
}
before, after := serve(old), serve(newReader)
client := &http.Client{Timeout: 25 * time.Second}
request := func(url, path string, actor int) (int, []byte, error) {
r, err := http.NewRequestWithContext(ctx, http.MethodGet, url+path, nil)
if err != nil {
return 0, nil, err
}
r.AddCookie(&http.Cookie{Name: sessionCookieName, Value: fixture.tokens[actor]})
resp, err := client.Do(r)
if err != nil {
return 0, nil, err
}
defer resp.Body.Close()
body, err := io.ReadAll(resp.Body)
return resp.StatusCode, body, err
}
paths := []string{
"/api/human-item-counts?pluginId=annotation-qa&stageIds=semantic_review",
"/api/dashboard/production?from=2026-11-01&to=2026-11-12",
"/api/dashboard/workforce?from=2026-11-01T00:00:00Z&to=2026-11-13T00:00:00Z&groupBy=member",
"/api/workforce-returns?from=2026-11-01T00:00:00Z&to=2026-11-13T00:00:00Z&limit=20&offset=0",
}
// Exact business JSON equality; only generated report timestamps are volatile.
for i, path := range paths {
a, ab, ae := request(before.URL, path, 0)
b, bb, be := request(after.URL, path, 0)
if ae != nil || be != nil || a != 200 || b != 200 {
t.Fatalf("route %d: old=%d/%v %.300s new=%d/%v %.300s", i, a, ae, ab, b, be, bb)
}
if !reflect.DeepEqual(isolatedBusinessJSON(t, ab), isolatedBusinessJSON(t, bb)) {
t.Fatalf("route %d business results differ: old %.500s new %.500s", i, ab, bb)
}
if i == 0 {
t.Logf("counter parity %s", ab)
}
}
// Keep output evidence independently identifiable rather than parsing t.Log.
report := map[string]any{"tasks": n, "projects": len(fixture.projects), "sourceDatabase": sdb.Name(), "targetDatabase": db.Name(), "cases": []any{}, "parity": true, "httpSessionAuth": true, "sourceMinimumPoolBothVariants": minPool, "apiQueryMaximumPool": 20, "projectorMaximumPool": 4, "builderMaximumPool": 100, "wireCompressionBothVariants": compression, "sourceReplyBytesKind": "logical BSON; not compressed wire bytes", "cache": "none; existing API snapshot/response caches unchanged"}
cases := []any{}
accepted := true
for _, mode := range []string{"old", "query"} {
url := before.URL
if mode == "query" {
url = after.URL
// Reacquire after the baseline measurement's idle period.
if err := q.Acquire(ctx); err != nil {
t.Fatal(err)
}
if _, err := q.RunOnce(ctx, false); err != nil {
t.Fatal(err)
}
close(projectorEnabled)
}
for _, concurrency := range []int{81, 200, 500} {
if concurrency > maxActors {
continue
}
name := fmt.Sprintf("%s-%d", mode, concurrency)
// Each variant pays the same session lookup cost; do not let the old
// phase warm a shared authentication cache for the query phase.
principalCacheMu.Lock()
principalCache = map[string]principalCacheEntry{}
principalCacheMu.Unlock()
scansBefore := isolatedSourceScans(t, sdb)
cpuBefore := isolatedSourceCPU(t)
mu.Lock()
phase = name
metrics[name] = &isolatedCommands{Commands: map[string]int64{}}
mu.Unlock()
t.Log("phase", name, time.Now().UTC().Format(time.RFC3339Nano))
started := time.Now()
latency := make([]float64, 0, concurrency*waves)
byRoute := map[string][]float64{}
errors := map[string]int{}
var firstError string
var responses int64
for wave := 0; wave < waves; wave++ {
var wg sync.WaitGroup
var results sync.Mutex
barrier := make(chan struct{})
for actor := 0; actor < concurrency; actor++ {
wg.Add(1)
go func(actor int) {
defer wg.Done()
<-barrier
path := paths[2+(actor+wave)%2]
if actor%81 == 0 {
path = paths[1]
} else if actor%81 >= 61 {
path = paths[0]
}
s := time.Now()
status, body, err := request(url, path, actor)
results.Lock()
defer results.Unlock()
ms := float64(time.Since(s).Microseconds()) / 1000
latency = append(latency, ms)
key := strings.Split(path, "?")[0]
byRoute[key] = append(byRoute[key], ms)
responses += int64(len(body))
if status != 200 || err != nil {
errors[fmt.Sprintf("%d/%v", status, err)]++
if firstError == "" {
firstError = fmt.Sprintf("%s: %d/%v %.300s", key, status, err, body)
}
}
}(actor)
}
close(barrier)
wg.Wait()
}
mu.Lock()
phase = ""
original := metrics[name]
m := &isolatedCommands{Commands: map[string]int64{}, RequestBytes: original.RequestBytes, ReplyBytes: original.ReplyBytes}
for cmd, count := range original.Commands {
m.Commands[cmd] = count
}
mu.Unlock()
cpu := isolatedSourceCPU(t) - cpuBefore
scans := isolatedSourceScans(t, sdb)
for key, before := range scansBefore {
scans[key] -= before
}
sort.Float64s(latency)
routes := map[string]any{}
for path, samples := range byRoute {
sort.Float64s(samples)
p95 := samples[(len(samples)*95+99)/100-1]
routes[path] = map[string]any{"requests": len(samples), "p95Ms": p95, "maxMs": samples[len(samples)-1]}
if mode == "query" && p95 > 2000 {
t.Errorf("%s %s p95 exceeds preliminary 2s bound", name, path)
}
}
result := map[string]any{"name": name, "seconds": time.Since(started).Seconds(), "requests": len(latency), "p50Ms": latency[(len(latency)-1)/2], "p95Ms": latency[(len(latency)-1)*95/100], "errors": errors, "firstError": firstError, "routes": routes, "responseBytes": responses, "sourceCPUSeconds": cpu, "sourceScans": scans, "source": m}
cases = append(cases, result)
encoded, _ := json.Marshal(result)
t.Log(string(encoded))
if len(errors) != 0 {
accepted = false
t.Errorf("%s has HTTP failures: %s", name, firstError)
break
}
if mode == "query" && latency[(len(latency)-1)*95/100] > 2000 {
t.Errorf("%s p95 exceeds preliminary 2s bound", name)
}
}
}
report["cases"] = cases
report["errorFree"] = accepted
report["capacityAccepted"] = false
report["bootstrapLeaseRecoveries"] = bootstrapRestarts
report["queryServing"] = newReader.QueryServingStatistics()
if newReader.QueryServingStatistics()["report_hits"] == 0 {
t.Error("production read did not use its background materialized report")
}
if actual := isolatedAnnotationDigest(t, sdb); actual != annotationDigest {
t.Fatal("read projection changed task status, version or annotation")
}
report["annotationsAndVersionsPreserved"] = true
targetStatsMu.Lock()
report["targetWorkerCommands"] = targetCommands
report["targetWorkerMaxRoundTripMs"] = targetMaxMillis
targetStatsMu.Unlock()
select {
case err := <-workerErrors:
t.Fatal("background worker failed", err)
default:
}
if path := os.Getenv("QUERY_COMPARE_REPORT"); path != "" {
targetStatsMu.Lock()
data, err := json.MarshalIndent(report, "", " ")
targetStatsMu.Unlock()
if err != nil {
t.Fatal(err)
}
if err := os.WriteFile(path, data, 0600); err != nil {
t.Fatal(err)
}
}
}
func isolatedAnnotationDigest(t *testing.T, db *mongo.Database) string {
t.Helper()
cur, err := db.Collection("humanItems").Find(t.Context(), bson.M{}, options.Find().SetProjection(bson.M{"_id": 0, "id": 1, "version": 1, "status": 1, "assignedUserId": 1, "subject.payload.annotationDraft": 1}).SetSort(bson.D{{Key: "id", Value: 1}}))
if err != nil {
t.Fatal(err)
}
defer cur.Close(t.Context())
hash := sha256.New()
for cur.Next(t.Context()) {
var row bson.M
if err := cur.Decode(&row); err != nil {
t.Fatal(err)
}
data, err := json.Marshal(row)
if err != nil {
t.Fatal(err)
}
_, _ = hash.Write(data)
}
if err := cur.Err(); err != nil {
t.Fatal(err)
}
return fmt.Sprintf("%x", hash.Sum(nil))
}
func isolatedSourceScans(t *testing.T, db *mongo.Database) map[string]int64 {
t.Helper()
var stats struct {
Network struct {
BytesIn int64 `bson:"bytesIn"`
BytesOut int64 `bson:"bytesOut"`
PhysicalBytesIn int64 `bson:"physicalBytesIn"`
PhysicalBytesOut int64 `bson:"physicalBytesOut"`
} `bson:"network"`
Metrics struct {
QueryExecutor struct {
Scanned int64 `bson:"scanned"`
ScannedObjects int64 `bson:"scannedObjects"`
} `bson:"queryExecutor"`
} `bson:"metrics"`
}
if err := db.Client().Database("admin").RunCommand(t.Context(), bson.M{"serverStatus": 1}).Decode(&stats); err != nil {
t.Fatal("synthetic source scan counters", err)
}
return map[string]int64{"indexEntries": stats.Metrics.QueryExecutor.Scanned, "documents": stats.Metrics.QueryExecutor.ScannedObjects, "networkLogicalBytesIn": stats.Network.BytesIn, "networkLogicalBytesOut": stats.Network.BytesOut, "networkPhysicalBytesIn": stats.Network.PhysicalBytesIn, "networkPhysicalBytesOut": stats.Network.PhysicalBytesOut}
}
func isolatedSourceCPU(t *testing.T) float64 {
t.Helper()
if os.Getenv("QUERY_COMPARE_IN_CLUSTER") == "1" {
if runtime.GOOS != "linux" || os.Getenv("NODE_NAME") != "10.189.16.4" || os.Getenv("POD_NAME") != "node4-query-fixture" {
t.Fatal("isolated source process identity rejected")
}
raw, err := os.ReadFile("/synthetic/mongod.pid")
if err != nil {
t.Fatal(err)
}
pid := strings.TrimSpace(string(raw))
if _, err := strconv.ParseUint(pid, 10, 32); err != nil {
t.Fatal("invalid synthetic source pid")
}
command, err := os.ReadFile("/proc/" + pid + "/cmdline")
if err != nil || !strings.Contains(string(command), "--dbpath\x00/synthetic/source\x00") || !strings.Contains(string(command), "--port\x0027119\x00") {
t.Fatal("source PID does not own the synthetic database")
}
stat, err := os.ReadFile("/proc/" + pid + "/stat")
if err != nil {
t.Fatal(err)
}
fields := strings.Fields(string(stat))
if len(fields) < 15 || fields[1] != "(mongod)" {
t.Fatal("unexpected synthetic process counters")
}
clock, err := exec.Command("getconf", "CLK_TCK").Output()
if err != nil {
t.Fatal(err)
}
hz, err := strconv.ParseFloat(strings.TrimSpace(string(clock)), 64)
if err != nil || hz <= 0 {
t.Fatal("invalid process clock")
}
user, e1 := strconv.ParseFloat(fields[13], 64)
system, e2 := strconv.ParseFloat(fields[14], 64)
if e1 != nil || e2 != nil {
t.Fatal("invalid process CPU counters")
}
return (user + system) / hz
}
raw, err := exec.Command("docker", "exec", "fanyu-query-replica-local-source-1", "cat", "/sys/fs/cgroup/cpu.stat").Output()
if err != nil {
t.Fatal("isolated source cgroup unavailable", err)
}
for _, line := range strings.Split(string(raw), "\n") {
fields := strings.Fields(line)
if len(fields) == 2 && fields[0] == "usage_usec" {
value, err := strconv.ParseFloat(fields[1], 64)
if err != nil {
t.Fatal(err)
}
return value / 1e6
}
}
t.Fatal("isolated source CPU counter missing")
return 0
}
type isolatedCommands struct {
Commands map[string]int64 `json:"commands"`
RequestBytes int64 `json:"requestBytes"`
ReplyBytes int64 `json:"replyBytes"`
}
type isolatedHTTPFixture struct {
org string
projects []string
tokens []string
}
func isolatedComparisonInt(t *testing.T, key string, fallback, min, max int) int {
t.Helper()
if os.Getenv(key) == "" {
return fallback
}
n, err := strconv.Atoi(os.Getenv(key))
if err != nil || n < min || n > max {
t.Fatalf("invalid %s", key)
}
return n
}
func isolatedBusinessJSON(t *testing.T, data []byte) any {
t.Helper()
var result any
if err := json.Unmarshal(data, &result); err != nil {
t.Fatal(err)
}
// Remove only named generation timestamps, never business dates or versions.
var walk func(any)
walk = func(v any) {
switch x := v.(type) {
case map[string]any:
delete(x, "asOf")
delete(x, "generatedAt")
for _, child := range x {
walk(child)
}
case []any:
for _, child := range x {
walk(child)
}
}
}
walk(result)
return result
}
func seedIsolatedHTTPComparison(t *testing.T, db *mongo.Database, s *store.MongoStore, n int) isolatedHTTPFixture {
t.Helper()
ctx := t.Context()
f := isolatedHTTPFixture{org: "query-synthetic-org"}
must := func(err error) {
t.Helper()
if err != nil {
t.Fatal(err)
}
}
must(s.UpsertOrganization(ctx, domain.OrganizationRecord{ID: f.org, Name: "Synthetic only", Slug: "query-synthetic", Status: domain.OrganizationStatusActive}))
for i := 0; i < 3; i++ {
must(s.UpsertSupplier(ctx, domain.SupplierRecord{ID: fmt.Sprintf("v%d", i), OrganizationID: f.org, Status: "active", Name: fmt.Sprintf("Synthetic supplier %d", i)}))
}
for i := 0; i < 20; i++ {
id := fmt.Sprintf("p%02d", i)
f.projects = append(f.projects, id)
must(s.UpsertProject(ctx, domain.ProjectRecord{ID: id, OrganizationID: f.org, Name: "Synthetic " + id, Status: domain.ProjectStatusActive}))
must(s.UpsertHumanBatch(ctx, domain.HumanBatchRecord{ID: "b" + id, RunID: "batch-run-" + id, GateNodeID: "gate", OrganizationID: f.org, ProjectID: id, PluginID: "annotation-qa", Status: domain.HumanBatchActive, Stages: domain.DefaultFanyuV5HumanStages()}))
}
_, err := s.SaveProductionStatsScope(ctx, f.org, productionstats.Scope{ProjectIDs: f.projects, Revision: 1}, 0)
must(err)
rows := map[string][]any{}
push := func(c string, d any) {
if c == "annotationTasks" {
raw, err := bson.Marshal(d)
must(err)
var doc bson.M
must(bson.Unmarshal(raw, &doc))
doc["sourceKey"] = doc["id"]
d = doc
}
rows[c] = append(rows[c], d)
}
flush := func() {
for c, docs := range rows {
if len(docs) > 0 {
_, err := db.Collection(c).InsertMany(ctx, docs)
must(err)
}
}
rows = map[string][]any{}
}
for i := 0; i < 500; i++ {
role := domain.UserRoleAnnotator
if i%81 == 0 {
role = domain.UserRoleAdmin
} else if i%81 >= 73 {
role = domain.UserRoleAcceptanceReviewer
} else if i%81 >= 61 {
role = domain.UserRoleQualityReviewer
}
uid, vendor := fmt.Sprintf("u%03d", i), fmt.Sprintf("v%d", i%3)
push("users", domain.UserRecord{ID: uid, OrganizationID: f.org, Role: role, Name: uid, Phone: fmt.Sprintf("138%08d", i), Status: domain.UserStatusActive, SupplierID: vendor})
token, err := randomSessionToken()
must(err)
f.tokens = append(f.tokens, token)
push("sessions", domain.SessionRecord{ID: "session-" + uid, UserID: uid, OrganizationID: f.org, TokenHash: sessionTokenHash(token), ExpiresAt: time.Now().Add(3 * time.Hour).UTC().Format(time.RFC3339), ExpiresAtTime: time.Now().Add(3 * time.Hour)})
for _, p := range f.projects {
push("projectMembers", domain.ProjectMemberRecord{ID: p + uid, OrganizationID: f.org, ProjectID: p, UserID: uid, SupplierID: vendor})
ar := domain.AnnotationRoleAnnotator
if role == domain.UserRoleQualityReviewer {
ar = domain.AnnotationRoleQualityReviewer
}
push("annotationMemberships", domain.ProjectAnnotationMembershipRecord{ID: p + uid, OrganizationID: f.org, ProjectID: p, UserID: uid, WorkgroupID: vendor, VendorID: vendor, AssignmentRole: ar})
}
}
flush()
for i := 0; i < n; i++ {
id := fmt.Sprintf("t%06d", i)
p, vendor, uid := f.projects[i%len(f.projects)], fmt.Sprintf("v%d", (i%60+1)%3), fmt.Sprintf("u%03d", i%60+1)
at := fmt.Sprintf("2026-11-%02dT10:00:00Z", i%10+1)
run := domain.RunRecord{ID: "r" + id, OrganizationID: f.org, ProjectID: p, TaskID: "c" + id, Status: domain.RunStatusWaitingForHuman, CaptureDirectories: []string{"SYNTHETIC/" + id}, CreatedAt: at}
task := domain.AnnotationTaskRecord{ID: id, OrganizationID: f.org, ProjectID: p, RunID: run.ID, ArtifactID: "SYNTHETIC/" + id + "/part0001", SourceVideo: "bos://synthetic.invalid/video.mp4", ClipEnd: 60, CurrentRevision: 1, Status: domain.AnnotationTaskStatusPendingQuality, SupplierID: vendor, WorkgroupID: vendor, VendorID: vendor, TemplateVersion: 5, CreatedAt: at, UpdatedAt: at}
item := domain.HumanItemRecord{ID: "i" + id, OrganizationID: f.org, ProjectID: p, BatchID: "b" + p, RunID: run.ID, LegacyTaskID: id, PluginID: "annotation-qa", CurrentStageID: domain.FanyuV5StageSemanticReview, Status: domain.HumanItemReady, SupplierID: vendor, WorkgroupID: vendor, VendorID: vendor, LatestAuthorUserID: uid, CurrentRevision: 1, Version: 1, CreatedAt: at, UpdatedAt: at, Subject: domain.HumanSubjectRef{ItemKey: task.ArtifactID, Payload: map[string]any{"durationSeconds": 60, "annotationDraft": strings.Repeat("synthetic protected annotation;", 128)}}}
push("runs", run)
push("captureRecords", domain.CaptureRecord{ID: run.TaskID, OrganizationID: f.org, ProjectID: p, Prefix: "SYNTHETIC/" + id, CaptureID: "SYNTHETIC/" + id, RunID: run.ID})
push("annotationTasks", task)
push("humanItems", item)
push("annotationRevisions", domain.AnnotationRevisionRecord{ID: "rev" + id, OrganizationID: f.org, TaskID: id, Revision: 1, AuthorUserID: uid, CountsAsOutput: true})
push("sliceRevisions", domain.SliceRevisionRecord{ID: "slice" + id, OrganizationID: f.org, TaskID: id, Revision: 1, Segments: []domain.SliceSegment{{ID: "s", Start: 0, End: 60}}})
push("annotationWorklogEvents", domain.AnnotationWorklogEventRecord{ID: "event" + id, OrganizationID: f.org, ProjectID: p, TaskID: id, UserID: uid, SupplierID: vendor, EventType: "combined_submitted", Revision: 1, OutputDelta: 1, CreatedAt: at})
if i%6 == 0 {
push("reviewOperations", domain.ReviewOperation{ID: "op" + id, OrganizationID: f.org, ProjectID: p, ItemID: item.ID, TaskID: id, StageID: "quality_review", Status: "completed", Decision: "rejected", CreatedAt: at, CompletedAt: at, ReviewRound: 1, Draft: domain.ReviewDraft{Problems: []domain.ReviewProblem{{ID: "problem", Scope: "slice", SliceID: "s", Start: 0, End: 10, TimeBasis: "capture-seconds", Text: "Synthetic return detail must survive"}}}, Effects: []domain.ReviewDecisionEffect{{Review: &domain.AnnotationReviewRecord{ID: "review" + id, TaskID: id, Revision: 1}}, {BeforeTask: &task}, {BeforeItem: &item, Item: &item}}})
}
if (i+1)%500 == 0 {
flush()
}
}
flush()
return f
}

View File

@ -12,11 +12,20 @@ import (
"droboticflow/internal/domain"
"droboticflow/internal/store"
"droboticflow/internal/testsupport"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
)
func TestQueryReplicaPoolPermissionRevocationAndSelfReviewLocal(t *testing.T) {
testQueryReplicaPermissionRevocationAndSelfReview(t, false)
}
func TestQueryReplicaCountOnlyPermissionRevocationAndSelfReviewLocal(t *testing.T) {
testQueryReplicaPermissionRevocationAndSelfReview(t, true)
}
func testQueryReplicaPermissionRevocationAndSelfReview(t *testing.T, countsOnly bool) {
if os.Getenv("QUERY_REPLICA_LOCAL_TEST") != "1" {
t.Skip("isolated local source and query replica required")
}
@ -24,7 +33,7 @@ func TestQueryReplicaPoolPermissionRevocationAndSelfReviewLocal(t *testing.T) {
ctx := t.Context()
clients := []*mongo.Client{}
stores := []*store.MongoStore{}
for _, uri := range []string{"mongodb://127.0.0.1:27119/?directConnection=true", "mongodb://127.0.0.1:27129/?replicaSet=query-read-rs"} {
for _, uri := range []string{"mongodb://127.0.0.1:27119/?directConnection=true", testsupport.QueryReplicaTargetURI(t)} {
c, e := mongo.Connect(ctx, options.Client().ApplyURI(uri))
if e != nil {
t.Fatal(e)
@ -108,7 +117,8 @@ func TestQueryReplicaPoolPermissionRevocationAndSelfReviewLocal(t *testing.T) {
if e = source.ConfigureQueryReplicaReports(target, time.Minute); e != nil {
t.Fatal(e)
}
source.EnableQueryReplicaPoolReads(true)
source.EnableQueryReplicaPoolReads(!countsOnly)
source.EnableQueryReplicaCountReads(countsOnly)
handler := NewHandler(source, nil)
request := func(path string) *httptest.ResponseRecorder {
r := httptest.NewRequest(http.MethodGet, path, nil)

View File

@ -36,6 +36,9 @@ type queryServingCounters struct{ reportHits, responseHits, searchHits, typesens
// QueryServingStatistics exposes counters only, never names or query contents.
func (s *MongoStore) QueryServingStatistics() map[string]uint64 {
if s.queryReplicaReads != nil && s.queryReplicaReads.target != nil {
return s.queryReplicaReads.target.QueryServingStatistics()
}
if s.queryServing == nil {
return map[string]uint64{}
}

View File

@ -325,6 +325,12 @@ func (q *QueryReplica) DrainStep(ctx context.Context, n string) (int, error) {
if e != nil {
return 0, e
}
// An empty poll has no target effect or source acknowledgement to fence.
// Keep lease fencing on actual changes and PublishReady, rather than writing
// the lease once per empty collection on every incremental pass.
if len(rows) == 0 {
return 0, nil
}
e = q.transaction(ctx, func(sc mongo.SessionContext) error { return q.apply(sc, n, rows) })
if e != nil {
return 0, e

View File

@ -19,7 +19,7 @@ func (s *MongoStore) ConfigureQueryReplicaFromEnvironment(ctx context.Context) (
if os.Getenv("QUERY_REPLICA_REPORTS") != "1" {
return func() {}, nil
}
s.queryReplicaReads = &queryReplicaReads{pool: os.Getenv("QUERY_REPLICA_POOLS") == "1"}
s.queryReplicaReads = &queryReplicaReads{pool: os.Getenv("QUERY_REPLICA_POOLS") == "1", counts: os.Getenv("QUERY_REPLICA_COUNTS") == "1"}
if os.Getenv("QUERY_SOURCE_TRACKING") != "1" {
return func() {}, errors.New("query reads require source writer tracking")
}
@ -50,5 +50,6 @@ func (s *MongoStore) ConfigureQueryReplicaFromEnvironment(ctx context.Context) (
}
s.queryReplicaReads = r
s.EnableQueryReplicaPoolReads(os.Getenv("QUERY_REPLICA_POOLS") == "1")
s.EnableQueryReplicaCountReads(os.Getenv("QUERY_REPLICA_COUNTS") == "1")
return func() { closeCache(); cleanup() }, nil
}

View File

@ -1,11 +1,37 @@
package store
import (
"context"
"droboticflow/internal/domain"
"errors"
"testing"
)
func TestQueryReplicaCountOnlyLeavesListsAndCoreReadsOnAuthority(t *testing.T) {
s := &MongoStore{queryReplicaReads: &queryReplicaReads{counts: true}}
plain := context.Background()
list := WithQueryReplicaPoolRead(plain)
count := WithQueryReplicaCountRead(list)
if s.UsesQueryReplicaPool(plain) || s.UsesQueryReplicaPool(list) || !s.UsesQueryReplicaPool(count) {
t.Fatal("count-only routing escaped its marked count request")
}
if target, err := s.poolReplica(list); target != nil || err != nil {
t.Fatal("unavailable count infrastructure blocked legacy lists", err)
}
if _, err := s.poolReplica(count); !errors.Is(err, ErrQueryReplicaNotReady) {
t.Fatal("explicit count mode silently fell back to authority", err)
}
s.EnableQueryReplicaPoolReads(true)
if !s.UsesQueryReplicaPool(list) || !s.UsesQueryReplicaPool(count) {
t.Fatal("legacy list+count switch lost compatibility")
}
s.EnableQueryReplicaPoolReads(false)
s.EnableQueryReplicaCountReads(false)
if s.UsesQueryReplicaPool(list) || s.UsesQueryReplicaPool(count) {
t.Fatal("disabled modes still routed")
}
}
func TestQueryReplicaCandidateCachePrivateMetadataAndIsolation(t *testing.T) {
original := domain.HumanItemRecord{ID: "i", LegacyAssignment: map[string]string{"role": "u"}, AccountHandoff: &domain.AccountTaskHandoff{Pending: true}, AcceptanceClaim: &domain.AcceptanceClaimAccess{UserID: "u", SupplierID: "s"}, Subject: domain.HumanSubjectRef{Payload: map[string]any{"annotationDraft": "never copy"}}}
safe := cloneQueryCountRow(original)

View File

@ -15,6 +15,7 @@ import (
"droboticflow/internal/domain"
"droboticflow/internal/productionstats"
"droboticflow/internal/testsupport"
"droboticflow/internal/workforcestats"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/bson/primitive"
@ -27,7 +28,11 @@ import (
func newLocalReplica(t *testing.T, monitors ...*event.CommandMonitor) (*MongoStore, *MongoStore, *QueryReplica) {
t.Helper()
source := newQuerySyncSource(t, monitors...)
c, e := mongo.Connect(t.Context(), options.Client().ApplyURI("mongodb://127.0.0.1:27129/?replicaSet=query-read-rs"))
targetOptions := options.Client().ApplyURI(testsupport.QueryReplicaTargetURI(t))
if len(monitors) > 1 {
targetOptions.SetMonitor(monitors[1])
}
c, e := mongo.Connect(t.Context(), targetOptions)
if e != nil {
t.Fatal(e)
}
@ -57,6 +62,47 @@ func newLocalReplica(t *testing.T, monitors ...*event.CommandMonitor) (*MongoSto
}
return source, target, q
}
func TestQueryReplicaEmptyDrainDoesNotWriteLocal(t *testing.T) {
var writes atomic.Int64
monitor := &event.CommandMonitor{Started: func(_ context.Context, e *event.CommandStartedEvent) {
switch e.CommandName {
case "update", "insert", "delete", "findAndModify", "commitTransaction":
writes.Add(1)
}
}}
_, _, q := newLocalReplica(t, nil, monitor)
writes.Store(0)
for _, collection := range replicaCollections() {
n, err := q.DrainStep(t.Context(), collection)
if err != nil || n != 0 {
t.Fatal(collection, n, err)
}
}
if got := writes.Load(); got != 0 {
t.Fatalf("empty incremental pass wrote %d target commands", got)
}
}
func TestQueryReplicaWorkforceCacheSeparatesDatabasesLocal(t *testing.T) {
source, target, projector := newLocalReplica(t)
finishLocalReplica(t, projector)
ctx := t.Context()
org := domain.NewID("cache-isolation")
q := workforcestats.Query{From: time.Date(2026, 11, 1, 0, 0, 0, 0, time.UTC), To: time.Date(2026, 11, 13, 0, 0, 0, 0, time.UTC)}
_, err := source.db.Collection("annotationWorklogEvents").InsertOne(ctx, bson.M{"id": "source-only", "organizationId": org, "projectId": "p", "taskId": "t", "userId": "u", "eventType": "combined_submitted", "createdAt": "2026-11-10T10:00:00Z"})
if err != nil {
t.Fatal(err)
}
warm, err := source.LoadWorkforceStatsCached(ctx, org, q)
if err != nil || len(warm.Events) != 1 {
t.Fatal("source fixture", len(warm.Events), err)
}
got, err := target.LoadWorkforceStatsCached(ctx, org, q)
if err != nil || len(got.Events) != 0 {
t.Fatal("query database reused authority snapshot", len(got.Events), err)
}
}
func finishLocalReplica(t *testing.T, q *QueryReplica) {
t.Helper()
for n := 0; n < 100; n++ {
@ -311,6 +357,52 @@ func TestQueryReplicaWorkforceDetailsParityLocal(t *testing.T) {
}
func TestQueryReplicaWorkforceIncrementalInvalidatesSnapshotLocal(t *testing.T) {
s, d, q := newLocalReplica(t)
ctx := t.Context()
seedWorkforceStats(t, s)
finishLocalReplica(t, q)
if err := s.ConfigureQueryReplicaReports(d, time.Minute); err != nil {
t.Fatal(err)
}
query := workforcestats.Query{ProjectID: "p", UserID: "u", MatchUser: true}
before, err := s.LoadWorkforceStatsCached(ctx, "workforce-test", query)
if err != nil {
t.Fatal(err)
}
event := domain.AnnotationWorklogEventRecord{ID: "incremental-rework", OrganizationID: "workforce-test", ProjectID: "p", TaskID: "task", UserID: "u", SupplierID: "v", EventType: "combined_submitted", SubmissionKind: "rework", Revision: 5, CreatedAt: "2026-09-28T12:03:00Z"}
if err = s.UpsertAnnotationWorklogEvent(ctx, event); err != nil {
t.Fatal(err)
}
// Restart the projector with its durable checkpoints and fencing identity.
restarted, err := NewQueryReplica(s, d)
if err != nil {
t.Fatal(err)
}
// Simulate the previous process stopping and its lease expiring.
if _, err = q.control().UpdateOne(ctx, bson.M{"_id": "lease"}, bson.M{"$set": bson.M{"until": time.Now().Add(-time.Second)}}); err != nil {
t.Fatal(err)
}
if err = restarted.Acquire(ctx); err != nil {
t.Fatal(err)
}
finishLocalReplica(t, restarted)
after, err := s.LoadWorkforceStatsCached(ctx, "workforce-test", query)
if err != nil {
t.Fatal(err)
}
if len(after.Events) != len(before.Events)+1 {
t.Fatalf("committed incremental event hidden by old snapshot: before=%d after=%d", len(before.Events), len(after.Events))
}
if _, err = q.DrainStep(ctx, "annotationWorklogEvents"); err != nil {
// Empty old-worker polls have no effect. Actual publication must fence.
t.Fatal(err)
}
if err = q.PublishReady(ctx); !errors.Is(err, ErrQueryReplicaLease) {
t.Fatal("restarted projector did not fence old publication", err)
}
}
func TestQueryReplicaPoolSearchCountsAndCurrentAuthorityLocal(t *testing.T) {
s, d, q := newLocalReplica(t)
ctx := t.Context()

View File

@ -18,8 +18,8 @@ func WithQueryReplicaPoolRead(ctx context.Context) context.Context {
return context.WithValue(ctx, queryReplicaPoolReadKey{}, &queryReplicaPoolBarrier{})
}
func (s *MongoStore) UsesQueryReplicaPool(ctx context.Context) bool {
_, v := ctx.Value(queryReplicaPoolReadKey{}).(*queryReplicaPoolBarrier)
return v && s.queryReplicaReads != nil && s.queryReplicaReads.pool
b, v := ctx.Value(queryReplicaPoolReadKey{}).(*queryReplicaPoolBarrier)
return v && s.queryReplicaReads != nil && (s.queryReplicaReads.pool || b.counts && s.queryReplicaReads.counts)
}
func (s *MongoStore) poolReplica(ctx context.Context) (*MongoStore, error) {
if !s.UsesQueryReplicaPool(ctx) {
@ -33,6 +33,14 @@ func (s *MongoStore) EnableQueryReplicaPoolReads(enabled bool) {
}
}
// Count-only rollout leaves list discovery and all business mutations on the
// authority. The legacy pool switch continues to enable both lists and counts.
func (s *MongoStore) EnableQueryReplicaCountReads(enabled bool) {
if s.queryReplicaReads != nil {
s.queryReplicaReads.counts = enabled
}
}
// Query candidates come from the read store; current task scope/status/version
// are fetched by bounded IDs before API authorization and canClaim computation.
// Missing/changed candidates keep an internal cursor placeholder so deletion

View File

@ -22,6 +22,7 @@ type queryReplicaReads struct {
verifyPermit chan struct{}
nextVerify time.Time
pool bool
counts bool
target *MongoStore
maxAge time.Duration
}

View File

@ -415,7 +415,18 @@ func (s *MongoStore) LoadWorkforceStatsCached(ctx context.Context, org string, q
if !q.To.IsZero() {
toKey = q.To.Unix() / 60
}
key := fmt.Sprintf("%s|%d|%d|%s|%s|%s|%v|%v|%s", org, fromKey, toKey, q.ProjectID, q.SupplierID, q.UserID, q.MatchUser, q.Details, strings.Join(allowed, ","))
epoch := int64(0)
if s.queryReplicaFactStore {
var err error
epoch, err = s.countFactEpoch(ctx)
if err != nil {
return workforcestats.Snapshot{}, err
}
}
// A committed fact change must invalidate the independent read snapshot,
// including when projector and API run in separate processes. Authority
// readers keep their existing TTL; this epoch is query-database metadata.
key := fmt.Sprintf("%p|%s|%d|%s|%d|%d|%s|%s|%s|%v|%v|%s", s.db.Client(), s.db.Name(), epoch, org, fromKey, toKey, q.ProjectID, q.SupplierID, q.UserID, q.MatchUser, q.Details, strings.Join(allowed, ","))
if snapshot, ok := workforceStatsCache.get(key, dashboardStatsCacheTTL); ok {
return snapshot, nil
}

View File

@ -0,0 +1,29 @@
package testsupport
import (
"fmt"
"os"
"testing"
)
// QueryReplicaTargetURI is test-only infrastructure. It deliberately does not
// accept arbitrary hosts, credentials or deployment Mongo environment values.
func QueryReplicaTargetURI(t testing.TB) string {
t.Helper()
const local = "mongodb://127.0.0.1:27129/?replicaSet=query-read-rs"
uri := os.Getenv("QUERY_REPLICA_TEST_TARGET_URI")
if uri == "" {
return local
}
if err := validateQueryReplicaTarget(uri, os.Getenv("QUERY_REPLICA_CLUSTER_TEST")); err != nil {
t.Fatal(err)
}
return uri
}
func validateQueryReplicaTarget(uri, enabled string) error {
if enabled != "1" || uri != "mongodb://127.0.0.1:27229/?directConnection=true&replicaSet=query-lab-rs" {
return fmt.Errorf("only the explicitly owned node4 synthetic query tunnel is allowed")
}
return nil
}

View File

@ -0,0 +1,27 @@
package testsupport
import "testing"
func TestQueryReplicaTargetRejectsUnapprovedConnections(t *testing.T) {
const allowed = "mongodb://127.0.0.1:27229/?directConnection=true&replicaSet=query-lab-rs"
for _, tc := range []struct{ uri, enabled string }{
{allowed, ""},
{"mongodb://10.189.16.4:27017", "1"},
{"mongodb://127.0.0.1:27018", "1"},
{allowed + "&authSource=admin", "1"},
} {
if validateQueryReplicaTarget(tc.uri, tc.enabled) == nil {
t.Fatal("accepted a connection outside the owned synthetic tunnel")
}
}
if err := validateQueryReplicaTarget(allowed, "1"); err != nil {
t.Fatal(err)
}
}
func TestQueryReplicaTargetDefaultsToLocalLab(t *testing.T) {
t.Setenv("QUERY_REPLICA_TEST_TARGET_URI", "")
if got := QueryReplicaTargetURI(t); got != "mongodb://127.0.0.1:27129/?replicaSet=query-read-rs" {
t.Fatal("default target changed")
}
}

View File

@ -0,0 +1,169 @@
#!/usr/bin/env python3
"""Create only the bounded node4 synthetic source/HTTP fixture, after preflight."""
import argparse
import hashlib
import gzip
import ipaddress
import json
from pathlib import Path
import subprocess
import uuid
NODE = "10.189.16.4"
NODE_UID = "ca3f254a-8452-4769-8929-6ec303905dbe"
NS = "droboticflow-query-lab"
NS_UID = "07d647bb-943c-472c-be0a-f6b1380d7105"
QUERY_UID = "19f5f918-9bb1-4e6a-a362-bb70afacee26"
IMAGE = "docker.io/library/mongo@sha256:02a0cc7939f5ed38f30f9bc714ef5f682d49baf9350c54acf302ce833087fe8a"
ROOT = Path(__file__).resolve().parents[2]
HARD = {"pods": "2", "requests.cpu": "2", "limits.cpu": "5", "requests.memory": "3Gi", "limits.memory": "5Gi", "services": "0", "persistentvolumeclaims": "0"}
def manifest():
env = [{"name": name, "valueFrom": {"fieldRef": {"fieldPath": field}}} for name, field in (("POD_UID", "metadata.uid"), ("POD_NAME", "metadata.name"), ("NODE_NAME", "spec.nodeName"))]
env += [{"name": "HOME", "value": "/tmp"}, {"name": "MONGOSH_DISABLE_TELEMETRY", "value": "1"}, {"name": "GOMAXPROCS", "value": "2"}, {"name": "GOMEMLIMIT", "value": "1536MiB"}]
command = """set -euo pipefail
mkdir -p /synthetic/source
mongod --dbpath /synthetic/source --port 27119 --bind_ip 127.0.0.1 --wiredTigerCacheSizeGB 1 --pidfilepath /synthetic/mongod.pid --logpath /synthetic/source.log --setParameter diagnosticDataCollectionEnabled=false &
pid=$!
stop() { kill -TERM "$pid" 2>/dev/null || true; wait "$pid" || true; }
trap stop EXIT
trap 'exit 0' TERM INT
while kill -0 "$pid" 2>/dev/null; do sleep 10 & wait $!; done
wait "$pid"
"""
pod = {"apiVersion": "v1", "kind": "Pod", "metadata": {"name": "node4-query-fixture", "namespace": NS, "labels": {"app": "node4-query-fixture", "purpose": "mongo-cpu-synthetic"}}, "spec": {"nodeName": NODE, "automountServiceAccountToken": False, "enableServiceLinks": False, "restartPolicy": "Never", "activeDeadlineSeconds": 7200, "terminationGracePeriodSeconds": 45, "securityContext": {"runAsUser": 999, "runAsGroup": 999, "runAsNonRoot": True, "fsGroup": 999}, "containers": [{"name": "lab", "image": IMAGE, "imagePullPolicy": "Never", "command": ["bash", "-c", command], "env": env, "resources": {"requests": {"cpu": "1", "memory": "2Gi", "ephemeral-storage": "32Mi"}, "limits": {"cpu": "4", "memory": "4Gi", "ephemeral-storage": "128Mi"}}, "securityContext": {"allowPrivilegeEscalation": False, "readOnlyRootFilesystem": True, "capabilities": {"drop": ["ALL"]}, "seccompProfile": {"type": "RuntimeDefault"}}, "readinessProbe": {"exec": {"command": ["mongosh", "--quiet", "--host", "127.0.0.1", "--port", "27119", "--eval", "quit(db.adminCommand({ping:1}).ok?0:1)"]}, "periodSeconds": 15, "timeoutSeconds": 5}, "volumeMounts": [{"name": "synthetic", "mountPath": "/synthetic"}, {"name": "tmp", "mountPath": "/tmp"}]}], "volumes": [{"name": "synthetic", "emptyDir": {"medium": "Memory", "sizeLimit": "2Gi"}}, {"name": "tmp", "emptyDir": {"medium": "Memory", "sizeLimit": "64Mi"}}]}}
ingress = {"apiVersion": "networking.k8s.io/v1", "kind": "NetworkPolicy", "metadata": {"name": "query-fixture-ingress", "namespace": NS}, "spec": {"podSelector": {"matchLabels": {"app": "node4-query25"}}, "policyTypes": ["Ingress"], "ingress": [{"from": [{"podSelector": {"matchLabels": {"app": "node4-query-fixture"}}}], "ports": [{"protocol": "TCP", "port": 27130}]}]}}
egress = {"apiVersion": "networking.k8s.io/v1", "kind": "NetworkPolicy", "metadata": {"name": "query-fixture-egress", "namespace": NS}, "spec": {"podSelector": {"matchLabels": {"app": "node4-query-fixture"}}, "policyTypes": ["Egress"], "egress": [{"to": [{"podSelector": {"matchLabels": {"app": "node4-query25"}}}], "ports": [{"protocol": "TCP", "port": 27130}]}]}}
return [ingress, egress, pod]
def main():
ap = argparse.ArgumentParser(description=__doc__)
ap.add_argument("--kubeconfig", required=True)
ap.add_argument("--output", required=True)
ap.add_argument("--api-binary", required=True)
ap.add_argument("--bridge-binary", required=True)
ap.add_argument("--execute", action="store_true")
ap.add_argument("--finish-existing", action="store_true", help="resume this recorded partial fixture without replacing resources")
ap.add_argument("--update-api-test", action="store_true", help="stage a new content-addressed test binary after its previous run ended")
ap.add_argument("--confirm-plan")
args = ap.parse_args()
out = Path(args.output).resolve()
out.mkdir(parents=True, exist_ok=True)
k = ["kubectl", "--kubeconfig", str(Path(args.kubeconfig).resolve())]
def call(*argv, data=None, timeout=30):
p = subprocess.run(k + list(argv), input=data, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=timeout)
if p.returncode:
raise RuntimeError(p.stderr.decode()[:2000])
return p.stdout.decode().strip()
def get(*argv):
return json.loads(call(*argv, "-o", "json"))
node = get("get", "node", NODE)
assert node["metadata"]["uid"] == NODE_UID
assert all(c["status"] == "False" for c in node["status"]["conditions"] if c["type"] in ("DiskPressure", "MemoryPressure", "PIDPressure"))
top = call("top", "node", NODE, "--no-headers").split()
assert float(top[2].rstrip("%")) <= 25 and float(top[4].rstrip("%")) <= 60, "insufficient idle headroom for synthetic fixture"
assert get("get", "namespace", NS)["metadata"]["uid"] == NS_UID
query = get("-n", NS, "get", "pod", "node4-query25")
assert query["metadata"]["uid"] == QUERY_UID and query["status"]["phase"] == "Running"
assert query["spec"]["nodeName"] == NODE
ip = query["status"]["podIP"]
assert ipaddress.IPv4Address(ip) in ipaddress.IPv4Network("10.189.16.0/24")
quota = get("-n", NS, "get", "resourcequota", "smoke-budget")
original_quota = {"pods": "1", "requests.cpu": "1", "limits.cpu": "1", "requests.memory": "1Gi", "limits.memory": "1Gi", "services": "0", "persistentvolumeclaims": "0"}
recorded = None
if args.finish_existing:
recorded = json.loads((out / "created.json").read_text())
previous = json.loads((out / "plan.json").read_text())
previous_digest = hashlib.sha256(json.dumps(previous, sort_keys=True).encode()).hexdigest()
assert recorded["planSHA256"] == previous_digest
assert quota["spec"]["hard"] == HARD
existing = get("-n", NS, "get", "pod", "node4-query-fixture")
assert existing["metadata"]["uid"] == recorded["fixtureUID"] and existing["spec"]["nodeName"] == NODE
assert existing["spec"]["containers"][0]["resources"]["limits"]["cpu"] == "4"
assert existing["spec"]["containers"][0]["resources"]["limits"]["memory"] == "4Gi"
for obj in manifest()[:2]:
assert get("-n", NS, "get", "networkpolicy", obj["metadata"]["name"])["spec"] == obj["spec"]
else:
assert quota["spec"]["hard"] == original_quota
for kind, name in (("pod", "node4-query-fixture"), ("networkpolicy", "query-fixture-ingress"), ("networkpolicy", "query-fixture-egress")):
assert not call("-n", NS, "get", kind, name, "--ignore-not-found", "-o", "name"), "existing object must be inspected, never replaced"
binaries = {"api": Path(args.api_binary).resolve(), "bridge": Path(args.bridge_binary).resolve()}
binary_root = (ROOT / "local-data/diagnostics/mongo-cpu/query25-http/bin").resolve()
for path in binaries.values():
assert path.parent == binary_root, "only this task's generated binaries may be copied"
header = path.read_bytes()[:20]
assert header[:4] == b"\x7fELF" and header[18:20] == b"\x3e\x00", "Linux amd64 ELF required"
hashes = {name: hashlib.sha256(path.read_bytes()).hexdigest() for name, path in binaries.items()}
if recorded:
assert hashes["bridge"] == previous["binarySHA256"]["bridge"], "resume must keep the verified bridge"
if not args.update_api_test:
assert hashes == previous["binarySHA256"], "resume must use the recorded binaries"
else:
call("-n", NS, "exec", "node4-query-fixture", "-c", "lab", "--", "sh", "-c", 'for p in /proc/[0-9]*/exe; do x=$(readlink "$p" 2>/dev/null || true); case "$x" in /synthetic/query-http*.test) exit 1;; esac; done')
objects = manifest()
plan = {"kind": "synthetic-in-cluster-comparison", "node": NODE, "namespaceUID": NS_UID, "queryUID": QUERY_UID, "queryIP": ip, "quotaVersion": quota["metadata"]["resourceVersion"], "originalQuota": original_quota, "temporaryQuota": HARD, "objects": objects, "binarySHA256": hashes, "customerData": "none", "newDisk": "none; source is bounded tmpfs", "productionChanges": "none", "queryBridgeStorage": "bounded /tmp memory volume; database noexec retained", "stop": "three degraded readiness/IO/CPU samples; delete only owned fixture", "rollback": "stop owned bridges, delete fixture and its two policies, CAS restore original quota; keep 25Gi data"}
if recorded:
plan["resumeFrom"] = recorded
raw = json.dumps(plan, sort_keys=True).encode()
digest = hashlib.sha256(raw).hexdigest()
if args.execute:
assert args.confirm_plan == digest, "preflight identity changed; inspect a freshly rendered plan"
(out / ("finish-plan.json" if recorded else "plan.json")).write_text(json.dumps(plan, indent=2))
(out / "objects.json").write_text(json.dumps({"apiVersion": "v1", "kind": "List", "items": objects}, indent=2))
print(json.dumps({"planSHA256": digest, "fixture": "node4-query-fixture", "maxCPU": 4, "maxMemory": "4Gi", "sourceStorage": "tmpfs <=2Gi", "queryStorageAndLimits": "unchanged"}), flush=True)
if not args.execute:
return
# No application release, production credential, PVC, Service or host mount.
if not recorded:
patch = [{"op": "test", "path": "/metadata/resourceVersion", "value": plan["quotaVersion"]}, {"op": "test", "path": "/spec/hard", "value": plan["originalQuota"]}, {"op": "replace", "path": "/spec/hard", "value": HARD}]
(out / "intent.json").write_text(json.dumps({"planSHA256": digest, "action": "temporary synthetic quota then server dry-run", "queryUID": QUERY_UID}))
call("-n", NS, "patch", "resourcequota", "smoke-budget", "--type=json", "-p", json.dumps(patch))
# Admission must see the temporary quota; the original budget is full.
call("create", "--dry-run=server", "-f", str(out / "objects.json"))
call("create", "-f", str(out / "objects.json"))
created = get("-n", NS, "get", "pod", "node4-query-fixture")
(out / "created.json").write_text(json.dumps({"fixtureUID": created["metadata"]["uid"], "planSHA256": digest}))
call("-n", NS, "wait", "--for=condition=Ready", "pod/node4-query-fixture", "--timeout=180s", timeout=190)
fixture = get("-n", NS, "get", "pod", "node4-query-fixture")
assert fixture["spec"]["nodeName"] == NODE
fuid = fixture["metadata"]["uid"]
assert str(uuid.UUID(fuid)) == fuid
assert get("-n", NS, "get", "pod", "node4-query25")["metadata"]["uid"] == QUERY_UID
qdir = "/tmp/query-http-lab/" + hashes["bridge"][:16]
def copy(pod, container, local, remote):
exists = subprocess.run(k + ["-n", NS, "exec", pod, "-c", container, "--", "test", "-f", remote], capture_output=True, timeout=30).returncode == 0
if exists:
assert call("-n", NS, "exec", pod, "-c", container, "--", "sha256sum", remote).split()[0] == hashlib.sha256(local.read_bytes()).hexdigest(), "existing fixture artifact differs"
return
call("-n", NS, "exec", pod, "-c", container, "--", "mkdir", "-p", str(Path(remote).parent))
command = "set -eu; umask 077; test ! -e " + remote + "; gzip -dc > " + remote + "; chmod 700 " + remote
p = subprocess.run(k + ["-n", NS, "exec", "-i", pod, "-c", container, "--", "sh", "-c", command], input=gzip.compress(local.read_bytes(), compresslevel=1), stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=600)
if p.returncode:
raise RuntimeError("owned fixture binary transfer failed")
actual = call("-n", NS, "exec", pod, "-c", container, "--", "sha256sum", remote).split()[0]
assert actual == hashlib.sha256(local.read_bytes()).hexdigest()
copy("node4-query25", "probe", binaries["bridge"], qdir + "/bridge")
copy("node4-query-fixture", "lab", binaries["bridge"], "/synthetic/bridge")
api_path = "/synthetic/query-http-" + hashes["api"][:16] + ".test"
copy("node4-query-fixture", "lab", binaries["api"], api_path)
# Fixed, preflight-derived private addresses and UID arguments only.
qcommand = "set -eu; if test -s " + qdir + "/bridge.pid; then p=$(cat " + qdir + "/bridge.pid); if kill -0 \"$p\" 2>/dev/null; then test \"$(readlink /proc/$p/exe)\" = " + qdir + "/bridge; exit 0; fi; fi; nohup " + qdir + "/bridge -listen " + ip + ":27130 -upstream 127.0.0.1:27129 -pod-uid " + QUERY_UID + " >" + qdir + "/bridge.log 2>&1 </dev/null & echo $! >" + qdir + "/bridge.pid"
call("-n", NS, "exec", "node4-query25", "-c", "probe", "--", "sh", "-c", qcommand)
fcommand = "set -eu; if test -s /synthetic/bridge.pid; then p=$(cat /synthetic/bridge.pid); if kill -0 \"$p\" 2>/dev/null; then test \"$(readlink /proc/$p/exe)\" = /synthetic/bridge; exit 0; fi; fi; nohup /synthetic/bridge -listen 127.0.0.1:27229 -upstream " + ip + ":27130 -pod-uid " + fuid + " >/synthetic/bridge.log 2>&1 </dev/null & echo $! >/synthetic/bridge.pid"
call("-n", NS, "exec", "node4-query-fixture", "-c", "lab", "--", "sh", "-c", fcommand)
call("-n", NS, "exec", "node4-query-fixture", "-c", "lab", "--", "mongosh", "--quiet", "mongodb://127.0.0.1:27229/?directConnection=true&replicaSet=query-lab-rs", "--eval", 'const m=db.getSiblingDB("query25_synthetic").marker.findOne({_id:"query25-confirmed"});quit(m&&m.value==="preserve-query25-v1"?0:2)')
owned = {"fixtureUID": fuid, "queryUID": QUERY_UID, "queryBridgeDirectory": qdir, "apiBinaryPath": api_path, "policies": {obj["metadata"]["name"]: get("-n", NS, "get", "networkpolicy", obj["metadata"]["name"])["metadata"]["uid"] for obj in objects[:2]}, "binarySHA256": hashes, "planSHA256": digest}
(out / "owned.json").write_text(json.dumps(owned, indent=2))
print("ISOLATED_FIXTURE_READY", flush=True)
if __name__ == "__main__":
main()

View File

@ -0,0 +1,220 @@
#!/usr/bin/env python3
"""Run only synthetic fixtures; sample both DB budgets and protect existing IO."""
import argparse
import hashlib
import json
import os
from pathlib import Path
import signal
import subprocess
import time
import urllib.request
NODE = "10.189.16.4"
NODE_UID = "ca3f254a-8452-4769-8929-6ec303905dbe"
NAMESPACE_UID = "07d647bb-943c-472c-be0a-f6b1380d7105"
CONTAINER = "fanyu-query-replica-local-source-1"
ROOT = Path(__file__).resolve().parents[2]
def run(args, timeout=20):
return subprocess.check_output(args, text=True, timeout=timeout).strip()
def source_fingerprint():
fingerprint = hashlib.sha256()
files = run(["git", "-C", str(ROOT), "ls-files", "-c", "-o", "--exclude-standard"]).splitlines()
for name in sorted(set(files)):
if name.startswith(("internal/", "cmd/", "scripts/")) or name in ("go.mod", "go.sum", "package-lock.json"):
path = ROOT / name
if path.is_file():
fingerprint.update(name.encode() + b"\0" + path.read_bytes())
return fingerprint.hexdigest()
def cgroup(pod):
uid = pod["metadata"]["uid"].replace("-", "_")
base = "/host/proc/1/root/sys/fs/cgroup/kubepods.slice/"
qos = pod["status"]["qosClass"]
if qos == "Guaranteed":
return base + "kubepods-pod" + uid + ".slice"
if qos not in ("Burstable", "BestEffort"):
raise RuntimeError("unknown cgroup layout")
part = "kubepods-" + qos.lower()
return base + part + ".slice/" + part + "-pod" + uid + ".slice"
def main():
ap = argparse.ArgumentParser(description=__doc__)
ap.add_argument("--kubeconfig", required=True)
ap.add_argument("--output", required=True)
ap.add_argument("--tasks", type=int, default=26218, choices=range(20, 30001))
ap.add_argument("--waves", type=int, default=2, choices=range(1, 101))
ap.add_argument("--max-actors", type=int, default=81, choices=(81, 200, 500))
ap.add_argument("--fixture-plan", help="owned.json from the bounded in-cluster fixture")
args = ap.parse_args()
out = Path(args.output).resolve()
out.mkdir(parents=True, exist_ok=False)
k = ["kubectl", "--kubeconfig", str(Path(args.kubeconfig).resolve())]
def get(*cmd):
return json.loads(run(k + list(cmd) + ["-o", "json"]))
node = get("get", "node", NODE)
assert node["metadata"]["uid"] == NODE_UID
ns = get("get", "namespace", "droboticflow-query-lab")
assert ns["metadata"]["uid"] == NAMESPACE_UID
query = get("-n", "droboticflow-query-lab", "get", "pod", "node4-query25")
assert query["status"]["phase"] == "Running"
assert query["spec"]["nodeName"] == NODE
assert all(query["spec"]["containers"][0]["resources"]["limits"].get(k) == v for k, v in {"cpu": "1", "memory": "1Gi"}.items())
fixture = None
owned = None
if args.fixture_plan:
owned = json.loads(Path(args.fixture_plan).read_text())
fixture = get("-n", "droboticflow-query-lab", "get", "pod", "node4-query-fixture")
assert fixture["metadata"]["uid"] == owned["fixtureUID"]
assert query["metadata"]["uid"] == owned["queryUID"]
assert fixture["status"]["phase"] == "Running" and fixture["spec"]["nodeName"] == NODE
budget = fixture["spec"]["containers"][0]["resources"]["limits"]
assert budget["cpu"] == "4" and budget["memory"] == "4Gi"
api_binary = owned.get("apiBinaryPath", "/synthetic/query-http.test")
assert api_binary.startswith("/synthetic/query-http") and api_binary.endswith(".test") and "/../" not in api_binary
actual = run(k + ["-n", "droboticflow-query-lab", "exec", "node4-query-fixture", "-c", "lab", "--", "sha256sum", api_binary]).split()[0]
assert actual == owned["binarySHA256"]["api"]
source = {"Id": fixture["metadata"]["uid"], "Image": fixture["status"]["containerStatuses"][0]["imageID"]}
else:
source = json.loads(run(["docker", "inspect", CONTAINER]))[0]
assert source["State"]["Running"]
assert source["HostConfig"]["NanoCpus"] == 4_000_000_000
assert source["HostConfig"]["Memory"] == 4_294_967_296
socket = run(["docker", "context", "inspect", "--format", "{{.Endpoints.docker.Host}}"])
assert socket.startswith("unix://"), "source must be a local Docker daemon"
protected = [p for p in get("get", "pods", "-A", "--field-selector", "spec.nodeName=" + NODE)["items"] if p["metadata"]["namespace"] != "droboticflow-query-lab"]
prod = get("-n", "droboticflow-fanyu", "get", "pods")
def pod_states(items):
return {p["metadata"]["uid"]: [s.get("restartCount", 0) for s in p["status"].get("containerStatuses", [])] for p in items}
states = pod_states(prod["items"])
# All interpolated paths are generated from validated Kubernetes UIDs/QoS,
# never arbitrary paths, credentials or user-provided command fragments.
script = "set -e; cat /host/proc/diskstats | awk '$3==\"vda\"'; stat -f --printf='AVAILABLE %a %S\\n' /host/proc/1/root; "
for p in protected:
script += "echo PROTECTED " + p["metadata"]["uid"] + "; cat " + cgroup(p) + "/io.pressure; "
for filename in ("cpu.stat", "memory.current", "memory.events", "io.stat", "io.max"):
script += "echo QUERY " + filename + "; cat " + cgroup(query) + "/" + filename + "; "
if fixture:
for filename in ("cpu.stat", "memory.current", "memory.events", "io.stat"):
script += "echo SOURCE " + filename + "; cat " + cgroup(fixture) + "/" + filename + "; "
previous = None
def sample():
nonlocal previous
started = time.monotonic()
with urllib.request.urlopen("http://120.48.180.80/api/ready", timeout=5) as response:
assert response.status == 200
response.read(65536)
ready = time.monotonic() - started
current = get("-n", "droboticflow-fanyu", "get", "pods")
assert pod_states(current["items"]) == states, "existing production Pod/restart set changed"
health = get("get", "node", NODE)
assert all(c["status"] == "False" for c in health["status"]["conditions"] if c["type"] in ("DiskPressure", "MemoryPressure", "PIDPressure"))
host = run(k + ["-n", "kube-system", "exec", "node-problem-detector-6v59v", "-c", "node-problem-detector", "--", "sh", "-c", script])
local = "included in host SOURCE fields" if fixture else run(["docker", "exec", CONTAINER, "sh", "-c", "cat /sys/fs/cgroup/cpu.stat; cat /sys/fs/cgroup/memory.current; cat /sys/fs/cgroup/io.stat"])
top = run(k + ["top", "node", NODE, "--no-headers"]).split()
cpu_pct = float(top[2].rstrip("%"))
if fixture:
current_fixture = get("-n", "droboticflow-query-lab", "get", "pod", "node4-query-fixture")
assert current_fixture["metadata"]["uid"] == owned["fixtureUID"] and current_fixture["status"]["phase"] == "Running", "owned source fixture stopped or was replaced"
available = next(line.split() for line in host.splitlines() if line.startswith("AVAILABLE "))
assert int(available[1]) * int(available[2]) >= 80_530_636_800, "node free space below 75Gi"
pressure = []
inside = False
for line in host.splitlines():
if line.startswith("PROTECTED "):
inside = True
elif line.startswith(("QUERY ", "SOURCE ")):
inside = False
elif inside and line.startswith("full "):
pressure.append(float(line.split("avg10=")[1].split()[0]))
assert len(pressure) == len(protected)
disk = [int(v) for v in next(line for line in host.splitlines() if line.split()[:3] == ["252", "0", "vda"]).split()[3:]]
record = {"time": time.time(), "readySeconds": ready, "sourceCgroup": local, "host": host, "nodeCPUPct": cpu_pct, "protectedFullAvg10": max(pressure, default=0), "disk": disk, "awaitMs": 0, "busyPct": 0}
if previous:
ops = disk[0] + disk[4] - previous["disk"][0] - previous["disk"][4]
record["awaitMs"] = (disk[3] + disk[7] - previous["disk"][3] - previous["disk"][7]) / max(ops, 1)
record["busyPct"] = (disk[9] - previous["disk"][9]) / max(record["time"] - previous["time"], 0.001) / 10
previous = record
with (out / "resources.jsonl").open("a") as f:
f.write(json.dumps(record) + "\n")
return record
baseline = [sample() for _ in range(3)]
limits = {"ready": max(1, max(s["readySeconds"] for s in baseline) * 2), "pressure": max(0.5, max(s["protectedFullAvg10"] for s in baseline) + 0.5), "await": max(10, max(s["awaitMs"] for s in baseline) * 3)}
manifest = {"sourceCommit": run(["git", "-C", str(ROOT), "rev-parse", "HEAD"]), "sourceFingerprint": source_fingerprint(), "sourceContainer": source["Id"], "sourceImage": source["Image"], "sourceCPU": 4, "sourceMemory": 4_294_967_296, "queryPodUID": query["metadata"]["uid"], "queryImage": query["spec"]["containers"][0]["image"], "queryCPU": 1, "queryMemory": "1Gi", "tasks": args.tasks, "waves": args.waves, "maxActors": args.max_actors, "wireCompressionBothVariants": os.getenv("QUERY_COMPARE_WIRE_COMPRESSION", ""), "minimumPoolBothVariants": os.getenv("QUERY_COMPARE_MIN_POOL", "0"), "limits": limits, "limitations": ["source Docker VM shares CPU with existing local containers", "target has 2MiB read / 1MiB write and 64/32 IOPS hard limits", "short comparison; not 30-minute three-round production capacity acceptance", "health probes do not establish customer workflow latency"]}
if fixture:
manifest["limitations"][0] = "source Mongo and native HTTP/projector/builder/client share one fixed 4CPU/4Gi Pod; source storage is bounded tmpfs"
manifest["sourceCPUCounter"] = "mongod process utime+stime; whole fixture Pod cgroup sampled separately"
manifest["binarySHA256"] = owned["binarySHA256"]
(out / "manifest.json").write_text(json.dumps(manifest, indent=2))
env = dict(os.environ, QUERY_REPLICA_ISOLATED_E2E="1", QUERY_REPLICA_CLUSTER_TEST="1", QUERY_REPLICA_TEST_TARGET_URI="mongodb://127.0.0.1:27229/?directConnection=true&replicaSet=query-lab-rs", QUERY_COMPARE_TASKS=str(args.tasks), QUERY_COMPARE_WAVES=str(args.waves), QUERY_COMPARE_MAX_ACTORS=str(args.max_actors), QUERY_COMPARE_REPORT=str(out / "comparison.json"))
argv = ["go", "test", "./internal/api", "-run", "^TestQueryReplicaIsolatedHTTPComparisonLocal$", "-count=1", "-timeout=45m", "-v"]
remote_report = "/synthetic/query-comparison-" + str(int(time.time())) + ".json"
if fixture:
settings = ["QUERY_REPLICA_ISOLATED_E2E=1", "QUERY_REPLICA_CLUSTER_TEST=1", "QUERY_REPLICA_TEST_TARGET_URI=mongodb://127.0.0.1:27229/?directConnection=true&replicaSet=query-lab-rs", "QUERY_COMPARE_IN_CLUSTER=1", "QUERY_COMPARE_TASKS=" + str(args.tasks), "QUERY_COMPARE_WAVES=" + str(args.waves), "QUERY_COMPARE_MAX_ACTORS=" + str(args.max_actors), "QUERY_COMPARE_REPORT=" + remote_report, "QUERY_COMPARE_WIRE_COMPRESSION=" + os.getenv("QUERY_COMPARE_WIRE_COMPRESSION", ""), "QUERY_COMPARE_MIN_POOL=" + os.getenv("QUERY_COMPARE_MIN_POOL", "0")]
argv = k + ["-n", "droboticflow-query-lab", "exec", "node4-query-fixture", "-c", "lab", "--", "env"] + settings + [api_binary, "-test.run=^TestQueryReplicaIsolatedHTTPComparisonLocal$", "-test.count=1", "-test.timeout=45m", "-test.v"]
with (out / "test.log").open("w") as log:
child = subprocess.Popen(argv, cwd=ROOT, env=env, stdout=log, stderr=subprocess.STDOUT, start_new_session=True)
bad = 0
try:
while child.poll() is None:
record = sample()
degraded = record["readySeconds"] > limits["ready"] or record["protectedFullAvg10"] > limits["pressure"] or record["awaitMs"] > limits["await"] or record["busyPct"] > 70 or record["nodeCPUPct"] > 75
bad = bad + 1 if degraded else 0
if bad >= 3:
raise RuntimeError("existing workload health/IO degraded for three samples")
time.sleep(5)
except BaseException as exc:
stopped = {"reason": str(exc), "syntheticDatabasesMayNeedOwnedCleanup": True, "cleanupErrors": []}
# Record the original stop even if the API or the child disappears
# during cleanup. Never delete a replacement Pod by name alone.
(out / "stopped.json").write_text(json.dumps(stopped))
try:
if fixture:
current = get("-n", "droboticflow-query-lab", "get", "pod", "node4-query-fixture")
if current["metadata"]["uid"] == owned["fixtureUID"]:
delete = out / "delete-owned-fixture.json"
delete.write_text(json.dumps({"apiVersion": "v1", "kind": "DeleteOptions", "preconditions": {"uid": owned["fixtureUID"]}}))
run(k + ["delete", "--raw", "/api/v1/namespaces/droboticflow-query-lab/pods/node4-query-fixture", "-f", str(delete)])
except Exception as cleanup_error:
stopped["cleanupErrors"].append(str(cleanup_error))
finally:
if child.poll() is None:
try:
os.killpg(child.pid, signal.SIGTERM)
except ProcessLookupError:
pass
try:
child.wait(timeout=30)
except subprocess.TimeoutExpired:
try:
os.killpg(child.pid, signal.SIGKILL)
except ProcessLookupError:
pass
child.wait(timeout=5)
(out / "stopped.json").write_text(json.dumps(stopped))
raise
sample()
if fixture:
result = subprocess.run(k + ["-n", "droboticflow-query-lab", "exec", "node4-query-fixture", "-c", "lab", "--", "cat", remote_report], capture_output=True, text=True, timeout=30)
if result.returncode == 0:
(out / "comparison.json").write_text(result.stdout)
changed = source_fingerprint() != manifest["sourceFingerprint"]
(out / "exit.json").write_text(json.dumps({"returncode": child.returncode, "sourceChanged": changed}))
raise SystemExit(child.returncode or (2 if changed else 0))
if __name__ == "__main__":
main()

View File

@ -0,0 +1,39 @@
import importlib.util
from pathlib import Path
import unittest
spec = importlib.util.spec_from_file_location("cluster_fixture", Path(__file__).with_name("query-replica-cluster-fixture.py"))
fixture = importlib.util.module_from_spec(spec)
spec.loader.exec_module(fixture)
class FixtureSafety(unittest.TestCase):
def test_source_is_memory_only_and_unprivileged(self):
pod = fixture.manifest()[-1]["spec"]
self.assertEqual(pod["nodeName"], "10.189.16.4")
self.assertFalse(pod["automountServiceAccountToken"])
self.assertFalse(pod.get("hostNetwork", False))
self.assertEqual(pod["activeDeadlineSeconds"], 7200)
for volume in pod["volumes"]:
self.assertEqual(volume["emptyDir"]["medium"], "Memory")
self.assertIn("sizeLimit", volume["emptyDir"])
self.assertNotIn("hostPath", volume)
container = pod["containers"][0]
self.assertEqual(container["resources"]["limits"]["cpu"], "4")
self.assertEqual(container["resources"]["limits"]["memory"], "4Gi")
self.assertFalse(container["securityContext"]["allowPrivilegeEscalation"])
self.assertTrue(container["securityContext"]["readOnlyRootFilesystem"])
self.assertEqual(container["imagePullPolicy"], "Never")
def test_network_is_only_fixture_to_query_private_port(self):
ingress, egress, _ = fixture.manifest()
self.assertEqual(ingress["spec"]["podSelector"]["matchLabels"], {"app": "node4-query25"})
self.assertEqual(ingress["spec"]["ingress"], [{"from": [{"podSelector": {"matchLabels": {"app": "node4-query-fixture"}}}], "ports": [{"protocol": "TCP", "port": 27130}]}])
self.assertEqual(egress["spec"]["egress"], [{"to": [{"podSelector": {"matchLabels": {"app": "node4-query25"}}}], "ports": [{"protocol": "TCP", "port": 27130}]}])
self.assertEqual(fixture.HARD["services"], "0")
self.assertEqual(fixture.HARD["persistentvolumeclaims"], "0")
self.assertEqual(fixture.HARD["pods"], "2")
if __name__ == "__main__":
unittest.main()

View File

@ -970,7 +970,9 @@
"name": "node4-query-pilot-guardrails",
"paths": [
"^scripts/local/node4-query-pilot/",
"^docs/operations/query-replica-node4-(isolation-plan|smoke-results)\\.md$"
"^docs/operations/query-replica-node4-(isolation-plan|smoke-results)\\.md$",
"^scripts/local/(query-replica-(cluster-fixture|isolated-compare)|test_query_replica_cluster_fixture)\\.py$",
"^docs/operations/query-replica-query25-comparison\\.md$"
],
"frontend": [],
"go": {},
@ -988,6 +990,10 @@
"-v"
],
"tests": true
},
{
"argv": ["python3", "-m", "unittest", "discover", "-s", "scripts/local", "-p", "test_query_replica_cluster_fixture.py", "-v"],
"tests": true
}
],
"warnings": ["仅执行离线安全回归;节点写回归属、Pod重建、满盘及容量仍须已授权的独立合成实测,不能以该检查替代。"]
@ -998,6 +1004,7 @@
"^internal/store/query_(replica|sync|tracked_collection|collection_interface).*\\.go$",
"^internal/api/query_replica.*\\.go$",
"^cmd/(query-replica|query-cdc-probe)/",
"^cmd/query-lab-bridge/",
"^scripts/test_query_replica\\.py$",
"^scripts/local/query-replica\\.compose\\.yml$",
"^Dockerfile\\.query-replica$",
@ -1007,10 +1014,12 @@
"go": {
"./internal/store": [
"^TestQueryReplicaCandidateCache",
"^TestQueryReplicaCountOnly",
"^TestQueryReplicaMissingConfiguration",
"^TestQueryReplicaRejectsSigned"
],
"./cmd/query-cdc-probe": ["^TestClassification"]
"./cmd/query-cdc-probe": ["^TestClassification"],
"./cmd/query-lab-bridge": ["^TestIsolatedBridge"]
},
"task": [
{