forked from longtao.wu/droboticflow
🧪 Validate isolated query replica routing and workload safety
This commit is contained in:
parent
fbaa8546c0
commit
e38c3be922
105
cmd/query-lab-bridge/main.go
Normal file
105
cmd/query-lab-bridge/main.go
Normal file
@ -0,0 +1,105 @@
|
||||
// query-lab-bridge is an isolated-fixture transport, never a production service.
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"flag"
|
||||
"io"
|
||||
"log"
|
||||
"net"
|
||||
"net/netip"
|
||||
"os"
|
||||
"os/signal"
|
||||
"sync"
|
||||
"syscall"
|
||||
"time"
|
||||
)
|
||||
|
||||
func allowedFlow(listen, upstream, node, actualUID, expectedUID string) bool {
|
||||
if node != "10.189.16.4" || actualUID == "" || actualUID != expectedUID {
|
||||
return false
|
||||
}
|
||||
l, le := netip.ParseAddrPort(listen)
|
||||
u, ue := netip.ParseAddrPort(upstream)
|
||||
if le != nil || ue != nil {
|
||||
return false
|
||||
}
|
||||
private := netip.MustParsePrefix("10.189.16.0/24")
|
||||
return l == netip.MustParseAddrPort("127.0.0.1:27229") && private.Contains(u.Addr()) && u.Port() == 27130 ||
|
||||
private.Contains(l.Addr()) && l.Port() == 27130 && u == netip.MustParseAddrPort("127.0.0.1:27129")
|
||||
}
|
||||
|
||||
// Do not embed TCPConn: promoted ReadFrom/WriteTo would bypass the deadlines
|
||||
// when io.Copy selects its optimized interfaces.
|
||||
type boundedConn struct{ conn *net.TCPConn }
|
||||
|
||||
func (c boundedConn) Read(p []byte) (int, error) {
|
||||
_ = c.conn.SetReadDeadline(time.Now().Add(time.Minute))
|
||||
return c.conn.Read(p)
|
||||
}
|
||||
func (c boundedConn) Write(p []byte) (int, error) {
|
||||
_ = c.conn.SetWriteDeadline(time.Now().Add(time.Minute))
|
||||
return c.conn.Write(p)
|
||||
}
|
||||
|
||||
func main() {
|
||||
listen := flag.String("listen", "", "fixed isolated listen address")
|
||||
upstream := flag.String("upstream", "", "fixed isolated query address")
|
||||
uid := flag.String("pod-uid", "", "preflight Pod identity")
|
||||
flag.Parse()
|
||||
if !allowedFlow(*listen, *upstream, os.Getenv("NODE_NAME"), os.Getenv("POD_UID"), *uid) {
|
||||
log.Fatal("isolated transport identity or address rejected")
|
||||
}
|
||||
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGTERM, os.Interrupt)
|
||||
defer stop()
|
||||
listener, err := net.Listen("tcp4", *listen)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
defer listener.Close()
|
||||
go func() { <-ctx.Done(); _ = listener.Close() }()
|
||||
var workers sync.WaitGroup
|
||||
defer workers.Wait()
|
||||
slots := make(chan struct{}, 128)
|
||||
for ctx.Err() == nil {
|
||||
conn, err := listener.Accept()
|
||||
if err != nil {
|
||||
if ctx.Err() != nil || errors.Is(err, net.ErrClosed) {
|
||||
return
|
||||
}
|
||||
log.Fatal("isolated accept failed")
|
||||
}
|
||||
select {
|
||||
case slots <- struct{}{}:
|
||||
default:
|
||||
_ = conn.Close()
|
||||
continue
|
||||
}
|
||||
workers.Add(1)
|
||||
go func(conn net.Conn) {
|
||||
defer workers.Done()
|
||||
defer func() { <-slots }()
|
||||
defer conn.Close()
|
||||
dialer := net.Dialer{Timeout: 3 * time.Second}
|
||||
remote, err := dialer.DialContext(ctx, "tcp4", *upstream)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer remote.Close()
|
||||
a, b := conn.(*net.TCPConn), remote.(*net.TCPConn)
|
||||
for _, c := range []*net.TCPConn{a, b} {
|
||||
_ = c.SetReadBuffer(64 << 10)
|
||||
_ = c.SetWriteBuffer(64 << 10)
|
||||
}
|
||||
localCtx, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
go func() { <-localCtx.Done(); _ = a.Close(); _ = b.Close() }()
|
||||
done := make(chan struct{})
|
||||
go func() { _, _ = io.Copy(boundedConn{b}, boundedConn{a}); _ = b.CloseWrite(); close(done) }()
|
||||
_, _ = io.Copy(boundedConn{a}, boundedConn{b})
|
||||
_ = a.CloseWrite()
|
||||
<-done
|
||||
}(conn)
|
||||
}
|
||||
}
|
||||
22
cmd/query-lab-bridge/main_test.go
Normal file
22
cmd/query-lab-bridge/main_test.go
Normal file
@ -0,0 +1,22 @@
|
||||
package main
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestIsolatedBridgeRejectsOtherNodesIdentitiesAndDestinations(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
listen, upstream, node, uid, expected string
|
||||
allowed bool
|
||||
}{
|
||||
{"127.0.0.1:27229", "10.189.16.46:27130", "10.189.16.4", "owned", "owned", true},
|
||||
{"10.189.16.46:27130", "127.0.0.1:27129", "10.189.16.4", "owned", "owned", true},
|
||||
{"0.0.0.0:27229", "10.189.16.46:27130", "10.189.16.4", "owned", "owned", false},
|
||||
{"127.0.0.1:27229", "8.8.8.8:27130", "10.189.16.4", "owned", "owned", false},
|
||||
{"127.0.0.1:27229", "10.189.16.46:27017", "10.189.16.4", "owned", "owned", false},
|
||||
{"127.0.0.1:27229", "10.189.16.46:27130", "10.189.16.5", "owned", "owned", false},
|
||||
{"127.0.0.1:27229", "10.189.16.46:27130", "10.189.16.4", "replaced", "owned", false},
|
||||
} {
|
||||
if got := allowedFlow(tc.listen, tc.upstream, tc.node, tc.uid, tc.expected); got != tc.allowed {
|
||||
t.Fatalf("unexpected bridge admission: %+v", tc)
|
||||
}
|
||||
}
|
||||
}
|
||||
97
docs/operations/query-replica-query25-comparison.md
Normal file
97
docs/operations/query-replica-query25-comparison.md
Normal file
@ -0,0 +1,97 @@
|
||||
# 25Gi独立查询库:合成增量与HTTP对照
|
||||
|
||||
状态:隔离验证中,未准备生产切读。未修改生产Mongo、客户记录、业务部署、资源或带宽;没有购买磁盘。
|
||||
|
||||
## 本批范围
|
||||
|
||||
工作树 `mongo-cpu`,分支 `codex/mongo-cpu`,任务基线 `fbaa8546`。开始时合入了当时 `origin/dev` 的两项前端提交,保留现有后端修复;后续新鲜度检查确认包含远端主线及线上构建。
|
||||
|
||||
- 复用`.4`节点25Gi受限查询Mongo,接入持久标记投影器和查询库Change Streams聚合器。
|
||||
- 使用真正的HTTP路由、会话Cookie与源库鉴权,对照任务计数、生产总览、个人看板和作业记录。验证程序不启动业务巡检/调度器。
|
||||
- 新增 `QUERY_REPLICA_COUNTS=1`,配合报告模式,仅迁计数。`QUERY_REPLICA_POOLS`仍关闭,普通列表及实际作业留在权威库;旧池开关启用列表和计数的行为保持兼容。
|
||||
- 夹具约2.6万任务、20项目、3供应商、500会话,包含大标注正文、切片版本、不可变提交及退回意见。规模对照固定数据和资源,前后都执行相同接口;计量包含新路径的增量轮询、确认与对账开销。
|
||||
|
||||
## 验证中修正的缺口
|
||||
|
||||
1. 人员快照缓存键缺少数据库/客户端身份,同进程对照可能串用源库与查询库的快照。已增加隔离维度;修复前后实际回归分别失败、通过。
|
||||
2. 投影已提交新事件并重启恢复后,人员快照仍可能沿用45秒旧缓存。查询库缓存现以事务提交的事实代次失效;源库既有TTL不变。故障用例已先复现,再修复通过。
|
||||
3. 空增量轮询对每个集合仍开启更新租约的事务。现无变化集合只读,实际变更和就绪发布仍验证租约/代次;旧投影器不能发布。真实Mongo回归确认空扫描不发目标写命令。
|
||||
4. 计数与列表原先共用一个开关,无法分步上线。已拆出计数开关并验证旧开关兼容、查询服务不可用时普通列表不被计数模式阻断。
|
||||
|
||||
没有为减少读取而移除服务端权限、版本或领取约束,也没有让查询库回写业务正文。
|
||||
|
||||
## 环境与计量边界
|
||||
|
||||
| 组件 | 固定隔离预算 |
|
||||
| --- | --- |
|
||||
| 合成源Mongo、HTTP/投影/聚合/客户端 | 同一临时Pod,上限4CPU/4Gi,申请1CPU/2Gi;Mongo WT1Gi;Go最多2核、软内存限额1536MiB |
|
||||
| 合成源数据 | 有界2Gi tmpfs;节点未启用交换盘;不是生产存储耐久性测试 |
|
||||
| 查询Mongo | 原25Gi EXT4承载文件,1CPU/1Gi,WT0.25Gi,独立RS;数据盘继续`noexec` |
|
||||
| 查询盘IO | loop和底层盘双层2MiB/s读、1MiB/s写,64/32 IOPS;配置保持不变 |
|
||||
| 网络 | 无Service、无公网入口;默认拒绝,仅放行源试验Pod到查询Pod的27130端口 |
|
||||
|
||||
源Mongo CPU使用进程`utime+stime`,另记录整个试验Pod的CPU、内存及IO,不能把共享4核说成Mongo独占4核。记录源Mongo扫描文档/索引、逻辑及物理网络字节、全部源连接的命令及BSON响应字节、API分路径延迟/错误,以及查询库资源。BSON响应字节不是压缩后的公网流量。
|
||||
|
||||
投影器源/目标连接池各4,聚合器独立连接,API实际配置入口创建20连接的majority查询连接;核心鉴权及写入保持源库。共享Valkey和Typesense本轮未启用。
|
||||
|
||||
只在新建随机合成数据库内写入;结束清理各自测试数据库。对全部任务的标注、状态和版本计算前后摘要,防止把数据丢失误算为加速。目标原有`query25_synthetic`耐久标记只读核对。
|
||||
|
||||
监控现有API健康、生产Pod UID/重启数、节点压力/剩余空间、既有Pod独立IO PSI、物理盘等待及节点CPU。持续异常停止自有合成负载,绝不重启或改配生产Pod。短健康探针不等于客户完整作业验收。
|
||||
|
||||
## 为什么弃用管理隧道作为容量依据
|
||||
|
||||
早期本机API经`kubectl port-forward`查询`.4`库。81并发读同一合成文档,1KiB响应p95约100ms;约90KiB未压缩响应有63次15秒超时。相同大响应使用zstd后p95约85ms,无错误。Mongo本身多数命令执行很快,客户端出现连接/传输等待。
|
||||
|
||||
完整接口即使预热和压缩仍有秒级延迟,因此不把这些结果归咎于原Mongo或当作集群容量结论。全部失败证据保留,未扩大到生产。后续将源库与驱动放到同一隔离网段,管理链路只接收小型日志和结果。
|
||||
|
||||
首次桥接程序放在已禁止执行的数据盘上,启动被阻止。恢复仅将2.3MiB桥接程序放到有界`/tmp`内存卷,复用既有Pod和网络策略;没有取消数据盘`noexec`或改变IO限额。
|
||||
|
||||
## 已完成的小样本
|
||||
|
||||
同网段100任务、81个真实会话、两轮请求:四条接口业务JSON一致,无HTTP错误,任务标注/状态/版本摘要一致,确实命中后台生产报表。
|
||||
|
||||
| 指标 | 原路径 | 查询库路径 |
|
||||
| --- | ---: | ---: |
|
||||
| 整体p95 | 708.8ms | 637.8ms |
|
||||
| 生产总览p95 | 3.0ms | 3.6ms |
|
||||
| 人员看板p95 | 209.5ms | 381.3ms |
|
||||
| 任务计数p95 | 724.6ms | 683.3ms |
|
||||
| 作业记录p95 | 210.0ms | 384.9ms |
|
||||
| 源Mongo CPU秒 | 2.91 | 2.42 |
|
||||
| 源扫描文档 | 912,234 | 806,993 |
|
||||
|
||||
小样本不是性能达标结论;新路径的人员看板和作业记录反而较慢,须继续看代表性规模与后台成本,不能只展示整体p95下降。
|
||||
|
||||
## 大规模回填的首轮停止与修正
|
||||
|
||||
`cluster-scale-81-1`在回填约198.7秒处因租约过期停止,未进入HTTP对照。测试当时为了加快初始化采用1000条批次,和正式投影器默认200条不同。隔离库日志显示一次majority提交等待约41.5秒,超过30秒租约;此前也出现5~24秒写确认等待。IO采样触及32写IOPS及约1MiB/s预算,不能据此说需要给API加核。
|
||||
|
||||
没有延长租约、放宽IO限额或忽略错误。后续夹具改用正式默认200条;遇到租约错误时最多三次尝试重新取得自有/过期租约,沿用已提交检查点,另一活跃持有者仍会阻止接管。首次失败与新一轮证据分别保留,不清空源标注或把失败列为通过。
|
||||
|
||||
首轮监控39个样本,既有API健康最大约48ms、既有Pod独立IO PSI均为0、节点CPU最高12%、物理盘平均等待最高约2.43ms。只证明该窗口未观察到现有服务恶化,不代表完整作业或长期容量已验收。
|
||||
|
||||
第二轮`cluster-scale-81-2`采用200条批次,10次回填循环约26.4秒,随后保护监控因连续三次既有Pod IO等待超限而停止并按UID删除合成源库Pod。受影响的是`.4`节点一个生产API Pod,其IO `full avg10`依次约0.9、6.75、3.7;同期物理盘忙碌约2~3%、平均等待约0.4~0.6ms,API健康探针正常。不能把低磁盘忙碌当作隔离已达标,也不能据此确认共盘写回就是唯一原因。
|
||||
|
||||
停测后该API Pod IO等待回到0,且其`io.max`为空,未被试验意外设置限额。尚需区分共享文件系统写回/元数据依赖与同期业务自身IO。当前不再对该盘做大规模写入,不放宽保护阈值。这次对照未完成,不形成源Mongo大规模减负或API速度达标结论。
|
||||
|
||||
临时源库Pod已停止;桥接程序已按PID/可执行文件身份停止;两条新增网络策略按UID删除;试验配额CAS恢复原1Pod/1CPU/1Gi。原25Gi库、数据、标记、资源与IO限额保留,生产Pod未修改。部分合成查询库数据保留供诊断,不拿源库已消失的数据宣称可恢复生产。
|
||||
|
||||
## 证据与仍待完成
|
||||
|
||||
原始文件位于本工作树忽略目录 `local-data/diagnostics/mongo-cpu/query25-http/`:
|
||||
|
||||
- `smoke-*`:本机/管理隧道的各次失败及后续对照;不覆盖为通过。
|
||||
- `transport-probe.jsonl`及对应诊断源码:大小响应与压缩的传输对照。
|
||||
- `cluster-fixture/`:原计划、部分创建记录、恢复计划、精确Pod/策略身份和二进制SHA256。
|
||||
- `cluster-smoke-1/`:同网段小样本的环境/源码摘要、资源采样、HTTP结果和退出状态。
|
||||
- `cluster-scale-*`:代表性规模后续结果;按原始返回状态判断,不以Pod就绪宣称验收。
|
||||
|
||||
仍须完成26,218任务对照、81/200/500用户及长稳态/冷启动/缓存丢失/积压恢复;真实验收聚合包、复杂项目/权限分布的代表性也需单列确认。当前合成批次数20,不能代替真实批次数量的容量证明。查询/缓存优化不得靠减少返回任务或过滤授权记录达标。
|
||||
|
||||
下一步先只读核对保护触发窗口的API自身IO与挂载/文件系统关系,制定可证实不影响业务的写入隔离/低速回填方案,再恢复代表性规模对照;不以延长租约、增加API副本、删除安全判断或采购磁盘代替定位。
|
||||
|
||||
本批任务级检查中,浏览器看板、生产统计Mongo专项、查询服务三组件、节点保护和离线夹具检查通过;完整查询副本runner因本机只有源库、没有其要求的本地查询副本而未启动,保留失败记录,不另起默认8Gi容器挤占共享开发环境。新增count-only权限撤销/自审的真实Mongo用例尚待执行。测试目标连接白名单的单元检查及前端全量格式检查通过。任务检查不是全绿,不能作为发布证据。
|
||||
|
||||
停止保护同时加固:先落盘原始停止原因,再按UID清理自有Pod;即使远端已消失或删除失败也继续终止本机测试子进程,并独立记录清理错误。远程临时桥接的本机端口转发已关闭。
|
||||
|
||||
普通列表与搜索迁移、生产47个源索引/写者覆盖和历史回填、生产资源配置及发布均未执行。达标后才准备相应生产计划,不以本地或隔离结果宣称线上容量已验收。
|
||||
@ -27,7 +27,8 @@
|
||||
| `QUERY_SOURCE_TRACKING=1` | API、切片处理器、校验器、Worker、协调器及写维护工具;启用同文档标记 |
|
||||
| `QUERY_READ_MONGO_URI` / `QUERY_READ_MONGO_DB` | 独立查询库;不得与权威部署相同 |
|
||||
| `QUERY_REPLICA_REPORTS=1` | API:计数/生产与人员看板/作业统计的目标读取;默认关闭 |
|
||||
| `QUERY_REPLICA_POOLS=1` | API:任务池候选、计数及管理员标注池;报告模式可用后另启用 |
|
||||
| `QUERY_REPLICA_COUNTS=1` | API:仅迁移带计数标记的任务池数量请求,普通列表和核心作业继续读权威库;需要报告模式 |
|
||||
| `QUERY_REPLICA_POOLS=1` | API:任务池候选、计数及管理员标注池;保留原开关行为,计数验收后再单独启用 |
|
||||
| `QUERY_SERVING_READS=1` / `QUERY_SERVING_NAMESPACE` | 查询库派生结果、不可变快照及缓存,namespace 必须隔离环境 |
|
||||
| `QUERY_VALKEY_URL` | 可选共享缓存。失联只回到查询库 |
|
||||
| `QUERY_TYPESENSE_URL` / `QUERY_TYPESENSE_API_KEY` | 可选搜索候选,后续独立验收;本批不要求启用 |
|
||||
|
||||
704
internal/api/query_replica_isolated_test.go
Normal file
704
internal/api/query_replica_isolated_test.go
Normal file
@ -0,0 +1,704 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"os/exec"
|
||||
"reflect"
|
||||
"runtime"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"droboticflow/internal/domain"
|
||||
"droboticflow/internal/productionstats"
|
||||
"droboticflow/internal/store"
|
||||
"droboticflow/internal/testsupport"
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
"go.mongodb.org/mongo-driver/event"
|
||||
"go.mongodb.org/mongo-driver/mongo"
|
||||
"go.mongodb.org/mongo-driver/mongo/options"
|
||||
"go.mongodb.org/mongo-driver/mongo/readconcern"
|
||||
"go.mongodb.org/mongo-driver/mongo/readpref"
|
||||
)
|
||||
|
||||
// This harness has no production URI input. Both DBs are newly named synthetic
|
||||
// fixtures, and the target must carry the independently verified node4 marker.
|
||||
// HTTP uses the actual session, ACL and count/report handlers. Business recovery
|
||||
// schedulers are deliberately not duplicated in the two benchmark handlers.
|
||||
func TestQueryReplicaIsolatedHTTPComparisonLocal(t *testing.T) {
|
||||
if os.Getenv("QUERY_REPLICA_ISOLATED_E2E") != "1" {
|
||||
t.Skip("explicit node4 synthetic comparison only")
|
||||
}
|
||||
if os.Getenv("QUERY_REPLICA_CLUSTER_TEST") != "1" {
|
||||
t.Fatal("cluster synthetic target must be explicitly selected")
|
||||
}
|
||||
t.Setenv("ANNOTATION_SOURCE_VALIDATION_MODE", "advisory")
|
||||
t.Setenv("QUERY_SERVING_NAMESPACE", "isolated-http-comparison")
|
||||
ctx := t.Context()
|
||||
n := isolatedComparisonInt(t, "QUERY_COMPARE_TASKS", 26218, 20, 30000)
|
||||
waves := isolatedComparisonInt(t, "QUERY_COMPARE_WAVES", 2, 1, 100)
|
||||
maxActors := isolatedComparisonInt(t, "QUERY_COMPARE_MAX_ACTORS", 81, 81, 500)
|
||||
minPool := isolatedComparisonInt(t, "QUERY_COMPARE_MIN_POOL", 0, 0, 100)
|
||||
compression := os.Getenv("QUERY_COMPARE_WIRE_COMPRESSION")
|
||||
if compression != "" && compression != "zstd" {
|
||||
t.Fatal("unsupported comparison compression")
|
||||
}
|
||||
var mu sync.Mutex
|
||||
var phase string
|
||||
inflight := map[int64]string{}
|
||||
metrics := map[string]*isolatedCommands{}
|
||||
var targetStatsMu sync.Mutex
|
||||
targetCommands := map[string]int64{}
|
||||
targetMaxMillis := map[string]float64{}
|
||||
targetMonitor := &event.CommandMonitor{Succeeded: func(_ context.Context, e *event.CommandSucceededEvent) {
|
||||
targetStatsMu.Lock()
|
||||
defer targetStatsMu.Unlock()
|
||||
targetCommands[e.CommandName]++
|
||||
ms := float64(e.DurationNanos) / 1e6
|
||||
targetMaxMillis[e.CommandName] = max(targetMaxMillis[e.CommandName], ms)
|
||||
}}
|
||||
monitor := &event.CommandMonitor{
|
||||
Started: func(_ context.Context, e *event.CommandStartedEvent) {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if phase != "" {
|
||||
m := metrics[phase]
|
||||
m.Commands[e.CommandName]++
|
||||
m.RequestBytes += int64(len(e.Command))
|
||||
inflight[e.RequestID] = phase
|
||||
}
|
||||
},
|
||||
Succeeded: func(_ context.Context, e *event.CommandSucceededEvent) {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if name := inflight[e.RequestID]; name != "" {
|
||||
metrics[name].ReplyBytes += int64(len(e.Reply))
|
||||
delete(inflight, e.RequestID)
|
||||
}
|
||||
},
|
||||
Failed: func(_ context.Context, e *event.CommandFailedEvent) {
|
||||
mu.Lock()
|
||||
delete(inflight, e.RequestID)
|
||||
mu.Unlock()
|
||||
},
|
||||
}
|
||||
connect := func(uri string, monitor *event.CommandMonitor) (*mongo.Database, *store.MongoStore) {
|
||||
opts := options.Client().ApplyURI(uri).SetMonitor(monitor).SetServerSelectionTimeout(5 * time.Second).SetMinPoolSize(uint64(minPool))
|
||||
if compression != "" {
|
||||
opts.SetCompressors([]string{compression})
|
||||
}
|
||||
c, err := mongo.Connect(ctx, opts)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
db := c.Database(domain.NewID("query_http_synthetic"))
|
||||
t.Cleanup(func() {
|
||||
clean, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
if err := db.Drop(clean); err != nil {
|
||||
t.Errorf("synthetic cleanup %s: %v", db.Name(), err)
|
||||
}
|
||||
_ = c.Disconnect(clean)
|
||||
})
|
||||
return db, store.NewMongoStore(db)
|
||||
}
|
||||
sdb, old := connect("mongodb://127.0.0.1:27119/?directConnection=true", monitor)
|
||||
db, _ := connect(testsupport.QueryReplicaTargetURI(t), targetMonitor)
|
||||
var marker bson.M
|
||||
if err := db.Client().Database("query25_synthetic").Collection("marker").FindOne(ctx, bson.M{"_id": "query25-confirmed"}).Decode(&marker); err != nil || marker["value"] != "preserve-query25-v1" {
|
||||
t.Fatalf("owned 25Gi target identity not confirmed: %v", err)
|
||||
}
|
||||
newReader := store.NewMongoStore(sdb)
|
||||
newReader.EnableQuerySourceTracking(true)
|
||||
if err := old.EnsureIndexes(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fixture := seedIsolatedHTTPComparison(t, sdb, old, n)
|
||||
annotationDigest := isolatedAnnotationDigest(t, sdb)
|
||||
peer := func(uri, name string, size uint64, monitor *event.CommandMonitor, majority bool) *store.MongoStore {
|
||||
opts := options.Client().ApplyURI(uri).SetMaxPoolSize(size).SetMonitor(monitor).SetServerSelectionTimeout(5 * time.Second)
|
||||
if compression != "" {
|
||||
opts.SetCompressors([]string{compression})
|
||||
}
|
||||
if majority {
|
||||
opts.SetReadPreference(readpref.Primary()).SetReadConcern(readconcern.Majority())
|
||||
}
|
||||
client, err := mongo.Connect(ctx, opts)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
stop, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
_ = client.Disconnect(stop)
|
||||
})
|
||||
return store.NewMongoStore(client.Database(name))
|
||||
}
|
||||
workerSource := peer("mongodb://127.0.0.1:27119/?directConnection=true", sdb.Name(), 4, monitor, false)
|
||||
workerTarget := peer(testsupport.QueryReplicaTargetURI(t), db.Name(), 4, targetMonitor, true)
|
||||
q, err := store.NewQueryReplica(workerSource, workerTarget)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, fn := range []func(context.Context) error{q.VerifyTopology, q.EnsureSourceIndexes, q.EnsureTargetIndexes, q.Acquire} {
|
||||
if err := fn(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
start := time.Now()
|
||||
q.BatchSize = isolatedComparisonInt(t, "QUERY_COMPARE_BOOTSTRAP_BATCH", 200, 20, 200)
|
||||
bootstrapRestarts := 0
|
||||
t.Logf("bootstrap starting source=%s target=%s batch=%d", sdb.Name(), db.Name(), q.BatchSize)
|
||||
for i := 0; ; i++ {
|
||||
ready, err := q.RunOnce(ctx, true)
|
||||
if err != nil {
|
||||
if errors.Is(err, store.ErrQueryReplicaLease) && bootstrapRestarts < 3 {
|
||||
// Reacquire only our own/expired lease, never another live owner.
|
||||
// Durable checkpoints, not a fresh database, resume slow commits.
|
||||
if err := q.Acquire(ctx); err != nil {
|
||||
t.Fatal("bootstrap lease recovery", err)
|
||||
}
|
||||
bootstrapRestarts++
|
||||
t.Logf("bootstrap lease recovered count=%d", bootstrapRestarts)
|
||||
continue
|
||||
}
|
||||
t.Fatal("bootstrap", err)
|
||||
}
|
||||
if ready {
|
||||
break
|
||||
}
|
||||
if i > n/q.BatchSize+100 {
|
||||
t.Fatal("bootstrap did not converge")
|
||||
}
|
||||
if i%10 == 0 {
|
||||
t.Logf("bootstrap pass=%d seconds=%.1f", i, time.Since(start).Seconds())
|
||||
}
|
||||
}
|
||||
t.Logf("bootstrap tasks=%d seconds=%.3f source=%s target=%s", n, time.Since(start).Seconds(), sdb.Name(), db.Name())
|
||||
q.BatchSize = 200
|
||||
builderTarget := peer(testsupport.QueryReplicaTargetURI(t), db.Name(), 100, targetMonitor, true)
|
||||
if err := builderTarget.VerifyQueryReplicaTarget(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
builderTarget.ConfigureQueryServing(nil, nil)
|
||||
if err := builderTarget.EnsureQueryReadModelIndexes(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
workerCtx, stopWorkers := context.WithCancel(ctx)
|
||||
defer stopWorkers()
|
||||
builder := store.NewQueryReadBuilder(builderTarget)
|
||||
|
||||
workerErrors := make(chan error, 2)
|
||||
var workers sync.WaitGroup
|
||||
workers.Add(2)
|
||||
go func() {
|
||||
defer workers.Done()
|
||||
if err := builder.Run(workerCtx); err != nil && workerCtx.Err() == nil {
|
||||
workerErrors <- err
|
||||
}
|
||||
}()
|
||||
// The measured new phase includes every poll, reconcile and acknowledgement.
|
||||
projectorEnabled := make(chan struct{})
|
||||
go func() {
|
||||
defer workers.Done()
|
||||
select {
|
||||
case <-projectorEnabled:
|
||||
case <-workerCtx.Done():
|
||||
return
|
||||
}
|
||||
tick := time.NewTicker(time.Second)
|
||||
defer tick.Stop()
|
||||
for workerCtx.Err() == nil {
|
||||
if _, err := q.RunOnce(workerCtx, false); err != nil && workerCtx.Err() == nil {
|
||||
workerErrors <- err
|
||||
return
|
||||
}
|
||||
select {
|
||||
case <-workerCtx.Done():
|
||||
case <-tick.C:
|
||||
}
|
||||
}
|
||||
}()
|
||||
defer func() { stopWorkers(); workers.Wait(); builder.Close() }()
|
||||
readyCtx, cancel := context.WithTimeout(ctx, 4*time.Minute)
|
||||
err = builder.WaitReady(readyCtx, fixture.org, fixture.projects)
|
||||
cancel()
|
||||
if err != nil {
|
||||
t.Fatal("query materialization", err)
|
||||
}
|
||||
// The projector is intentionally paused for the authority measurement. A
|
||||
// long initial build must not leave that fixture's watermark expired before
|
||||
// the first paired HTTP read, even though its source has had no changes.
|
||||
if err := q.Acquire(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := q.PublishReady(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Use the actual API configuration entry point (20-connection, majority
|
||||
// target client), separate from projector and builder connection pools.
|
||||
t.Setenv("QUERY_REPLICA_REPORTS", "1")
|
||||
t.Setenv("QUERY_REPLICA_POOLS", "0")
|
||||
t.Setenv("QUERY_REPLICA_COUNTS", "1")
|
||||
t.Setenv("QUERY_SOURCE_TRACKING", "1")
|
||||
uri := testsupport.QueryReplicaTargetURI(t)
|
||||
if compression != "" {
|
||||
uri += "&compressors=" + compression
|
||||
}
|
||||
t.Setenv("QUERY_READ_MONGO_URI", uri)
|
||||
t.Setenv("QUERY_READ_MONGO_DB", db.Name())
|
||||
t.Setenv("QUERY_SERVING_READS", "1")
|
||||
t.Setenv("QUERY_VALKEY_URL", "")
|
||||
t.Setenv("QUERY_TYPESENSE_URL", "")
|
||||
closeReplica, err := newReader.ConfigureQueryReplicaFromEnvironment(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(closeReplica)
|
||||
serve := func(s store.Store) *httptest.Server {
|
||||
h := &server{store: s, requireSessionAuth: true, authLimiter: newRateLimiter(20, time.Minute), externalWriteLimiter: newRateLimiter(120, time.Minute), dashboardResponses: dashboardResponseCache{now: time.Now}}
|
||||
ts := httptest.NewServer(h.routes())
|
||||
t.Cleanup(ts.Close)
|
||||
return ts
|
||||
}
|
||||
before, after := serve(old), serve(newReader)
|
||||
client := &http.Client{Timeout: 25 * time.Second}
|
||||
request := func(url, path string, actor int) (int, []byte, error) {
|
||||
r, err := http.NewRequestWithContext(ctx, http.MethodGet, url+path, nil)
|
||||
if err != nil {
|
||||
return 0, nil, err
|
||||
}
|
||||
r.AddCookie(&http.Cookie{Name: sessionCookieName, Value: fixture.tokens[actor]})
|
||||
resp, err := client.Do(r)
|
||||
if err != nil {
|
||||
return 0, nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
return resp.StatusCode, body, err
|
||||
}
|
||||
paths := []string{
|
||||
"/api/human-item-counts?pluginId=annotation-qa&stageIds=semantic_review",
|
||||
"/api/dashboard/production?from=2026-11-01&to=2026-11-12",
|
||||
"/api/dashboard/workforce?from=2026-11-01T00:00:00Z&to=2026-11-13T00:00:00Z&groupBy=member",
|
||||
"/api/workforce-returns?from=2026-11-01T00:00:00Z&to=2026-11-13T00:00:00Z&limit=20&offset=0",
|
||||
}
|
||||
// Exact business JSON equality; only generated report timestamps are volatile.
|
||||
for i, path := range paths {
|
||||
a, ab, ae := request(before.URL, path, 0)
|
||||
b, bb, be := request(after.URL, path, 0)
|
||||
if ae != nil || be != nil || a != 200 || b != 200 {
|
||||
t.Fatalf("route %d: old=%d/%v %.300s new=%d/%v %.300s", i, a, ae, ab, b, be, bb)
|
||||
}
|
||||
if !reflect.DeepEqual(isolatedBusinessJSON(t, ab), isolatedBusinessJSON(t, bb)) {
|
||||
t.Fatalf("route %d business results differ: old %.500s new %.500s", i, ab, bb)
|
||||
}
|
||||
if i == 0 {
|
||||
t.Logf("counter parity %s", ab)
|
||||
}
|
||||
}
|
||||
// Keep output evidence independently identifiable rather than parsing t.Log.
|
||||
report := map[string]any{"tasks": n, "projects": len(fixture.projects), "sourceDatabase": sdb.Name(), "targetDatabase": db.Name(), "cases": []any{}, "parity": true, "httpSessionAuth": true, "sourceMinimumPoolBothVariants": minPool, "apiQueryMaximumPool": 20, "projectorMaximumPool": 4, "builderMaximumPool": 100, "wireCompressionBothVariants": compression, "sourceReplyBytesKind": "logical BSON; not compressed wire bytes", "cache": "none; existing API snapshot/response caches unchanged"}
|
||||
cases := []any{}
|
||||
accepted := true
|
||||
for _, mode := range []string{"old", "query"} {
|
||||
url := before.URL
|
||||
if mode == "query" {
|
||||
url = after.URL
|
||||
// Reacquire after the baseline measurement's idle period.
|
||||
if err := q.Acquire(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := q.RunOnce(ctx, false); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
close(projectorEnabled)
|
||||
}
|
||||
for _, concurrency := range []int{81, 200, 500} {
|
||||
if concurrency > maxActors {
|
||||
continue
|
||||
}
|
||||
name := fmt.Sprintf("%s-%d", mode, concurrency)
|
||||
// Each variant pays the same session lookup cost; do not let the old
|
||||
// phase warm a shared authentication cache for the query phase.
|
||||
principalCacheMu.Lock()
|
||||
principalCache = map[string]principalCacheEntry{}
|
||||
principalCacheMu.Unlock()
|
||||
scansBefore := isolatedSourceScans(t, sdb)
|
||||
cpuBefore := isolatedSourceCPU(t)
|
||||
mu.Lock()
|
||||
phase = name
|
||||
metrics[name] = &isolatedCommands{Commands: map[string]int64{}}
|
||||
mu.Unlock()
|
||||
t.Log("phase", name, time.Now().UTC().Format(time.RFC3339Nano))
|
||||
started := time.Now()
|
||||
latency := make([]float64, 0, concurrency*waves)
|
||||
byRoute := map[string][]float64{}
|
||||
errors := map[string]int{}
|
||||
var firstError string
|
||||
var responses int64
|
||||
for wave := 0; wave < waves; wave++ {
|
||||
var wg sync.WaitGroup
|
||||
var results sync.Mutex
|
||||
barrier := make(chan struct{})
|
||||
for actor := 0; actor < concurrency; actor++ {
|
||||
wg.Add(1)
|
||||
go func(actor int) {
|
||||
defer wg.Done()
|
||||
<-barrier
|
||||
path := paths[2+(actor+wave)%2]
|
||||
if actor%81 == 0 {
|
||||
path = paths[1]
|
||||
} else if actor%81 >= 61 {
|
||||
path = paths[0]
|
||||
}
|
||||
s := time.Now()
|
||||
status, body, err := request(url, path, actor)
|
||||
results.Lock()
|
||||
defer results.Unlock()
|
||||
ms := float64(time.Since(s).Microseconds()) / 1000
|
||||
latency = append(latency, ms)
|
||||
key := strings.Split(path, "?")[0]
|
||||
byRoute[key] = append(byRoute[key], ms)
|
||||
responses += int64(len(body))
|
||||
if status != 200 || err != nil {
|
||||
errors[fmt.Sprintf("%d/%v", status, err)]++
|
||||
if firstError == "" {
|
||||
firstError = fmt.Sprintf("%s: %d/%v %.300s", key, status, err, body)
|
||||
}
|
||||
}
|
||||
}(actor)
|
||||
}
|
||||
close(barrier)
|
||||
wg.Wait()
|
||||
}
|
||||
mu.Lock()
|
||||
phase = ""
|
||||
original := metrics[name]
|
||||
m := &isolatedCommands{Commands: map[string]int64{}, RequestBytes: original.RequestBytes, ReplyBytes: original.ReplyBytes}
|
||||
for cmd, count := range original.Commands {
|
||||
m.Commands[cmd] = count
|
||||
}
|
||||
mu.Unlock()
|
||||
cpu := isolatedSourceCPU(t) - cpuBefore
|
||||
scans := isolatedSourceScans(t, sdb)
|
||||
for key, before := range scansBefore {
|
||||
scans[key] -= before
|
||||
}
|
||||
sort.Float64s(latency)
|
||||
routes := map[string]any{}
|
||||
for path, samples := range byRoute {
|
||||
sort.Float64s(samples)
|
||||
p95 := samples[(len(samples)*95+99)/100-1]
|
||||
routes[path] = map[string]any{"requests": len(samples), "p95Ms": p95, "maxMs": samples[len(samples)-1]}
|
||||
if mode == "query" && p95 > 2000 {
|
||||
t.Errorf("%s %s p95 exceeds preliminary 2s bound", name, path)
|
||||
}
|
||||
}
|
||||
result := map[string]any{"name": name, "seconds": time.Since(started).Seconds(), "requests": len(latency), "p50Ms": latency[(len(latency)-1)/2], "p95Ms": latency[(len(latency)-1)*95/100], "errors": errors, "firstError": firstError, "routes": routes, "responseBytes": responses, "sourceCPUSeconds": cpu, "sourceScans": scans, "source": m}
|
||||
cases = append(cases, result)
|
||||
encoded, _ := json.Marshal(result)
|
||||
t.Log(string(encoded))
|
||||
if len(errors) != 0 {
|
||||
accepted = false
|
||||
t.Errorf("%s has HTTP failures: %s", name, firstError)
|
||||
break
|
||||
}
|
||||
if mode == "query" && latency[(len(latency)-1)*95/100] > 2000 {
|
||||
t.Errorf("%s p95 exceeds preliminary 2s bound", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
report["cases"] = cases
|
||||
report["errorFree"] = accepted
|
||||
report["capacityAccepted"] = false
|
||||
report["bootstrapLeaseRecoveries"] = bootstrapRestarts
|
||||
report["queryServing"] = newReader.QueryServingStatistics()
|
||||
if newReader.QueryServingStatistics()["report_hits"] == 0 {
|
||||
t.Error("production read did not use its background materialized report")
|
||||
}
|
||||
if actual := isolatedAnnotationDigest(t, sdb); actual != annotationDigest {
|
||||
t.Fatal("read projection changed task status, version or annotation")
|
||||
}
|
||||
report["annotationsAndVersionsPreserved"] = true
|
||||
targetStatsMu.Lock()
|
||||
report["targetWorkerCommands"] = targetCommands
|
||||
report["targetWorkerMaxRoundTripMs"] = targetMaxMillis
|
||||
targetStatsMu.Unlock()
|
||||
select {
|
||||
case err := <-workerErrors:
|
||||
t.Fatal("background worker failed", err)
|
||||
default:
|
||||
}
|
||||
if path := os.Getenv("QUERY_COMPARE_REPORT"); path != "" {
|
||||
targetStatsMu.Lock()
|
||||
data, err := json.MarshalIndent(report, "", " ")
|
||||
targetStatsMu.Unlock()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(path, data, 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func isolatedAnnotationDigest(t *testing.T, db *mongo.Database) string {
|
||||
t.Helper()
|
||||
cur, err := db.Collection("humanItems").Find(t.Context(), bson.M{}, options.Find().SetProjection(bson.M{"_id": 0, "id": 1, "version": 1, "status": 1, "assignedUserId": 1, "subject.payload.annotationDraft": 1}).SetSort(bson.D{{Key: "id", Value: 1}}))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer cur.Close(t.Context())
|
||||
hash := sha256.New()
|
||||
for cur.Next(t.Context()) {
|
||||
var row bson.M
|
||||
if err := cur.Decode(&row); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
data, err := json.Marshal(row)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, _ = hash.Write(data)
|
||||
}
|
||||
if err := cur.Err(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return fmt.Sprintf("%x", hash.Sum(nil))
|
||||
}
|
||||
|
||||
func isolatedSourceScans(t *testing.T, db *mongo.Database) map[string]int64 {
|
||||
t.Helper()
|
||||
var stats struct {
|
||||
Network struct {
|
||||
BytesIn int64 `bson:"bytesIn"`
|
||||
BytesOut int64 `bson:"bytesOut"`
|
||||
PhysicalBytesIn int64 `bson:"physicalBytesIn"`
|
||||
PhysicalBytesOut int64 `bson:"physicalBytesOut"`
|
||||
} `bson:"network"`
|
||||
Metrics struct {
|
||||
QueryExecutor struct {
|
||||
Scanned int64 `bson:"scanned"`
|
||||
ScannedObjects int64 `bson:"scannedObjects"`
|
||||
} `bson:"queryExecutor"`
|
||||
} `bson:"metrics"`
|
||||
}
|
||||
if err := db.Client().Database("admin").RunCommand(t.Context(), bson.M{"serverStatus": 1}).Decode(&stats); err != nil {
|
||||
t.Fatal("synthetic source scan counters", err)
|
||||
}
|
||||
return map[string]int64{"indexEntries": stats.Metrics.QueryExecutor.Scanned, "documents": stats.Metrics.QueryExecutor.ScannedObjects, "networkLogicalBytesIn": stats.Network.BytesIn, "networkLogicalBytesOut": stats.Network.BytesOut, "networkPhysicalBytesIn": stats.Network.PhysicalBytesIn, "networkPhysicalBytesOut": stats.Network.PhysicalBytesOut}
|
||||
}
|
||||
|
||||
func isolatedSourceCPU(t *testing.T) float64 {
|
||||
t.Helper()
|
||||
if os.Getenv("QUERY_COMPARE_IN_CLUSTER") == "1" {
|
||||
if runtime.GOOS != "linux" || os.Getenv("NODE_NAME") != "10.189.16.4" || os.Getenv("POD_NAME") != "node4-query-fixture" {
|
||||
t.Fatal("isolated source process identity rejected")
|
||||
}
|
||||
raw, err := os.ReadFile("/synthetic/mongod.pid")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
pid := strings.TrimSpace(string(raw))
|
||||
if _, err := strconv.ParseUint(pid, 10, 32); err != nil {
|
||||
t.Fatal("invalid synthetic source pid")
|
||||
}
|
||||
command, err := os.ReadFile("/proc/" + pid + "/cmdline")
|
||||
if err != nil || !strings.Contains(string(command), "--dbpath\x00/synthetic/source\x00") || !strings.Contains(string(command), "--port\x0027119\x00") {
|
||||
t.Fatal("source PID does not own the synthetic database")
|
||||
}
|
||||
stat, err := os.ReadFile("/proc/" + pid + "/stat")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fields := strings.Fields(string(stat))
|
||||
if len(fields) < 15 || fields[1] != "(mongod)" {
|
||||
t.Fatal("unexpected synthetic process counters")
|
||||
}
|
||||
clock, err := exec.Command("getconf", "CLK_TCK").Output()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
hz, err := strconv.ParseFloat(strings.TrimSpace(string(clock)), 64)
|
||||
if err != nil || hz <= 0 {
|
||||
t.Fatal("invalid process clock")
|
||||
}
|
||||
user, e1 := strconv.ParseFloat(fields[13], 64)
|
||||
system, e2 := strconv.ParseFloat(fields[14], 64)
|
||||
if e1 != nil || e2 != nil {
|
||||
t.Fatal("invalid process CPU counters")
|
||||
}
|
||||
return (user + system) / hz
|
||||
}
|
||||
raw, err := exec.Command("docker", "exec", "fanyu-query-replica-local-source-1", "cat", "/sys/fs/cgroup/cpu.stat").Output()
|
||||
if err != nil {
|
||||
t.Fatal("isolated source cgroup unavailable", err)
|
||||
}
|
||||
for _, line := range strings.Split(string(raw), "\n") {
|
||||
fields := strings.Fields(line)
|
||||
if len(fields) == 2 && fields[0] == "usage_usec" {
|
||||
value, err := strconv.ParseFloat(fields[1], 64)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return value / 1e6
|
||||
}
|
||||
}
|
||||
t.Fatal("isolated source CPU counter missing")
|
||||
return 0
|
||||
}
|
||||
|
||||
type isolatedCommands struct {
|
||||
Commands map[string]int64 `json:"commands"`
|
||||
RequestBytes int64 `json:"requestBytes"`
|
||||
ReplyBytes int64 `json:"replyBytes"`
|
||||
}
|
||||
|
||||
type isolatedHTTPFixture struct {
|
||||
org string
|
||||
projects []string
|
||||
tokens []string
|
||||
}
|
||||
|
||||
func isolatedComparisonInt(t *testing.T, key string, fallback, min, max int) int {
|
||||
t.Helper()
|
||||
if os.Getenv(key) == "" {
|
||||
return fallback
|
||||
}
|
||||
n, err := strconv.Atoi(os.Getenv(key))
|
||||
if err != nil || n < min || n > max {
|
||||
t.Fatalf("invalid %s", key)
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
func isolatedBusinessJSON(t *testing.T, data []byte) any {
|
||||
t.Helper()
|
||||
var result any
|
||||
if err := json.Unmarshal(data, &result); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Remove only named generation timestamps, never business dates or versions.
|
||||
var walk func(any)
|
||||
walk = func(v any) {
|
||||
switch x := v.(type) {
|
||||
case map[string]any:
|
||||
delete(x, "asOf")
|
||||
delete(x, "generatedAt")
|
||||
for _, child := range x {
|
||||
walk(child)
|
||||
}
|
||||
case []any:
|
||||
for _, child := range x {
|
||||
walk(child)
|
||||
}
|
||||
}
|
||||
}
|
||||
walk(result)
|
||||
return result
|
||||
}
|
||||
|
||||
func seedIsolatedHTTPComparison(t *testing.T, db *mongo.Database, s *store.MongoStore, n int) isolatedHTTPFixture {
|
||||
t.Helper()
|
||||
ctx := t.Context()
|
||||
f := isolatedHTTPFixture{org: "query-synthetic-org"}
|
||||
must := func(err error) {
|
||||
t.Helper()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
must(s.UpsertOrganization(ctx, domain.OrganizationRecord{ID: f.org, Name: "Synthetic only", Slug: "query-synthetic", Status: domain.OrganizationStatusActive}))
|
||||
for i := 0; i < 3; i++ {
|
||||
must(s.UpsertSupplier(ctx, domain.SupplierRecord{ID: fmt.Sprintf("v%d", i), OrganizationID: f.org, Status: "active", Name: fmt.Sprintf("Synthetic supplier %d", i)}))
|
||||
}
|
||||
for i := 0; i < 20; i++ {
|
||||
id := fmt.Sprintf("p%02d", i)
|
||||
f.projects = append(f.projects, id)
|
||||
must(s.UpsertProject(ctx, domain.ProjectRecord{ID: id, OrganizationID: f.org, Name: "Synthetic " + id, Status: domain.ProjectStatusActive}))
|
||||
must(s.UpsertHumanBatch(ctx, domain.HumanBatchRecord{ID: "b" + id, RunID: "batch-run-" + id, GateNodeID: "gate", OrganizationID: f.org, ProjectID: id, PluginID: "annotation-qa", Status: domain.HumanBatchActive, Stages: domain.DefaultFanyuV5HumanStages()}))
|
||||
}
|
||||
_, err := s.SaveProductionStatsScope(ctx, f.org, productionstats.Scope{ProjectIDs: f.projects, Revision: 1}, 0)
|
||||
must(err)
|
||||
rows := map[string][]any{}
|
||||
push := func(c string, d any) {
|
||||
if c == "annotationTasks" {
|
||||
raw, err := bson.Marshal(d)
|
||||
must(err)
|
||||
var doc bson.M
|
||||
must(bson.Unmarshal(raw, &doc))
|
||||
doc["sourceKey"] = doc["id"]
|
||||
d = doc
|
||||
}
|
||||
rows[c] = append(rows[c], d)
|
||||
}
|
||||
flush := func() {
|
||||
for c, docs := range rows {
|
||||
if len(docs) > 0 {
|
||||
_, err := db.Collection(c).InsertMany(ctx, docs)
|
||||
must(err)
|
||||
}
|
||||
}
|
||||
rows = map[string][]any{}
|
||||
}
|
||||
for i := 0; i < 500; i++ {
|
||||
role := domain.UserRoleAnnotator
|
||||
if i%81 == 0 {
|
||||
role = domain.UserRoleAdmin
|
||||
} else if i%81 >= 73 {
|
||||
role = domain.UserRoleAcceptanceReviewer
|
||||
} else if i%81 >= 61 {
|
||||
role = domain.UserRoleQualityReviewer
|
||||
}
|
||||
uid, vendor := fmt.Sprintf("u%03d", i), fmt.Sprintf("v%d", i%3)
|
||||
push("users", domain.UserRecord{ID: uid, OrganizationID: f.org, Role: role, Name: uid, Phone: fmt.Sprintf("138%08d", i), Status: domain.UserStatusActive, SupplierID: vendor})
|
||||
token, err := randomSessionToken()
|
||||
must(err)
|
||||
f.tokens = append(f.tokens, token)
|
||||
push("sessions", domain.SessionRecord{ID: "session-" + uid, UserID: uid, OrganizationID: f.org, TokenHash: sessionTokenHash(token), ExpiresAt: time.Now().Add(3 * time.Hour).UTC().Format(time.RFC3339), ExpiresAtTime: time.Now().Add(3 * time.Hour)})
|
||||
for _, p := range f.projects {
|
||||
push("projectMembers", domain.ProjectMemberRecord{ID: p + uid, OrganizationID: f.org, ProjectID: p, UserID: uid, SupplierID: vendor})
|
||||
ar := domain.AnnotationRoleAnnotator
|
||||
if role == domain.UserRoleQualityReviewer {
|
||||
ar = domain.AnnotationRoleQualityReviewer
|
||||
}
|
||||
push("annotationMemberships", domain.ProjectAnnotationMembershipRecord{ID: p + uid, OrganizationID: f.org, ProjectID: p, UserID: uid, WorkgroupID: vendor, VendorID: vendor, AssignmentRole: ar})
|
||||
}
|
||||
}
|
||||
flush()
|
||||
for i := 0; i < n; i++ {
|
||||
id := fmt.Sprintf("t%06d", i)
|
||||
p, vendor, uid := f.projects[i%len(f.projects)], fmt.Sprintf("v%d", (i%60+1)%3), fmt.Sprintf("u%03d", i%60+1)
|
||||
at := fmt.Sprintf("2026-11-%02dT10:00:00Z", i%10+1)
|
||||
run := domain.RunRecord{ID: "r" + id, OrganizationID: f.org, ProjectID: p, TaskID: "c" + id, Status: domain.RunStatusWaitingForHuman, CaptureDirectories: []string{"SYNTHETIC/" + id}, CreatedAt: at}
|
||||
task := domain.AnnotationTaskRecord{ID: id, OrganizationID: f.org, ProjectID: p, RunID: run.ID, ArtifactID: "SYNTHETIC/" + id + "/part0001", SourceVideo: "bos://synthetic.invalid/video.mp4", ClipEnd: 60, CurrentRevision: 1, Status: domain.AnnotationTaskStatusPendingQuality, SupplierID: vendor, WorkgroupID: vendor, VendorID: vendor, TemplateVersion: 5, CreatedAt: at, UpdatedAt: at}
|
||||
item := domain.HumanItemRecord{ID: "i" + id, OrganizationID: f.org, ProjectID: p, BatchID: "b" + p, RunID: run.ID, LegacyTaskID: id, PluginID: "annotation-qa", CurrentStageID: domain.FanyuV5StageSemanticReview, Status: domain.HumanItemReady, SupplierID: vendor, WorkgroupID: vendor, VendorID: vendor, LatestAuthorUserID: uid, CurrentRevision: 1, Version: 1, CreatedAt: at, UpdatedAt: at, Subject: domain.HumanSubjectRef{ItemKey: task.ArtifactID, Payload: map[string]any{"durationSeconds": 60, "annotationDraft": strings.Repeat("synthetic protected annotation;", 128)}}}
|
||||
push("runs", run)
|
||||
push("captureRecords", domain.CaptureRecord{ID: run.TaskID, OrganizationID: f.org, ProjectID: p, Prefix: "SYNTHETIC/" + id, CaptureID: "SYNTHETIC/" + id, RunID: run.ID})
|
||||
push("annotationTasks", task)
|
||||
push("humanItems", item)
|
||||
push("annotationRevisions", domain.AnnotationRevisionRecord{ID: "rev" + id, OrganizationID: f.org, TaskID: id, Revision: 1, AuthorUserID: uid, CountsAsOutput: true})
|
||||
push("sliceRevisions", domain.SliceRevisionRecord{ID: "slice" + id, OrganizationID: f.org, TaskID: id, Revision: 1, Segments: []domain.SliceSegment{{ID: "s", Start: 0, End: 60}}})
|
||||
push("annotationWorklogEvents", domain.AnnotationWorklogEventRecord{ID: "event" + id, OrganizationID: f.org, ProjectID: p, TaskID: id, UserID: uid, SupplierID: vendor, EventType: "combined_submitted", Revision: 1, OutputDelta: 1, CreatedAt: at})
|
||||
if i%6 == 0 {
|
||||
push("reviewOperations", domain.ReviewOperation{ID: "op" + id, OrganizationID: f.org, ProjectID: p, ItemID: item.ID, TaskID: id, StageID: "quality_review", Status: "completed", Decision: "rejected", CreatedAt: at, CompletedAt: at, ReviewRound: 1, Draft: domain.ReviewDraft{Problems: []domain.ReviewProblem{{ID: "problem", Scope: "slice", SliceID: "s", Start: 0, End: 10, TimeBasis: "capture-seconds", Text: "Synthetic return detail must survive"}}}, Effects: []domain.ReviewDecisionEffect{{Review: &domain.AnnotationReviewRecord{ID: "review" + id, TaskID: id, Revision: 1}}, {BeforeTask: &task}, {BeforeItem: &item, Item: &item}}})
|
||||
}
|
||||
if (i+1)%500 == 0 {
|
||||
flush()
|
||||
}
|
||||
}
|
||||
flush()
|
||||
return f
|
||||
}
|
||||
@ -12,11 +12,20 @@ import (
|
||||
|
||||
"droboticflow/internal/domain"
|
||||
"droboticflow/internal/store"
|
||||
"droboticflow/internal/testsupport"
|
||||
"go.mongodb.org/mongo-driver/mongo"
|
||||
"go.mongodb.org/mongo-driver/mongo/options"
|
||||
)
|
||||
|
||||
func TestQueryReplicaPoolPermissionRevocationAndSelfReviewLocal(t *testing.T) {
|
||||
testQueryReplicaPermissionRevocationAndSelfReview(t, false)
|
||||
}
|
||||
|
||||
func TestQueryReplicaCountOnlyPermissionRevocationAndSelfReviewLocal(t *testing.T) {
|
||||
testQueryReplicaPermissionRevocationAndSelfReview(t, true)
|
||||
}
|
||||
|
||||
func testQueryReplicaPermissionRevocationAndSelfReview(t *testing.T, countsOnly bool) {
|
||||
if os.Getenv("QUERY_REPLICA_LOCAL_TEST") != "1" {
|
||||
t.Skip("isolated local source and query replica required")
|
||||
}
|
||||
@ -24,7 +33,7 @@ func TestQueryReplicaPoolPermissionRevocationAndSelfReviewLocal(t *testing.T) {
|
||||
ctx := t.Context()
|
||||
clients := []*mongo.Client{}
|
||||
stores := []*store.MongoStore{}
|
||||
for _, uri := range []string{"mongodb://127.0.0.1:27119/?directConnection=true", "mongodb://127.0.0.1:27129/?replicaSet=query-read-rs"} {
|
||||
for _, uri := range []string{"mongodb://127.0.0.1:27119/?directConnection=true", testsupport.QueryReplicaTargetURI(t)} {
|
||||
c, e := mongo.Connect(ctx, options.Client().ApplyURI(uri))
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
@ -108,7 +117,8 @@ func TestQueryReplicaPoolPermissionRevocationAndSelfReviewLocal(t *testing.T) {
|
||||
if e = source.ConfigureQueryReplicaReports(target, time.Minute); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
source.EnableQueryReplicaPoolReads(true)
|
||||
source.EnableQueryReplicaPoolReads(!countsOnly)
|
||||
source.EnableQueryReplicaCountReads(countsOnly)
|
||||
handler := NewHandler(source, nil)
|
||||
request := func(path string) *httptest.ResponseRecorder {
|
||||
r := httptest.NewRequest(http.MethodGet, path, nil)
|
||||
|
||||
@ -36,6 +36,9 @@ type queryServingCounters struct{ reportHits, responseHits, searchHits, typesens
|
||||
|
||||
// QueryServingStatistics exposes counters only, never names or query contents.
|
||||
func (s *MongoStore) QueryServingStatistics() map[string]uint64 {
|
||||
if s.queryReplicaReads != nil && s.queryReplicaReads.target != nil {
|
||||
return s.queryReplicaReads.target.QueryServingStatistics()
|
||||
}
|
||||
if s.queryServing == nil {
|
||||
return map[string]uint64{}
|
||||
}
|
||||
|
||||
@ -325,6 +325,12 @@ func (q *QueryReplica) DrainStep(ctx context.Context, n string) (int, error) {
|
||||
if e != nil {
|
||||
return 0, e
|
||||
}
|
||||
// An empty poll has no target effect or source acknowledgement to fence.
|
||||
// Keep lease fencing on actual changes and PublishReady, rather than writing
|
||||
// the lease once per empty collection on every incremental pass.
|
||||
if len(rows) == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
e = q.transaction(ctx, func(sc mongo.SessionContext) error { return q.apply(sc, n, rows) })
|
||||
if e != nil {
|
||||
return 0, e
|
||||
|
||||
@ -19,7 +19,7 @@ func (s *MongoStore) ConfigureQueryReplicaFromEnvironment(ctx context.Context) (
|
||||
if os.Getenv("QUERY_REPLICA_REPORTS") != "1" {
|
||||
return func() {}, nil
|
||||
}
|
||||
s.queryReplicaReads = &queryReplicaReads{pool: os.Getenv("QUERY_REPLICA_POOLS") == "1"}
|
||||
s.queryReplicaReads = &queryReplicaReads{pool: os.Getenv("QUERY_REPLICA_POOLS") == "1", counts: os.Getenv("QUERY_REPLICA_COUNTS") == "1"}
|
||||
if os.Getenv("QUERY_SOURCE_TRACKING") != "1" {
|
||||
return func() {}, errors.New("query reads require source writer tracking")
|
||||
}
|
||||
@ -50,5 +50,6 @@ func (s *MongoStore) ConfigureQueryReplicaFromEnvironment(ctx context.Context) (
|
||||
}
|
||||
s.queryReplicaReads = r
|
||||
s.EnableQueryReplicaPoolReads(os.Getenv("QUERY_REPLICA_POOLS") == "1")
|
||||
s.EnableQueryReplicaCountReads(os.Getenv("QUERY_REPLICA_COUNTS") == "1")
|
||||
return func() { closeCache(); cleanup() }, nil
|
||||
}
|
||||
|
||||
@ -1,11 +1,37 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"droboticflow/internal/domain"
|
||||
"errors"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestQueryReplicaCountOnlyLeavesListsAndCoreReadsOnAuthority(t *testing.T) {
|
||||
s := &MongoStore{queryReplicaReads: &queryReplicaReads{counts: true}}
|
||||
plain := context.Background()
|
||||
list := WithQueryReplicaPoolRead(plain)
|
||||
count := WithQueryReplicaCountRead(list)
|
||||
if s.UsesQueryReplicaPool(plain) || s.UsesQueryReplicaPool(list) || !s.UsesQueryReplicaPool(count) {
|
||||
t.Fatal("count-only routing escaped its marked count request")
|
||||
}
|
||||
if target, err := s.poolReplica(list); target != nil || err != nil {
|
||||
t.Fatal("unavailable count infrastructure blocked legacy lists", err)
|
||||
}
|
||||
if _, err := s.poolReplica(count); !errors.Is(err, ErrQueryReplicaNotReady) {
|
||||
t.Fatal("explicit count mode silently fell back to authority", err)
|
||||
}
|
||||
s.EnableQueryReplicaPoolReads(true)
|
||||
if !s.UsesQueryReplicaPool(list) || !s.UsesQueryReplicaPool(count) {
|
||||
t.Fatal("legacy list+count switch lost compatibility")
|
||||
}
|
||||
s.EnableQueryReplicaPoolReads(false)
|
||||
s.EnableQueryReplicaCountReads(false)
|
||||
if s.UsesQueryReplicaPool(list) || s.UsesQueryReplicaPool(count) {
|
||||
t.Fatal("disabled modes still routed")
|
||||
}
|
||||
}
|
||||
|
||||
func TestQueryReplicaCandidateCachePrivateMetadataAndIsolation(t *testing.T) {
|
||||
original := domain.HumanItemRecord{ID: "i", LegacyAssignment: map[string]string{"role": "u"}, AccountHandoff: &domain.AccountTaskHandoff{Pending: true}, AcceptanceClaim: &domain.AcceptanceClaimAccess{UserID: "u", SupplierID: "s"}, Subject: domain.HumanSubjectRef{Payload: map[string]any{"annotationDraft": "never copy"}}}
|
||||
safe := cloneQueryCountRow(original)
|
||||
|
||||
@ -15,6 +15,7 @@ import (
|
||||
|
||||
"droboticflow/internal/domain"
|
||||
"droboticflow/internal/productionstats"
|
||||
"droboticflow/internal/testsupport"
|
||||
"droboticflow/internal/workforcestats"
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
"go.mongodb.org/mongo-driver/bson/primitive"
|
||||
@ -27,7 +28,11 @@ import (
|
||||
func newLocalReplica(t *testing.T, monitors ...*event.CommandMonitor) (*MongoStore, *MongoStore, *QueryReplica) {
|
||||
t.Helper()
|
||||
source := newQuerySyncSource(t, monitors...)
|
||||
c, e := mongo.Connect(t.Context(), options.Client().ApplyURI("mongodb://127.0.0.1:27129/?replicaSet=query-read-rs"))
|
||||
targetOptions := options.Client().ApplyURI(testsupport.QueryReplicaTargetURI(t))
|
||||
if len(monitors) > 1 {
|
||||
targetOptions.SetMonitor(monitors[1])
|
||||
}
|
||||
c, e := mongo.Connect(t.Context(), targetOptions)
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
@ -57,6 +62,47 @@ func newLocalReplica(t *testing.T, monitors ...*event.CommandMonitor) (*MongoSto
|
||||
}
|
||||
return source, target, q
|
||||
}
|
||||
|
||||
func TestQueryReplicaEmptyDrainDoesNotWriteLocal(t *testing.T) {
|
||||
var writes atomic.Int64
|
||||
monitor := &event.CommandMonitor{Started: func(_ context.Context, e *event.CommandStartedEvent) {
|
||||
switch e.CommandName {
|
||||
case "update", "insert", "delete", "findAndModify", "commitTransaction":
|
||||
writes.Add(1)
|
||||
}
|
||||
}}
|
||||
_, _, q := newLocalReplica(t, nil, monitor)
|
||||
writes.Store(0)
|
||||
for _, collection := range replicaCollections() {
|
||||
n, err := q.DrainStep(t.Context(), collection)
|
||||
if err != nil || n != 0 {
|
||||
t.Fatal(collection, n, err)
|
||||
}
|
||||
}
|
||||
if got := writes.Load(); got != 0 {
|
||||
t.Fatalf("empty incremental pass wrote %d target commands", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestQueryReplicaWorkforceCacheSeparatesDatabasesLocal(t *testing.T) {
|
||||
source, target, projector := newLocalReplica(t)
|
||||
finishLocalReplica(t, projector)
|
||||
ctx := t.Context()
|
||||
org := domain.NewID("cache-isolation")
|
||||
q := workforcestats.Query{From: time.Date(2026, 11, 1, 0, 0, 0, 0, time.UTC), To: time.Date(2026, 11, 13, 0, 0, 0, 0, time.UTC)}
|
||||
_, err := source.db.Collection("annotationWorklogEvents").InsertOne(ctx, bson.M{"id": "source-only", "organizationId": org, "projectId": "p", "taskId": "t", "userId": "u", "eventType": "combined_submitted", "createdAt": "2026-11-10T10:00:00Z"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
warm, err := source.LoadWorkforceStatsCached(ctx, org, q)
|
||||
if err != nil || len(warm.Events) != 1 {
|
||||
t.Fatal("source fixture", len(warm.Events), err)
|
||||
}
|
||||
got, err := target.LoadWorkforceStatsCached(ctx, org, q)
|
||||
if err != nil || len(got.Events) != 0 {
|
||||
t.Fatal("query database reused authority snapshot", len(got.Events), err)
|
||||
}
|
||||
}
|
||||
func finishLocalReplica(t *testing.T, q *QueryReplica) {
|
||||
t.Helper()
|
||||
for n := 0; n < 100; n++ {
|
||||
@ -311,6 +357,52 @@ func TestQueryReplicaWorkforceDetailsParityLocal(t *testing.T) {
|
||||
|
||||
}
|
||||
|
||||
func TestQueryReplicaWorkforceIncrementalInvalidatesSnapshotLocal(t *testing.T) {
|
||||
s, d, q := newLocalReplica(t)
|
||||
ctx := t.Context()
|
||||
seedWorkforceStats(t, s)
|
||||
finishLocalReplica(t, q)
|
||||
if err := s.ConfigureQueryReplicaReports(d, time.Minute); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
query := workforcestats.Query{ProjectID: "p", UserID: "u", MatchUser: true}
|
||||
before, err := s.LoadWorkforceStatsCached(ctx, "workforce-test", query)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
event := domain.AnnotationWorklogEventRecord{ID: "incremental-rework", OrganizationID: "workforce-test", ProjectID: "p", TaskID: "task", UserID: "u", SupplierID: "v", EventType: "combined_submitted", SubmissionKind: "rework", Revision: 5, CreatedAt: "2026-09-28T12:03:00Z"}
|
||||
if err = s.UpsertAnnotationWorklogEvent(ctx, event); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Restart the projector with its durable checkpoints and fencing identity.
|
||||
restarted, err := NewQueryReplica(s, d)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Simulate the previous process stopping and its lease expiring.
|
||||
if _, err = q.control().UpdateOne(ctx, bson.M{"_id": "lease"}, bson.M{"$set": bson.M{"until": time.Now().Add(-time.Second)}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = restarted.Acquire(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
finishLocalReplica(t, restarted)
|
||||
after, err := s.LoadWorkforceStatsCached(ctx, "workforce-test", query)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(after.Events) != len(before.Events)+1 {
|
||||
t.Fatalf("committed incremental event hidden by old snapshot: before=%d after=%d", len(before.Events), len(after.Events))
|
||||
}
|
||||
if _, err = q.DrainStep(ctx, "annotationWorklogEvents"); err != nil {
|
||||
// Empty old-worker polls have no effect. Actual publication must fence.
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = q.PublishReady(ctx); !errors.Is(err, ErrQueryReplicaLease) {
|
||||
t.Fatal("restarted projector did not fence old publication", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestQueryReplicaPoolSearchCountsAndCurrentAuthorityLocal(t *testing.T) {
|
||||
s, d, q := newLocalReplica(t)
|
||||
ctx := t.Context()
|
||||
|
||||
@ -18,8 +18,8 @@ func WithQueryReplicaPoolRead(ctx context.Context) context.Context {
|
||||
return context.WithValue(ctx, queryReplicaPoolReadKey{}, &queryReplicaPoolBarrier{})
|
||||
}
|
||||
func (s *MongoStore) UsesQueryReplicaPool(ctx context.Context) bool {
|
||||
_, v := ctx.Value(queryReplicaPoolReadKey{}).(*queryReplicaPoolBarrier)
|
||||
return v && s.queryReplicaReads != nil && s.queryReplicaReads.pool
|
||||
b, v := ctx.Value(queryReplicaPoolReadKey{}).(*queryReplicaPoolBarrier)
|
||||
return v && s.queryReplicaReads != nil && (s.queryReplicaReads.pool || b.counts && s.queryReplicaReads.counts)
|
||||
}
|
||||
func (s *MongoStore) poolReplica(ctx context.Context) (*MongoStore, error) {
|
||||
if !s.UsesQueryReplicaPool(ctx) {
|
||||
@ -33,6 +33,14 @@ func (s *MongoStore) EnableQueryReplicaPoolReads(enabled bool) {
|
||||
}
|
||||
}
|
||||
|
||||
// Count-only rollout leaves list discovery and all business mutations on the
|
||||
// authority. The legacy pool switch continues to enable both lists and counts.
|
||||
func (s *MongoStore) EnableQueryReplicaCountReads(enabled bool) {
|
||||
if s.queryReplicaReads != nil {
|
||||
s.queryReplicaReads.counts = enabled
|
||||
}
|
||||
}
|
||||
|
||||
// Query candidates come from the read store; current task scope/status/version
|
||||
// are fetched by bounded IDs before API authorization and canClaim computation.
|
||||
// Missing/changed candidates keep an internal cursor placeholder so deletion
|
||||
|
||||
@ -22,6 +22,7 @@ type queryReplicaReads struct {
|
||||
verifyPermit chan struct{}
|
||||
nextVerify time.Time
|
||||
pool bool
|
||||
counts bool
|
||||
target *MongoStore
|
||||
maxAge time.Duration
|
||||
}
|
||||
|
||||
@ -415,7 +415,18 @@ func (s *MongoStore) LoadWorkforceStatsCached(ctx context.Context, org string, q
|
||||
if !q.To.IsZero() {
|
||||
toKey = q.To.Unix() / 60
|
||||
}
|
||||
key := fmt.Sprintf("%s|%d|%d|%s|%s|%s|%v|%v|%s", org, fromKey, toKey, q.ProjectID, q.SupplierID, q.UserID, q.MatchUser, q.Details, strings.Join(allowed, ","))
|
||||
epoch := int64(0)
|
||||
if s.queryReplicaFactStore {
|
||||
var err error
|
||||
epoch, err = s.countFactEpoch(ctx)
|
||||
if err != nil {
|
||||
return workforcestats.Snapshot{}, err
|
||||
}
|
||||
}
|
||||
// A committed fact change must invalidate the independent read snapshot,
|
||||
// including when projector and API run in separate processes. Authority
|
||||
// readers keep their existing TTL; this epoch is query-database metadata.
|
||||
key := fmt.Sprintf("%p|%s|%d|%s|%d|%d|%s|%s|%s|%v|%v|%s", s.db.Client(), s.db.Name(), epoch, org, fromKey, toKey, q.ProjectID, q.SupplierID, q.UserID, q.MatchUser, q.Details, strings.Join(allowed, ","))
|
||||
if snapshot, ok := workforceStatsCache.get(key, dashboardStatsCacheTTL); ok {
|
||||
return snapshot, nil
|
||||
}
|
||||
|
||||
29
internal/testsupport/query_replica_lab.go
Normal file
29
internal/testsupport/query_replica_lab.go
Normal file
@ -0,0 +1,29 @@
|
||||
package testsupport
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// QueryReplicaTargetURI is test-only infrastructure. It deliberately does not
|
||||
// accept arbitrary hosts, credentials or deployment Mongo environment values.
|
||||
func QueryReplicaTargetURI(t testing.TB) string {
|
||||
t.Helper()
|
||||
const local = "mongodb://127.0.0.1:27129/?replicaSet=query-read-rs"
|
||||
uri := os.Getenv("QUERY_REPLICA_TEST_TARGET_URI")
|
||||
if uri == "" {
|
||||
return local
|
||||
}
|
||||
if err := validateQueryReplicaTarget(uri, os.Getenv("QUERY_REPLICA_CLUSTER_TEST")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return uri
|
||||
}
|
||||
|
||||
func validateQueryReplicaTarget(uri, enabled string) error {
|
||||
if enabled != "1" || uri != "mongodb://127.0.0.1:27229/?directConnection=true&replicaSet=query-lab-rs" {
|
||||
return fmt.Errorf("only the explicitly owned node4 synthetic query tunnel is allowed")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
27
internal/testsupport/query_replica_lab_test.go
Normal file
27
internal/testsupport/query_replica_lab_test.go
Normal file
@ -0,0 +1,27 @@
|
||||
package testsupport
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestQueryReplicaTargetRejectsUnapprovedConnections(t *testing.T) {
|
||||
const allowed = "mongodb://127.0.0.1:27229/?directConnection=true&replicaSet=query-lab-rs"
|
||||
for _, tc := range []struct{ uri, enabled string }{
|
||||
{allowed, ""},
|
||||
{"mongodb://10.189.16.4:27017", "1"},
|
||||
{"mongodb://127.0.0.1:27018", "1"},
|
||||
{allowed + "&authSource=admin", "1"},
|
||||
} {
|
||||
if validateQueryReplicaTarget(tc.uri, tc.enabled) == nil {
|
||||
t.Fatal("accepted a connection outside the owned synthetic tunnel")
|
||||
}
|
||||
}
|
||||
if err := validateQueryReplicaTarget(allowed, "1"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestQueryReplicaTargetDefaultsToLocalLab(t *testing.T) {
|
||||
t.Setenv("QUERY_REPLICA_TEST_TARGET_URI", "")
|
||||
if got := QueryReplicaTargetURI(t); got != "mongodb://127.0.0.1:27129/?replicaSet=query-read-rs" {
|
||||
t.Fatal("default target changed")
|
||||
}
|
||||
}
|
||||
169
scripts/local/query-replica-cluster-fixture.py
Normal file
169
scripts/local/query-replica-cluster-fixture.py
Normal file
@ -0,0 +1,169 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Create only the bounded node4 synthetic source/HTTP fixture, after preflight."""
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import gzip
|
||||
import ipaddress
|
||||
import json
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import uuid
|
||||
|
||||
NODE = "10.189.16.4"
|
||||
NODE_UID = "ca3f254a-8452-4769-8929-6ec303905dbe"
|
||||
NS = "droboticflow-query-lab"
|
||||
NS_UID = "07d647bb-943c-472c-be0a-f6b1380d7105"
|
||||
QUERY_UID = "19f5f918-9bb1-4e6a-a362-bb70afacee26"
|
||||
IMAGE = "docker.io/library/mongo@sha256:02a0cc7939f5ed38f30f9bc714ef5f682d49baf9350c54acf302ce833087fe8a"
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
HARD = {"pods": "2", "requests.cpu": "2", "limits.cpu": "5", "requests.memory": "3Gi", "limits.memory": "5Gi", "services": "0", "persistentvolumeclaims": "0"}
|
||||
|
||||
|
||||
def manifest():
|
||||
env = [{"name": name, "valueFrom": {"fieldRef": {"fieldPath": field}}} for name, field in (("POD_UID", "metadata.uid"), ("POD_NAME", "metadata.name"), ("NODE_NAME", "spec.nodeName"))]
|
||||
env += [{"name": "HOME", "value": "/tmp"}, {"name": "MONGOSH_DISABLE_TELEMETRY", "value": "1"}, {"name": "GOMAXPROCS", "value": "2"}, {"name": "GOMEMLIMIT", "value": "1536MiB"}]
|
||||
command = """set -euo pipefail
|
||||
mkdir -p /synthetic/source
|
||||
mongod --dbpath /synthetic/source --port 27119 --bind_ip 127.0.0.1 --wiredTigerCacheSizeGB 1 --pidfilepath /synthetic/mongod.pid --logpath /synthetic/source.log --setParameter diagnosticDataCollectionEnabled=false &
|
||||
pid=$!
|
||||
stop() { kill -TERM "$pid" 2>/dev/null || true; wait "$pid" || true; }
|
||||
trap stop EXIT
|
||||
trap 'exit 0' TERM INT
|
||||
while kill -0 "$pid" 2>/dev/null; do sleep 10 & wait $!; done
|
||||
wait "$pid"
|
||||
"""
|
||||
pod = {"apiVersion": "v1", "kind": "Pod", "metadata": {"name": "node4-query-fixture", "namespace": NS, "labels": {"app": "node4-query-fixture", "purpose": "mongo-cpu-synthetic"}}, "spec": {"nodeName": NODE, "automountServiceAccountToken": False, "enableServiceLinks": False, "restartPolicy": "Never", "activeDeadlineSeconds": 7200, "terminationGracePeriodSeconds": 45, "securityContext": {"runAsUser": 999, "runAsGroup": 999, "runAsNonRoot": True, "fsGroup": 999}, "containers": [{"name": "lab", "image": IMAGE, "imagePullPolicy": "Never", "command": ["bash", "-c", command], "env": env, "resources": {"requests": {"cpu": "1", "memory": "2Gi", "ephemeral-storage": "32Mi"}, "limits": {"cpu": "4", "memory": "4Gi", "ephemeral-storage": "128Mi"}}, "securityContext": {"allowPrivilegeEscalation": False, "readOnlyRootFilesystem": True, "capabilities": {"drop": ["ALL"]}, "seccompProfile": {"type": "RuntimeDefault"}}, "readinessProbe": {"exec": {"command": ["mongosh", "--quiet", "--host", "127.0.0.1", "--port", "27119", "--eval", "quit(db.adminCommand({ping:1}).ok?0:1)"]}, "periodSeconds": 15, "timeoutSeconds": 5}, "volumeMounts": [{"name": "synthetic", "mountPath": "/synthetic"}, {"name": "tmp", "mountPath": "/tmp"}]}], "volumes": [{"name": "synthetic", "emptyDir": {"medium": "Memory", "sizeLimit": "2Gi"}}, {"name": "tmp", "emptyDir": {"medium": "Memory", "sizeLimit": "64Mi"}}]}}
|
||||
ingress = {"apiVersion": "networking.k8s.io/v1", "kind": "NetworkPolicy", "metadata": {"name": "query-fixture-ingress", "namespace": NS}, "spec": {"podSelector": {"matchLabels": {"app": "node4-query25"}}, "policyTypes": ["Ingress"], "ingress": [{"from": [{"podSelector": {"matchLabels": {"app": "node4-query-fixture"}}}], "ports": [{"protocol": "TCP", "port": 27130}]}]}}
|
||||
egress = {"apiVersion": "networking.k8s.io/v1", "kind": "NetworkPolicy", "metadata": {"name": "query-fixture-egress", "namespace": NS}, "spec": {"podSelector": {"matchLabels": {"app": "node4-query-fixture"}}, "policyTypes": ["Egress"], "egress": [{"to": [{"podSelector": {"matchLabels": {"app": "node4-query25"}}}], "ports": [{"protocol": "TCP", "port": 27130}]}]}}
|
||||
return [ingress, egress, pod]
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description=__doc__)
|
||||
ap.add_argument("--kubeconfig", required=True)
|
||||
ap.add_argument("--output", required=True)
|
||||
ap.add_argument("--api-binary", required=True)
|
||||
ap.add_argument("--bridge-binary", required=True)
|
||||
ap.add_argument("--execute", action="store_true")
|
||||
ap.add_argument("--finish-existing", action="store_true", help="resume this recorded partial fixture without replacing resources")
|
||||
ap.add_argument("--update-api-test", action="store_true", help="stage a new content-addressed test binary after its previous run ended")
|
||||
ap.add_argument("--confirm-plan")
|
||||
args = ap.parse_args()
|
||||
out = Path(args.output).resolve()
|
||||
out.mkdir(parents=True, exist_ok=True)
|
||||
k = ["kubectl", "--kubeconfig", str(Path(args.kubeconfig).resolve())]
|
||||
|
||||
def call(*argv, data=None, timeout=30):
|
||||
p = subprocess.run(k + list(argv), input=data, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=timeout)
|
||||
if p.returncode:
|
||||
raise RuntimeError(p.stderr.decode()[:2000])
|
||||
return p.stdout.decode().strip()
|
||||
|
||||
def get(*argv):
|
||||
return json.loads(call(*argv, "-o", "json"))
|
||||
|
||||
node = get("get", "node", NODE)
|
||||
assert node["metadata"]["uid"] == NODE_UID
|
||||
assert all(c["status"] == "False" for c in node["status"]["conditions"] if c["type"] in ("DiskPressure", "MemoryPressure", "PIDPressure"))
|
||||
top = call("top", "node", NODE, "--no-headers").split()
|
||||
assert float(top[2].rstrip("%")) <= 25 and float(top[4].rstrip("%")) <= 60, "insufficient idle headroom for synthetic fixture"
|
||||
assert get("get", "namespace", NS)["metadata"]["uid"] == NS_UID
|
||||
query = get("-n", NS, "get", "pod", "node4-query25")
|
||||
assert query["metadata"]["uid"] == QUERY_UID and query["status"]["phase"] == "Running"
|
||||
assert query["spec"]["nodeName"] == NODE
|
||||
ip = query["status"]["podIP"]
|
||||
assert ipaddress.IPv4Address(ip) in ipaddress.IPv4Network("10.189.16.0/24")
|
||||
quota = get("-n", NS, "get", "resourcequota", "smoke-budget")
|
||||
original_quota = {"pods": "1", "requests.cpu": "1", "limits.cpu": "1", "requests.memory": "1Gi", "limits.memory": "1Gi", "services": "0", "persistentvolumeclaims": "0"}
|
||||
recorded = None
|
||||
if args.finish_existing:
|
||||
recorded = json.loads((out / "created.json").read_text())
|
||||
previous = json.loads((out / "plan.json").read_text())
|
||||
previous_digest = hashlib.sha256(json.dumps(previous, sort_keys=True).encode()).hexdigest()
|
||||
assert recorded["planSHA256"] == previous_digest
|
||||
assert quota["spec"]["hard"] == HARD
|
||||
existing = get("-n", NS, "get", "pod", "node4-query-fixture")
|
||||
assert existing["metadata"]["uid"] == recorded["fixtureUID"] and existing["spec"]["nodeName"] == NODE
|
||||
assert existing["spec"]["containers"][0]["resources"]["limits"]["cpu"] == "4"
|
||||
assert existing["spec"]["containers"][0]["resources"]["limits"]["memory"] == "4Gi"
|
||||
for obj in manifest()[:2]:
|
||||
assert get("-n", NS, "get", "networkpolicy", obj["metadata"]["name"])["spec"] == obj["spec"]
|
||||
else:
|
||||
assert quota["spec"]["hard"] == original_quota
|
||||
for kind, name in (("pod", "node4-query-fixture"), ("networkpolicy", "query-fixture-ingress"), ("networkpolicy", "query-fixture-egress")):
|
||||
assert not call("-n", NS, "get", kind, name, "--ignore-not-found", "-o", "name"), "existing object must be inspected, never replaced"
|
||||
binaries = {"api": Path(args.api_binary).resolve(), "bridge": Path(args.bridge_binary).resolve()}
|
||||
binary_root = (ROOT / "local-data/diagnostics/mongo-cpu/query25-http/bin").resolve()
|
||||
for path in binaries.values():
|
||||
assert path.parent == binary_root, "only this task's generated binaries may be copied"
|
||||
header = path.read_bytes()[:20]
|
||||
assert header[:4] == b"\x7fELF" and header[18:20] == b"\x3e\x00", "Linux amd64 ELF required"
|
||||
hashes = {name: hashlib.sha256(path.read_bytes()).hexdigest() for name, path in binaries.items()}
|
||||
if recorded:
|
||||
assert hashes["bridge"] == previous["binarySHA256"]["bridge"], "resume must keep the verified bridge"
|
||||
if not args.update_api_test:
|
||||
assert hashes == previous["binarySHA256"], "resume must use the recorded binaries"
|
||||
else:
|
||||
call("-n", NS, "exec", "node4-query-fixture", "-c", "lab", "--", "sh", "-c", 'for p in /proc/[0-9]*/exe; do x=$(readlink "$p" 2>/dev/null || true); case "$x" in /synthetic/query-http*.test) exit 1;; esac; done')
|
||||
objects = manifest()
|
||||
plan = {"kind": "synthetic-in-cluster-comparison", "node": NODE, "namespaceUID": NS_UID, "queryUID": QUERY_UID, "queryIP": ip, "quotaVersion": quota["metadata"]["resourceVersion"], "originalQuota": original_quota, "temporaryQuota": HARD, "objects": objects, "binarySHA256": hashes, "customerData": "none", "newDisk": "none; source is bounded tmpfs", "productionChanges": "none", "queryBridgeStorage": "bounded /tmp memory volume; database noexec retained", "stop": "three degraded readiness/IO/CPU samples; delete only owned fixture", "rollback": "stop owned bridges, delete fixture and its two policies, CAS restore original quota; keep 25Gi data"}
|
||||
if recorded:
|
||||
plan["resumeFrom"] = recorded
|
||||
raw = json.dumps(plan, sort_keys=True).encode()
|
||||
digest = hashlib.sha256(raw).hexdigest()
|
||||
if args.execute:
|
||||
assert args.confirm_plan == digest, "preflight identity changed; inspect a freshly rendered plan"
|
||||
(out / ("finish-plan.json" if recorded else "plan.json")).write_text(json.dumps(plan, indent=2))
|
||||
(out / "objects.json").write_text(json.dumps({"apiVersion": "v1", "kind": "List", "items": objects}, indent=2))
|
||||
print(json.dumps({"planSHA256": digest, "fixture": "node4-query-fixture", "maxCPU": 4, "maxMemory": "4Gi", "sourceStorage": "tmpfs <=2Gi", "queryStorageAndLimits": "unchanged"}), flush=True)
|
||||
if not args.execute:
|
||||
return
|
||||
# No application release, production credential, PVC, Service or host mount.
|
||||
if not recorded:
|
||||
patch = [{"op": "test", "path": "/metadata/resourceVersion", "value": plan["quotaVersion"]}, {"op": "test", "path": "/spec/hard", "value": plan["originalQuota"]}, {"op": "replace", "path": "/spec/hard", "value": HARD}]
|
||||
(out / "intent.json").write_text(json.dumps({"planSHA256": digest, "action": "temporary synthetic quota then server dry-run", "queryUID": QUERY_UID}))
|
||||
call("-n", NS, "patch", "resourcequota", "smoke-budget", "--type=json", "-p", json.dumps(patch))
|
||||
# Admission must see the temporary quota; the original budget is full.
|
||||
call("create", "--dry-run=server", "-f", str(out / "objects.json"))
|
||||
call("create", "-f", str(out / "objects.json"))
|
||||
created = get("-n", NS, "get", "pod", "node4-query-fixture")
|
||||
(out / "created.json").write_text(json.dumps({"fixtureUID": created["metadata"]["uid"], "planSHA256": digest}))
|
||||
call("-n", NS, "wait", "--for=condition=Ready", "pod/node4-query-fixture", "--timeout=180s", timeout=190)
|
||||
fixture = get("-n", NS, "get", "pod", "node4-query-fixture")
|
||||
assert fixture["spec"]["nodeName"] == NODE
|
||||
fuid = fixture["metadata"]["uid"]
|
||||
assert str(uuid.UUID(fuid)) == fuid
|
||||
assert get("-n", NS, "get", "pod", "node4-query25")["metadata"]["uid"] == QUERY_UID
|
||||
qdir = "/tmp/query-http-lab/" + hashes["bridge"][:16]
|
||||
|
||||
def copy(pod, container, local, remote):
|
||||
exists = subprocess.run(k + ["-n", NS, "exec", pod, "-c", container, "--", "test", "-f", remote], capture_output=True, timeout=30).returncode == 0
|
||||
if exists:
|
||||
assert call("-n", NS, "exec", pod, "-c", container, "--", "sha256sum", remote).split()[0] == hashlib.sha256(local.read_bytes()).hexdigest(), "existing fixture artifact differs"
|
||||
return
|
||||
call("-n", NS, "exec", pod, "-c", container, "--", "mkdir", "-p", str(Path(remote).parent))
|
||||
command = "set -eu; umask 077; test ! -e " + remote + "; gzip -dc > " + remote + "; chmod 700 " + remote
|
||||
p = subprocess.run(k + ["-n", NS, "exec", "-i", pod, "-c", container, "--", "sh", "-c", command], input=gzip.compress(local.read_bytes(), compresslevel=1), stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=600)
|
||||
if p.returncode:
|
||||
raise RuntimeError("owned fixture binary transfer failed")
|
||||
actual = call("-n", NS, "exec", pod, "-c", container, "--", "sha256sum", remote).split()[0]
|
||||
assert actual == hashlib.sha256(local.read_bytes()).hexdigest()
|
||||
|
||||
copy("node4-query25", "probe", binaries["bridge"], qdir + "/bridge")
|
||||
copy("node4-query-fixture", "lab", binaries["bridge"], "/synthetic/bridge")
|
||||
api_path = "/synthetic/query-http-" + hashes["api"][:16] + ".test"
|
||||
copy("node4-query-fixture", "lab", binaries["api"], api_path)
|
||||
# Fixed, preflight-derived private addresses and UID arguments only.
|
||||
qcommand = "set -eu; if test -s " + qdir + "/bridge.pid; then p=$(cat " + qdir + "/bridge.pid); if kill -0 \"$p\" 2>/dev/null; then test \"$(readlink /proc/$p/exe)\" = " + qdir + "/bridge; exit 0; fi; fi; nohup " + qdir + "/bridge -listen " + ip + ":27130 -upstream 127.0.0.1:27129 -pod-uid " + QUERY_UID + " >" + qdir + "/bridge.log 2>&1 </dev/null & echo $! >" + qdir + "/bridge.pid"
|
||||
call("-n", NS, "exec", "node4-query25", "-c", "probe", "--", "sh", "-c", qcommand)
|
||||
fcommand = "set -eu; if test -s /synthetic/bridge.pid; then p=$(cat /synthetic/bridge.pid); if kill -0 \"$p\" 2>/dev/null; then test \"$(readlink /proc/$p/exe)\" = /synthetic/bridge; exit 0; fi; fi; nohup /synthetic/bridge -listen 127.0.0.1:27229 -upstream " + ip + ":27130 -pod-uid " + fuid + " >/synthetic/bridge.log 2>&1 </dev/null & echo $! >/synthetic/bridge.pid"
|
||||
call("-n", NS, "exec", "node4-query-fixture", "-c", "lab", "--", "sh", "-c", fcommand)
|
||||
call("-n", NS, "exec", "node4-query-fixture", "-c", "lab", "--", "mongosh", "--quiet", "mongodb://127.0.0.1:27229/?directConnection=true&replicaSet=query-lab-rs", "--eval", 'const m=db.getSiblingDB("query25_synthetic").marker.findOne({_id:"query25-confirmed"});quit(m&&m.value==="preserve-query25-v1"?0:2)')
|
||||
owned = {"fixtureUID": fuid, "queryUID": QUERY_UID, "queryBridgeDirectory": qdir, "apiBinaryPath": api_path, "policies": {obj["metadata"]["name"]: get("-n", NS, "get", "networkpolicy", obj["metadata"]["name"])["metadata"]["uid"] for obj in objects[:2]}, "binarySHA256": hashes, "planSHA256": digest}
|
||||
(out / "owned.json").write_text(json.dumps(owned, indent=2))
|
||||
print("ISOLATED_FIXTURE_READY", flush=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
220
scripts/local/query-replica-isolated-compare.py
Normal file
220
scripts/local/query-replica-isolated-compare.py
Normal file
@ -0,0 +1,220 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run only synthetic fixtures; sample both DB budgets and protect existing IO."""
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import signal
|
||||
import subprocess
|
||||
import time
|
||||
import urllib.request
|
||||
|
||||
NODE = "10.189.16.4"
|
||||
NODE_UID = "ca3f254a-8452-4769-8929-6ec303905dbe"
|
||||
NAMESPACE_UID = "07d647bb-943c-472c-be0a-f6b1380d7105"
|
||||
CONTAINER = "fanyu-query-replica-local-source-1"
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
|
||||
|
||||
def run(args, timeout=20):
|
||||
return subprocess.check_output(args, text=True, timeout=timeout).strip()
|
||||
|
||||
|
||||
def source_fingerprint():
|
||||
fingerprint = hashlib.sha256()
|
||||
files = run(["git", "-C", str(ROOT), "ls-files", "-c", "-o", "--exclude-standard"]).splitlines()
|
||||
for name in sorted(set(files)):
|
||||
if name.startswith(("internal/", "cmd/", "scripts/")) or name in ("go.mod", "go.sum", "package-lock.json"):
|
||||
path = ROOT / name
|
||||
if path.is_file():
|
||||
fingerprint.update(name.encode() + b"\0" + path.read_bytes())
|
||||
return fingerprint.hexdigest()
|
||||
|
||||
|
||||
def cgroup(pod):
|
||||
uid = pod["metadata"]["uid"].replace("-", "_")
|
||||
base = "/host/proc/1/root/sys/fs/cgroup/kubepods.slice/"
|
||||
qos = pod["status"]["qosClass"]
|
||||
if qos == "Guaranteed":
|
||||
return base + "kubepods-pod" + uid + ".slice"
|
||||
if qos not in ("Burstable", "BestEffort"):
|
||||
raise RuntimeError("unknown cgroup layout")
|
||||
part = "kubepods-" + qos.lower()
|
||||
return base + part + ".slice/" + part + "-pod" + uid + ".slice"
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description=__doc__)
|
||||
ap.add_argument("--kubeconfig", required=True)
|
||||
ap.add_argument("--output", required=True)
|
||||
ap.add_argument("--tasks", type=int, default=26218, choices=range(20, 30001))
|
||||
ap.add_argument("--waves", type=int, default=2, choices=range(1, 101))
|
||||
ap.add_argument("--max-actors", type=int, default=81, choices=(81, 200, 500))
|
||||
ap.add_argument("--fixture-plan", help="owned.json from the bounded in-cluster fixture")
|
||||
args = ap.parse_args()
|
||||
out = Path(args.output).resolve()
|
||||
out.mkdir(parents=True, exist_ok=False)
|
||||
k = ["kubectl", "--kubeconfig", str(Path(args.kubeconfig).resolve())]
|
||||
|
||||
def get(*cmd):
|
||||
return json.loads(run(k + list(cmd) + ["-o", "json"]))
|
||||
|
||||
node = get("get", "node", NODE)
|
||||
assert node["metadata"]["uid"] == NODE_UID
|
||||
ns = get("get", "namespace", "droboticflow-query-lab")
|
||||
assert ns["metadata"]["uid"] == NAMESPACE_UID
|
||||
query = get("-n", "droboticflow-query-lab", "get", "pod", "node4-query25")
|
||||
assert query["status"]["phase"] == "Running"
|
||||
assert query["spec"]["nodeName"] == NODE
|
||||
assert all(query["spec"]["containers"][0]["resources"]["limits"].get(k) == v for k, v in {"cpu": "1", "memory": "1Gi"}.items())
|
||||
fixture = None
|
||||
owned = None
|
||||
if args.fixture_plan:
|
||||
owned = json.loads(Path(args.fixture_plan).read_text())
|
||||
fixture = get("-n", "droboticflow-query-lab", "get", "pod", "node4-query-fixture")
|
||||
assert fixture["metadata"]["uid"] == owned["fixtureUID"]
|
||||
assert query["metadata"]["uid"] == owned["queryUID"]
|
||||
assert fixture["status"]["phase"] == "Running" and fixture["spec"]["nodeName"] == NODE
|
||||
budget = fixture["spec"]["containers"][0]["resources"]["limits"]
|
||||
assert budget["cpu"] == "4" and budget["memory"] == "4Gi"
|
||||
api_binary = owned.get("apiBinaryPath", "/synthetic/query-http.test")
|
||||
assert api_binary.startswith("/synthetic/query-http") and api_binary.endswith(".test") and "/../" not in api_binary
|
||||
actual = run(k + ["-n", "droboticflow-query-lab", "exec", "node4-query-fixture", "-c", "lab", "--", "sha256sum", api_binary]).split()[0]
|
||||
assert actual == owned["binarySHA256"]["api"]
|
||||
source = {"Id": fixture["metadata"]["uid"], "Image": fixture["status"]["containerStatuses"][0]["imageID"]}
|
||||
else:
|
||||
source = json.loads(run(["docker", "inspect", CONTAINER]))[0]
|
||||
assert source["State"]["Running"]
|
||||
assert source["HostConfig"]["NanoCpus"] == 4_000_000_000
|
||||
assert source["HostConfig"]["Memory"] == 4_294_967_296
|
||||
socket = run(["docker", "context", "inspect", "--format", "{{.Endpoints.docker.Host}}"])
|
||||
assert socket.startswith("unix://"), "source must be a local Docker daemon"
|
||||
protected = [p for p in get("get", "pods", "-A", "--field-selector", "spec.nodeName=" + NODE)["items"] if p["metadata"]["namespace"] != "droboticflow-query-lab"]
|
||||
prod = get("-n", "droboticflow-fanyu", "get", "pods")
|
||||
|
||||
def pod_states(items):
|
||||
return {p["metadata"]["uid"]: [s.get("restartCount", 0) for s in p["status"].get("containerStatuses", [])] for p in items}
|
||||
|
||||
states = pod_states(prod["items"])
|
||||
# All interpolated paths are generated from validated Kubernetes UIDs/QoS,
|
||||
# never arbitrary paths, credentials or user-provided command fragments.
|
||||
script = "set -e; cat /host/proc/diskstats | awk '$3==\"vda\"'; stat -f --printf='AVAILABLE %a %S\\n' /host/proc/1/root; "
|
||||
for p in protected:
|
||||
script += "echo PROTECTED " + p["metadata"]["uid"] + "; cat " + cgroup(p) + "/io.pressure; "
|
||||
for filename in ("cpu.stat", "memory.current", "memory.events", "io.stat", "io.max"):
|
||||
script += "echo QUERY " + filename + "; cat " + cgroup(query) + "/" + filename + "; "
|
||||
if fixture:
|
||||
for filename in ("cpu.stat", "memory.current", "memory.events", "io.stat"):
|
||||
script += "echo SOURCE " + filename + "; cat " + cgroup(fixture) + "/" + filename + "; "
|
||||
previous = None
|
||||
|
||||
def sample():
|
||||
nonlocal previous
|
||||
started = time.monotonic()
|
||||
with urllib.request.urlopen("http://120.48.180.80/api/ready", timeout=5) as response:
|
||||
assert response.status == 200
|
||||
response.read(65536)
|
||||
ready = time.monotonic() - started
|
||||
current = get("-n", "droboticflow-fanyu", "get", "pods")
|
||||
assert pod_states(current["items"]) == states, "existing production Pod/restart set changed"
|
||||
health = get("get", "node", NODE)
|
||||
assert all(c["status"] == "False" for c in health["status"]["conditions"] if c["type"] in ("DiskPressure", "MemoryPressure", "PIDPressure"))
|
||||
host = run(k + ["-n", "kube-system", "exec", "node-problem-detector-6v59v", "-c", "node-problem-detector", "--", "sh", "-c", script])
|
||||
local = "included in host SOURCE fields" if fixture else run(["docker", "exec", CONTAINER, "sh", "-c", "cat /sys/fs/cgroup/cpu.stat; cat /sys/fs/cgroup/memory.current; cat /sys/fs/cgroup/io.stat"])
|
||||
top = run(k + ["top", "node", NODE, "--no-headers"]).split()
|
||||
cpu_pct = float(top[2].rstrip("%"))
|
||||
if fixture:
|
||||
current_fixture = get("-n", "droboticflow-query-lab", "get", "pod", "node4-query-fixture")
|
||||
assert current_fixture["metadata"]["uid"] == owned["fixtureUID"] and current_fixture["status"]["phase"] == "Running", "owned source fixture stopped or was replaced"
|
||||
available = next(line.split() for line in host.splitlines() if line.startswith("AVAILABLE "))
|
||||
assert int(available[1]) * int(available[2]) >= 80_530_636_800, "node free space below 75Gi"
|
||||
pressure = []
|
||||
inside = False
|
||||
for line in host.splitlines():
|
||||
if line.startswith("PROTECTED "):
|
||||
inside = True
|
||||
elif line.startswith(("QUERY ", "SOURCE ")):
|
||||
inside = False
|
||||
elif inside and line.startswith("full "):
|
||||
pressure.append(float(line.split("avg10=")[1].split()[0]))
|
||||
assert len(pressure) == len(protected)
|
||||
disk = [int(v) for v in next(line for line in host.splitlines() if line.split()[:3] == ["252", "0", "vda"]).split()[3:]]
|
||||
record = {"time": time.time(), "readySeconds": ready, "sourceCgroup": local, "host": host, "nodeCPUPct": cpu_pct, "protectedFullAvg10": max(pressure, default=0), "disk": disk, "awaitMs": 0, "busyPct": 0}
|
||||
if previous:
|
||||
ops = disk[0] + disk[4] - previous["disk"][0] - previous["disk"][4]
|
||||
record["awaitMs"] = (disk[3] + disk[7] - previous["disk"][3] - previous["disk"][7]) / max(ops, 1)
|
||||
record["busyPct"] = (disk[9] - previous["disk"][9]) / max(record["time"] - previous["time"], 0.001) / 10
|
||||
previous = record
|
||||
with (out / "resources.jsonl").open("a") as f:
|
||||
f.write(json.dumps(record) + "\n")
|
||||
return record
|
||||
|
||||
baseline = [sample() for _ in range(3)]
|
||||
limits = {"ready": max(1, max(s["readySeconds"] for s in baseline) * 2), "pressure": max(0.5, max(s["protectedFullAvg10"] for s in baseline) + 0.5), "await": max(10, max(s["awaitMs"] for s in baseline) * 3)}
|
||||
manifest = {"sourceCommit": run(["git", "-C", str(ROOT), "rev-parse", "HEAD"]), "sourceFingerprint": source_fingerprint(), "sourceContainer": source["Id"], "sourceImage": source["Image"], "sourceCPU": 4, "sourceMemory": 4_294_967_296, "queryPodUID": query["metadata"]["uid"], "queryImage": query["spec"]["containers"][0]["image"], "queryCPU": 1, "queryMemory": "1Gi", "tasks": args.tasks, "waves": args.waves, "maxActors": args.max_actors, "wireCompressionBothVariants": os.getenv("QUERY_COMPARE_WIRE_COMPRESSION", ""), "minimumPoolBothVariants": os.getenv("QUERY_COMPARE_MIN_POOL", "0"), "limits": limits, "limitations": ["source Docker VM shares CPU with existing local containers", "target has 2MiB read / 1MiB write and 64/32 IOPS hard limits", "short comparison; not 30-minute three-round production capacity acceptance", "health probes do not establish customer workflow latency"]}
|
||||
if fixture:
|
||||
manifest["limitations"][0] = "source Mongo and native HTTP/projector/builder/client share one fixed 4CPU/4Gi Pod; source storage is bounded tmpfs"
|
||||
manifest["sourceCPUCounter"] = "mongod process utime+stime; whole fixture Pod cgroup sampled separately"
|
||||
manifest["binarySHA256"] = owned["binarySHA256"]
|
||||
(out / "manifest.json").write_text(json.dumps(manifest, indent=2))
|
||||
env = dict(os.environ, QUERY_REPLICA_ISOLATED_E2E="1", QUERY_REPLICA_CLUSTER_TEST="1", QUERY_REPLICA_TEST_TARGET_URI="mongodb://127.0.0.1:27229/?directConnection=true&replicaSet=query-lab-rs", QUERY_COMPARE_TASKS=str(args.tasks), QUERY_COMPARE_WAVES=str(args.waves), QUERY_COMPARE_MAX_ACTORS=str(args.max_actors), QUERY_COMPARE_REPORT=str(out / "comparison.json"))
|
||||
argv = ["go", "test", "./internal/api", "-run", "^TestQueryReplicaIsolatedHTTPComparisonLocal$", "-count=1", "-timeout=45m", "-v"]
|
||||
remote_report = "/synthetic/query-comparison-" + str(int(time.time())) + ".json"
|
||||
if fixture:
|
||||
settings = ["QUERY_REPLICA_ISOLATED_E2E=1", "QUERY_REPLICA_CLUSTER_TEST=1", "QUERY_REPLICA_TEST_TARGET_URI=mongodb://127.0.0.1:27229/?directConnection=true&replicaSet=query-lab-rs", "QUERY_COMPARE_IN_CLUSTER=1", "QUERY_COMPARE_TASKS=" + str(args.tasks), "QUERY_COMPARE_WAVES=" + str(args.waves), "QUERY_COMPARE_MAX_ACTORS=" + str(args.max_actors), "QUERY_COMPARE_REPORT=" + remote_report, "QUERY_COMPARE_WIRE_COMPRESSION=" + os.getenv("QUERY_COMPARE_WIRE_COMPRESSION", ""), "QUERY_COMPARE_MIN_POOL=" + os.getenv("QUERY_COMPARE_MIN_POOL", "0")]
|
||||
argv = k + ["-n", "droboticflow-query-lab", "exec", "node4-query-fixture", "-c", "lab", "--", "env"] + settings + [api_binary, "-test.run=^TestQueryReplicaIsolatedHTTPComparisonLocal$", "-test.count=1", "-test.timeout=45m", "-test.v"]
|
||||
with (out / "test.log").open("w") as log:
|
||||
child = subprocess.Popen(argv, cwd=ROOT, env=env, stdout=log, stderr=subprocess.STDOUT, start_new_session=True)
|
||||
bad = 0
|
||||
try:
|
||||
while child.poll() is None:
|
||||
record = sample()
|
||||
degraded = record["readySeconds"] > limits["ready"] or record["protectedFullAvg10"] > limits["pressure"] or record["awaitMs"] > limits["await"] or record["busyPct"] > 70 or record["nodeCPUPct"] > 75
|
||||
bad = bad + 1 if degraded else 0
|
||||
if bad >= 3:
|
||||
raise RuntimeError("existing workload health/IO degraded for three samples")
|
||||
time.sleep(5)
|
||||
except BaseException as exc:
|
||||
stopped = {"reason": str(exc), "syntheticDatabasesMayNeedOwnedCleanup": True, "cleanupErrors": []}
|
||||
# Record the original stop even if the API or the child disappears
|
||||
# during cleanup. Never delete a replacement Pod by name alone.
|
||||
(out / "stopped.json").write_text(json.dumps(stopped))
|
||||
try:
|
||||
if fixture:
|
||||
current = get("-n", "droboticflow-query-lab", "get", "pod", "node4-query-fixture")
|
||||
if current["metadata"]["uid"] == owned["fixtureUID"]:
|
||||
delete = out / "delete-owned-fixture.json"
|
||||
delete.write_text(json.dumps({"apiVersion": "v1", "kind": "DeleteOptions", "preconditions": {"uid": owned["fixtureUID"]}}))
|
||||
run(k + ["delete", "--raw", "/api/v1/namespaces/droboticflow-query-lab/pods/node4-query-fixture", "-f", str(delete)])
|
||||
except Exception as cleanup_error:
|
||||
stopped["cleanupErrors"].append(str(cleanup_error))
|
||||
finally:
|
||||
if child.poll() is None:
|
||||
try:
|
||||
os.killpg(child.pid, signal.SIGTERM)
|
||||
except ProcessLookupError:
|
||||
pass
|
||||
try:
|
||||
child.wait(timeout=30)
|
||||
except subprocess.TimeoutExpired:
|
||||
try:
|
||||
os.killpg(child.pid, signal.SIGKILL)
|
||||
except ProcessLookupError:
|
||||
pass
|
||||
child.wait(timeout=5)
|
||||
(out / "stopped.json").write_text(json.dumps(stopped))
|
||||
raise
|
||||
sample()
|
||||
if fixture:
|
||||
result = subprocess.run(k + ["-n", "droboticflow-query-lab", "exec", "node4-query-fixture", "-c", "lab", "--", "cat", remote_report], capture_output=True, text=True, timeout=30)
|
||||
if result.returncode == 0:
|
||||
(out / "comparison.json").write_text(result.stdout)
|
||||
changed = source_fingerprint() != manifest["sourceFingerprint"]
|
||||
(out / "exit.json").write_text(json.dumps({"returncode": child.returncode, "sourceChanged": changed}))
|
||||
raise SystemExit(child.returncode or (2 if changed else 0))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
39
scripts/local/test_query_replica_cluster_fixture.py
Normal file
39
scripts/local/test_query_replica_cluster_fixture.py
Normal file
@ -0,0 +1,39 @@
|
||||
import importlib.util
|
||||
from pathlib import Path
|
||||
import unittest
|
||||
|
||||
spec = importlib.util.spec_from_file_location("cluster_fixture", Path(__file__).with_name("query-replica-cluster-fixture.py"))
|
||||
fixture = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(fixture)
|
||||
|
||||
|
||||
class FixtureSafety(unittest.TestCase):
|
||||
def test_source_is_memory_only_and_unprivileged(self):
|
||||
pod = fixture.manifest()[-1]["spec"]
|
||||
self.assertEqual(pod["nodeName"], "10.189.16.4")
|
||||
self.assertFalse(pod["automountServiceAccountToken"])
|
||||
self.assertFalse(pod.get("hostNetwork", False))
|
||||
self.assertEqual(pod["activeDeadlineSeconds"], 7200)
|
||||
for volume in pod["volumes"]:
|
||||
self.assertEqual(volume["emptyDir"]["medium"], "Memory")
|
||||
self.assertIn("sizeLimit", volume["emptyDir"])
|
||||
self.assertNotIn("hostPath", volume)
|
||||
container = pod["containers"][0]
|
||||
self.assertEqual(container["resources"]["limits"]["cpu"], "4")
|
||||
self.assertEqual(container["resources"]["limits"]["memory"], "4Gi")
|
||||
self.assertFalse(container["securityContext"]["allowPrivilegeEscalation"])
|
||||
self.assertTrue(container["securityContext"]["readOnlyRootFilesystem"])
|
||||
self.assertEqual(container["imagePullPolicy"], "Never")
|
||||
|
||||
def test_network_is_only_fixture_to_query_private_port(self):
|
||||
ingress, egress, _ = fixture.manifest()
|
||||
self.assertEqual(ingress["spec"]["podSelector"]["matchLabels"], {"app": "node4-query25"})
|
||||
self.assertEqual(ingress["spec"]["ingress"], [{"from": [{"podSelector": {"matchLabels": {"app": "node4-query-fixture"}}}], "ports": [{"protocol": "TCP", "port": 27130}]}])
|
||||
self.assertEqual(egress["spec"]["egress"], [{"to": [{"podSelector": {"matchLabels": {"app": "node4-query25"}}}], "ports": [{"protocol": "TCP", "port": 27130}]}])
|
||||
self.assertEqual(fixture.HARD["services"], "0")
|
||||
self.assertEqual(fixture.HARD["persistentvolumeclaims"], "0")
|
||||
self.assertEqual(fixture.HARD["pods"], "2")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@ -970,7 +970,9 @@
|
||||
"name": "node4-query-pilot-guardrails",
|
||||
"paths": [
|
||||
"^scripts/local/node4-query-pilot/",
|
||||
"^docs/operations/query-replica-node4-(isolation-plan|smoke-results)\\.md$"
|
||||
"^docs/operations/query-replica-node4-(isolation-plan|smoke-results)\\.md$",
|
||||
"^scripts/local/(query-replica-(cluster-fixture|isolated-compare)|test_query_replica_cluster_fixture)\\.py$",
|
||||
"^docs/operations/query-replica-query25-comparison\\.md$"
|
||||
],
|
||||
"frontend": [],
|
||||
"go": {},
|
||||
@ -988,6 +990,10 @@
|
||||
"-v"
|
||||
],
|
||||
"tests": true
|
||||
},
|
||||
{
|
||||
"argv": ["python3", "-m", "unittest", "discover", "-s", "scripts/local", "-p", "test_query_replica_cluster_fixture.py", "-v"],
|
||||
"tests": true
|
||||
}
|
||||
],
|
||||
"warnings": ["仅执行离线安全回归;节点写回归属、Pod重建、满盘及容量仍须已授权的独立合成实测,不能以该检查替代。"]
|
||||
@ -998,6 +1004,7 @@
|
||||
"^internal/store/query_(replica|sync|tracked_collection|collection_interface).*\\.go$",
|
||||
"^internal/api/query_replica.*\\.go$",
|
||||
"^cmd/(query-replica|query-cdc-probe)/",
|
||||
"^cmd/query-lab-bridge/",
|
||||
"^scripts/test_query_replica\\.py$",
|
||||
"^scripts/local/query-replica\\.compose\\.yml$",
|
||||
"^Dockerfile\\.query-replica$",
|
||||
@ -1007,10 +1014,12 @@
|
||||
"go": {
|
||||
"./internal/store": [
|
||||
"^TestQueryReplicaCandidateCache",
|
||||
"^TestQueryReplicaCountOnly",
|
||||
"^TestQueryReplicaMissingConfiguration",
|
||||
"^TestQueryReplicaRejectsSigned"
|
||||
],
|
||||
"./cmd/query-cdc-probe": ["^TestClassification"]
|
||||
"./cmd/query-cdc-probe": ["^TestClassification"],
|
||||
"./cmd/query-lab-bridge": ["^TestIsolatedBridge"]
|
||||
},
|
||||
"task": [
|
||||
{
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user