✨ Serve periodic query publications with live authorization
This commit is contained in:
parent
f3328f148b
commit
50228cdb21
@ -24,7 +24,22 @@ func main() {
|
||||
once := flag.Bool("once", false, "one bounded pass only")
|
||||
interval := flag.Duration("interval", time.Second, "minimum delay between bounded passes")
|
||||
flag.Parse()
|
||||
if !*writers || *interval < 100*time.Millisecond || os.Getenv("QUERY_SERVING_NAMESPACE") == "" {
|
||||
mode := os.Getenv("QUERY_REPLICA_REFRESH_MODE")
|
||||
if mode != "" && mode != "continuous" && mode != "periodic" {
|
||||
log.Fatal("invalid query refresh mode")
|
||||
}
|
||||
if mode == "periodic" {
|
||||
explicit := false
|
||||
flag.Visit(func(f *flag.Flag) {
|
||||
if f.Name == "interval" {
|
||||
explicit = true
|
||||
}
|
||||
})
|
||||
if !explicit {
|
||||
*interval = store.QueryPeriodicSyncInterval
|
||||
}
|
||||
}
|
||||
if !*writers || *interval < 100*time.Millisecond || *interval > 5*time.Minute || os.Getenv("QUERY_SERVING_NAMESPACE") == "" {
|
||||
log.Fatal("writer coverage, namespace and bounded polling interval required")
|
||||
}
|
||||
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
||||
@ -76,10 +91,16 @@ func main() {
|
||||
}
|
||||
return
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-time.After(*interval):
|
||||
// Drain bounded backlog immediately, then idle without repeated source
|
||||
// scans. Keep the original lease alive only in the independent query DB.
|
||||
if ready || mode != "periodic" {
|
||||
if e := q.WaitNextPass(ctx, *interval); e != nil {
|
||||
if ctx.Err() == nil {
|
||||
log.Print("projector lease unavailable; checkpoint retained")
|
||||
os.Exit(1)
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -39,3 +39,16 @@
|
||||
人员统计复用原历史计算器生成无正文贡献,按不可变版本发布。完整新版本提交前始终保留上一版本;重启清理未提交暂存,已提交历史按有界宽限清理。历史供应商归属和正式操作抑制旧审查仍依查询范围解析,日期边界保留纳秒。后台当前使用有界组织快照计算与差量写入,尚非逐任务分区增量计算;超预算保留旧版本,不截断数据、不增加核心作业准入条件。
|
||||
|
||||
作业记录在查询库筛选计数及分页后,仅补选中页详情与权威文字,来源指纹不一致时拒绝拼接。旧任务/未首次物化的库保留读库历史兼容路径。极少数等价时间字符串的排序统一为绝对时间和稳定身份,避免偏移页漂移。以上实现仍需完整规模性能及持久化确认压力验收,不能将本地功能测试当作上线承诺。
|
||||
|
||||
|
||||
## 周期刷新与延迟展示(2026-11-13,用户批准,本地实施)
|
||||
|
||||
显式 `QUERY_REPLICA_REFRESH_MODE=periodic` 启用新契约;缺省或 `continuous` 保持原兼容语义,报告/任务池读取开关仍分别默认关闭。原库按持久变更标记增量同步:每次追平后空闲约1分钟,待处理批次继续有界排空;空闲只在查询库续租,不反复扫描原库。历史仍限速、可恢复地一次回填,低频对账保留。
|
||||
|
||||
查询库内变化按组织合并,搜索重建至少间隔1分钟、统计至少间隔3分钟,初次构建立即执行。原有变更观察/租约栅栏保留,减少心跳、预热和历史清理的轮询次数;权限不进入缓存。当前统计仍是按受影响组织读取快照、归约后差量写入,不宣称已经实现逐任务增量计算。重启会重新核对并重建,未完成暂存不能替换已发布版本。
|
||||
|
||||
展示优先读取已多数提交的结果;允许同步水位/报表计算落后,15分钟数据年龄边界与绑定、schema、未来时间检查仍保留。源同步水位失效不会回扫原库。统计刷新待处理或失败时继续提供时间边界内的旧完整版本;新模式下生产报表/常规人员统计首次发布前返回未就绪,不因浏览器请求启动全历史计算。历史少见查询语义仍保留查询库兼容计算路径,不回源扫描。人员统计返回可选 `asOf`,生产报表沿用既有时间字段;时间为结果实际计算时间,不冒充实时。
|
||||
|
||||
计数展示不再读取原库脏任务/删除意图做即时补偿,因此数量是查询库已提交事实的结果,不保证包含刚刚发生的操作。批次权限/策略、账号、成员、供应商权限继续实时校验;列表维持本页当前归属/版本读取与原服务端可见性/`canClaim`,领取、保存、送检、质检、验收和后台调度均沿原权威路径。此变更不允许旧缓存绕过权限,也不允许以旧任务列表重复领取。
|
||||
|
||||
本次只在合成隔离环境验证,不切换生产、不修改客户数据。固定原请求超时、租约和多数提交要求;延迟展示是独立业务契约,不是延长请求超时或降低持久性来通过测试。验证必须单列本机与25Gi共享盘结果,以及冷构建、故障恢复和持续变化期间的覆盖限制。
|
||||
|
||||
@ -108,3 +108,48 @@ QUERY_REPLICA_LOCAL_TEST=1 QUERY_REPLICA_SCALE_LOCAL=1 go test ./internal/store
|
||||
本批只读存储盘点:当前注册的 StorageClass 为静态 `csi-bos`,现有80Gi本地卷和200Gi BOS卷均已占用,未做修改。隔离验收仍需独立SSD持久卷,不能将现有BOS挂载视为已满足数据库文件系统要求。MongoDB要求目录fsync等文件系统语义,并推荐WiredTiger使用XFS,参见[官方生产说明](https://www.mongodb.com/docs/manual/administration/production-notes/)。节点allocatable不等于空闲资源;尚未创建Pod、PV、StorageClass或Secret。
|
||||
|
||||
定稿性能复测中,单项目500并发曾有70次就绪检查失败。已将重复的水位读取合并为仅在途共享(无跨请求TTL、无权限缓存),以真实Mongo命令监控证明100个同时检查只读一次;相同夹具复测新路径所有档位均零错误。旧路径仍有超时,因此组合对照仍标为失败,而非性能验收通过。最终全部项目500并发p95528ms仍高于暖态500ms目标;81/200档通过的只是存储计数访问器,完整HTTP/业务授权及长稳态仍待验收。
|
||||
|
||||
|
||||
## 周期展示模式(本地实施,未切生产)
|
||||
|
||||
`QUERY_REPLICA_REFRESH_MODE=periodic` 需要在 API、query-replica 与 query-read-builder 一致配置。它不启用任何原先关闭的报告/计数/任务池读取开关,也不替代写者覆盖和首次完整同步验收。无该设置时保留原连续模式。
|
||||
|
||||
- 投影器追平后约60秒再做增量读取,积压按原200条有界批次连续排空;空闲每10秒只在独立查询库续原30秒租约。失权或三秒确认失败退出并保留断点,不扫描原库保活,不延长租约。
|
||||
- 变化按组织合并,搜索至少60秒、统计至少180秒计算一次;首次构建立即开始。统计复用旧归约器并差量发布,尚非逐任务分区计算;64MiB/250,000行背景快照上限仍在。
|
||||
- 展示读已提交结果,刷新期间保留旧完整版本。同步水位与结果年龄的正常延迟允许到15分钟边界;原绑定/schema/未来时间校验保留,超界报告未就绪且不回源全扫。计数不再逐请求追读原库脏任务;批次策略、用户/成员/供应商权限仍即时读取,列表本页归属/版本和全部正式操作仍走权威路径。
|
||||
- 人员统计可选 `asOf` 与既有生产报表时间标识实际发布数据时间;不是请求时间。首次无已发布结果时不由客户请求触发组织全量计算。旧模式和历史少见查询兼容路径保留。
|
||||
|
||||
本轮证据保存在 `local-data/diagnostics/mongo-cpu/periodic-refresh/`:
|
||||
|
||||
1. `red.log` 先复现已发布两分钟的结果被同步延迟拒绝、单次计数追加3次原库追平读取;对应回归修复后通过。生命周期覆盖旧完整版本、变化合并、未提交暂存恢复、边界过期和来源隔离;真实HTTP权限撤销、未同步批次策略变化、自审禁止及标注保留通过。
|
||||
2. 首次26218任务本机对照 `http-26218.log` 因统一就绪条件仍要求秒级观察水位而失败,原记录保留。修改仅对新周期模式采用已发布代次及年龄的就绪规则,原四分钟等待/请求超时及持久性不变。
|
||||
3. `http-26218-published.json`:26218任务、20项目、81账号、两轮162请求,新路径零错误,总p95=1325.065ms(计数1335.207、人员统计553.760、作业记录309.176、生产总览40.549ms)。旧路径16次MaxTimeMS失败,因此整体测试失败,不能从这轮报告严格等成功降载比例;人员统计混合冷热p95也不作为暖态500ms目标通过。
|
||||
4. `http-26218-matched.json`:相同数据、81账号、162请求,新旧均限制8个在途请求,仅为等成功降载/逐账号结果补充;双方零错误、162份结果逐条匹配、无差异。源Mongo累计CPU 15.272151→2.314956秒(下降84.84%);扫描文档10,078,564→1,450,972(下降85.60%);逻辑BSON返回字节下降78.91%。该轮整体p95旧710.792ms、新785.101ms,不能宣称低并发整体延迟下降;新路径人员统计99.604、生产5.574、作业记录61.113、计数811.500ms。
|
||||
|
||||
两轮均固定源Mongo4CPU/4Gi、查询Mongo1CPU/1Gi、WT缓存0.25Gi;Go HTTP/构建进程在本机,两个Mongo共享Docker主机,没有`.4`节点共享盘IO限额。包含请求测量期间的实际同步/对账/确认开销,但短时不包含长期持续变化/缓存失效所有周期。新旧源CPU为累计用时,不能当作线上CPU百分比;逻辑BSON字节不能当作公网视频带宽。余下较多源读取是当前授权/旧授权兼容路径,未引入跨请求权限缓存来换性能。
|
||||
|
||||
共享盘25Gi冷构建和完整HTTP对照、长期稳定性与暖态500ms仍单列待验收;本机证据不替代它们。不改生产数据、路由或资源,不因本地通过准备部署。
|
||||
|
||||
|
||||
### 共享盘冷构建及未采用实验
|
||||
|
||||
`periodic-refresh/cluster-26218` 在原25Gi、1CPU/1Gi、2MiB/s读/1MiB/s写及64/32 IOPS下实际运行:历史复制372.203秒;搜索78.042秒;统计加载/归约2.435秒,确认写入17,720行/26,559,500字节,统计阶段95.638秒,生产/统计读表总阶段112.633秒后发布。随后产出快照约47秒仍在构建时触及原四分钟就绪预算,取消;未进入HTTP阶段,没有本轮p95/CPU降载比,`coverageComplete=false`。终止时取消上下文会令元数据读取均失败,不能把终端打印的全false理解成已发布版本丢失;后续诊断保留最后一次有效就绪状态。
|
||||
|
||||
该轮生产健康保护未触发,最终健康无错误;源夹具/策略/桥接按UID/PID清理,配额CAS恢复,25Gi标记保留。没有客户记录或生产路由变更。
|
||||
|
||||
曾尝试让初次产出快照与搜索重叠、共用原写入预算;小回归及竞态验证通过,但同规模本机1Gi查询Mongo出现OOM(exit137),对应 `overlap-oom.json`、`overlap-experiment.patch` 和 `http-26218-overlap-matched.log`。不能将并行视为安全优化,也未证明并行是内存峰值的唯一原因。**最终代码撤回该调度,保留搜索/冷快照串行保护,不增资源、不降持久性、不放宽超时。** 本机Mongo按原容器/配额恢复,仅删除该次精确随机测试库。该实验未推送、未在公司集群执行。
|
||||
|
||||
正式任务检查 `verify-1794585127732124000/report.json` 中 Go基线、看板/刷新浏览器、统计稳定性、查询服务race与完整Mongo/Valkey副本race均通过;系统Python缺pytest项用现有根目录venv复跑37通过/4个真实交付夹具缺失跳过。额外原始API全包仍失败,分类依现有Go门禁:68项均在基线、0新增;不手工扩失败基线。原任务失败/跳过记录均保留,不将整项标成全绿。
|
||||
|
||||
周期刷新/已发布结果读取的实现和本机降载证据可以继续评估;共享盘冷容量仍不通过,因此不准备生产发布。后续优先把旧产出快照重复历史读取/构建开销独立量化并减少,不以并发堆积、提高IO限额、扩内存或延长等待来冒充当前配置达标。
|
||||
|
||||
|
||||
### 最终串行候选的补充验证与剩余风险
|
||||
|
||||
- 撤回重叠调度后,`http-final-matched.log` 在管理员人员统计首次聚合出现一次 MaxTimeMSExpired,未形成完整对照。慢日志元数据 `final-fact-slow-metadata.json` 显示规划约5.8ms、执行CPU约834ms;不能据此认定规划是根因,也不能用上一次成功结果掩盖本次失败。
|
||||
- 增加了可选启动命令监控接线,使实际API查询客户端也能记录预算,默认运行不安装监控。此前预算探针只接到构建客户端而未捕获HTTP事实聚合,该空报告不能作为API预算证据。
|
||||
- 带诊断对照 `http-diagnostic-matched.json` 新路径162请求零错误、160份可配对业务结果一致,但旧路径2次失败;整体仍失败,不能据此重新报告严格降载比例。该次仅保留两个明确本机合成库,供快速复查:源 `query_http_synthetic-58b6b07eb71b63db7f4541fefa653bfa`、查询 `query_http_synthetic-96fa522d81f68b11f10d0dd4f8394644`;客户数据不在其中。
|
||||
- 使用该保留夹具、真实HTTP/会话/管理员授权重放3次均200,首次实际聚合约1518ms,后续命中缓存。`replay-budget.json` 确认发送maxTimeMS=8000,上下文未额外缩短预算。临时合成会话已清理。单次冷态失败仍未稳定复现,不能标为已修复;本机容器与宿主时钟相差约19秒的观测也不能直接归因为该错误,未更改全局时钟或其他工作树的资源。
|
||||
- 最终快速检查 `verify-1794589491130862000/report.json` 所选检查通过;最终完整副本race/cache记录 `check-5b8d9540efff4b01825a878e91549000.log` 无缺失/跳过/源码变化。新增串行调度保护回归保留,失败的并行实验日志保留。前述原始任务检查的交付夹具跳过、原始API既有失败仍如实保留,不将最终状态写成整体验收通过。
|
||||
|
||||
本批没有推送、PR、生产应用或数据变更;独立查询库周期读取开关仍默认关闭。下一步先稳定复现冷态统计超时,并减少旧产出快照和统计的重复读/持久化确认轮次;达到原配置下的完整正确性、速度与资源对照后再准备上线。
|
||||
|
||||
@ -85,7 +85,7 @@ func (server *server) populateDashboardWorkforce(ctx context.Context, requestOrg
|
||||
return err
|
||||
}
|
||||
summary.AudienceScope = scope
|
||||
summary.Workforce = domain.DashboardWorkforceSummary{From: from.Format(time.RFC3339Nano), To: to.Format(time.RFC3339Nano), Totals: report.Totals, Items: report.Items}
|
||||
summary.Workforce = domain.DashboardWorkforceSummary{AsOf: report.AsOf, From: from.Format(time.RFC3339Nano), To: to.Format(time.RFC3339Nano), Totals: report.Totals, Items: report.Items}
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
55
internal/api/query_periodic_fixture_test.go
Normal file
55
internal/api/query_periodic_fixture_test.go
Normal file
@ -0,0 +1,55 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"os/exec"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// A separate opt-in for same-size LOCAL synthetic HTTP comparison. It cannot
|
||||
// waive the owned marker / node guards of the company-cluster harness.
|
||||
func verifyPeriodicLocalHTTPFixture(t *testing.T) {
|
||||
t.Helper()
|
||||
if os.Getenv("QUERY_REPLICA_CLUSTER_TEST") != "" || os.Getenv("QUERY_REPLICA_TEST_TARGET_URI") != "" || os.Getenv("QUERY_COMPARE_IN_CLUSTER") != "" {
|
||||
t.Fatal("local comparison cannot select a cluster/alternate URI")
|
||||
}
|
||||
endpoint, err := exec.Command("docker", "context", "inspect", "--format", "{{.Endpoints.docker.Host}}").Output()
|
||||
if err != nil || !strings.HasPrefix(strings.TrimSpace(string(endpoint)), "unix://") {
|
||||
t.Fatal("owned local Docker socket required")
|
||||
}
|
||||
for _, name := range []string{"source", "query"} {
|
||||
raw, err := exec.Command("docker", "inspect", "fanyu-query-replica-local-"+name+"-1").Output()
|
||||
if err != nil {
|
||||
t.Fatal("local Mongo fixture unavailable")
|
||||
}
|
||||
var d []struct {
|
||||
State struct{ Running bool }
|
||||
Config struct {
|
||||
Labels map[string]string
|
||||
Cmd []string
|
||||
}
|
||||
HostConfig struct {
|
||||
NanoCpus int64
|
||||
Memory int64
|
||||
PortBindings map[string][]struct{ HostIp, HostPort string }
|
||||
}
|
||||
}
|
||||
if json.Unmarshal(raw, &d) != nil || len(d) != 1 || !d[0].State.Running || d[0].Config.Labels["com.docker.compose.project"] != "fanyu-query-replica-local" {
|
||||
t.Fatal("unowned synthetic container")
|
||||
}
|
||||
cpu, memory, port := int64(4e9), int64(4<<30), "27119"
|
||||
if name == "query" {
|
||||
cpu, memory, port = 1e9, 1<<30, "27129"
|
||||
}
|
||||
bindings := d[0].HostConfig.PortBindings[port+"/tcp"]
|
||||
if d[0].HostConfig.NanoCpus != cpu || d[0].HostConfig.Memory != memory || len(bindings) != 1 || bindings[0].HostIp != "127.0.0.1" || bindings[0].HostPort != port {
|
||||
t.Fatal("local fixture resources or loopback binding changed")
|
||||
}
|
||||
command := " " + strings.Join(d[0].Config.Cmd, " ")
|
||||
if !strings.Contains(command, " --port "+port) || (name == "query" && (!strings.Contains(command, " --replSet query-read-rs") || !strings.Contains(command, " --wiredTigerCacheSizeGB 0.25"))) {
|
||||
t.Fatal("synthetic Mongo command/cache identity changed")
|
||||
}
|
||||
}
|
||||
}
|
||||
130
internal/api/query_periodic_replay_test.go
Normal file
130
internal/api/query_periodic_replay_test.go
Normal file
@ -0,0 +1,130 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"droboticflow/internal/domain"
|
||||
"droboticflow/internal/store"
|
||||
"droboticflow/internal/testsupport"
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
"go.mongodb.org/mongo-driver/event"
|
||||
"go.mongodb.org/mongo-driver/mongo"
|
||||
"go.mongodb.org/mongo-driver/mongo/options"
|
||||
)
|
||||
|
||||
// Diagnostic replay of an explicitly retained LOCAL synthetic fixture. It
|
||||
// neither bootstraps data nor runs workers. Normal verification does not opt in.
|
||||
func TestQueryPeriodicRetainedBudgetReplayLocal(t *testing.T) {
|
||||
if os.Getenv("QUERY_PERIODIC_REPLAY") != "1" {
|
||||
t.Skip("explicit retained local synthetic fixture required")
|
||||
}
|
||||
verifyPeriodicLocalHTTPFixture(t)
|
||||
sourceDB, targetDB := os.Getenv("QUERY_REPLAY_SOURCE"), os.Getenv("QUERY_REPLAY_TARGET")
|
||||
for _, n := range []string{sourceDB, targetDB} {
|
||||
if !strings.HasPrefix(n, "query_http_synthetic-") || len(n) != 53 {
|
||||
t.Fatal("unowned diagnostic database name")
|
||||
}
|
||||
}
|
||||
t.Setenv("QUERY_SERVING_NAMESPACE", "isolated-http-comparison")
|
||||
t.Setenv("QUERY_REPLICA_REPORTS", "1")
|
||||
t.Setenv("QUERY_SOURCE_TRACKING", "1")
|
||||
t.Setenv("QUERY_SERVING_READS", "1")
|
||||
t.Setenv("QUERY_REPLICA_REFRESH_MODE", "periodic")
|
||||
t.Setenv("QUERY_READ_MONGO_URI", testsupport.QueryReplicaTargetURI(t))
|
||||
t.Setenv("QUERY_READ_MONGO_DB", targetDB)
|
||||
t.Setenv("QUERY_VALKEY_URL", "")
|
||||
t.Setenv("QUERY_TYPESENSE_URL", "")
|
||||
c, err := mongo.Connect(t.Context(), options.Client().ApplyURI("mongodb://127.0.0.1:27119/?directConnection=true"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer c.Disconnect(context.Background())
|
||||
source := store.NewMongoStore(c.Database(sourceDB))
|
||||
org, err := source.GetOrganization(t.Context(), "query-synthetic-org")
|
||||
if err != nil || org == nil || org.Name != "Synthetic only" {
|
||||
t.Fatal("synthetic organization identity unavailable", err)
|
||||
}
|
||||
var mu sync.Mutex
|
||||
trace := []map[string]any{}
|
||||
pending := map[int64]int{}
|
||||
monitor := &event.CommandMonitor{Started: func(ctx context.Context, e *event.CommandStartedEvent) {
|
||||
name, _ := e.Command.Lookup("aggregate").StringValueOK()
|
||||
if name != "queryWorkforceFacts" {
|
||||
return
|
||||
}
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
mt, present := e.Command.Lookup("maxTimeMS").AsInt64OK()
|
||||
left := int64(-1)
|
||||
if d, ok := ctx.Deadline(); ok {
|
||||
left = time.Until(d).Milliseconds()
|
||||
}
|
||||
pending[e.RequestID] = len(trace)
|
||||
trace = append(trace, map[string]any{"appTime": time.Now().UTC().Format(time.RFC3339Nano), "maxTimeMS": mt, "maxTimePresent": present, "contextRemainingMS": left})
|
||||
}, Succeeded: func(_ context.Context, e *event.CommandSucceededEvent) {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if i, ok := pending[e.RequestID]; ok {
|
||||
trace[i]["durationMS"] = float64(e.DurationNanos) / 1e6
|
||||
trace[i]["failed"] = false
|
||||
}
|
||||
}, Failed: func(_ context.Context, e *event.CommandFailedEvent) {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if i, ok := pending[e.RequestID]; ok {
|
||||
trace[i]["durationMS"] = float64(e.DurationNanos) / 1e6
|
||||
trace[i]["failed"] = true
|
||||
}
|
||||
}}
|
||||
defer func() {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
raw, _ := json.MarshalIndent(trace, "", " ")
|
||||
if p := os.Getenv("QUERY_REPLAY_REPORT"); p != "" {
|
||||
if err := os.WriteFile(p, raw, 0600); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}
|
||||
}()
|
||||
closeReplica, err := source.ConfigureQueryReplicaFromEnvironment(t.Context(), monitor)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer closeReplica()
|
||||
token, err := randomSessionToken()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
id := domain.NewID("synthetic-replay-session")
|
||||
if _, err = source.CreateSession(t.Context(), domain.SessionRecord{ID: id, UserID: "u000", OrganizationID: "query-synthetic-org", TokenHash: sessionTokenHash(token), ExpiresAt: time.Now().Add(time.Hour).UTC().Format(time.RFC3339), ExpiresAtTime: time.Now().Add(time.Hour)}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer c.Database(sourceDB).Collection("sessions").DeleteOne(context.Background(), bson.M{"id": id})
|
||||
s := &server{store: source, requireSessionAuth: true, authLimiter: newRateLimiter(20, time.Minute), externalWriteLimiter: newRateLimiter(120, time.Minute), dashboardResponses: dashboardResponseCache{now: time.Now}}
|
||||
ts := httptest.NewServer(s.routes())
|
||||
defer ts.Close()
|
||||
client := &http.Client{Timeout: 25 * time.Second}
|
||||
for i := 0; i < 3; i++ {
|
||||
r, err := http.NewRequestWithContext(t.Context(), http.MethodGet, ts.URL+"/api/dashboard/workforce?from=2026-11-01T00:00:00Z&to=2026-11-13T00:00:00Z&groupBy=member", nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r.AddCookie(&http.Cookie{Name: sessionCookieName, Value: token})
|
||||
res, err := client.Do(r)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
res.Body.Close()
|
||||
if res.StatusCode != 200 {
|
||||
t.Fatalf("admin aggregate replay status=%d", res.StatusCode)
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -37,10 +37,13 @@ import (
|
||||
// HTTP uses the actual session, ACL and count/report handlers. Business recovery
|
||||
// schedulers are deliberately not duplicated in the two benchmark handlers.
|
||||
func TestQueryReplicaIsolatedHTTPComparisonLocal(t *testing.T) {
|
||||
if os.Getenv("QUERY_REPLICA_ISOLATED_E2E") != "1" {
|
||||
local := os.Getenv("QUERY_PERIODIC_HTTP_LOCAL") == "1"
|
||||
if os.Getenv("QUERY_REPLICA_ISOLATED_E2E") != "1" && !local {
|
||||
t.Skip("explicit node4 synthetic comparison only")
|
||||
}
|
||||
if os.Getenv("QUERY_REPLICA_CLUSTER_TEST") != "1" {
|
||||
if local {
|
||||
verifyPeriodicLocalHTTPFixture(t)
|
||||
} else if os.Getenv("QUERY_REPLICA_CLUSTER_TEST") != "1" {
|
||||
t.Fatal("cluster synthetic target must be explicitly selected")
|
||||
}
|
||||
t.Setenv("ANNOTATION_SOURCE_VALIDATION_MODE", "advisory")
|
||||
@ -49,6 +52,10 @@ func TestQueryReplicaIsolatedHTTPComparisonLocal(t *testing.T) {
|
||||
n := isolatedComparisonInt(t, "QUERY_COMPARE_TASKS", 26218, 20, 30000)
|
||||
waves := isolatedComparisonInt(t, "QUERY_COMPARE_WAVES", 2, 1, 100)
|
||||
maxActors := isolatedComparisonInt(t, "QUERY_COMPARE_MAX_ACTORS", 81, 81, 500)
|
||||
maxInflight := isolatedComparisonInt(t, "QUERY_COMPARE_CPU_MAX_INFLIGHT", 0, 0, 8)
|
||||
if maxInflight > 0 && !local {
|
||||
t.Fatal("supplemental matched CPU comparison is local-only")
|
||||
}
|
||||
minPool := isolatedComparisonInt(t, "QUERY_COMPARE_MIN_POOL", 0, 0, 100)
|
||||
compression := os.Getenv("QUERY_COMPARE_WIRE_COMPRESSION")
|
||||
if compression != "" && compression != "zstd" {
|
||||
@ -59,14 +66,60 @@ func TestQueryReplicaIsolatedHTTPComparisonLocal(t *testing.T) {
|
||||
inflight := map[int64]string{}
|
||||
metrics := map[string]*isolatedCommands{}
|
||||
var targetStatsMu sync.Mutex
|
||||
var factTrace []map[string]any
|
||||
factPending := map[int64]int{}
|
||||
if path := os.Getenv("QUERY_COMPARE_DIAGNOSTIC_REPORT"); local && path != "" {
|
||||
defer func() {
|
||||
targetStatsMu.Lock()
|
||||
defer targetStatsMu.Unlock()
|
||||
raw, _ := json.MarshalIndent(factTrace, "", " ")
|
||||
if err := os.WriteFile(path, raw, 0600); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}()
|
||||
}
|
||||
targetCommands := map[string]int64{}
|
||||
targetMaxMillis := map[string]float64{}
|
||||
targetMonitor := &event.CommandMonitor{Succeeded: func(_ context.Context, e *event.CommandSucceededEvent) {
|
||||
targetMonitor := &event.CommandMonitor{Started: func(ctx context.Context, e *event.CommandStartedEvent) {
|
||||
if !local {
|
||||
return
|
||||
}
|
||||
name, _ := e.Command.Lookup("aggregate").StringValueOK()
|
||||
if name != "queryWorkforceFacts" {
|
||||
return
|
||||
}
|
||||
targetStatsMu.Lock()
|
||||
defer targetStatsMu.Unlock()
|
||||
if len(factTrace) >= 512 {
|
||||
return
|
||||
}
|
||||
maxTime, present := e.Command.Lookup("maxTimeMS").AsInt64OK()
|
||||
remaining := int64(-1)
|
||||
if deadline, ok := ctx.Deadline(); ok {
|
||||
remaining = time.Until(deadline).Milliseconds()
|
||||
}
|
||||
factPending[e.RequestID] = len(factTrace)
|
||||
factTrace = append(factTrace, map[string]any{"appTimeUTC": time.Now().UTC().Format(time.RFC3339Nano), "wireMaxTimeMS": maxTime, "wireMaxTimePresent": present, "contextRemainingMS": remaining})
|
||||
}, Succeeded: func(_ context.Context, e *event.CommandSucceededEvent) {
|
||||
targetStatsMu.Lock()
|
||||
defer targetStatsMu.Unlock()
|
||||
targetCommands[e.CommandName]++
|
||||
ms := float64(e.DurationNanos) / 1e6
|
||||
targetMaxMillis[e.CommandName] = max(targetMaxMillis[e.CommandName], ms)
|
||||
if i, ok := factPending[e.RequestID]; ok {
|
||||
factTrace[i]["durationMS"] = ms
|
||||
factTrace[i]["failed"] = false
|
||||
delete(factPending, e.RequestID)
|
||||
}
|
||||
}, Failed: func(_ context.Context, e *event.CommandFailedEvent) {
|
||||
targetStatsMu.Lock()
|
||||
defer targetStatsMu.Unlock()
|
||||
if i, ok := factPending[e.RequestID]; ok {
|
||||
factTrace[i]["durationMS"] = float64(e.DurationNanos) / 1e6
|
||||
factTrace[i]["failed"] = true
|
||||
factTrace[i]["maxTimeFailure"] = strings.Contains(e.Failure, "MaxTimeMS")
|
||||
delete(factPending, e.RequestID)
|
||||
}
|
||||
}}
|
||||
monitor := &event.CommandMonitor{
|
||||
Started: func(_ context.Context, e *event.CommandStartedEvent) {
|
||||
@ -116,7 +169,9 @@ func TestQueryReplicaIsolatedHTTPComparisonLocal(t *testing.T) {
|
||||
t.Cleanup(func() {
|
||||
clean, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
if err := db.Drop(clean); err != nil {
|
||||
if local && os.Getenv("QUERY_COMPARE_KEEP_FAILED_FIXTURES") == "1" && t.Failed() {
|
||||
t.Logf("retained owned diagnostic fixture %s", db.Name())
|
||||
} else if err := db.Drop(clean); err != nil {
|
||||
t.Errorf("synthetic cleanup %s: %v", db.Name(), err)
|
||||
}
|
||||
_ = c.Disconnect(clean)
|
||||
@ -125,9 +180,11 @@ func TestQueryReplicaIsolatedHTTPComparisonLocal(t *testing.T) {
|
||||
}
|
||||
sdb, old := connect("mongodb://127.0.0.1:27119/?directConnection=true", monitor)
|
||||
db, _ := connect(testsupport.QueryReplicaTargetURI(t), targetMonitor)
|
||||
var marker bson.M
|
||||
if err := db.Client().Database("query25_synthetic").Collection("marker").FindOne(ctx, bson.M{"_id": "query25-confirmed"}).Decode(&marker); err != nil || marker["value"] != "preserve-query25-v1" {
|
||||
t.Fatalf("owned 25Gi target identity not confirmed: %v", err)
|
||||
if !local {
|
||||
var marker bson.M
|
||||
if err := db.Client().Database("query25_synthetic").Collection("marker").FindOne(ctx, bson.M{"_id": "query25-confirmed"}).Decode(&marker); err != nil || marker["value"] != "preserve-query25-v1" {
|
||||
t.Fatalf("owned 25Gi target identity not confirmed: %v", err)
|
||||
}
|
||||
}
|
||||
newReader := store.NewMongoStore(sdb)
|
||||
newReader.EnableQuerySourceTracking(true)
|
||||
@ -202,6 +259,9 @@ func TestQueryReplicaIsolatedHTTPComparisonLocal(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
builderTarget.ConfigureQueryServing(nil, nil)
|
||||
if err := builderTarget.ConfigureQueryRefreshMode(os.Getenv("QUERY_REPLICA_REFRESH_MODE")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := builderTarget.EnsureQueryReadModelIndexes(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@ -227,16 +287,18 @@ func TestQueryReplicaIsolatedHTTPComparisonLocal(t *testing.T) {
|
||||
case <-workerCtx.Done():
|
||||
return
|
||||
}
|
||||
tick := time.NewTicker(time.Second)
|
||||
defer tick.Stop()
|
||||
interval := time.Second
|
||||
if os.Getenv("QUERY_REPLICA_REFRESH_MODE") == "periodic" {
|
||||
interval = store.QueryPeriodicSyncInterval
|
||||
}
|
||||
for workerCtx.Err() == nil {
|
||||
if _, err := q.RunOnce(workerCtx, false); err != nil && workerCtx.Err() == nil {
|
||||
workerErrors <- err
|
||||
return
|
||||
}
|
||||
select {
|
||||
case <-workerCtx.Done():
|
||||
case <-tick.C:
|
||||
if err := q.WaitNextPass(workerCtx, interval); err != nil && workerCtx.Err() == nil {
|
||||
workerErrors <- err
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
@ -271,7 +333,7 @@ func TestQueryReplicaIsolatedHTTPComparisonLocal(t *testing.T) {
|
||||
t.Setenv("QUERY_SERVING_READS", "1")
|
||||
t.Setenv("QUERY_VALKEY_URL", "")
|
||||
t.Setenv("QUERY_TYPESENSE_URL", "")
|
||||
closeReplica, err := newReader.ConfigureQueryReplicaFromEnvironment(ctx)
|
||||
closeReplica, err := newReader.ConfigureQueryReplicaFromEnvironment(ctx, targetMonitor)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@ -320,8 +382,13 @@ func TestQueryReplicaIsolatedHTTPComparisonLocal(t *testing.T) {
|
||||
}
|
||||
// Keep output evidence independently identifiable rather than parsing t.Log.
|
||||
report := map[string]any{"tasks": n, "projects": len(fixture.projects), "sourceDatabase": sdb.Name(), "targetDatabase": db.Name(), "cases": []any{}, "parity": true, "httpSessionAuth": true, "sourceMinimumPoolBothVariants": minPool, "apiQueryMaximumPool": 20, "projectorMaximumPool": 4, "builderMaximumPool": 100, "wireCompressionBothVariants": compression, "sourceReplyBytesKind": "logical BSON; not compressed wire bytes", "cache": "none; existing API snapshot/response caches unchanged"}
|
||||
report["refreshMode"] = os.Getenv("QUERY_REPLICA_REFRESH_MODE")
|
||||
report["localDocker"] = local
|
||||
report["targetMetricsIncludeAPI"] = true
|
||||
report["supplementalMaxInFlight"] = maxInflight
|
||||
cases := []any{}
|
||||
accepted := true
|
||||
expectedResults := map[string]any{}
|
||||
for _, mode := range []string{"old", "query"} {
|
||||
url := before.URL
|
||||
if mode == "query" {
|
||||
@ -358,6 +425,11 @@ func TestQueryReplicaIsolatedHTTPComparisonLocal(t *testing.T) {
|
||||
errors := map[string]int{}
|
||||
var firstError string
|
||||
var responses int64
|
||||
matchedResults, mismatches := 0, 0
|
||||
var slots chan struct{}
|
||||
if maxInflight > 0 {
|
||||
slots = make(chan struct{}, maxInflight)
|
||||
}
|
||||
for wave := 0; wave < waves; wave++ {
|
||||
var wg sync.WaitGroup
|
||||
var results sync.Mutex
|
||||
@ -367,6 +439,10 @@ func TestQueryReplicaIsolatedHTTPComparisonLocal(t *testing.T) {
|
||||
go func(actor int) {
|
||||
defer wg.Done()
|
||||
<-barrier
|
||||
if slots != nil {
|
||||
slots <- struct{}{}
|
||||
defer func() { <-slots }()
|
||||
}
|
||||
path := paths[2+(actor+wave)%2]
|
||||
if actor%81 == 0 {
|
||||
path = paths[1]
|
||||
@ -382,6 +458,18 @@ func TestQueryReplicaIsolatedHTTPComparisonLocal(t *testing.T) {
|
||||
key := strings.Split(path, "?")[0]
|
||||
byRoute[key] = append(byRoute[key], ms)
|
||||
responses += int64(len(body))
|
||||
if status == 200 && err == nil {
|
||||
id := fmt.Sprintf("%d/%d/%s", wave, actor, path)
|
||||
value := isolatedBusinessJSON(t, body)
|
||||
if mode == "old" {
|
||||
expectedResults[id] = value
|
||||
} else if expected, ok := expectedResults[id]; ok {
|
||||
matchedResults++
|
||||
if !reflect.DeepEqual(expected, value) {
|
||||
mismatches++
|
||||
}
|
||||
}
|
||||
}
|
||||
if status != 200 || err != nil {
|
||||
errors[fmt.Sprintf("%d/%v", status, err)]++
|
||||
if firstError == "" {
|
||||
@ -396,7 +484,10 @@ func TestQueryReplicaIsolatedHTTPComparisonLocal(t *testing.T) {
|
||||
mu.Lock()
|
||||
phase = ""
|
||||
original := metrics[name]
|
||||
m := &isolatedCommands{Commands: map[string]int64{}, RequestBytes: original.RequestBytes, ReplyBytes: original.ReplyBytes}
|
||||
m := &isolatedCommands{Commands: map[string]int64{}, Collections: map[string]int64{}, RequestBytes: original.RequestBytes, ReplyBytes: original.ReplyBytes}
|
||||
for collection, count := range original.Collections {
|
||||
m.Collections[collection] = count
|
||||
}
|
||||
for cmd, count := range original.Commands {
|
||||
m.Commands[cmd] = count
|
||||
}
|
||||
@ -417,6 +508,11 @@ func TestQueryReplicaIsolatedHTTPComparisonLocal(t *testing.T) {
|
||||
}
|
||||
}
|
||||
result := map[string]any{"name": name, "seconds": time.Since(started).Seconds(), "requests": len(latency), "p50Ms": latency[(len(latency)-1)/2], "p95Ms": latency[(len(latency)-1)*95/100], "errors": errors, "firstError": firstError, "routes": routes, "responseBytes": responses, "sourceCPUSeconds": cpu, "sourceScans": scans, "source": m}
|
||||
result["matchedBusinessResults"], result["businessMismatches"] = matchedResults, mismatches
|
||||
if mode == "query" && mismatches > 0 {
|
||||
t.Errorf("%d per-actor business results differed", mismatches)
|
||||
accepted = false
|
||||
}
|
||||
cases = append(cases, result)
|
||||
encoded, _ := json.Marshal(result)
|
||||
t.Log(string(encoded))
|
||||
|
||||
@ -25,7 +25,11 @@ func TestQueryReplicaCountOnlyPermissionRevocationAndSelfReviewLocal(t *testing.
|
||||
testQueryReplicaPermissionRevocationAndSelfReview(t, true)
|
||||
}
|
||||
|
||||
func testQueryReplicaPermissionRevocationAndSelfReview(t *testing.T, countsOnly bool) {
|
||||
func TestQueryPeriodicPermissionRevocationAndSelfReviewLocal(t *testing.T) {
|
||||
testQueryReplicaPermissionRevocationAndSelfReview(t, false, true)
|
||||
}
|
||||
|
||||
func testQueryReplicaPermissionRevocationAndSelfReview(t *testing.T, countsOnly bool, periodic ...bool) {
|
||||
if os.Getenv("QUERY_REPLICA_LOCAL_TEST") != "1" {
|
||||
t.Skip("isolated local source and query replica required")
|
||||
}
|
||||
@ -49,6 +53,11 @@ func testQueryReplicaPermissionRevocationAndSelfReview(t *testing.T, countsOnly
|
||||
stores = append(stores, store.NewMongoStore(db))
|
||||
}
|
||||
source, target := stores[0], stores[1]
|
||||
if len(periodic) > 0 && periodic[0] {
|
||||
if err := target.ConfigureQueryRefreshMode("periodic"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
_ = clients
|
||||
source.EnableQuerySourceTracking(true)
|
||||
if e := source.EnsureIndexes(ctx); e != nil {
|
||||
|
||||
@ -139,6 +139,7 @@ func (s *server) handleDashboardWorkforce(w http.ResponseWriter, r *http.Request
|
||||
workforcestats.ApplyRates(&report.Items[i])
|
||||
}
|
||||
renderJSON(w, http.StatusOK, map[string]any{
|
||||
"asOf": report.AsOf,
|
||||
"from": q.From.Format(time.RFC3339Nano),
|
||||
"to": q.To.Format(time.RFC3339Nano),
|
||||
"groupBy": q.GroupBy,
|
||||
|
||||
@ -1235,6 +1235,7 @@ type DashboardWorkforceMetric struct {
|
||||
}
|
||||
|
||||
type DashboardWorkforceSummary struct {
|
||||
AsOf string `json:"asOf,omitempty"`
|
||||
From string `json:"from,omitempty"`
|
||||
To string `json:"to,omitempty"`
|
||||
Totals DashboardWorkforceMetric `json:"totals"`
|
||||
|
||||
@ -24,6 +24,7 @@ type MongoStore struct {
|
||||
querySyncEnabled atomic.Bool
|
||||
queryReplicaReads *queryReplicaReads
|
||||
queryReplicaFactStore bool
|
||||
queryPeriodic bool
|
||||
queryBuildWrites *queryBuildWriteGate
|
||||
queryReplicaCounts queryCountCache
|
||||
queryReplicaBatchFacts queryBatchFacts
|
||||
|
||||
76
internal/store/query_periodic.go
Normal file
76
internal/store/query_periodic.go
Normal file
@ -0,0 +1,76 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
"go.mongodb.org/mongo-driver/mongo"
|
||||
)
|
||||
|
||||
// Explicit opt-in; authority writes and permission decisions never use this
|
||||
// display policy. The bounds describe data age, NOT longer request deadlines.
|
||||
const (
|
||||
QueryPeriodicSyncInterval = time.Minute
|
||||
queryPeriodicBuildInterval = 3 * time.Minute
|
||||
queryPeriodicDisplayMaxAge = 15 * time.Minute
|
||||
)
|
||||
|
||||
func (s *MongoStore) ConfigureQueryRefreshMode(mode string) error {
|
||||
switch mode {
|
||||
case "", "continuous":
|
||||
s.queryPeriodic = false
|
||||
case "periodic":
|
||||
s.queryPeriodic = true
|
||||
default:
|
||||
return errors.New("invalid query refresh mode")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *MongoStore) periodicQueryReads() bool {
|
||||
return s.queryReplicaReads != nil && s.queryReplicaReads.target != nil && s.queryReplicaReads.target.queryPeriodic
|
||||
}
|
||||
|
||||
func (s *MongoStore) periodicQueryStore() bool { return s.queryPeriodic && s.queryReplicaFactStore }
|
||||
|
||||
func (s *MongoStore) querySearchPublicationReady(st querySearchState, h QueryModelHead, exists bool, revision int64, now time.Time) bool {
|
||||
if !exists || h.Generation == "" || now.Before(h.AsOf) {
|
||||
return false
|
||||
}
|
||||
if s.periodicQueryStore() && now.Sub(h.AsOf) <= queryPeriodicDisplayMaxAge {
|
||||
return true
|
||||
}
|
||||
return h.Revision == revision && h.GlobalRevision == st.Global && now.Sub(st.ObservedAt) >= 0 && now.Sub(st.ObservedAt) <= 5*time.Second
|
||||
}
|
||||
|
||||
func queryBuildDue(now, last time.Time, interval time.Duration) bool {
|
||||
return last.IsZero() || !now.Before(last.Add(interval))
|
||||
}
|
||||
|
||||
// The projector's lease remains 30s. Idle only touches its query-store lease,
|
||||
// never scans authority or extends an expired/unconfirmed lease.
|
||||
func (q *QueryReplica) WaitNextPass(ctx context.Context, interval time.Duration) error {
|
||||
if interval < 100*time.Millisecond || interval > 5*time.Minute {
|
||||
return errors.New("invalid projector interval")
|
||||
}
|
||||
timer := time.NewTimer(interval)
|
||||
defer timer.Stop()
|
||||
heartbeat := time.NewTicker(10 * time.Second)
|
||||
defer heartbeat.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-timer.C:
|
||||
return nil
|
||||
case <-heartbeat.C:
|
||||
short, stop := context.WithTimeout(ctx, 3*time.Second)
|
||||
err := q.transaction(short, func(_ mongo.SessionContext) error { return nil })
|
||||
stop()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
323
internal/store/query_periodic_local_test.go
Normal file
323
internal/store/query_periodic_local_test.go
Normal file
@ -0,0 +1,323 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"reflect"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"droboticflow/internal/domain"
|
||||
"droboticflow/internal/productionstats"
|
||||
"droboticflow/internal/workforcestats"
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
"go.mongodb.org/mongo-driver/event"
|
||||
)
|
||||
|
||||
func TestQueryPeriodicPublishedReportIgnoresPendingRefreshLocal(t *testing.T) {
|
||||
if os.Getenv("QUERY_REPLICA_LOCAL_TEST") != "1" {
|
||||
t.Skip("owned synthetic replica required")
|
||||
}
|
||||
var sourceReads atomic.Int32
|
||||
monitor := &event.CommandMonitor{Started: func(_ context.Context, e *event.CommandStartedEvent) {
|
||||
if e.CommandName == "find" || e.CommandName == "aggregate" {
|
||||
sourceReads.Add(1)
|
||||
}
|
||||
}}
|
||||
source, target, projector := newLocalReplica(t, monitor, nil)
|
||||
finishLocalReplica(t, projector)
|
||||
ctx := t.Context()
|
||||
target.queryPeriodic = true
|
||||
target.ConfigureQueryServing(nil, nil)
|
||||
if err := source.ConfigureQueryReplicaReports(target, 30*time.Second); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
lease, ok, err := target.acquireQueryModelLease(ctx)
|
||||
if err != nil || !ok {
|
||||
t.Fatal(ok, err)
|
||||
}
|
||||
asOf := time.Now().UTC().Add(-2 * time.Minute).Truncate(time.Millisecond)
|
||||
head, err := target.stageQueryModel(ctx, queryModelKey("production", "o", nil), productionstats.Report{}, asOf)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = target.publishQueryModelHead(ctx, lease, queryModelKey("production", "o", nil), head); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// A change / failed observer must not hide an already published generation.
|
||||
_, err = target.queryControl().UpdateOne(ctx, bson.M{"_id": "global"}, bson.M{"$inc": bson.M{"productionRevision": 1}, "$set": bson.M{"observedAt": time.Time{}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err = target.db.Collection(queryReplicaControl).UpdateOne(ctx, bson.M{"_id": "state"}, bson.M{"$set": bson.M{"observedAt": asOf}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sourceReads.Store(0)
|
||||
_, got, err := source.LoadProductionReport(ctx, "o", nil)
|
||||
if err != nil || got != asOf.Format(time.RFC3339Nano) {
|
||||
t.Fatalf("published result hidden by pending refresh: asOf=%q err=%v", got, err)
|
||||
}
|
||||
if sourceReads.Load() != 0 {
|
||||
t.Fatalf("display triggered %d authority reads", sourceReads.Load())
|
||||
}
|
||||
// No unpublished/raw fallback when an independent report has never existed.
|
||||
_, _, err = source.LoadProductionReport(ctx, "missing", nil)
|
||||
if err == nil {
|
||||
t.Fatal("cold display calculated a report instead of requiring a publication")
|
||||
}
|
||||
// Hard staleness bound and binding checks are not disabled by delayed display.
|
||||
_, err = target.db.Collection(queryReplicaControl).UpdateOne(ctx, bson.M{"_id": "state"}, bson.M{"$set": bson.M{"observedAt": time.Now().Add(-16 * time.Minute)}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, _, err = source.LoadProductionReport(ctx, "o", nil); err == nil {
|
||||
t.Fatal("expired sync certificate accepted")
|
||||
}
|
||||
}
|
||||
|
||||
func TestQueryPeriodicFactsRemainReadableThroughFailureAndRestartLocal(t *testing.T) {
|
||||
if os.Getenv("QUERY_REPLICA_LOCAL_TEST") != "1" {
|
||||
t.Skip("owned synthetic replica required")
|
||||
}
|
||||
source, target, p := newLocalReplica(t)
|
||||
seedWorkforceStats(t, source)
|
||||
if err := source.UpsertOrganization(t.Context(), domain.OrganizationRecord{ID: "workforce-test", Status: domain.OrganizationStatusActive}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
finishLocalReplica(t, p)
|
||||
target.queryPeriodic = true
|
||||
target.ConfigureQueryServing(nil, nil)
|
||||
if err := target.EnsureQueryReadModelIndexes(t.Context()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := source.ConfigureQueryReplicaReports(target, time.Minute); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
q := workforcestats.Query{Mode: domain.RunModeAll}
|
||||
old, err := source.LoadWorkforceStats(t.Context(), "workforce-test", q)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := workforcestats.Build(old, q)
|
||||
want.Details = nil
|
||||
b := NewQueryReadBuilder(target)
|
||||
ctx, stop := context.WithCancel(t.Context())
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- b.Run(ctx) }()
|
||||
stopped := false
|
||||
defer func() {
|
||||
if !stopped {
|
||||
stop()
|
||||
<-done
|
||||
}
|
||||
b.Close()
|
||||
}()
|
||||
ready, finish := context.WithTimeout(t.Context(), 20*time.Second)
|
||||
err = b.WaitReady(ready, "workforce-test", nil)
|
||||
finish()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
head, exists, err := target.workforceHead(t.Context(), "workforce-test")
|
||||
if err != nil || !exists {
|
||||
t.Fatal(exists, err)
|
||||
}
|
||||
// New work enters the independent replica; display must retain the old
|
||||
// complete facts while the periodic computation is pending.
|
||||
err = source.UpsertAnnotationWorklogEvent(t.Context(), domain.AnnotationWorklogEventRecord{ID: "new-periodic-event", OrganizationID: "workforce-test", ProjectID: "p", TaskID: "task", UserID: "u", EventType: "combined_submitted", CreatedAt: "2026-09-28T16:00:00Z"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
finishLocalReplica(t, p)
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
for {
|
||||
b.mu.Lock()
|
||||
pending := b.dirty["workforce-test"]
|
||||
b.mu.Unlock()
|
||||
if pending {
|
||||
break
|
||||
}
|
||||
var state struct {
|
||||
Revision int64 `bson:"productionRevision"`
|
||||
}
|
||||
_ = target.queryControl().FindOne(t.Context(), bson.M{"_id": "global"}).Decode(&state)
|
||||
if state.Revision > head.SourceRevision {
|
||||
break
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatal("change was not observed")
|
||||
}
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
}
|
||||
got, err := source.LoadWorkforceReportCached(t.Context(), "workforce-test", q)
|
||||
if err != nil {
|
||||
t.Fatal("pending refresh hid published facts", err)
|
||||
}
|
||||
got.Details = nil
|
||||
if got.AsOf == "" {
|
||||
t.Fatal("published report has no data timestamp")
|
||||
}
|
||||
got.AsOf = ""
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Fatal("pending report mixed old and new generations")
|
||||
}
|
||||
after, _, err := target.workforceHead(t.Context(), "workforce-test")
|
||||
if err != nil || after.FactEpoch != head.FactEpoch {
|
||||
t.Fatal("periodic changes rebuilt immediately", err)
|
||||
}
|
||||
// Simulate a failed future publication; the committed pointer is unchanged.
|
||||
_, err = target.db.Collection(workforceFactCollection).InsertOne(t.Context(), bson.M{"_id": "unfinished", "organizationId": "workforce-test", "fromEpoch": head.FactEpoch + 1, "kind": "event"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
stop()
|
||||
<-done
|
||||
stopped = true
|
||||
lease, ok, err := target.acquireQueryModelLease(t.Context())
|
||||
if err != nil || !ok {
|
||||
t.Fatal(ok, err)
|
||||
}
|
||||
if err = target.rebuildWorkforceFacts(t.Context(), lease, "workforce-test", head.SourceRevision+1); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
latest, err := source.LoadWorkforceStats(t.Context(), "workforce-test", q)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want = workforcestats.Build(latest, q)
|
||||
want.Details = nil
|
||||
got, err = source.LoadWorkforceReportCached(t.Context(), "workforce-test", q)
|
||||
got.Details = nil
|
||||
got.AsOf = ""
|
||||
if err != nil || !reflect.DeepEqual(got, want) {
|
||||
t.Fatal("restart failed to publish the updated correct facts", err)
|
||||
}
|
||||
if n, err := target.db.Collection(workforceFactCollection).CountDocuments(t.Context(), bson.M{"_id": "unfinished"}); err != nil || n != 0 {
|
||||
t.Fatal("unfinished stage survived recovery", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestQueryPeriodicCountsDoNotOverlayDirtyAuthorityLocal(t *testing.T) {
|
||||
if os.Getenv("QUERY_REPLICA_LOCAL_TEST") != "1" {
|
||||
t.Skip("owned synthetic replica required")
|
||||
}
|
||||
var reads atomic.Int32
|
||||
m := &event.CommandMonitor{Started: func(_ context.Context, e *event.CommandStartedEvent) {
|
||||
if e.CommandName == "find" || e.CommandName == "aggregate" {
|
||||
reads.Add(1)
|
||||
}
|
||||
}}
|
||||
source, target, p := newLocalReplica(t, m, nil)
|
||||
finishLocalReplica(t, p)
|
||||
target.queryPeriodic = true
|
||||
if err := source.ConfigureQueryReplicaReports(target, time.Minute); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
source.EnableQueryReplicaCountReads(true)
|
||||
ctx := WithQueryReplicaCountRead(t.Context())
|
||||
reads.Store(0)
|
||||
b, err := source.countReplicaBarrier(ctx, "o")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if b == nil || len(b.ids) != 0 || reads.Load() != 0 {
|
||||
t.Fatalf("display still chasing authority: barrier=%v reads=%d", b, reads.Load())
|
||||
}
|
||||
// Batch access remains authoritative: permissions/policy are not snapshots.
|
||||
if err := source.UpsertHumanBatch(t.Context(), domain.HumanBatchRecord{ID: "b", OrganizationID: "o", ProjectID: "p"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
reads.Store(0)
|
||||
rows, err := source.replicaBatchAccess(ctx, target, "o", []string{"b"})
|
||||
if err != nil || len(rows) != 1 || reads.Load() == 0 {
|
||||
t.Fatalf("current batch access was cached: rows=%d reads=%d err=%v", len(rows), reads.Load(), err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestQueryPeriodicIdleKeepsAuthorityQuietAndRejectsSuccessorLocal(t *testing.T) {
|
||||
if os.Getenv("QUERY_REPLICA_LOCAL_TEST") != "1" {
|
||||
t.Skip("owned synthetic replica required")
|
||||
}
|
||||
var reads atomic.Int32
|
||||
monitor := &event.CommandMonitor{Started: func(_ context.Context, e *event.CommandStartedEvent) { reads.Add(1) }}
|
||||
_, target, p := newLocalReplica(t, monitor, nil)
|
||||
reads.Store(0)
|
||||
if err := p.WaitNextPass(t.Context(), 100*time.Millisecond); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if reads.Load() != 0 {
|
||||
t.Fatal("idle interval queried authority", reads.Load())
|
||||
}
|
||||
_, err := p.control().UpdateOne(t.Context(), bson.M{"_id": "lease"}, bson.M{"$set": bson.M{"owner": "successor", "epoch": p.epoch + 1, "until": time.Now().Add(time.Minute)}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, stop := context.WithTimeout(t.Context(), 12*time.Second)
|
||||
defer stop()
|
||||
if err = p.WaitNextPass(ctx, 11*time.Second); !errors.Is(err, ErrQueryReplicaLease) {
|
||||
t.Fatal("idle heartbeat did not stop after losing ownership", err)
|
||||
}
|
||||
var state struct {
|
||||
Owner string `bson:"owner"`
|
||||
}
|
||||
if err = target.db.Collection(queryReplicaControl).FindOne(t.Context(), bson.M{"_id": "lease"}).Decode(&state); err != nil || state.Owner != "successor" {
|
||||
t.Fatal("old worker overwrote successor", state.Owner, err)
|
||||
}
|
||||
if reads.Load() != 0 {
|
||||
t.Fatal("idle heartbeat read authority", reads.Load())
|
||||
}
|
||||
}
|
||||
|
||||
func TestQueryPeriodicColdBuildKeepsSearchAndSnapshotSerializedLocal(t *testing.T) {
|
||||
if os.Getenv("QUERY_REPLICA_LOCAL_TEST") != "1" {
|
||||
t.Skip("owned synthetic replica required")
|
||||
}
|
||||
var sequence, firstOutput, firstModel atomic.Int32
|
||||
inserted := make(chan struct{}, 1)
|
||||
monitor := &event.CommandMonitor{Started: func(_ context.Context, e *event.CommandStartedEvent) {
|
||||
if e.CommandName == "find" {
|
||||
if name, _ := e.Command.Lookup("find").StringValueOK(); name == "annotationOutputSnapshotState" {
|
||||
firstOutput.CompareAndSwap(0, sequence.Add(1))
|
||||
}
|
||||
}
|
||||
if e.CommandName == "insert" {
|
||||
if name, _ := e.Command.Lookup("insert").StringValueOK(); name == "queryReadModelData" {
|
||||
firstModel.CompareAndSwap(0, sequence.Add(1))
|
||||
select {
|
||||
case inserted <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
}
|
||||
}
|
||||
}}
|
||||
source, target, p := newLocalReplica(t, nil, monitor)
|
||||
ctx := t.Context()
|
||||
for _, err := range []error{source.UpsertOrganization(ctx, domain.OrganizationRecord{ID: "o", Status: domain.OrganizationStatusActive}), source.UpsertProject(ctx, domain.ProjectRecord{ID: "p", OrganizationID: "o", Name: "p"}), source.UpsertHumanItem(ctx, domain.HumanItemRecord{ID: "i", OrganizationID: "o", ProjectID: "p", PluginID: "annotation-qa", CurrentStageID: "quality_review", Status: domain.HumanItemReady, Subject: domain.HumanSubjectRef{ItemKey: "capture/part0001"}})} {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if _, err := source.SaveProductionStatsScope(ctx, "o", productionstats.Scope{ProjectIDs: []string{"p"}, Revision: 1}, 0); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
finishLocalReplica(t, p)
|
||||
target.queryPeriodic = true
|
||||
target.ConfigureQueryServing(nil, nil)
|
||||
b := NewQueryReadBuilder(target)
|
||||
defer b.Close()
|
||||
worker, stop := context.WithCancel(ctx)
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- b.Run(worker) }()
|
||||
defer func() { stop(); <-done }()
|
||||
select {
|
||||
case <-inserted:
|
||||
case <-time.After(10 * time.Second):
|
||||
t.Fatal("search fixture did not write")
|
||||
}
|
||||
if firstOutput.Load() != 0 && firstOutput.Load() < firstModel.Load() {
|
||||
t.Fatalf("cold full scans overlapped on the bounded query store: output=%d search=%d", firstOutput.Load(), firstModel.Load())
|
||||
}
|
||||
}
|
||||
56
internal/store/query_periodic_test.go
Normal file
56
internal/store/query_periodic_test.go
Normal file
@ -0,0 +1,56 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestQueryPeriodicScheduleCoalescesAndKeepsInitialBuildImmediate(t *testing.T) {
|
||||
now := time.Date(2026, 11, 13, 0, 0, 0, 0, time.UTC)
|
||||
if !queryBuildDue(now, time.Time{}, queryPeriodicBuildInterval) {
|
||||
t.Fatal("cold build delayed")
|
||||
}
|
||||
for i := 1; i < 180; i++ {
|
||||
if queryBuildDue(now.Add(time.Duration(i)*time.Second), now, queryPeriodicBuildInterval) {
|
||||
t.Fatal("repeated changes caused early rebuild", i)
|
||||
}
|
||||
}
|
||||
if !queryBuildDue(now.Add(3*time.Minute), now, queryPeriodicBuildInterval) {
|
||||
t.Fatal("pending changes never scheduled")
|
||||
}
|
||||
s := &MongoStore{}
|
||||
for _, mode := range []string{"periodic", "continuous", ""} {
|
||||
if err := s.ConfigureQueryRefreshMode(mode); err != nil || s.queryPeriodic != (mode == "periodic") {
|
||||
t.Fatal(mode, err)
|
||||
}
|
||||
}
|
||||
if s.ConfigureQueryRefreshMode("unknown") == nil {
|
||||
t.Fatal("unknown refresh mode accepted")
|
||||
}
|
||||
}
|
||||
|
||||
func TestQueryPeriodicReadinessUsesPublishedAgeNotLiveBarrier(t *testing.T) {
|
||||
now := time.Now().UTC()
|
||||
h := QueryModelHead{Generation: "published", AsOf: now.Add(-2 * time.Minute), Revision: 1, GlobalRevision: 1}
|
||||
st := querySearchState{Global: 2, ObservedAt: time.Time{}}
|
||||
target := &MongoStore{queryPeriodic: true, queryReplicaFactStore: true}
|
||||
if !target.querySearchPublicationReady(st, h, true, 2, now) {
|
||||
t.Fatal("published search blocked by pending observer")
|
||||
}
|
||||
if target.querySearchPublicationReady(st, h, false, 2, now) {
|
||||
t.Fatal("missing publication accepted")
|
||||
}
|
||||
h.AsOf = now.Add(-16 * time.Minute)
|
||||
if target.querySearchPublicationReady(st, h, true, 2, now) {
|
||||
t.Fatal("expired generation accepted")
|
||||
}
|
||||
h.AsOf = now.Add(time.Second)
|
||||
if target.querySearchPublicationReady(st, h, true, 2, now) {
|
||||
t.Fatal("future generation accepted")
|
||||
}
|
||||
h.AsOf = now.Add(-time.Minute)
|
||||
authority := &MongoStore{queryPeriodic: true}
|
||||
if authority.querySearchPublicationReady(st, h, true, 2, now) {
|
||||
t.Fatal("display mode changed authority search behavior")
|
||||
}
|
||||
}
|
||||
@ -45,12 +45,16 @@ func (b *QueryReadBuilder) Close() { b.reader.Close() }
|
||||
func (b *QueryReadBuilder) WaitReady(ctx context.Context, org string, projects []string) error {
|
||||
tick := time.NewTicker(500 * time.Millisecond)
|
||||
defer tick.Stop()
|
||||
lastState := "unobserved"
|
||||
for {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return fmt.Errorf("query readiness last=%s: %w", lastState, err)
|
||||
}
|
||||
productionHead, production := b.store.readQueryModelHead(ctx, queryModelKey("production", org, projects), 5*time.Minute)
|
||||
st, err := b.store.querySearchState(ctx)
|
||||
production = production && productionHead.GlobalRevision == st.Global
|
||||
production = production && (b.store.periodicQueryStore() || productionHead.GlobalRevision == st.Global)
|
||||
h, exists := st.Heads[queryModelKey("search", org, nil)]
|
||||
search := err == nil && exists && h.Generation != "" && h.Revision == st.Revisions[queryModelKey("search", org, nil)] && h.GlobalRevision == st.Global && time.Since(st.ObservedAt) >= 0 && time.Since(st.ObservedAt) < 5*time.Second
|
||||
search := err == nil && b.store.querySearchPublicationReady(st, h, exists, st.Revisions[queryModelKey("search", org, nil)], time.Now())
|
||||
output := len(projects) == 0
|
||||
if !output {
|
||||
health, e := b.reader.Inspect(ctx, org, projects)
|
||||
@ -64,9 +68,12 @@ func (b *QueryReadBuilder) WaitReady(ctx context.Context, org string, projects [
|
||||
if production && search && output && workforce {
|
||||
return nil
|
||||
}
|
||||
if ctx.Err() == nil {
|
||||
lastState = fmt.Sprintf("production=%t search=%t output=%t workforce=%t", production, search, output, workforce)
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
return fmt.Errorf("query readiness last=%s: %w", lastState, ctx.Err())
|
||||
case <-tick.C:
|
||||
}
|
||||
}
|
||||
@ -296,12 +303,16 @@ func (b *QueryReadBuilder) run(ctx context.Context, started chan<- struct{}) err
|
||||
go func() {
|
||||
defer close(watchExited)
|
||||
lastHeartbeat, lastBarrier, observed := time.Time{}, time.Time{}, time.Time{}
|
||||
barrierInterval := time.Second
|
||||
if b.store.queryPeriodic {
|
||||
barrierInterval = 10 * time.Second
|
||||
}
|
||||
pendingChange := false
|
||||
for watchCtx.Err() == nil {
|
||||
// An empty filtered batch is not proof that the stream reached
|
||||
// the oplog end. Publish a private marker, then advance proof
|
||||
// only when that exact immutable update delta is observed.
|
||||
if time.Since(lastBarrier) >= time.Second {
|
||||
if time.Since(lastBarrier) >= barrierInterval {
|
||||
now := time.Now().UTC()
|
||||
result, e := b.store.queryControl().UpdateOne(watchCtx, queryLeaseFilter(l), bson.M{"$set": bson.M{"watchBarrier": now}})
|
||||
if e != nil {
|
||||
@ -335,7 +346,7 @@ func (b *QueryReadBuilder) run(ctx context.Context, started chan<- struct{}) err
|
||||
failWatch(e)
|
||||
return
|
||||
}
|
||||
if observed.IsZero() || time.Since(lastHeartbeat) < time.Second {
|
||||
if observed.IsZero() || time.Since(lastHeartbeat) < barrierInterval {
|
||||
continue
|
||||
}
|
||||
if e := b.recordObserved(watchCtx, l, observed, pendingChange); e != nil {
|
||||
@ -363,6 +374,7 @@ func (b *QueryReadBuilder) run(ctx context.Context, started chan<- struct{}) err
|
||||
defer tick.Stop()
|
||||
pendingStats, pendingSearch := map[string]bool{}, map[string]bool{}
|
||||
lastStats := map[string]time.Time{}
|
||||
lastSearch := map[string]time.Time{}
|
||||
searchRetries := map[string]queryBuildRetry{}
|
||||
statsRetries := map[string]queryBuildRetry{}
|
||||
var scopes map[string][]string
|
||||
@ -370,6 +382,13 @@ func (b *QueryReadBuilder) run(ctx context.Context, started chan<- struct{}) err
|
||||
lastScopeRead := time.Time{}
|
||||
lastCleanup := time.Now()
|
||||
lastWorkforceCleanup := time.Now()
|
||||
lastOutputWarm := time.Time{}
|
||||
statsInterval, searchInterval, scopeInterval := 30*time.Second, time.Duration(0), 30*time.Second
|
||||
maintenanceInterval, outputInterval := 5*time.Second, time.Second
|
||||
if b.store.queryPeriodic {
|
||||
statsInterval, searchInterval, scopeInterval = queryPeriodicBuildInterval, QueryPeriodicSyncInterval, QueryPeriodicSyncInterval
|
||||
maintenanceInterval, outputInterval = time.Minute, time.Minute
|
||||
}
|
||||
for {
|
||||
b.mu.Lock()
|
||||
all, allSearch := b.all, b.allSearch
|
||||
@ -382,7 +401,7 @@ func (b *QueryReadBuilder) run(ctx context.Context, started chan<- struct{}) err
|
||||
b.dirty, b.searchDirty = map[string]bool{}, map[string]bool{}
|
||||
b.all, b.allSearch = false, false
|
||||
b.mu.Unlock()
|
||||
if refreshScopes || all || time.Since(lastScopeRead) > 30*time.Second {
|
||||
if refreshScopes || all || time.Since(lastScopeRead) > scopeInterval {
|
||||
var e error
|
||||
scopes, e = b.scopes(ctx)
|
||||
if e != nil {
|
||||
@ -410,9 +429,13 @@ func (b *QueryReadBuilder) run(ctx context.Context, started chan<- struct{}) err
|
||||
if time.Now().Before(searchRetries[org].Next) {
|
||||
continue
|
||||
}
|
||||
if !queryBuildDue(time.Now(), lastSearch[org], searchInterval) {
|
||||
continue
|
||||
}
|
||||
if e := b.store.buildQuerySearchModel(ctx, l, org); e == nil {
|
||||
delete(searchRetries, org)
|
||||
delete(pendingSearch, org)
|
||||
lastSearch[org] = time.Now()
|
||||
} else {
|
||||
retry := searchRetries[org]
|
||||
retry.fail(5*time.Second, time.Minute)
|
||||
@ -435,7 +458,7 @@ func (b *QueryReadBuilder) run(ctx context.Context, started chan<- struct{}) err
|
||||
delete(pendingStats, org)
|
||||
continue
|
||||
}
|
||||
if time.Since(lastStats[org]) < 30*time.Second {
|
||||
if !queryBuildDue(time.Now(), lastStats[org], statsInterval) {
|
||||
continue
|
||||
}
|
||||
if time.Now().Before(statsRetries[org].Next) {
|
||||
@ -459,17 +482,20 @@ func (b *QueryReadBuilder) run(ctx context.Context, started chan<- struct{}) err
|
||||
finish()
|
||||
lastCleanup = time.Now()
|
||||
}
|
||||
if b.store.queryReplicaFactStore && time.Since(lastWorkforceCleanup) > 5*time.Second {
|
||||
if b.store.queryReplicaFactStore && time.Since(lastWorkforceCleanup) > maintenanceInterval {
|
||||
cleanup, finish := context.WithTimeout(ctx, 2*time.Second)
|
||||
_ = b.store.maintainWorkforceVersions(cleanup, l)
|
||||
finish()
|
||||
lastWorkforceCleanup = time.Now()
|
||||
}
|
||||
// Prewarm due generations even when no browser is connected.
|
||||
for org, ids := range scopes {
|
||||
if len(ids) > 0 {
|
||||
_, _ = b.reader.Prewarm(ctx, org, ids)
|
||||
if queryBuildDue(time.Now(), lastOutputWarm, outputInterval) {
|
||||
for org, ids := range scopes {
|
||||
if len(ids) > 0 {
|
||||
_, _ = b.reader.Prewarm(ctx, org, ids)
|
||||
}
|
||||
}
|
||||
lastOutputWarm = time.Now()
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
|
||||
@ -142,6 +142,9 @@ func (s *MongoStore) readQueryModelHead(ctx context.Context, key string, maxAge
|
||||
h, ok := st.Heads[key]
|
||||
age := time.Since(h.AsOf)
|
||||
verifiedUnchanged := st.SourceRevision == h.SourceRevision && time.Since(st.ObservedAt) >= 0 && time.Since(st.ObservedAt) <= 5*time.Second
|
||||
if s.periodicQueryStore() {
|
||||
maxAge = queryPeriodicDisplayMaxAge
|
||||
}
|
||||
return h, ok && h.Generation != "" && age >= 0 && (age <= maxAge || verifiedUnchanged)
|
||||
}
|
||||
func (s *MongoStore) publishQueryModelHead(ctx context.Context, l queryModelLease, key string, h QueryModelHead) error {
|
||||
@ -255,6 +258,9 @@ func (s *MongoStore) LoadProductionReport(ctx context.Context, org string, proje
|
||||
}
|
||||
}
|
||||
}
|
||||
if s.periodicQueryStore() {
|
||||
return productionstats.Report{}, "", ErrQueryReplicaNotReady
|
||||
}
|
||||
snapshot, err := s.LoadProductionStatsCached(ctx, org, projects)
|
||||
if err != nil {
|
||||
return productionstats.Report{}, "", err
|
||||
|
||||
@ -40,6 +40,11 @@ func (s *MongoStore) countReplicaBarrier(ctx context.Context, org string) (*quer
|
||||
b.org = org
|
||||
b.ids = map[string]bool{}
|
||||
b.batches = map[string]bool{}
|
||||
// Periodic display reports committed query facts. Pending business
|
||||
// changes wait for the next projector cycle; do not chase them per reader.
|
||||
if s.periodicQueryReads() {
|
||||
return
|
||||
}
|
||||
// Acknowledgement follows the majority-committed target transaction. Thus a
|
||||
// clean source record is already present in the query DB; pending records
|
||||
// are overlaid from authority. Never assume a dirty record is absent.
|
||||
@ -183,6 +188,9 @@ func (s *MongoStore) visitReplicaPendingCounts(ctx context.Context, org string,
|
||||
}
|
||||
|
||||
func (s *MongoStore) replicaBatchAccess(ctx context.Context, target *MongoStore, org string, ids []string) ([]domain.HumanBatchRecord, error) {
|
||||
if s.periodicQueryReads() {
|
||||
return s.listAuthoritativeBatchAccess(ctx, org, ids)
|
||||
}
|
||||
b, e := s.countReplicaBarrier(ctx, org)
|
||||
if e != nil {
|
||||
return nil, e
|
||||
|
||||
@ -6,6 +6,7 @@ import (
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"go.mongodb.org/mongo-driver/event"
|
||||
"go.mongodb.org/mongo-driver/mongo"
|
||||
"go.mongodb.org/mongo-driver/mongo/options"
|
||||
"go.mongodb.org/mongo-driver/mongo/readconcern"
|
||||
@ -15,7 +16,7 @@ import (
|
||||
// ConfigureQueryReplicaFromEnvironment is an entry-point operation. Invalid or
|
||||
// unavailable reporting infrastructure cannot disable existing work mutations.
|
||||
// Explicit serving mode fails closed for reports, with no source-scan fallback.
|
||||
func (s *MongoStore) ConfigureQueryReplicaFromEnvironment(ctx context.Context) (func(), error) {
|
||||
func (s *MongoStore) ConfigureQueryReplicaFromEnvironment(ctx context.Context, monitors ...*event.CommandMonitor) (func(), error) {
|
||||
if os.Getenv("QUERY_REPLICA_REPORTS") != "1" {
|
||||
return func() {}, nil
|
||||
}
|
||||
@ -27,7 +28,14 @@ func (s *MongoStore) ConfigureQueryReplicaFromEnvironment(ctx context.Context) (
|
||||
if uri == "" || db == "" || os.Getenv("QUERY_SERVING_NAMESPACE") == "" {
|
||||
return func() {}, errors.New("explicit query database and namespace required")
|
||||
}
|
||||
c, e := mongo.Connect(ctx, options.Client().ApplyURI(uri).SetReadPreference(readpref.Primary()).SetReadConcern(readconcern.Majority()).SetMaxPoolSize(20).SetServerSelectionTimeout(2*time.Second))
|
||||
if len(monitors) > 1 {
|
||||
return func() {}, errors.New("only one query command monitor is supported")
|
||||
}
|
||||
opts := options.Client().ApplyURI(uri).SetReadPreference(readpref.Primary()).SetReadConcern(readconcern.Majority()).SetMaxPoolSize(20).SetServerSelectionTimeout(2 * time.Second)
|
||||
if len(monitors) == 1 {
|
||||
opts.SetMonitor(monitors[0])
|
||||
}
|
||||
c, e := mongo.Connect(ctx, opts)
|
||||
if e != nil {
|
||||
return func() {}, errors.New("query database connection unavailable")
|
||||
}
|
||||
|
||||
@ -120,7 +120,11 @@ func (s *MongoStore) reportReplica(ctx context.Context) (*MongoStore, error) {
|
||||
return nil, e
|
||||
}
|
||||
age := time.Since(state.ObservedAt)
|
||||
if e != nil || !state.Ready || state.Schema != queryReplicaSchema || state.Binding.SourceDatabase != s.db.Name() || state.Binding.Namespace != queryReplicaNamespace(s.db.Name()) || age < 0 || age > s.queryReplicaReads.maxAge {
|
||||
maxAge := s.queryReplicaReads.maxAge
|
||||
if s.periodicQueryReads() {
|
||||
maxAge = queryPeriodicDisplayMaxAge
|
||||
}
|
||||
if e != nil || !state.Ready || state.Schema != queryReplicaSchema || state.Binding.SourceDatabase != s.db.Name() || state.Binding.Namespace != queryReplicaNamespace(s.db.Name()) || age < 0 || age > maxAge {
|
||||
return nil, ErrQueryReplicaNotReady
|
||||
}
|
||||
return s.queryReplicaReads.target, nil
|
||||
|
||||
@ -194,7 +194,7 @@ func (s *MongoStore) querySearchCandidates(ctx context.Context, org, search, cap
|
||||
}
|
||||
key := queryModelKey("search", org, nil)
|
||||
h, ok := st.Heads[key]
|
||||
if !ok || h.Generation == "" || h.Revision != st.Revisions[key] || h.GlobalRevision != st.Global || time.Since(st.ObservedAt) > 5*time.Second || time.Since(st.ObservedAt) < 0 {
|
||||
if !s.querySearchPublicationReady(st, h, ok, st.Revisions[key], time.Now()) {
|
||||
return nil, false
|
||||
}
|
||||
cacheKey := queryserving.Key("search-candidates/v1", s.db.Name(), org, h.Generation, search, capture)
|
||||
|
||||
@ -11,6 +11,9 @@ import (
|
||||
// different deployments even when their database names happen to be identical.
|
||||
func (s *MongoStore) ConfigureQueryServingFromEnvironment() (func(), error) {
|
||||
close := func() {}
|
||||
if err := s.ConfigureQueryRefreshMode(os.Getenv("QUERY_REPLICA_REFRESH_MODE")); err != nil {
|
||||
return close, err
|
||||
}
|
||||
if os.Getenv("QUERY_SERVING_READS") != "1" {
|
||||
return close, nil
|
||||
}
|
||||
|
||||
@ -74,7 +74,11 @@ func (s *MongoStore) workforceHead(ctx context.Context, org string) (QueryModelH
|
||||
return h, false, nil
|
||||
}
|
||||
unchanged := h.SourceRevision == state.Revision && !state.Observed.IsZero() && time.Since(state.Observed) >= 0 && time.Since(state.Observed) <= 5*time.Second
|
||||
if h.Collection != workforceFactCollection || h.FactEpoch < 1 || h.Generation != workforceFactSchema || time.Since(h.AsOf) < 0 || (!unchanged && time.Since(h.AsOf) > dashboardStatsCacheTTL) {
|
||||
maxAge := dashboardStatsCacheTTL
|
||||
if s.periodicQueryStore() {
|
||||
maxAge = queryPeriodicDisplayMaxAge
|
||||
}
|
||||
if h.Collection != workforceFactCollection || h.FactEpoch < 1 || h.Generation != workforceFactSchema || time.Since(h.AsOf) < 0 || (!unchanged && time.Since(h.AsOf) > maxAge) {
|
||||
return h, true, ErrQueryReplicaNotReady
|
||||
}
|
||||
return h, true, nil
|
||||
@ -546,10 +550,16 @@ func (s *MongoStore) loadWorkforceFactReport(ctx context.Context, org string, q
|
||||
return workforcestats.Report{}, false, nil
|
||||
}
|
||||
h, exists, err := s.workforceHead(ctx, org)
|
||||
if s.periodicQueryStore() && !exists && err == nil {
|
||||
return workforcestats.Report{}, true, ErrQueryReplicaNotReady
|
||||
}
|
||||
if err != nil || !exists {
|
||||
return workforcestats.Report{}, exists, err
|
||||
}
|
||||
key := queryserving.Key(workforceFactSchema, s.db.Name(), org, h.FactEpoch, q)
|
||||
if s.periodicQueryStore() {
|
||||
key = queryserving.Key("periodic-report/v1", key)
|
||||
}
|
||||
if raw, ok := workforceReportCache.get(key, dashboardStatsCacheTTL); ok {
|
||||
var report workforcestats.Report
|
||||
if json.Unmarshal(raw, &report) == nil {
|
||||
@ -579,6 +589,9 @@ func (s *MongoStore) loadWorkforceFactReport(ctx context.Context, org string, q
|
||||
return workforcestats.Report{}, true, err
|
||||
}
|
||||
report := workforcestats.Report{Items: []domain.DashboardWorkforceMetric{}}
|
||||
if s.periodicQueryStore() {
|
||||
report.AsOf = h.AsOf.UTC().Format(time.RFC3339Nano)
|
||||
}
|
||||
if len(rows) > 0 {
|
||||
for _, m := range rows[0].Items {
|
||||
var v domain.DashboardWorkforceMetric
|
||||
|
||||
@ -41,6 +41,7 @@ type Snapshot struct {
|
||||
}
|
||||
|
||||
type Report struct {
|
||||
AsOf string `json:"asOf,omitempty"`
|
||||
Items []domain.DashboardWorkforceMetric
|
||||
Totals domain.DashboardWorkforceMetric
|
||||
Details []domain.WorkforceReturnDetail
|
||||
|
||||
@ -130,6 +130,7 @@ def main():
|
||||
ap.add_argument("--max-actors", type=int, default=81, choices=(81, 200, 500))
|
||||
ap.add_argument("--fixture-plan", help="owned.json from the bounded in-cluster fixture")
|
||||
ap.add_argument("--cold-write-probe", action="store_true", help="bounded 8MiB synthetic diagnostic; never HTTP capacity evidence")
|
||||
ap.add_argument("--periodic-refresh", action="store_true", help="explicit periodic display policy; same load and safety limits")
|
||||
ap.add_argument("--cold-pace-kib", type=int, choices=(0, 128), default=0)
|
||||
args = ap.parse_args()
|
||||
assert args.cold_write_probe or args.cold_pace_kib == 0
|
||||
@ -244,6 +245,7 @@ def main():
|
||||
limits = {"ready": max(1, max(s["readySeconds"] for s in baseline) * 2), "pressure": max(0.5, max(s["protectedFullAvg10"] for s in baseline) + 0.5), "await": max(10, max(s["awaitMs"] for s in baseline) * 3)}
|
||||
manifest = {"sourceCommit": run(["git", "-C", str(ROOT), "rev-parse", "HEAD"]), "sourceFingerprint": source_fingerprint(), "sourceContainer": source["Id"], "sourceImage": source["Image"], "sourceCPU": 4, "sourceMemory": 4_294_967_296, "queryPodUID": query["metadata"]["uid"], "queryImage": query["spec"]["containers"][0]["image"], "queryCPU": 1, "queryMemory": "1Gi", "tasks": args.tasks, "waves": args.waves, "maxActors": args.max_actors, "wireCompressionBothVariants": os.getenv("QUERY_COMPARE_WIRE_COMPRESSION", ""), "minimumPoolBothVariants": os.getenv("QUERY_COMPARE_MIN_POOL", "0"), "limits": limits, "limitations": ["source Docker VM shares CPU with existing local containers", "target has 2MiB read / 1MiB write and 64/32 IOPS hard limits", "short comparison; not 30-minute three-round production capacity acceptance", "health probes do not establish customer workflow latency"]}
|
||||
manifest["baselineSamples"] = len(baseline)
|
||||
manifest["refreshMode"] = "periodic" if args.periodic_refresh else "continuous"
|
||||
manifest["baselineSeconds"] = baseline[-1]["time"] - baseline[0]["time"]
|
||||
manifest["diagnostic"] = {"kind": "cold-write-probe", "payloadBytes": 8 << 20, "paceKiB": args.cold_pace_kib} if args.cold_write_probe else None
|
||||
if fixture:
|
||||
@ -252,12 +254,14 @@ def main():
|
||||
manifest["binarySHA256"] = owned["binarySHA256"]
|
||||
(out / "manifest.json").write_text(json.dumps(manifest, indent=2))
|
||||
env = dict(os.environ, QUERY_REPLICA_ISOLATED_E2E="1", QUERY_REPLICA_CLUSTER_TEST="1", QUERY_REPLICA_TEST_TARGET_URI="mongodb://127.0.0.1:27229/?directConnection=true&replicaSet=query-lab-rs", QUERY_COMPARE_TASKS=str(args.tasks), QUERY_COMPARE_WAVES=str(args.waves), QUERY_COMPARE_MAX_ACTORS=str(args.max_actors), QUERY_COMPARE_REPORT=str(out / "comparison.json"))
|
||||
env["QUERY_REPLICA_REFRESH_MODE"] = manifest["refreshMode"]
|
||||
argv = ["go", "test", "./internal/api", "-run", "^TestQueryReplicaIsolatedHTTPComparisonLocal$", "-count=1", "-timeout=45m", "-v"]
|
||||
remote_report = "/synthetic/query-comparison-" + str(int(time.time())) + ".json"
|
||||
if fixture:
|
||||
settings = ["TMPDIR=/synthetic/tmp", "QUERY_REPLICA_ISOLATED_E2E=1", "QUERY_REPLICA_CLUSTER_TEST=1", "QUERY_REPLICA_TEST_TARGET_URI=mongodb://127.0.0.1:27229/?directConnection=true&replicaSet=query-lab-rs", "QUERY_COMPARE_IN_CLUSTER=1", "QUERY_COMPARE_TASKS=" + str(args.tasks), "QUERY_COMPARE_WAVES=" + str(args.waves), "QUERY_COMPARE_MAX_ACTORS=" + str(args.max_actors), "QUERY_COMPARE_REPORT=" + remote_report, "QUERY_COMPARE_WIRE_COMPRESSION=" + os.getenv("QUERY_COMPARE_WIRE_COMPRESSION", ""), "QUERY_COMPARE_MIN_POOL=" + os.getenv("QUERY_COMPARE_MIN_POOL", "0")]
|
||||
if args.cold_write_probe:
|
||||
settings += ["QUERY_REPLICA_COLD_WRITE_PROBE=1", "QUERY_COLD_PACE_KIB=" + str(args.cold_pace_kib)]
|
||||
settings += ["QUERY_REPLICA_REFRESH_MODE=" + manifest["refreshMode"]]
|
||||
argv = k + ["-n", "droboticflow-query-lab", "exec", "node4-query-fixture", "-c", "lab", "--", "env"] + settings + [api_binary, "-test.run=^" + test_name + "$", "-test.count=1", "-test.timeout=45m", "-test.v"]
|
||||
with (out / "test.log").open("w") as log:
|
||||
child = subprocess.Popen(argv, cwd=ROOT, env=env, stdout=log, stderr=subprocess.STDOUT, start_new_session=True)
|
||||
|
||||
@ -24,6 +24,7 @@ CASES = ('internal/store/query_replica_local_test.go',
|
||||
'internal/store/workforce_report_cache_local_test.go',
|
||||
'internal/store/workforce_facts_local_test.go',
|
||||
'internal/store/query_cold_write_local_test.go',
|
||||
'internal/store/query_periodic_local_test.go',
|
||||
'internal/store/query_pool_buckets_local_test.go')
|
||||
|
||||
|
||||
|
||||
@ -1021,6 +1021,8 @@
|
||||
"^internal/store/workforce_(report_cache|heap_diagnostic|facts|return_page).*\\.go$",
|
||||
"^internal/store/query_pool_buckets.*\\.go$",
|
||||
"^internal/store/query_cold_write.*\\.go$",
|
||||
"^internal/store/query_periodic.*\\.go$",
|
||||
"^internal/api/query_periodic.*\\.go$",
|
||||
"^internal/api/query_fact_routes_test\\.go$",
|
||||
"^internal/api/workforce_compact_report_test\\.go$",
|
||||
"^internal/testsupport/query_replica_lab.*\\.go$",
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user