diff --git a/cmd/query-replica/main.go b/cmd/query-replica/main.go index 39ccc1e7..29d32b0b 100644 --- a/cmd/query-replica/main.go +++ b/cmd/query-replica/main.go @@ -24,7 +24,22 @@ func main() { once := flag.Bool("once", false, "one bounded pass only") interval := flag.Duration("interval", time.Second, "minimum delay between bounded passes") flag.Parse() - if !*writers || *interval < 100*time.Millisecond || os.Getenv("QUERY_SERVING_NAMESPACE") == "" { + mode := os.Getenv("QUERY_REPLICA_REFRESH_MODE") + if mode != "" && mode != "continuous" && mode != "periodic" { + log.Fatal("invalid query refresh mode") + } + if mode == "periodic" { + explicit := false + flag.Visit(func(f *flag.Flag) { + if f.Name == "interval" { + explicit = true + } + }) + if !explicit { + *interval = store.QueryPeriodicSyncInterval + } + } + if !*writers || *interval < 100*time.Millisecond || *interval > 5*time.Minute || os.Getenv("QUERY_SERVING_NAMESPACE") == "" { log.Fatal("writer coverage, namespace and bounded polling interval required") } ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) @@ -76,10 +91,16 @@ func main() { } return } - select { - case <-ctx.Done(): - return - case <-time.After(*interval): + // Drain bounded backlog immediately, then idle without repeated source + // scans. Keep the original lease alive only in the independent query DB. + if ready || mode != "periodic" { + if e := q.WaitNextPass(ctx, *interval); e != nil { + if ctx.Err() == nil { + log.Print("projector lease unavailable; checkpoint retained") + os.Exit(1) + } + return + } } } } diff --git a/docs/adr/0014-independent-query-database.md b/docs/adr/0014-independent-query-database.md index 04c9ec2e..532dee7c 100644 --- a/docs/adr/0014-independent-query-database.md +++ b/docs/adr/0014-independent-query-database.md @@ -39,3 +39,16 @@ 人员统计复用原历史计算器生成无正文贡献,按不可变版本发布。完整新版本提交前始终保留上一版本;重启清理未提交暂存,已提交历史按有界宽限清理。历史供应商归属和正式操作抑制旧审查仍依查询范围解析,日期边界保留纳秒。后台当前使用有界组织快照计算与差量写入,尚非逐任务分区增量计算;超预算保留旧版本,不截断数据、不增加核心作业准入条件。 作业记录在查询库筛选计数及分页后,仅补选中页详情与权威文字,来源指纹不一致时拒绝拼接。旧任务/未首次物化的库保留读库历史兼容路径。极少数等价时间字符串的排序统一为绝对时间和稳定身份,避免偏移页漂移。以上实现仍需完整规模性能及持久化确认压力验收,不能将本地功能测试当作上线承诺。 + + +## 周期刷新与延迟展示(2026-11-13,用户批准,本地实施) + +显式 `QUERY_REPLICA_REFRESH_MODE=periodic` 启用新契约;缺省或 `continuous` 保持原兼容语义,报告/任务池读取开关仍分别默认关闭。原库按持久变更标记增量同步:每次追平后空闲约1分钟,待处理批次继续有界排空;空闲只在查询库续租,不反复扫描原库。历史仍限速、可恢复地一次回填,低频对账保留。 + +查询库内变化按组织合并,搜索重建至少间隔1分钟、统计至少间隔3分钟,初次构建立即执行。原有变更观察/租约栅栏保留,减少心跳、预热和历史清理的轮询次数;权限不进入缓存。当前统计仍是按受影响组织读取快照、归约后差量写入,不宣称已经实现逐任务增量计算。重启会重新核对并重建,未完成暂存不能替换已发布版本。 + +展示优先读取已多数提交的结果;允许同步水位/报表计算落后,15分钟数据年龄边界与绑定、schema、未来时间检查仍保留。源同步水位失效不会回扫原库。统计刷新待处理或失败时继续提供时间边界内的旧完整版本;新模式下生产报表/常规人员统计首次发布前返回未就绪,不因浏览器请求启动全历史计算。历史少见查询语义仍保留查询库兼容计算路径,不回源扫描。人员统计返回可选 `asOf`,生产报表沿用既有时间字段;时间为结果实际计算时间,不冒充实时。 + +计数展示不再读取原库脏任务/删除意图做即时补偿,因此数量是查询库已提交事实的结果,不保证包含刚刚发生的操作。批次权限/策略、账号、成员、供应商权限继续实时校验;列表维持本页当前归属/版本读取与原服务端可见性/`canClaim`,领取、保存、送检、质检、验收和后台调度均沿原权威路径。此变更不允许旧缓存绕过权限,也不允许以旧任务列表重复领取。 + +本次只在合成隔离环境验证,不切换生产、不修改客户数据。固定原请求超时、租约和多数提交要求;延迟展示是独立业务契约,不是延长请求超时或降低持久性来通过测试。验证必须单列本机与25Gi共享盘结果,以及冷构建、故障恢复和持续变化期间的覆盖限制。 diff --git a/docs/operations/query-replica.md b/docs/operations/query-replica.md index 760892e2..c3bbfe04 100644 --- a/docs/operations/query-replica.md +++ b/docs/operations/query-replica.md @@ -108,3 +108,48 @@ QUERY_REPLICA_LOCAL_TEST=1 QUERY_REPLICA_SCALE_LOCAL=1 go test ./internal/store 本批只读存储盘点:当前注册的 StorageClass 为静态 `csi-bos`,现有80Gi本地卷和200Gi BOS卷均已占用,未做修改。隔离验收仍需独立SSD持久卷,不能将现有BOS挂载视为已满足数据库文件系统要求。MongoDB要求目录fsync等文件系统语义,并推荐WiredTiger使用XFS,参见[官方生产说明](https://www.mongodb.com/docs/manual/administration/production-notes/)。节点allocatable不等于空闲资源;尚未创建Pod、PV、StorageClass或Secret。 定稿性能复测中,单项目500并发曾有70次就绪检查失败。已将重复的水位读取合并为仅在途共享(无跨请求TTL、无权限缓存),以真实Mongo命令监控证明100个同时检查只读一次;相同夹具复测新路径所有档位均零错误。旧路径仍有超时,因此组合对照仍标为失败,而非性能验收通过。最终全部项目500并发p95528ms仍高于暖态500ms目标;81/200档通过的只是存储计数访问器,完整HTTP/业务授权及长稳态仍待验收。 + + +## 周期展示模式(本地实施,未切生产) + +`QUERY_REPLICA_REFRESH_MODE=periodic` 需要在 API、query-replica 与 query-read-builder 一致配置。它不启用任何原先关闭的报告/计数/任务池读取开关,也不替代写者覆盖和首次完整同步验收。无该设置时保留原连续模式。 + +- 投影器追平后约60秒再做增量读取,积压按原200条有界批次连续排空;空闲每10秒只在独立查询库续原30秒租约。失权或三秒确认失败退出并保留断点,不扫描原库保活,不延长租约。 +- 变化按组织合并,搜索至少60秒、统计至少180秒计算一次;首次构建立即开始。统计复用旧归约器并差量发布,尚非逐任务分区计算;64MiB/250,000行背景快照上限仍在。 +- 展示读已提交结果,刷新期间保留旧完整版本。同步水位与结果年龄的正常延迟允许到15分钟边界;原绑定/schema/未来时间校验保留,超界报告未就绪且不回源全扫。计数不再逐请求追读原库脏任务;批次策略、用户/成员/供应商权限仍即时读取,列表本页归属/版本和全部正式操作仍走权威路径。 +- 人员统计可选 `asOf` 与既有生产报表时间标识实际发布数据时间;不是请求时间。首次无已发布结果时不由客户请求触发组织全量计算。旧模式和历史少见查询兼容路径保留。 + +本轮证据保存在 `local-data/diagnostics/mongo-cpu/periodic-refresh/`: + +1. `red.log` 先复现已发布两分钟的结果被同步延迟拒绝、单次计数追加3次原库追平读取;对应回归修复后通过。生命周期覆盖旧完整版本、变化合并、未提交暂存恢复、边界过期和来源隔离;真实HTTP权限撤销、未同步批次策略变化、自审禁止及标注保留通过。 +2. 首次26218任务本机对照 `http-26218.log` 因统一就绪条件仍要求秒级观察水位而失败,原记录保留。修改仅对新周期模式采用已发布代次及年龄的就绪规则,原四分钟等待/请求超时及持久性不变。 +3. `http-26218-published.json`:26218任务、20项目、81账号、两轮162请求,新路径零错误,总p95=1325.065ms(计数1335.207、人员统计553.760、作业记录309.176、生产总览40.549ms)。旧路径16次MaxTimeMS失败,因此整体测试失败,不能从这轮报告严格等成功降载比例;人员统计混合冷热p95也不作为暖态500ms目标通过。 +4. `http-26218-matched.json`:相同数据、81账号、162请求,新旧均限制8个在途请求,仅为等成功降载/逐账号结果补充;双方零错误、162份结果逐条匹配、无差异。源Mongo累计CPU 15.272151→2.314956秒(下降84.84%);扫描文档10,078,564→1,450,972(下降85.60%);逻辑BSON返回字节下降78.91%。该轮整体p95旧710.792ms、新785.101ms,不能宣称低并发整体延迟下降;新路径人员统计99.604、生产5.574、作业记录61.113、计数811.500ms。 + +两轮均固定源Mongo4CPU/4Gi、查询Mongo1CPU/1Gi、WT缓存0.25Gi;Go HTTP/构建进程在本机,两个Mongo共享Docker主机,没有`.4`节点共享盘IO限额。包含请求测量期间的实际同步/对账/确认开销,但短时不包含长期持续变化/缓存失效所有周期。新旧源CPU为累计用时,不能当作线上CPU百分比;逻辑BSON字节不能当作公网视频带宽。余下较多源读取是当前授权/旧授权兼容路径,未引入跨请求权限缓存来换性能。 + +共享盘25Gi冷构建和完整HTTP对照、长期稳定性与暖态500ms仍单列待验收;本机证据不替代它们。不改生产数据、路由或资源,不因本地通过准备部署。 + + +### 共享盘冷构建及未采用实验 + +`periodic-refresh/cluster-26218` 在原25Gi、1CPU/1Gi、2MiB/s读/1MiB/s写及64/32 IOPS下实际运行:历史复制372.203秒;搜索78.042秒;统计加载/归约2.435秒,确认写入17,720行/26,559,500字节,统计阶段95.638秒,生产/统计读表总阶段112.633秒后发布。随后产出快照约47秒仍在构建时触及原四分钟就绪预算,取消;未进入HTTP阶段,没有本轮p95/CPU降载比,`coverageComplete=false`。终止时取消上下文会令元数据读取均失败,不能把终端打印的全false理解成已发布版本丢失;后续诊断保留最后一次有效就绪状态。 + +该轮生产健康保护未触发,最终健康无错误;源夹具/策略/桥接按UID/PID清理,配额CAS恢复,25Gi标记保留。没有客户记录或生产路由变更。 + +曾尝试让初次产出快照与搜索重叠、共用原写入预算;小回归及竞态验证通过,但同规模本机1Gi查询Mongo出现OOM(exit137),对应 `overlap-oom.json`、`overlap-experiment.patch` 和 `http-26218-overlap-matched.log`。不能将并行视为安全优化,也未证明并行是内存峰值的唯一原因。**最终代码撤回该调度,保留搜索/冷快照串行保护,不增资源、不降持久性、不放宽超时。** 本机Mongo按原容器/配额恢复,仅删除该次精确随机测试库。该实验未推送、未在公司集群执行。 + +正式任务检查 `verify-1794585127732124000/report.json` 中 Go基线、看板/刷新浏览器、统计稳定性、查询服务race与完整Mongo/Valkey副本race均通过;系统Python缺pytest项用现有根目录venv复跑37通过/4个真实交付夹具缺失跳过。额外原始API全包仍失败,分类依现有Go门禁:68项均在基线、0新增;不手工扩失败基线。原任务失败/跳过记录均保留,不将整项标成全绿。 + +周期刷新/已发布结果读取的实现和本机降载证据可以继续评估;共享盘冷容量仍不通过,因此不准备生产发布。后续优先把旧产出快照重复历史读取/构建开销独立量化并减少,不以并发堆积、提高IO限额、扩内存或延长等待来冒充当前配置达标。 + + +### 最终串行候选的补充验证与剩余风险 + +- 撤回重叠调度后,`http-final-matched.log` 在管理员人员统计首次聚合出现一次 MaxTimeMSExpired,未形成完整对照。慢日志元数据 `final-fact-slow-metadata.json` 显示规划约5.8ms、执行CPU约834ms;不能据此认定规划是根因,也不能用上一次成功结果掩盖本次失败。 +- 增加了可选启动命令监控接线,使实际API查询客户端也能记录预算,默认运行不安装监控。此前预算探针只接到构建客户端而未捕获HTTP事实聚合,该空报告不能作为API预算证据。 +- 带诊断对照 `http-diagnostic-matched.json` 新路径162请求零错误、160份可配对业务结果一致,但旧路径2次失败;整体仍失败,不能据此重新报告严格降载比例。该次仅保留两个明确本机合成库,供快速复查:源 `query_http_synthetic-58b6b07eb71b63db7f4541fefa653bfa`、查询 `query_http_synthetic-96fa522d81f68b11f10d0dd4f8394644`;客户数据不在其中。 +- 使用该保留夹具、真实HTTP/会话/管理员授权重放3次均200,首次实际聚合约1518ms,后续命中缓存。`replay-budget.json` 确认发送maxTimeMS=8000,上下文未额外缩短预算。临时合成会话已清理。单次冷态失败仍未稳定复现,不能标为已修复;本机容器与宿主时钟相差约19秒的观测也不能直接归因为该错误,未更改全局时钟或其他工作树的资源。 +- 最终快速检查 `verify-1794589491130862000/report.json` 所选检查通过;最终完整副本race/cache记录 `check-5b8d9540efff4b01825a878e91549000.log` 无缺失/跳过/源码变化。新增串行调度保护回归保留,失败的并行实验日志保留。前述原始任务检查的交付夹具跳过、原始API既有失败仍如实保留,不将最终状态写成整体验收通过。 + +本批没有推送、PR、生产应用或数据变更;独立查询库周期读取开关仍默认关闭。下一步先稳定复现冷态统计超时,并减少旧产出快照和统计的重复读/持久化确认轮次;达到原配置下的完整正确性、速度与资源对照后再准备上线。 diff --git a/internal/api/dashboard_workforce.go b/internal/api/dashboard_workforce.go index e3cec0d1..e93867b5 100644 --- a/internal/api/dashboard_workforce.go +++ b/internal/api/dashboard_workforce.go @@ -85,7 +85,7 @@ func (server *server) populateDashboardWorkforce(ctx context.Context, requestOrg return err } summary.AudienceScope = scope - summary.Workforce = domain.DashboardWorkforceSummary{From: from.Format(time.RFC3339Nano), To: to.Format(time.RFC3339Nano), Totals: report.Totals, Items: report.Items} + summary.Workforce = domain.DashboardWorkforceSummary{AsOf: report.AsOf, From: from.Format(time.RFC3339Nano), To: to.Format(time.RFC3339Nano), Totals: report.Totals, Items: report.Items} return nil } diff --git a/internal/api/query_periodic_fixture_test.go b/internal/api/query_periodic_fixture_test.go new file mode 100644 index 00000000..a056dea3 --- /dev/null +++ b/internal/api/query_periodic_fixture_test.go @@ -0,0 +1,55 @@ +package api + +import ( + "encoding/json" + "os" + "os/exec" + "strings" + "testing" +) + +// A separate opt-in for same-size LOCAL synthetic HTTP comparison. It cannot +// waive the owned marker / node guards of the company-cluster harness. +func verifyPeriodicLocalHTTPFixture(t *testing.T) { + t.Helper() + if os.Getenv("QUERY_REPLICA_CLUSTER_TEST") != "" || os.Getenv("QUERY_REPLICA_TEST_TARGET_URI") != "" || os.Getenv("QUERY_COMPARE_IN_CLUSTER") != "" { + t.Fatal("local comparison cannot select a cluster/alternate URI") + } + endpoint, err := exec.Command("docker", "context", "inspect", "--format", "{{.Endpoints.docker.Host}}").Output() + if err != nil || !strings.HasPrefix(strings.TrimSpace(string(endpoint)), "unix://") { + t.Fatal("owned local Docker socket required") + } + for _, name := range []string{"source", "query"} { + raw, err := exec.Command("docker", "inspect", "fanyu-query-replica-local-"+name+"-1").Output() + if err != nil { + t.Fatal("local Mongo fixture unavailable") + } + var d []struct { + State struct{ Running bool } + Config struct { + Labels map[string]string + Cmd []string + } + HostConfig struct { + NanoCpus int64 + Memory int64 + PortBindings map[string][]struct{ HostIp, HostPort string } + } + } + if json.Unmarshal(raw, &d) != nil || len(d) != 1 || !d[0].State.Running || d[0].Config.Labels["com.docker.compose.project"] != "fanyu-query-replica-local" { + t.Fatal("unowned synthetic container") + } + cpu, memory, port := int64(4e9), int64(4<<30), "27119" + if name == "query" { + cpu, memory, port = 1e9, 1<<30, "27129" + } + bindings := d[0].HostConfig.PortBindings[port+"/tcp"] + if d[0].HostConfig.NanoCpus != cpu || d[0].HostConfig.Memory != memory || len(bindings) != 1 || bindings[0].HostIp != "127.0.0.1" || bindings[0].HostPort != port { + t.Fatal("local fixture resources or loopback binding changed") + } + command := " " + strings.Join(d[0].Config.Cmd, " ") + if !strings.Contains(command, " --port "+port) || (name == "query" && (!strings.Contains(command, " --replSet query-read-rs") || !strings.Contains(command, " --wiredTigerCacheSizeGB 0.25"))) { + t.Fatal("synthetic Mongo command/cache identity changed") + } + } +} diff --git a/internal/api/query_periodic_replay_test.go b/internal/api/query_periodic_replay_test.go new file mode 100644 index 00000000..792eb9e8 --- /dev/null +++ b/internal/api/query_periodic_replay_test.go @@ -0,0 +1,130 @@ +package api + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "strings" + "sync" + "testing" + "time" + + "droboticflow/internal/domain" + "droboticflow/internal/store" + "droboticflow/internal/testsupport" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/event" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" +) + +// Diagnostic replay of an explicitly retained LOCAL synthetic fixture. It +// neither bootstraps data nor runs workers. Normal verification does not opt in. +func TestQueryPeriodicRetainedBudgetReplayLocal(t *testing.T) { + if os.Getenv("QUERY_PERIODIC_REPLAY") != "1" { + t.Skip("explicit retained local synthetic fixture required") + } + verifyPeriodicLocalHTTPFixture(t) + sourceDB, targetDB := os.Getenv("QUERY_REPLAY_SOURCE"), os.Getenv("QUERY_REPLAY_TARGET") + for _, n := range []string{sourceDB, targetDB} { + if !strings.HasPrefix(n, "query_http_synthetic-") || len(n) != 53 { + t.Fatal("unowned diagnostic database name") + } + } + t.Setenv("QUERY_SERVING_NAMESPACE", "isolated-http-comparison") + t.Setenv("QUERY_REPLICA_REPORTS", "1") + t.Setenv("QUERY_SOURCE_TRACKING", "1") + t.Setenv("QUERY_SERVING_READS", "1") + t.Setenv("QUERY_REPLICA_REFRESH_MODE", "periodic") + t.Setenv("QUERY_READ_MONGO_URI", testsupport.QueryReplicaTargetURI(t)) + t.Setenv("QUERY_READ_MONGO_DB", targetDB) + t.Setenv("QUERY_VALKEY_URL", "") + t.Setenv("QUERY_TYPESENSE_URL", "") + c, err := mongo.Connect(t.Context(), options.Client().ApplyURI("mongodb://127.0.0.1:27119/?directConnection=true")) + if err != nil { + t.Fatal(err) + } + defer c.Disconnect(context.Background()) + source := store.NewMongoStore(c.Database(sourceDB)) + org, err := source.GetOrganization(t.Context(), "query-synthetic-org") + if err != nil || org == nil || org.Name != "Synthetic only" { + t.Fatal("synthetic organization identity unavailable", err) + } + var mu sync.Mutex + trace := []map[string]any{} + pending := map[int64]int{} + monitor := &event.CommandMonitor{Started: func(ctx context.Context, e *event.CommandStartedEvent) { + name, _ := e.Command.Lookup("aggregate").StringValueOK() + if name != "queryWorkforceFacts" { + return + } + mu.Lock() + defer mu.Unlock() + mt, present := e.Command.Lookup("maxTimeMS").AsInt64OK() + left := int64(-1) + if d, ok := ctx.Deadline(); ok { + left = time.Until(d).Milliseconds() + } + pending[e.RequestID] = len(trace) + trace = append(trace, map[string]any{"appTime": time.Now().UTC().Format(time.RFC3339Nano), "maxTimeMS": mt, "maxTimePresent": present, "contextRemainingMS": left}) + }, Succeeded: func(_ context.Context, e *event.CommandSucceededEvent) { + mu.Lock() + defer mu.Unlock() + if i, ok := pending[e.RequestID]; ok { + trace[i]["durationMS"] = float64(e.DurationNanos) / 1e6 + trace[i]["failed"] = false + } + }, Failed: func(_ context.Context, e *event.CommandFailedEvent) { + mu.Lock() + defer mu.Unlock() + if i, ok := pending[e.RequestID]; ok { + trace[i]["durationMS"] = float64(e.DurationNanos) / 1e6 + trace[i]["failed"] = true + } + }} + defer func() { + mu.Lock() + defer mu.Unlock() + raw, _ := json.MarshalIndent(trace, "", " ") + if p := os.Getenv("QUERY_REPLAY_REPORT"); p != "" { + if err := os.WriteFile(p, raw, 0600); err != nil { + t.Error(err) + } + } + }() + closeReplica, err := source.ConfigureQueryReplicaFromEnvironment(t.Context(), monitor) + if err != nil { + t.Fatal(err) + } + defer closeReplica() + token, err := randomSessionToken() + if err != nil { + t.Fatal(err) + } + id := domain.NewID("synthetic-replay-session") + if _, err = source.CreateSession(t.Context(), domain.SessionRecord{ID: id, UserID: "u000", OrganizationID: "query-synthetic-org", TokenHash: sessionTokenHash(token), ExpiresAt: time.Now().Add(time.Hour).UTC().Format(time.RFC3339), ExpiresAtTime: time.Now().Add(time.Hour)}); err != nil { + t.Fatal(err) + } + defer c.Database(sourceDB).Collection("sessions").DeleteOne(context.Background(), bson.M{"id": id}) + s := &server{store: source, requireSessionAuth: true, authLimiter: newRateLimiter(20, time.Minute), externalWriteLimiter: newRateLimiter(120, time.Minute), dashboardResponses: dashboardResponseCache{now: time.Now}} + ts := httptest.NewServer(s.routes()) + defer ts.Close() + client := &http.Client{Timeout: 25 * time.Second} + for i := 0; i < 3; i++ { + r, err := http.NewRequestWithContext(t.Context(), http.MethodGet, ts.URL+"/api/dashboard/workforce?from=2026-11-01T00:00:00Z&to=2026-11-13T00:00:00Z&groupBy=member", nil) + if err != nil { + t.Fatal(err) + } + r.AddCookie(&http.Cookie{Name: sessionCookieName, Value: token}) + res, err := client.Do(r) + if err != nil { + t.Fatal(err) + } + res.Body.Close() + if res.StatusCode != 200 { + t.Fatalf("admin aggregate replay status=%d", res.StatusCode) + } + } +} diff --git a/internal/api/query_replica_isolated_test.go b/internal/api/query_replica_isolated_test.go index 9ca3491c..14e411b0 100644 --- a/internal/api/query_replica_isolated_test.go +++ b/internal/api/query_replica_isolated_test.go @@ -37,10 +37,13 @@ import ( // HTTP uses the actual session, ACL and count/report handlers. Business recovery // schedulers are deliberately not duplicated in the two benchmark handlers. func TestQueryReplicaIsolatedHTTPComparisonLocal(t *testing.T) { - if os.Getenv("QUERY_REPLICA_ISOLATED_E2E") != "1" { + local := os.Getenv("QUERY_PERIODIC_HTTP_LOCAL") == "1" + if os.Getenv("QUERY_REPLICA_ISOLATED_E2E") != "1" && !local { t.Skip("explicit node4 synthetic comparison only") } - if os.Getenv("QUERY_REPLICA_CLUSTER_TEST") != "1" { + if local { + verifyPeriodicLocalHTTPFixture(t) + } else if os.Getenv("QUERY_REPLICA_CLUSTER_TEST") != "1" { t.Fatal("cluster synthetic target must be explicitly selected") } t.Setenv("ANNOTATION_SOURCE_VALIDATION_MODE", "advisory") @@ -49,6 +52,10 @@ func TestQueryReplicaIsolatedHTTPComparisonLocal(t *testing.T) { n := isolatedComparisonInt(t, "QUERY_COMPARE_TASKS", 26218, 20, 30000) waves := isolatedComparisonInt(t, "QUERY_COMPARE_WAVES", 2, 1, 100) maxActors := isolatedComparisonInt(t, "QUERY_COMPARE_MAX_ACTORS", 81, 81, 500) + maxInflight := isolatedComparisonInt(t, "QUERY_COMPARE_CPU_MAX_INFLIGHT", 0, 0, 8) + if maxInflight > 0 && !local { + t.Fatal("supplemental matched CPU comparison is local-only") + } minPool := isolatedComparisonInt(t, "QUERY_COMPARE_MIN_POOL", 0, 0, 100) compression := os.Getenv("QUERY_COMPARE_WIRE_COMPRESSION") if compression != "" && compression != "zstd" { @@ -59,14 +66,60 @@ func TestQueryReplicaIsolatedHTTPComparisonLocal(t *testing.T) { inflight := map[int64]string{} metrics := map[string]*isolatedCommands{} var targetStatsMu sync.Mutex + var factTrace []map[string]any + factPending := map[int64]int{} + if path := os.Getenv("QUERY_COMPARE_DIAGNOSTIC_REPORT"); local && path != "" { + defer func() { + targetStatsMu.Lock() + defer targetStatsMu.Unlock() + raw, _ := json.MarshalIndent(factTrace, "", " ") + if err := os.WriteFile(path, raw, 0600); err != nil { + t.Error(err) + } + }() + } targetCommands := map[string]int64{} targetMaxMillis := map[string]float64{} - targetMonitor := &event.CommandMonitor{Succeeded: func(_ context.Context, e *event.CommandSucceededEvent) { + targetMonitor := &event.CommandMonitor{Started: func(ctx context.Context, e *event.CommandStartedEvent) { + if !local { + return + } + name, _ := e.Command.Lookup("aggregate").StringValueOK() + if name != "queryWorkforceFacts" { + return + } + targetStatsMu.Lock() + defer targetStatsMu.Unlock() + if len(factTrace) >= 512 { + return + } + maxTime, present := e.Command.Lookup("maxTimeMS").AsInt64OK() + remaining := int64(-1) + if deadline, ok := ctx.Deadline(); ok { + remaining = time.Until(deadline).Milliseconds() + } + factPending[e.RequestID] = len(factTrace) + factTrace = append(factTrace, map[string]any{"appTimeUTC": time.Now().UTC().Format(time.RFC3339Nano), "wireMaxTimeMS": maxTime, "wireMaxTimePresent": present, "contextRemainingMS": remaining}) + }, Succeeded: func(_ context.Context, e *event.CommandSucceededEvent) { targetStatsMu.Lock() defer targetStatsMu.Unlock() targetCommands[e.CommandName]++ ms := float64(e.DurationNanos) / 1e6 targetMaxMillis[e.CommandName] = max(targetMaxMillis[e.CommandName], ms) + if i, ok := factPending[e.RequestID]; ok { + factTrace[i]["durationMS"] = ms + factTrace[i]["failed"] = false + delete(factPending, e.RequestID) + } + }, Failed: func(_ context.Context, e *event.CommandFailedEvent) { + targetStatsMu.Lock() + defer targetStatsMu.Unlock() + if i, ok := factPending[e.RequestID]; ok { + factTrace[i]["durationMS"] = float64(e.DurationNanos) / 1e6 + factTrace[i]["failed"] = true + factTrace[i]["maxTimeFailure"] = strings.Contains(e.Failure, "MaxTimeMS") + delete(factPending, e.RequestID) + } }} monitor := &event.CommandMonitor{ Started: func(_ context.Context, e *event.CommandStartedEvent) { @@ -116,7 +169,9 @@ func TestQueryReplicaIsolatedHTTPComparisonLocal(t *testing.T) { t.Cleanup(func() { clean, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() - if err := db.Drop(clean); err != nil { + if local && os.Getenv("QUERY_COMPARE_KEEP_FAILED_FIXTURES") == "1" && t.Failed() { + t.Logf("retained owned diagnostic fixture %s", db.Name()) + } else if err := db.Drop(clean); err != nil { t.Errorf("synthetic cleanup %s: %v", db.Name(), err) } _ = c.Disconnect(clean) @@ -125,9 +180,11 @@ func TestQueryReplicaIsolatedHTTPComparisonLocal(t *testing.T) { } sdb, old := connect("mongodb://127.0.0.1:27119/?directConnection=true", monitor) db, _ := connect(testsupport.QueryReplicaTargetURI(t), targetMonitor) - var marker bson.M - if err := db.Client().Database("query25_synthetic").Collection("marker").FindOne(ctx, bson.M{"_id": "query25-confirmed"}).Decode(&marker); err != nil || marker["value"] != "preserve-query25-v1" { - t.Fatalf("owned 25Gi target identity not confirmed: %v", err) + if !local { + var marker bson.M + if err := db.Client().Database("query25_synthetic").Collection("marker").FindOne(ctx, bson.M{"_id": "query25-confirmed"}).Decode(&marker); err != nil || marker["value"] != "preserve-query25-v1" { + t.Fatalf("owned 25Gi target identity not confirmed: %v", err) + } } newReader := store.NewMongoStore(sdb) newReader.EnableQuerySourceTracking(true) @@ -202,6 +259,9 @@ func TestQueryReplicaIsolatedHTTPComparisonLocal(t *testing.T) { t.Fatal(err) } builderTarget.ConfigureQueryServing(nil, nil) + if err := builderTarget.ConfigureQueryRefreshMode(os.Getenv("QUERY_REPLICA_REFRESH_MODE")); err != nil { + t.Fatal(err) + } if err := builderTarget.EnsureQueryReadModelIndexes(ctx); err != nil { t.Fatal(err) } @@ -227,16 +287,18 @@ func TestQueryReplicaIsolatedHTTPComparisonLocal(t *testing.T) { case <-workerCtx.Done(): return } - tick := time.NewTicker(time.Second) - defer tick.Stop() + interval := time.Second + if os.Getenv("QUERY_REPLICA_REFRESH_MODE") == "periodic" { + interval = store.QueryPeriodicSyncInterval + } for workerCtx.Err() == nil { if _, err := q.RunOnce(workerCtx, false); err != nil && workerCtx.Err() == nil { workerErrors <- err return } - select { - case <-workerCtx.Done(): - case <-tick.C: + if err := q.WaitNextPass(workerCtx, interval); err != nil && workerCtx.Err() == nil { + workerErrors <- err + return } } }() @@ -271,7 +333,7 @@ func TestQueryReplicaIsolatedHTTPComparisonLocal(t *testing.T) { t.Setenv("QUERY_SERVING_READS", "1") t.Setenv("QUERY_VALKEY_URL", "") t.Setenv("QUERY_TYPESENSE_URL", "") - closeReplica, err := newReader.ConfigureQueryReplicaFromEnvironment(ctx) + closeReplica, err := newReader.ConfigureQueryReplicaFromEnvironment(ctx, targetMonitor) if err != nil { t.Fatal(err) } @@ -320,8 +382,13 @@ func TestQueryReplicaIsolatedHTTPComparisonLocal(t *testing.T) { } // Keep output evidence independently identifiable rather than parsing t.Log. report := map[string]any{"tasks": n, "projects": len(fixture.projects), "sourceDatabase": sdb.Name(), "targetDatabase": db.Name(), "cases": []any{}, "parity": true, "httpSessionAuth": true, "sourceMinimumPoolBothVariants": minPool, "apiQueryMaximumPool": 20, "projectorMaximumPool": 4, "builderMaximumPool": 100, "wireCompressionBothVariants": compression, "sourceReplyBytesKind": "logical BSON; not compressed wire bytes", "cache": "none; existing API snapshot/response caches unchanged"} + report["refreshMode"] = os.Getenv("QUERY_REPLICA_REFRESH_MODE") + report["localDocker"] = local + report["targetMetricsIncludeAPI"] = true + report["supplementalMaxInFlight"] = maxInflight cases := []any{} accepted := true + expectedResults := map[string]any{} for _, mode := range []string{"old", "query"} { url := before.URL if mode == "query" { @@ -358,6 +425,11 @@ func TestQueryReplicaIsolatedHTTPComparisonLocal(t *testing.T) { errors := map[string]int{} var firstError string var responses int64 + matchedResults, mismatches := 0, 0 + var slots chan struct{} + if maxInflight > 0 { + slots = make(chan struct{}, maxInflight) + } for wave := 0; wave < waves; wave++ { var wg sync.WaitGroup var results sync.Mutex @@ -367,6 +439,10 @@ func TestQueryReplicaIsolatedHTTPComparisonLocal(t *testing.T) { go func(actor int) { defer wg.Done() <-barrier + if slots != nil { + slots <- struct{}{} + defer func() { <-slots }() + } path := paths[2+(actor+wave)%2] if actor%81 == 0 { path = paths[1] @@ -382,6 +458,18 @@ func TestQueryReplicaIsolatedHTTPComparisonLocal(t *testing.T) { key := strings.Split(path, "?")[0] byRoute[key] = append(byRoute[key], ms) responses += int64(len(body)) + if status == 200 && err == nil { + id := fmt.Sprintf("%d/%d/%s", wave, actor, path) + value := isolatedBusinessJSON(t, body) + if mode == "old" { + expectedResults[id] = value + } else if expected, ok := expectedResults[id]; ok { + matchedResults++ + if !reflect.DeepEqual(expected, value) { + mismatches++ + } + } + } if status != 200 || err != nil { errors[fmt.Sprintf("%d/%v", status, err)]++ if firstError == "" { @@ -396,7 +484,10 @@ func TestQueryReplicaIsolatedHTTPComparisonLocal(t *testing.T) { mu.Lock() phase = "" original := metrics[name] - m := &isolatedCommands{Commands: map[string]int64{}, RequestBytes: original.RequestBytes, ReplyBytes: original.ReplyBytes} + m := &isolatedCommands{Commands: map[string]int64{}, Collections: map[string]int64{}, RequestBytes: original.RequestBytes, ReplyBytes: original.ReplyBytes} + for collection, count := range original.Collections { + m.Collections[collection] = count + } for cmd, count := range original.Commands { m.Commands[cmd] = count } @@ -417,6 +508,11 @@ func TestQueryReplicaIsolatedHTTPComparisonLocal(t *testing.T) { } } result := map[string]any{"name": name, "seconds": time.Since(started).Seconds(), "requests": len(latency), "p50Ms": latency[(len(latency)-1)/2], "p95Ms": latency[(len(latency)-1)*95/100], "errors": errors, "firstError": firstError, "routes": routes, "responseBytes": responses, "sourceCPUSeconds": cpu, "sourceScans": scans, "source": m} + result["matchedBusinessResults"], result["businessMismatches"] = matchedResults, mismatches + if mode == "query" && mismatches > 0 { + t.Errorf("%d per-actor business results differed", mismatches) + accepted = false + } cases = append(cases, result) encoded, _ := json.Marshal(result) t.Log(string(encoded)) diff --git a/internal/api/query_replica_local_test.go b/internal/api/query_replica_local_test.go index acadbd1a..10700ccb 100644 --- a/internal/api/query_replica_local_test.go +++ b/internal/api/query_replica_local_test.go @@ -25,7 +25,11 @@ func TestQueryReplicaCountOnlyPermissionRevocationAndSelfReviewLocal(t *testing. testQueryReplicaPermissionRevocationAndSelfReview(t, true) } -func testQueryReplicaPermissionRevocationAndSelfReview(t *testing.T, countsOnly bool) { +func TestQueryPeriodicPermissionRevocationAndSelfReviewLocal(t *testing.T) { + testQueryReplicaPermissionRevocationAndSelfReview(t, false, true) +} + +func testQueryReplicaPermissionRevocationAndSelfReview(t *testing.T, countsOnly bool, periodic ...bool) { if os.Getenv("QUERY_REPLICA_LOCAL_TEST") != "1" { t.Skip("isolated local source and query replica required") } @@ -49,6 +53,11 @@ func testQueryReplicaPermissionRevocationAndSelfReview(t *testing.T, countsOnly stores = append(stores, store.NewMongoStore(db)) } source, target := stores[0], stores[1] + if len(periodic) > 0 && periodic[0] { + if err := target.ConfigureQueryRefreshMode("periodic"); err != nil { + t.Fatal(err) + } + } _ = clients source.EnableQuerySourceTracking(true) if e := source.EnsureIndexes(ctx); e != nil { diff --git a/internal/api/workforce_statistics.go b/internal/api/workforce_statistics.go index 671d14e7..d0f6a9c3 100644 --- a/internal/api/workforce_statistics.go +++ b/internal/api/workforce_statistics.go @@ -139,6 +139,7 @@ func (s *server) handleDashboardWorkforce(w http.ResponseWriter, r *http.Request workforcestats.ApplyRates(&report.Items[i]) } renderJSON(w, http.StatusOK, map[string]any{ + "asOf": report.AsOf, "from": q.From.Format(time.RFC3339Nano), "to": q.To.Format(time.RFC3339Nano), "groupBy": q.GroupBy, diff --git a/internal/domain/types.go b/internal/domain/types.go index e1becc38..6690ffe7 100644 --- a/internal/domain/types.go +++ b/internal/domain/types.go @@ -1235,6 +1235,7 @@ type DashboardWorkforceMetric struct { } type DashboardWorkforceSummary struct { + AsOf string `json:"asOf,omitempty"` From string `json:"from,omitempty"` To string `json:"to,omitempty"` Totals DashboardWorkforceMetric `json:"totals"` diff --git a/internal/store/mongo.go b/internal/store/mongo.go index ada8ce0c..b9166ec0 100644 --- a/internal/store/mongo.go +++ b/internal/store/mongo.go @@ -24,6 +24,7 @@ type MongoStore struct { querySyncEnabled atomic.Bool queryReplicaReads *queryReplicaReads queryReplicaFactStore bool + queryPeriodic bool queryBuildWrites *queryBuildWriteGate queryReplicaCounts queryCountCache queryReplicaBatchFacts queryBatchFacts diff --git a/internal/store/query_periodic.go b/internal/store/query_periodic.go new file mode 100644 index 00000000..149d2940 --- /dev/null +++ b/internal/store/query_periodic.go @@ -0,0 +1,76 @@ +package store + +import ( + "context" + "errors" + "time" + + "go.mongodb.org/mongo-driver/mongo" +) + +// Explicit opt-in; authority writes and permission decisions never use this +// display policy. The bounds describe data age, NOT longer request deadlines. +const ( + QueryPeriodicSyncInterval = time.Minute + queryPeriodicBuildInterval = 3 * time.Minute + queryPeriodicDisplayMaxAge = 15 * time.Minute +) + +func (s *MongoStore) ConfigureQueryRefreshMode(mode string) error { + switch mode { + case "", "continuous": + s.queryPeriodic = false + case "periodic": + s.queryPeriodic = true + default: + return errors.New("invalid query refresh mode") + } + return nil +} + +func (s *MongoStore) periodicQueryReads() bool { + return s.queryReplicaReads != nil && s.queryReplicaReads.target != nil && s.queryReplicaReads.target.queryPeriodic +} + +func (s *MongoStore) periodicQueryStore() bool { return s.queryPeriodic && s.queryReplicaFactStore } + +func (s *MongoStore) querySearchPublicationReady(st querySearchState, h QueryModelHead, exists bool, revision int64, now time.Time) bool { + if !exists || h.Generation == "" || now.Before(h.AsOf) { + return false + } + if s.periodicQueryStore() && now.Sub(h.AsOf) <= queryPeriodicDisplayMaxAge { + return true + } + return h.Revision == revision && h.GlobalRevision == st.Global && now.Sub(st.ObservedAt) >= 0 && now.Sub(st.ObservedAt) <= 5*time.Second +} + +func queryBuildDue(now, last time.Time, interval time.Duration) bool { + return last.IsZero() || !now.Before(last.Add(interval)) +} + +// The projector's lease remains 30s. Idle only touches its query-store lease, +// never scans authority or extends an expired/unconfirmed lease. +func (q *QueryReplica) WaitNextPass(ctx context.Context, interval time.Duration) error { + if interval < 100*time.Millisecond || interval > 5*time.Minute { + return errors.New("invalid projector interval") + } + timer := time.NewTimer(interval) + defer timer.Stop() + heartbeat := time.NewTicker(10 * time.Second) + defer heartbeat.Stop() + for { + select { + case <-ctx.Done(): + return ctx.Err() + case <-timer.C: + return nil + case <-heartbeat.C: + short, stop := context.WithTimeout(ctx, 3*time.Second) + err := q.transaction(short, func(_ mongo.SessionContext) error { return nil }) + stop() + if err != nil { + return err + } + } + } +} diff --git a/internal/store/query_periodic_local_test.go b/internal/store/query_periodic_local_test.go new file mode 100644 index 00000000..b8b97334 --- /dev/null +++ b/internal/store/query_periodic_local_test.go @@ -0,0 +1,323 @@ +package store + +import ( + "context" + "errors" + "os" + "reflect" + "sync/atomic" + "testing" + "time" + + "droboticflow/internal/domain" + "droboticflow/internal/productionstats" + "droboticflow/internal/workforcestats" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/event" +) + +func TestQueryPeriodicPublishedReportIgnoresPendingRefreshLocal(t *testing.T) { + if os.Getenv("QUERY_REPLICA_LOCAL_TEST") != "1" { + t.Skip("owned synthetic replica required") + } + var sourceReads atomic.Int32 + monitor := &event.CommandMonitor{Started: func(_ context.Context, e *event.CommandStartedEvent) { + if e.CommandName == "find" || e.CommandName == "aggregate" { + sourceReads.Add(1) + } + }} + source, target, projector := newLocalReplica(t, monitor, nil) + finishLocalReplica(t, projector) + ctx := t.Context() + target.queryPeriodic = true + target.ConfigureQueryServing(nil, nil) + if err := source.ConfigureQueryReplicaReports(target, 30*time.Second); err != nil { + t.Fatal(err) + } + lease, ok, err := target.acquireQueryModelLease(ctx) + if err != nil || !ok { + t.Fatal(ok, err) + } + asOf := time.Now().UTC().Add(-2 * time.Minute).Truncate(time.Millisecond) + head, err := target.stageQueryModel(ctx, queryModelKey("production", "o", nil), productionstats.Report{}, asOf) + if err != nil { + t.Fatal(err) + } + if err = target.publishQueryModelHead(ctx, lease, queryModelKey("production", "o", nil), head); err != nil { + t.Fatal(err) + } + // A change / failed observer must not hide an already published generation. + _, err = target.queryControl().UpdateOne(ctx, bson.M{"_id": "global"}, bson.M{"$inc": bson.M{"productionRevision": 1}, "$set": bson.M{"observedAt": time.Time{}}}) + if err != nil { + t.Fatal(err) + } + _, err = target.db.Collection(queryReplicaControl).UpdateOne(ctx, bson.M{"_id": "state"}, bson.M{"$set": bson.M{"observedAt": asOf}}) + if err != nil { + t.Fatal(err) + } + sourceReads.Store(0) + _, got, err := source.LoadProductionReport(ctx, "o", nil) + if err != nil || got != asOf.Format(time.RFC3339Nano) { + t.Fatalf("published result hidden by pending refresh: asOf=%q err=%v", got, err) + } + if sourceReads.Load() != 0 { + t.Fatalf("display triggered %d authority reads", sourceReads.Load()) + } + // No unpublished/raw fallback when an independent report has never existed. + _, _, err = source.LoadProductionReport(ctx, "missing", nil) + if err == nil { + t.Fatal("cold display calculated a report instead of requiring a publication") + } + // Hard staleness bound and binding checks are not disabled by delayed display. + _, err = target.db.Collection(queryReplicaControl).UpdateOne(ctx, bson.M{"_id": "state"}, bson.M{"$set": bson.M{"observedAt": time.Now().Add(-16 * time.Minute)}}) + if err != nil { + t.Fatal(err) + } + if _, _, err = source.LoadProductionReport(ctx, "o", nil); err == nil { + t.Fatal("expired sync certificate accepted") + } +} + +func TestQueryPeriodicFactsRemainReadableThroughFailureAndRestartLocal(t *testing.T) { + if os.Getenv("QUERY_REPLICA_LOCAL_TEST") != "1" { + t.Skip("owned synthetic replica required") + } + source, target, p := newLocalReplica(t) + seedWorkforceStats(t, source) + if err := source.UpsertOrganization(t.Context(), domain.OrganizationRecord{ID: "workforce-test", Status: domain.OrganizationStatusActive}); err != nil { + t.Fatal(err) + } + finishLocalReplica(t, p) + target.queryPeriodic = true + target.ConfigureQueryServing(nil, nil) + if err := target.EnsureQueryReadModelIndexes(t.Context()); err != nil { + t.Fatal(err) + } + if err := source.ConfigureQueryReplicaReports(target, time.Minute); err != nil { + t.Fatal(err) + } + q := workforcestats.Query{Mode: domain.RunModeAll} + old, err := source.LoadWorkforceStats(t.Context(), "workforce-test", q) + if err != nil { + t.Fatal(err) + } + want := workforcestats.Build(old, q) + want.Details = nil + b := NewQueryReadBuilder(target) + ctx, stop := context.WithCancel(t.Context()) + done := make(chan error, 1) + go func() { done <- b.Run(ctx) }() + stopped := false + defer func() { + if !stopped { + stop() + <-done + } + b.Close() + }() + ready, finish := context.WithTimeout(t.Context(), 20*time.Second) + err = b.WaitReady(ready, "workforce-test", nil) + finish() + if err != nil { + t.Fatal(err) + } + head, exists, err := target.workforceHead(t.Context(), "workforce-test") + if err != nil || !exists { + t.Fatal(exists, err) + } + // New work enters the independent replica; display must retain the old + // complete facts while the periodic computation is pending. + err = source.UpsertAnnotationWorklogEvent(t.Context(), domain.AnnotationWorklogEventRecord{ID: "new-periodic-event", OrganizationID: "workforce-test", ProjectID: "p", TaskID: "task", UserID: "u", EventType: "combined_submitted", CreatedAt: "2026-09-28T16:00:00Z"}) + if err != nil { + t.Fatal(err) + } + finishLocalReplica(t, p) + deadline := time.Now().Add(5 * time.Second) + for { + b.mu.Lock() + pending := b.dirty["workforce-test"] + b.mu.Unlock() + if pending { + break + } + var state struct { + Revision int64 `bson:"productionRevision"` + } + _ = target.queryControl().FindOne(t.Context(), bson.M{"_id": "global"}).Decode(&state) + if state.Revision > head.SourceRevision { + break + } + if time.Now().After(deadline) { + t.Fatal("change was not observed") + } + time.Sleep(20 * time.Millisecond) + } + got, err := source.LoadWorkforceReportCached(t.Context(), "workforce-test", q) + if err != nil { + t.Fatal("pending refresh hid published facts", err) + } + got.Details = nil + if got.AsOf == "" { + t.Fatal("published report has no data timestamp") + } + got.AsOf = "" + if !reflect.DeepEqual(got, want) { + t.Fatal("pending report mixed old and new generations") + } + after, _, err := target.workforceHead(t.Context(), "workforce-test") + if err != nil || after.FactEpoch != head.FactEpoch { + t.Fatal("periodic changes rebuilt immediately", err) + } + // Simulate a failed future publication; the committed pointer is unchanged. + _, err = target.db.Collection(workforceFactCollection).InsertOne(t.Context(), bson.M{"_id": "unfinished", "organizationId": "workforce-test", "fromEpoch": head.FactEpoch + 1, "kind": "event"}) + if err != nil { + t.Fatal(err) + } + stop() + <-done + stopped = true + lease, ok, err := target.acquireQueryModelLease(t.Context()) + if err != nil || !ok { + t.Fatal(ok, err) + } + if err = target.rebuildWorkforceFacts(t.Context(), lease, "workforce-test", head.SourceRevision+1); err != nil { + t.Fatal(err) + } + latest, err := source.LoadWorkforceStats(t.Context(), "workforce-test", q) + if err != nil { + t.Fatal(err) + } + want = workforcestats.Build(latest, q) + want.Details = nil + got, err = source.LoadWorkforceReportCached(t.Context(), "workforce-test", q) + got.Details = nil + got.AsOf = "" + if err != nil || !reflect.DeepEqual(got, want) { + t.Fatal("restart failed to publish the updated correct facts", err) + } + if n, err := target.db.Collection(workforceFactCollection).CountDocuments(t.Context(), bson.M{"_id": "unfinished"}); err != nil || n != 0 { + t.Fatal("unfinished stage survived recovery", err) + } +} + +func TestQueryPeriodicCountsDoNotOverlayDirtyAuthorityLocal(t *testing.T) { + if os.Getenv("QUERY_REPLICA_LOCAL_TEST") != "1" { + t.Skip("owned synthetic replica required") + } + var reads atomic.Int32 + m := &event.CommandMonitor{Started: func(_ context.Context, e *event.CommandStartedEvent) { + if e.CommandName == "find" || e.CommandName == "aggregate" { + reads.Add(1) + } + }} + source, target, p := newLocalReplica(t, m, nil) + finishLocalReplica(t, p) + target.queryPeriodic = true + if err := source.ConfigureQueryReplicaReports(target, time.Minute); err != nil { + t.Fatal(err) + } + source.EnableQueryReplicaCountReads(true) + ctx := WithQueryReplicaCountRead(t.Context()) + reads.Store(0) + b, err := source.countReplicaBarrier(ctx, "o") + if err != nil { + t.Fatal(err) + } + if b == nil || len(b.ids) != 0 || reads.Load() != 0 { + t.Fatalf("display still chasing authority: barrier=%v reads=%d", b, reads.Load()) + } + // Batch access remains authoritative: permissions/policy are not snapshots. + if err := source.UpsertHumanBatch(t.Context(), domain.HumanBatchRecord{ID: "b", OrganizationID: "o", ProjectID: "p"}); err != nil { + t.Fatal(err) + } + reads.Store(0) + rows, err := source.replicaBatchAccess(ctx, target, "o", []string{"b"}) + if err != nil || len(rows) != 1 || reads.Load() == 0 { + t.Fatalf("current batch access was cached: rows=%d reads=%d err=%v", len(rows), reads.Load(), err) + } +} + +func TestQueryPeriodicIdleKeepsAuthorityQuietAndRejectsSuccessorLocal(t *testing.T) { + if os.Getenv("QUERY_REPLICA_LOCAL_TEST") != "1" { + t.Skip("owned synthetic replica required") + } + var reads atomic.Int32 + monitor := &event.CommandMonitor{Started: func(_ context.Context, e *event.CommandStartedEvent) { reads.Add(1) }} + _, target, p := newLocalReplica(t, monitor, nil) + reads.Store(0) + if err := p.WaitNextPass(t.Context(), 100*time.Millisecond); err != nil { + t.Fatal(err) + } + if reads.Load() != 0 { + t.Fatal("idle interval queried authority", reads.Load()) + } + _, err := p.control().UpdateOne(t.Context(), bson.M{"_id": "lease"}, bson.M{"$set": bson.M{"owner": "successor", "epoch": p.epoch + 1, "until": time.Now().Add(time.Minute)}}) + if err != nil { + t.Fatal(err) + } + ctx, stop := context.WithTimeout(t.Context(), 12*time.Second) + defer stop() + if err = p.WaitNextPass(ctx, 11*time.Second); !errors.Is(err, ErrQueryReplicaLease) { + t.Fatal("idle heartbeat did not stop after losing ownership", err) + } + var state struct { + Owner string `bson:"owner"` + } + if err = target.db.Collection(queryReplicaControl).FindOne(t.Context(), bson.M{"_id": "lease"}).Decode(&state); err != nil || state.Owner != "successor" { + t.Fatal("old worker overwrote successor", state.Owner, err) + } + if reads.Load() != 0 { + t.Fatal("idle heartbeat read authority", reads.Load()) + } +} + +func TestQueryPeriodicColdBuildKeepsSearchAndSnapshotSerializedLocal(t *testing.T) { + if os.Getenv("QUERY_REPLICA_LOCAL_TEST") != "1" { + t.Skip("owned synthetic replica required") + } + var sequence, firstOutput, firstModel atomic.Int32 + inserted := make(chan struct{}, 1) + monitor := &event.CommandMonitor{Started: func(_ context.Context, e *event.CommandStartedEvent) { + if e.CommandName == "find" { + if name, _ := e.Command.Lookup("find").StringValueOK(); name == "annotationOutputSnapshotState" { + firstOutput.CompareAndSwap(0, sequence.Add(1)) + } + } + if e.CommandName == "insert" { + if name, _ := e.Command.Lookup("insert").StringValueOK(); name == "queryReadModelData" { + firstModel.CompareAndSwap(0, sequence.Add(1)) + select { + case inserted <- struct{}{}: + default: + } + } + } + }} + source, target, p := newLocalReplica(t, nil, monitor) + ctx := t.Context() + for _, err := range []error{source.UpsertOrganization(ctx, domain.OrganizationRecord{ID: "o", Status: domain.OrganizationStatusActive}), source.UpsertProject(ctx, domain.ProjectRecord{ID: "p", OrganizationID: "o", Name: "p"}), source.UpsertHumanItem(ctx, domain.HumanItemRecord{ID: "i", OrganizationID: "o", ProjectID: "p", PluginID: "annotation-qa", CurrentStageID: "quality_review", Status: domain.HumanItemReady, Subject: domain.HumanSubjectRef{ItemKey: "capture/part0001"}})} { + if err != nil { + t.Fatal(err) + } + } + if _, err := source.SaveProductionStatsScope(ctx, "o", productionstats.Scope{ProjectIDs: []string{"p"}, Revision: 1}, 0); err != nil { + t.Fatal(err) + } + finishLocalReplica(t, p) + target.queryPeriodic = true + target.ConfigureQueryServing(nil, nil) + b := NewQueryReadBuilder(target) + defer b.Close() + worker, stop := context.WithCancel(ctx) + done := make(chan error, 1) + go func() { done <- b.Run(worker) }() + defer func() { stop(); <-done }() + select { + case <-inserted: + case <-time.After(10 * time.Second): + t.Fatal("search fixture did not write") + } + if firstOutput.Load() != 0 && firstOutput.Load() < firstModel.Load() { + t.Fatalf("cold full scans overlapped on the bounded query store: output=%d search=%d", firstOutput.Load(), firstModel.Load()) + } +} diff --git a/internal/store/query_periodic_test.go b/internal/store/query_periodic_test.go new file mode 100644 index 00000000..4f1a7208 --- /dev/null +++ b/internal/store/query_periodic_test.go @@ -0,0 +1,56 @@ +package store + +import ( + "testing" + "time" +) + +func TestQueryPeriodicScheduleCoalescesAndKeepsInitialBuildImmediate(t *testing.T) { + now := time.Date(2026, 11, 13, 0, 0, 0, 0, time.UTC) + if !queryBuildDue(now, time.Time{}, queryPeriodicBuildInterval) { + t.Fatal("cold build delayed") + } + for i := 1; i < 180; i++ { + if queryBuildDue(now.Add(time.Duration(i)*time.Second), now, queryPeriodicBuildInterval) { + t.Fatal("repeated changes caused early rebuild", i) + } + } + if !queryBuildDue(now.Add(3*time.Minute), now, queryPeriodicBuildInterval) { + t.Fatal("pending changes never scheduled") + } + s := &MongoStore{} + for _, mode := range []string{"periodic", "continuous", ""} { + if err := s.ConfigureQueryRefreshMode(mode); err != nil || s.queryPeriodic != (mode == "periodic") { + t.Fatal(mode, err) + } + } + if s.ConfigureQueryRefreshMode("unknown") == nil { + t.Fatal("unknown refresh mode accepted") + } +} + +func TestQueryPeriodicReadinessUsesPublishedAgeNotLiveBarrier(t *testing.T) { + now := time.Now().UTC() + h := QueryModelHead{Generation: "published", AsOf: now.Add(-2 * time.Minute), Revision: 1, GlobalRevision: 1} + st := querySearchState{Global: 2, ObservedAt: time.Time{}} + target := &MongoStore{queryPeriodic: true, queryReplicaFactStore: true} + if !target.querySearchPublicationReady(st, h, true, 2, now) { + t.Fatal("published search blocked by pending observer") + } + if target.querySearchPublicationReady(st, h, false, 2, now) { + t.Fatal("missing publication accepted") + } + h.AsOf = now.Add(-16 * time.Minute) + if target.querySearchPublicationReady(st, h, true, 2, now) { + t.Fatal("expired generation accepted") + } + h.AsOf = now.Add(time.Second) + if target.querySearchPublicationReady(st, h, true, 2, now) { + t.Fatal("future generation accepted") + } + h.AsOf = now.Add(-time.Minute) + authority := &MongoStore{queryPeriodic: true} + if authority.querySearchPublicationReady(st, h, true, 2, now) { + t.Fatal("display mode changed authority search behavior") + } +} diff --git a/internal/store/query_read_builder.go b/internal/store/query_read_builder.go index 92b37c1a..e31b93f8 100644 --- a/internal/store/query_read_builder.go +++ b/internal/store/query_read_builder.go @@ -45,12 +45,16 @@ func (b *QueryReadBuilder) Close() { b.reader.Close() } func (b *QueryReadBuilder) WaitReady(ctx context.Context, org string, projects []string) error { tick := time.NewTicker(500 * time.Millisecond) defer tick.Stop() + lastState := "unobserved" for { + if err := ctx.Err(); err != nil { + return fmt.Errorf("query readiness last=%s: %w", lastState, err) + } productionHead, production := b.store.readQueryModelHead(ctx, queryModelKey("production", org, projects), 5*time.Minute) st, err := b.store.querySearchState(ctx) - production = production && productionHead.GlobalRevision == st.Global + production = production && (b.store.periodicQueryStore() || productionHead.GlobalRevision == st.Global) h, exists := st.Heads[queryModelKey("search", org, nil)] - search := err == nil && exists && h.Generation != "" && h.Revision == st.Revisions[queryModelKey("search", org, nil)] && h.GlobalRevision == st.Global && time.Since(st.ObservedAt) >= 0 && time.Since(st.ObservedAt) < 5*time.Second + search := err == nil && b.store.querySearchPublicationReady(st, h, exists, st.Revisions[queryModelKey("search", org, nil)], time.Now()) output := len(projects) == 0 if !output { health, e := b.reader.Inspect(ctx, org, projects) @@ -64,9 +68,12 @@ func (b *QueryReadBuilder) WaitReady(ctx context.Context, org string, projects [ if production && search && output && workforce { return nil } + if ctx.Err() == nil { + lastState = fmt.Sprintf("production=%t search=%t output=%t workforce=%t", production, search, output, workforce) + } select { case <-ctx.Done(): - return ctx.Err() + return fmt.Errorf("query readiness last=%s: %w", lastState, ctx.Err()) case <-tick.C: } } @@ -296,12 +303,16 @@ func (b *QueryReadBuilder) run(ctx context.Context, started chan<- struct{}) err go func() { defer close(watchExited) lastHeartbeat, lastBarrier, observed := time.Time{}, time.Time{}, time.Time{} + barrierInterval := time.Second + if b.store.queryPeriodic { + barrierInterval = 10 * time.Second + } pendingChange := false for watchCtx.Err() == nil { // An empty filtered batch is not proof that the stream reached // the oplog end. Publish a private marker, then advance proof // only when that exact immutable update delta is observed. - if time.Since(lastBarrier) >= time.Second { + if time.Since(lastBarrier) >= barrierInterval { now := time.Now().UTC() result, e := b.store.queryControl().UpdateOne(watchCtx, queryLeaseFilter(l), bson.M{"$set": bson.M{"watchBarrier": now}}) if e != nil { @@ -335,7 +346,7 @@ func (b *QueryReadBuilder) run(ctx context.Context, started chan<- struct{}) err failWatch(e) return } - if observed.IsZero() || time.Since(lastHeartbeat) < time.Second { + if observed.IsZero() || time.Since(lastHeartbeat) < barrierInterval { continue } if e := b.recordObserved(watchCtx, l, observed, pendingChange); e != nil { @@ -363,6 +374,7 @@ func (b *QueryReadBuilder) run(ctx context.Context, started chan<- struct{}) err defer tick.Stop() pendingStats, pendingSearch := map[string]bool{}, map[string]bool{} lastStats := map[string]time.Time{} + lastSearch := map[string]time.Time{} searchRetries := map[string]queryBuildRetry{} statsRetries := map[string]queryBuildRetry{} var scopes map[string][]string @@ -370,6 +382,13 @@ func (b *QueryReadBuilder) run(ctx context.Context, started chan<- struct{}) err lastScopeRead := time.Time{} lastCleanup := time.Now() lastWorkforceCleanup := time.Now() + lastOutputWarm := time.Time{} + statsInterval, searchInterval, scopeInterval := 30*time.Second, time.Duration(0), 30*time.Second + maintenanceInterval, outputInterval := 5*time.Second, time.Second + if b.store.queryPeriodic { + statsInterval, searchInterval, scopeInterval = queryPeriodicBuildInterval, QueryPeriodicSyncInterval, QueryPeriodicSyncInterval + maintenanceInterval, outputInterval = time.Minute, time.Minute + } for { b.mu.Lock() all, allSearch := b.all, b.allSearch @@ -382,7 +401,7 @@ func (b *QueryReadBuilder) run(ctx context.Context, started chan<- struct{}) err b.dirty, b.searchDirty = map[string]bool{}, map[string]bool{} b.all, b.allSearch = false, false b.mu.Unlock() - if refreshScopes || all || time.Since(lastScopeRead) > 30*time.Second { + if refreshScopes || all || time.Since(lastScopeRead) > scopeInterval { var e error scopes, e = b.scopes(ctx) if e != nil { @@ -410,9 +429,13 @@ func (b *QueryReadBuilder) run(ctx context.Context, started chan<- struct{}) err if time.Now().Before(searchRetries[org].Next) { continue } + if !queryBuildDue(time.Now(), lastSearch[org], searchInterval) { + continue + } if e := b.store.buildQuerySearchModel(ctx, l, org); e == nil { delete(searchRetries, org) delete(pendingSearch, org) + lastSearch[org] = time.Now() } else { retry := searchRetries[org] retry.fail(5*time.Second, time.Minute) @@ -435,7 +458,7 @@ func (b *QueryReadBuilder) run(ctx context.Context, started chan<- struct{}) err delete(pendingStats, org) continue } - if time.Since(lastStats[org]) < 30*time.Second { + if !queryBuildDue(time.Now(), lastStats[org], statsInterval) { continue } if time.Now().Before(statsRetries[org].Next) { @@ -459,17 +482,20 @@ func (b *QueryReadBuilder) run(ctx context.Context, started chan<- struct{}) err finish() lastCleanup = time.Now() } - if b.store.queryReplicaFactStore && time.Since(lastWorkforceCleanup) > 5*time.Second { + if b.store.queryReplicaFactStore && time.Since(lastWorkforceCleanup) > maintenanceInterval { cleanup, finish := context.WithTimeout(ctx, 2*time.Second) _ = b.store.maintainWorkforceVersions(cleanup, l) finish() lastWorkforceCleanup = time.Now() } // Prewarm due generations even when no browser is connected. - for org, ids := range scopes { - if len(ids) > 0 { - _, _ = b.reader.Prewarm(ctx, org, ids) + if queryBuildDue(time.Now(), lastOutputWarm, outputInterval) { + for org, ids := range scopes { + if len(ids) > 0 { + _, _ = b.reader.Prewarm(ctx, org, ids) + } } + lastOutputWarm = time.Now() } select { case <-ctx.Done(): diff --git a/internal/store/query_read_models.go b/internal/store/query_read_models.go index ce192e59..d5b164fb 100644 --- a/internal/store/query_read_models.go +++ b/internal/store/query_read_models.go @@ -142,6 +142,9 @@ func (s *MongoStore) readQueryModelHead(ctx context.Context, key string, maxAge h, ok := st.Heads[key] age := time.Since(h.AsOf) verifiedUnchanged := st.SourceRevision == h.SourceRevision && time.Since(st.ObservedAt) >= 0 && time.Since(st.ObservedAt) <= 5*time.Second + if s.periodicQueryStore() { + maxAge = queryPeriodicDisplayMaxAge + } return h, ok && h.Generation != "" && age >= 0 && (age <= maxAge || verifiedUnchanged) } func (s *MongoStore) publishQueryModelHead(ctx context.Context, l queryModelLease, key string, h QueryModelHead) error { @@ -255,6 +258,9 @@ func (s *MongoStore) LoadProductionReport(ctx context.Context, org string, proje } } } + if s.periodicQueryStore() { + return productionstats.Report{}, "", ErrQueryReplicaNotReady + } snapshot, err := s.LoadProductionStatsCached(ctx, org, projects) if err != nil { return productionstats.Report{}, "", err diff --git a/internal/store/query_replica_barrier.go b/internal/store/query_replica_barrier.go index 2866c32f..7f772172 100644 --- a/internal/store/query_replica_barrier.go +++ b/internal/store/query_replica_barrier.go @@ -40,6 +40,11 @@ func (s *MongoStore) countReplicaBarrier(ctx context.Context, org string) (*quer b.org = org b.ids = map[string]bool{} b.batches = map[string]bool{} + // Periodic display reports committed query facts. Pending business + // changes wait for the next projector cycle; do not chase them per reader. + if s.periodicQueryReads() { + return + } // Acknowledgement follows the majority-committed target transaction. Thus a // clean source record is already present in the query DB; pending records // are overlaid from authority. Never assume a dirty record is absent. @@ -183,6 +188,9 @@ func (s *MongoStore) visitReplicaPendingCounts(ctx context.Context, org string, } func (s *MongoStore) replicaBatchAccess(ctx context.Context, target *MongoStore, org string, ids []string) ([]domain.HumanBatchRecord, error) { + if s.periodicQueryReads() { + return s.listAuthoritativeBatchAccess(ctx, org, ids) + } b, e := s.countReplicaBarrier(ctx, org) if e != nil { return nil, e diff --git a/internal/store/query_replica_config.go b/internal/store/query_replica_config.go index e288cf29..ecc15298 100644 --- a/internal/store/query_replica_config.go +++ b/internal/store/query_replica_config.go @@ -6,6 +6,7 @@ import ( "os" "time" + "go.mongodb.org/mongo-driver/event" "go.mongodb.org/mongo-driver/mongo" "go.mongodb.org/mongo-driver/mongo/options" "go.mongodb.org/mongo-driver/mongo/readconcern" @@ -15,7 +16,7 @@ import ( // ConfigureQueryReplicaFromEnvironment is an entry-point operation. Invalid or // unavailable reporting infrastructure cannot disable existing work mutations. // Explicit serving mode fails closed for reports, with no source-scan fallback. -func (s *MongoStore) ConfigureQueryReplicaFromEnvironment(ctx context.Context) (func(), error) { +func (s *MongoStore) ConfigureQueryReplicaFromEnvironment(ctx context.Context, monitors ...*event.CommandMonitor) (func(), error) { if os.Getenv("QUERY_REPLICA_REPORTS") != "1" { return func() {}, nil } @@ -27,7 +28,14 @@ func (s *MongoStore) ConfigureQueryReplicaFromEnvironment(ctx context.Context) ( if uri == "" || db == "" || os.Getenv("QUERY_SERVING_NAMESPACE") == "" { return func() {}, errors.New("explicit query database and namespace required") } - c, e := mongo.Connect(ctx, options.Client().ApplyURI(uri).SetReadPreference(readpref.Primary()).SetReadConcern(readconcern.Majority()).SetMaxPoolSize(20).SetServerSelectionTimeout(2*time.Second)) + if len(monitors) > 1 { + return func() {}, errors.New("only one query command monitor is supported") + } + opts := options.Client().ApplyURI(uri).SetReadPreference(readpref.Primary()).SetReadConcern(readconcern.Majority()).SetMaxPoolSize(20).SetServerSelectionTimeout(2 * time.Second) + if len(monitors) == 1 { + opts.SetMonitor(monitors[0]) + } + c, e := mongo.Connect(ctx, opts) if e != nil { return func() {}, errors.New("query database connection unavailable") } diff --git a/internal/store/query_replica_reads.go b/internal/store/query_replica_reads.go index 35001a0d..d8bf5b8c 100644 --- a/internal/store/query_replica_reads.go +++ b/internal/store/query_replica_reads.go @@ -120,7 +120,11 @@ func (s *MongoStore) reportReplica(ctx context.Context) (*MongoStore, error) { return nil, e } age := time.Since(state.ObservedAt) - if e != nil || !state.Ready || state.Schema != queryReplicaSchema || state.Binding.SourceDatabase != s.db.Name() || state.Binding.Namespace != queryReplicaNamespace(s.db.Name()) || age < 0 || age > s.queryReplicaReads.maxAge { + maxAge := s.queryReplicaReads.maxAge + if s.periodicQueryReads() { + maxAge = queryPeriodicDisplayMaxAge + } + if e != nil || !state.Ready || state.Schema != queryReplicaSchema || state.Binding.SourceDatabase != s.db.Name() || state.Binding.Namespace != queryReplicaNamespace(s.db.Name()) || age < 0 || age > maxAge { return nil, ErrQueryReplicaNotReady } return s.queryReplicaReads.target, nil diff --git a/internal/store/query_search_model.go b/internal/store/query_search_model.go index 1243d550..34b7202f 100644 --- a/internal/store/query_search_model.go +++ b/internal/store/query_search_model.go @@ -194,7 +194,7 @@ func (s *MongoStore) querySearchCandidates(ctx context.Context, org, search, cap } key := queryModelKey("search", org, nil) h, ok := st.Heads[key] - if !ok || h.Generation == "" || h.Revision != st.Revisions[key] || h.GlobalRevision != st.Global || time.Since(st.ObservedAt) > 5*time.Second || time.Since(st.ObservedAt) < 0 { + if !s.querySearchPublicationReady(st, h, ok, st.Revisions[key], time.Now()) { return nil, false } cacheKey := queryserving.Key("search-candidates/v1", s.db.Name(), org, h.Generation, search, capture) diff --git a/internal/store/query_serving_config.go b/internal/store/query_serving_config.go index 156f3b83..fbb745c2 100644 --- a/internal/store/query_serving_config.go +++ b/internal/store/query_serving_config.go @@ -11,6 +11,9 @@ import ( // different deployments even when their database names happen to be identical. func (s *MongoStore) ConfigureQueryServingFromEnvironment() (func(), error) { close := func() {} + if err := s.ConfigureQueryRefreshMode(os.Getenv("QUERY_REPLICA_REFRESH_MODE")); err != nil { + return close, err + } if os.Getenv("QUERY_SERVING_READS") != "1" { return close, nil } diff --git a/internal/store/workforce_facts.go b/internal/store/workforce_facts.go index e162c117..18627ff8 100644 --- a/internal/store/workforce_facts.go +++ b/internal/store/workforce_facts.go @@ -74,7 +74,11 @@ func (s *MongoStore) workforceHead(ctx context.Context, org string) (QueryModelH return h, false, nil } unchanged := h.SourceRevision == state.Revision && !state.Observed.IsZero() && time.Since(state.Observed) >= 0 && time.Since(state.Observed) <= 5*time.Second - if h.Collection != workforceFactCollection || h.FactEpoch < 1 || h.Generation != workforceFactSchema || time.Since(h.AsOf) < 0 || (!unchanged && time.Since(h.AsOf) > dashboardStatsCacheTTL) { + maxAge := dashboardStatsCacheTTL + if s.periodicQueryStore() { + maxAge = queryPeriodicDisplayMaxAge + } + if h.Collection != workforceFactCollection || h.FactEpoch < 1 || h.Generation != workforceFactSchema || time.Since(h.AsOf) < 0 || (!unchanged && time.Since(h.AsOf) > maxAge) { return h, true, ErrQueryReplicaNotReady } return h, true, nil @@ -546,10 +550,16 @@ func (s *MongoStore) loadWorkforceFactReport(ctx context.Context, org string, q return workforcestats.Report{}, false, nil } h, exists, err := s.workforceHead(ctx, org) + if s.periodicQueryStore() && !exists && err == nil { + return workforcestats.Report{}, true, ErrQueryReplicaNotReady + } if err != nil || !exists { return workforcestats.Report{}, exists, err } key := queryserving.Key(workforceFactSchema, s.db.Name(), org, h.FactEpoch, q) + if s.periodicQueryStore() { + key = queryserving.Key("periodic-report/v1", key) + } if raw, ok := workforceReportCache.get(key, dashboardStatsCacheTTL); ok { var report workforcestats.Report if json.Unmarshal(raw, &report) == nil { @@ -579,6 +589,9 @@ func (s *MongoStore) loadWorkforceFactReport(ctx context.Context, org string, q return workforcestats.Report{}, true, err } report := workforcestats.Report{Items: []domain.DashboardWorkforceMetric{}} + if s.periodicQueryStore() { + report.AsOf = h.AsOf.UTC().Format(time.RFC3339Nano) + } if len(rows) > 0 { for _, m := range rows[0].Items { var v domain.DashboardWorkforceMetric diff --git a/internal/workforcestats/types.go b/internal/workforcestats/types.go index 0f77a51d..acbdbb17 100644 --- a/internal/workforcestats/types.go +++ b/internal/workforcestats/types.go @@ -41,6 +41,7 @@ type Snapshot struct { } type Report struct { + AsOf string `json:"asOf,omitempty"` Items []domain.DashboardWorkforceMetric Totals domain.DashboardWorkforceMetric Details []domain.WorkforceReturnDetail diff --git a/scripts/local/query-replica-isolated-compare.py b/scripts/local/query-replica-isolated-compare.py index 66033973..c0afe164 100644 --- a/scripts/local/query-replica-isolated-compare.py +++ b/scripts/local/query-replica-isolated-compare.py @@ -130,6 +130,7 @@ def main(): ap.add_argument("--max-actors", type=int, default=81, choices=(81, 200, 500)) ap.add_argument("--fixture-plan", help="owned.json from the bounded in-cluster fixture") ap.add_argument("--cold-write-probe", action="store_true", help="bounded 8MiB synthetic diagnostic; never HTTP capacity evidence") + ap.add_argument("--periodic-refresh", action="store_true", help="explicit periodic display policy; same load and safety limits") ap.add_argument("--cold-pace-kib", type=int, choices=(0, 128), default=0) args = ap.parse_args() assert args.cold_write_probe or args.cold_pace_kib == 0 @@ -244,6 +245,7 @@ def main(): limits = {"ready": max(1, max(s["readySeconds"] for s in baseline) * 2), "pressure": max(0.5, max(s["protectedFullAvg10"] for s in baseline) + 0.5), "await": max(10, max(s["awaitMs"] for s in baseline) * 3)} manifest = {"sourceCommit": run(["git", "-C", str(ROOT), "rev-parse", "HEAD"]), "sourceFingerprint": source_fingerprint(), "sourceContainer": source["Id"], "sourceImage": source["Image"], "sourceCPU": 4, "sourceMemory": 4_294_967_296, "queryPodUID": query["metadata"]["uid"], "queryImage": query["spec"]["containers"][0]["image"], "queryCPU": 1, "queryMemory": "1Gi", "tasks": args.tasks, "waves": args.waves, "maxActors": args.max_actors, "wireCompressionBothVariants": os.getenv("QUERY_COMPARE_WIRE_COMPRESSION", ""), "minimumPoolBothVariants": os.getenv("QUERY_COMPARE_MIN_POOL", "0"), "limits": limits, "limitations": ["source Docker VM shares CPU with existing local containers", "target has 2MiB read / 1MiB write and 64/32 IOPS hard limits", "short comparison; not 30-minute three-round production capacity acceptance", "health probes do not establish customer workflow latency"]} manifest["baselineSamples"] = len(baseline) + manifest["refreshMode"] = "periodic" if args.periodic_refresh else "continuous" manifest["baselineSeconds"] = baseline[-1]["time"] - baseline[0]["time"] manifest["diagnostic"] = {"kind": "cold-write-probe", "payloadBytes": 8 << 20, "paceKiB": args.cold_pace_kib} if args.cold_write_probe else None if fixture: @@ -252,12 +254,14 @@ def main(): manifest["binarySHA256"] = owned["binarySHA256"] (out / "manifest.json").write_text(json.dumps(manifest, indent=2)) env = dict(os.environ, QUERY_REPLICA_ISOLATED_E2E="1", QUERY_REPLICA_CLUSTER_TEST="1", QUERY_REPLICA_TEST_TARGET_URI="mongodb://127.0.0.1:27229/?directConnection=true&replicaSet=query-lab-rs", QUERY_COMPARE_TASKS=str(args.tasks), QUERY_COMPARE_WAVES=str(args.waves), QUERY_COMPARE_MAX_ACTORS=str(args.max_actors), QUERY_COMPARE_REPORT=str(out / "comparison.json")) + env["QUERY_REPLICA_REFRESH_MODE"] = manifest["refreshMode"] argv = ["go", "test", "./internal/api", "-run", "^TestQueryReplicaIsolatedHTTPComparisonLocal$", "-count=1", "-timeout=45m", "-v"] remote_report = "/synthetic/query-comparison-" + str(int(time.time())) + ".json" if fixture: settings = ["TMPDIR=/synthetic/tmp", "QUERY_REPLICA_ISOLATED_E2E=1", "QUERY_REPLICA_CLUSTER_TEST=1", "QUERY_REPLICA_TEST_TARGET_URI=mongodb://127.0.0.1:27229/?directConnection=true&replicaSet=query-lab-rs", "QUERY_COMPARE_IN_CLUSTER=1", "QUERY_COMPARE_TASKS=" + str(args.tasks), "QUERY_COMPARE_WAVES=" + str(args.waves), "QUERY_COMPARE_MAX_ACTORS=" + str(args.max_actors), "QUERY_COMPARE_REPORT=" + remote_report, "QUERY_COMPARE_WIRE_COMPRESSION=" + os.getenv("QUERY_COMPARE_WIRE_COMPRESSION", ""), "QUERY_COMPARE_MIN_POOL=" + os.getenv("QUERY_COMPARE_MIN_POOL", "0")] if args.cold_write_probe: settings += ["QUERY_REPLICA_COLD_WRITE_PROBE=1", "QUERY_COLD_PACE_KIB=" + str(args.cold_pace_kib)] + settings += ["QUERY_REPLICA_REFRESH_MODE=" + manifest["refreshMode"]] argv = k + ["-n", "droboticflow-query-lab", "exec", "node4-query-fixture", "-c", "lab", "--", "env"] + settings + [api_binary, "-test.run=^" + test_name + "$", "-test.count=1", "-test.timeout=45m", "-test.v"] with (out / "test.log").open("w") as log: child = subprocess.Popen(argv, cwd=ROOT, env=env, stdout=log, stderr=subprocess.STDOUT, start_new_session=True) diff --git a/scripts/test_query_replica.py b/scripts/test_query_replica.py index ccc6bc02..050c63bc 100644 --- a/scripts/test_query_replica.py +++ b/scripts/test_query_replica.py @@ -24,6 +24,7 @@ CASES = ('internal/store/query_replica_local_test.go', 'internal/store/workforce_report_cache_local_test.go', 'internal/store/workforce_facts_local_test.go', 'internal/store/query_cold_write_local_test.go', + 'internal/store/query_periodic_local_test.go', 'internal/store/query_pool_buckets_local_test.go') diff --git a/scripts/verification-scopes.json b/scripts/verification-scopes.json index b04ae7ca..40a65b9b 100644 --- a/scripts/verification-scopes.json +++ b/scripts/verification-scopes.json @@ -1021,6 +1021,8 @@ "^internal/store/workforce_(report_cache|heap_diagnostic|facts|return_page).*\\.go$", "^internal/store/query_pool_buckets.*\\.go$", "^internal/store/query_cold_write.*\\.go$", + "^internal/store/query_periodic.*\\.go$", + "^internal/api/query_periodic.*\\.go$", "^internal/api/query_fact_routes_test\\.go$", "^internal/api/workforce_compact_report_test\\.go$", "^internal/testsupport/query_replica_lab.*\\.go$",