⚡ Publish query reports and coalesce task pool reads
This commit is contained in:
parent
3e59fa19a9
commit
fc7786190f
@ -17,7 +17,7 @@ import (
|
||||
)
|
||||
|
||||
func allowedFlow(listen, upstream, node, actualUID, expectedUID string) bool {
|
||||
if node != "10.189.16.4" || actualUID == "" || actualUID != expectedUID {
|
||||
if (node != "10.189.16.4" && node != "10.189.16.33") || actualUID == "" || actualUID != expectedUID {
|
||||
return false
|
||||
}
|
||||
l, le := netip.ParseAddrPort(listen)
|
||||
@ -27,7 +27,7 @@ func allowedFlow(listen, upstream, node, actualUID, expectedUID string) bool {
|
||||
}
|
||||
private := netip.MustParsePrefix("10.189.16.0/24")
|
||||
return l == netip.MustParseAddrPort("127.0.0.1:27229") && private.Contains(u.Addr()) && u.Port() == 27130 ||
|
||||
private.Contains(l.Addr()) && l.Port() == 27130 && u == netip.MustParseAddrPort("127.0.0.1:27129")
|
||||
node == "10.189.16.4" && private.Contains(l.Addr()) && l.Port() == 27130 && u == netip.MustParseAddrPort("127.0.0.1:27129")
|
||||
}
|
||||
|
||||
// Do not embed TCPConn: promoted ReadFrom/WriteTo would bypass the deadlines
|
||||
|
||||
@ -9,6 +9,8 @@ func TestIsolatedBridgeRejectsOtherNodesIdentitiesAndDestinations(t *testing.T)
|
||||
}{
|
||||
{"127.0.0.1:27229", "10.189.16.46:27130", "10.189.16.4", "owned", "owned", true},
|
||||
{"10.189.16.46:27130", "127.0.0.1:27129", "10.189.16.4", "owned", "owned", true},
|
||||
{"127.0.0.1:27229", "10.189.16.46:27130", "10.189.16.33", "owned", "owned", true},
|
||||
{"10.189.16.46:27130", "127.0.0.1:27129", "10.189.16.33", "owned", "owned", false},
|
||||
{"0.0.0.0:27229", "10.189.16.46:27130", "10.189.16.4", "owned", "owned", false},
|
||||
{"127.0.0.1:27229", "8.8.8.8:27130", "10.189.16.4", "owned", "owned", false},
|
||||
{"127.0.0.1:27229", "10.189.16.46:27017", "10.189.16.4", "owned", "owned", false},
|
||||
|
||||
@ -52,3 +52,17 @@
|
||||
计数展示不再读取原库脏任务/删除意图做即时补偿,因此数量是查询库已提交事实的结果,不保证包含刚刚发生的操作。批次权限/策略、账号、成员、供应商权限继续实时校验;列表维持本页当前归属/版本读取与原服务端可见性/`canClaim`,领取、保存、送检、质检、验收和后台调度均沿原权威路径。此变更不允许旧缓存绕过权限,也不允许以旧任务列表重复领取。
|
||||
|
||||
本次只在合成隔离环境验证,不切换生产、不修改客户数据。固定原请求超时、租约和多数提交要求;延迟展示是独立业务契约,不是延长请求超时或降低持久性来通过测试。验证必须单列本机与25Gi共享盘结果,以及冷构建、故障恢复和持续变化期间的覆盖限制。
|
||||
|
||||
## 最终展示结果后台发布(2026-11-14,隔离实施)
|
||||
|
||||
在周期模式外再显式启用 `QUERY_PUBLISHED_RESULTS=1`,人员看板和作业记录汇总把原归约器的完整结果发布到查询库,并在提交后写入专用 Valkey。HTTP 不因普通缓存未命中重新聚合历史。Valkey 丢失或不可达时读取同一查询库中的完整成品;缺少成品返回 HTTP 202 `building`,客户端保留加载状态并在既有30秒总预算内等待,离开页面取消请求。权限失败和其他错误不伪装成准备状态。
|
||||
|
||||
成品键包含查询库、组织、当前服务端权限范围和精确筛选;每次请求仍实时建立权限范围,缓存不保存授权结论。默认滚动日期保存为后台计算配方,发布时记录实际区间;显式日期保持纳秒边界,避免每次请求的当前时间产生不同键。事实版本、构建租约和绑定在事务提交时重新核验;旧完整版本保留到新版本成功,失效或错误不集中回源原库。
|
||||
|
||||
缓存信封带已提交事实证书、摘要与到期时间,热读无需额外查查询库水位;首次读取仍验证物理绑定。最长展示年龄仍为15分钟,不能通过重新包装旧成品延长。成品生成受每轮16项/8秒调度预算约束;单组织登记最多1024种活跃查询,24小时后清理闲置登记。超过查询描述或12MiB成品边界的历史特殊结果保留查询库原兼容聚合,不截断结果,不回扫权威库。
|
||||
|
||||
该机制只接管派生展示。领取、标注、切片、送检、正式质检及验收不等待成品生成,也不读取缓存决定权限、任务归属或版本。独立 Valkey 是可丢弃缓存,持久结果继续由查询 Mongo 的原多数提交与日志持久性保证;尚未切换生产入口。
|
||||
|
||||
准备状态通过显式媒体类型协商;旧客户端收到未就绪错误而非成功的非报表JSON。活跃配方的登记触碰合并到15分钟一次,后台统计刷新周期不变。任务池只共享候选身份,个人任务/审查历史维持当前权威查询;权限目录的逐项目重复读合并为请求内批量读,继续复用原授权种子规则,不形成共享权限缓存。
|
||||
|
||||
候选身份的冷缓存查询合并仅作用于相同数据库、组织、事实代次和完整筛选,沿用既有共享名称读取的8秒工作上限及独立取消规则;完成后的长效成品仍只在共享缓存/查询库中。每个请求各自取当前任务与运行服务端权限校验,合并结果不带授权结论、正文或可领取标志。
|
||||
|
||||
@ -153,3 +153,43 @@ QUERY_REPLICA_LOCAL_TEST=1 QUERY_REPLICA_SCALE_LOCAL=1 go test ./internal/store
|
||||
- 最终快速检查 `verify-1794589491130862000/report.json` 所选检查通过;最终完整副本race/cache记录 `check-5b8d9540efff4b01825a878e91549000.log` 无缺失/跳过/源码变化。新增串行调度保护回归保留,失败的并行实验日志保留。前述原始任务检查的交付夹具跳过、原始API既有失败仍如实保留,不将最终状态写成整体验收通过。
|
||||
|
||||
本批没有推送、PR、生产应用或数据变更;独立查询库周期读取开关仍默认关闭。下一步先稳定复现冷态统计超时,并减少旧产出快照和统计的重复读/持久化确认轮次;达到原配置下的完整正确性、速度与资源对照后再准备上线。
|
||||
|
||||
## 成品发布与专用缓存隔离验证(2026-11-14 UTC)
|
||||
|
||||
本批基线 `3e59fa19`,仅在 `mongo-cpu` 工作树和已授权合成命名空间实施。没有切换生产查询入口、修改客户数据或购买磁盘。
|
||||
|
||||
- `QUERY_PUBLISHED_RESULTS=1` 与周期模式组合启用。常规人员/工作记录汇总由后台登记、归约、事务发布,再放入专用 Valkey;前台优先读取完整成品。缓存丢失、连接失败或API进程重启后只读查询库成品,不触发权威库历史聚合。明细仍在查询库分页后补当前页正文。
|
||||
- 首次未准备好时,新客户端通过 `Accept: application/vnd.droboticflow.prepared+json` 协商 HTTP 202;旧客户端得到503,避免将准备信封当成空报告。完成后的响应保持旧格式。等待、重试、切换页面取消均沿原30秒预算,不延长接口或聚合超时。
|
||||
- 已登记范围的活跃时间最多每15分钟触碰一次,减少面板访问引发的查询库写入;这不改变3分钟的后台统计刷新。每组织最多1024个活跃配方、24小时闲置清理,尚未证明500人任意筛选组合的容量。
|
||||
- Valkey 位于 `.33` 的独立实验命名空间,限2核/4Gi、maxmemory 2Gi,仅允许指定合成夹具访问。它是可丢弃派生缓存;持久成品仍由查询Mongo保证。该单Pod、无持久化缓存配置不是生产高可用方案。
|
||||
- 查询Mongo在 `.4` 保留25Gi盘,使用4核/8Gi、WiredTiger 2Gi;读限4MiB/128 IOPS,写依次试验4MiB/128 IOPS、8MiB/256 IOPS。初始化器、运行守护、loop与底层设备限额同步更新,Pod重建后核实实际值。多数提交、日志确认、原有超时和磁盘边界均未降低。
|
||||
- 任务池共享缓存仅保存本页候选ID和游标身份;返回前按ID取当前归属、状态、版本,服务端继续执行可见性与 `canClaim`。“我的任务”和个人审查历史维持权威读取,避免刚领取/提交后因异步发现滞后而消失。权限材料改为请求内批量读取,不跨请求缓存授权结论;默认组织、旧授权、撤销、禁用账号、异常历史项目ID均与原算法对照。
|
||||
|
||||
证据根目录:`local-data/diagnostics/mongo-cpu/published-results-20261114/`(本工作树忽略目录)。`write4-products-pool-v4`、`write8-products-pool-v4` 是同一二进制、相同26,218任务/81人/两轮的IO对照,均结果等价、无新业务错误。该窗口源库CPU工作量约下降90%、文档扫描约下降93%;它是合成源进程的CPU秒数对照,不是生产Mongo即时CPU百分比。首次复制约55秒→27秒,统计物化约118秒→112秒,说明增加写IO对首次复制帮助更明显,不能将所有查询收益归于扩资源。
|
||||
|
||||
`query4-write8` 保存前一4MiB配置及UID回退步骤。Pod重建保留原验证标记,并将一份合成结果以majority+j写入后验证逐字节和SHA恢复;这不代替完整生产备份恢复演练。两轮采样中受保护Pod的IO full PSI为0,生产就绪、Pod身份/重启集合正常;只是该窗口未观察到影响。
|
||||
|
||||
首次测试失败记录保留:旧节点夹具检查拒绝`.33`、旧汇总计量混入合法分页聚合、2Gi缓存的BSON Long比较错误,分别修正后重测。真实浏览器发现加载时仍同时展示空表提示,已修复;准备、空结果、权限错误和退出取消均验证。任务门禁原600秒整体预算耗尽后按原单项超时续跑;浏览器清理超时单项原样重跑通过。完整API包失败通过官方Go基线门禁核验,无新增失败;未扩大已知失败名单。具体来源、环境、命令和各次失败保留在 `verify-task-console.log`、`verify-continuation.log` 和其引用的JSON报告中。
|
||||
|
||||
v5的广范围搜索及冷暖读取对照记录于 `final-products-pool-v5`;后续v6修复了该轮暴露的冷缓存并发重复查询。本批不是30分钟三轮、200/500人、持续写入积压或节点故障容量验收,未准备生产切读。上线前仍须明确生产HA/备份、所有写者标记覆盖、源索引与限速回填范围,并重新核对当前 `dev` 和实际线上版本。
|
||||
|
||||
最终 `final-products-pool-v6` 使用相同26,218任务/81人两轮主读取及20名审查账号的五类任务池读取(含广范围命中、无结果搜索)。主读取162次、任务池200次新旧结果一致;查询库/cache故障回退验证通过。大量命中搜索在v5为1.52秒,定位同条件冷缓存并发重复查询后增加有界在途合并;独立取消与本页当前版本读取保留,v6降至约664毫秒。不是通过少返回任务、放宽权限或增加超时取得结果。
|
||||
|
||||
| 读取 | 旧路径 p95 | 最终查询路径 p95 |
|
||||
| --- | ---: | ---: |
|
||||
| 人员看板 | 17,524ms | 75ms |
|
||||
| 作业记录分页 | 17,618ms | 407ms |
|
||||
| 任务池计数 | 18,394ms | 986ms |
|
||||
| 可领取列表 | 370ms | 384ms |
|
||||
| 大量命中搜索 | 781ms | 664ms |
|
||||
| 无结果搜索 | 5,670ms | 297ms |
|
||||
|
||||
普通列表未观察到速度提升,不宣称所有读取均变快。“我的任务/已审”仍为权威路径,约4~5毫秒。生产总览仅2次暖读,不能据此承诺该角色的持续容量。源库处理同批主读取的CPU耗时23.74→2.40秒,扫描9,887,630→914,716文档;此对照包含测量窗口的后台读写,但一次回填和首次物化另列,不能替代30分钟持续同步负载。
|
||||
|
||||
首次复制约30秒、查询库初始物化约107秒,登记的81个成品另需约8秒准备;后台未完成前没有“瞬间冷启动”承诺。暖读与缓存故障均有独立记录。最终窗口受保护业务IO full PSI为0、健康最大约160毫秒、物理盘平均等待采样最大约4.55毫秒、节点CPU最高6%,无生产Pod身份/重启变化。数据库结果及标注/版本摘要保持一致。
|
||||
|
||||
临时合成源Pod和桥接在测后按UID清理,查询库配额恢复。实验Pod继续保留原自动到期机制,25Gi数据盘不因到期删除。专用Valkey曾正常到期,已按原参数重建并在最后一轮从空缓存重测;没有取消到期机制或扩大生产入口。下一阶段须补充真实验收聚合包形态、持续变更和200/500人验证,再制定生产影子读取与切换计划。
|
||||
|
||||
最终集成报告:`local-data/test-results/verify-1794636683705497000/report.json`,源码未在运行中变化、无未映射文件。41项中39项通过,包含隔离副本/缓存/Mongo专项、源码校验不入队、标注/切片浏览器、搜索及面板、生产前端构建和静态启动。API整包原始检查仍报告已有失败,最终官方Go基线门禁确认68项均在70项既有基线中,无新增;store整包PASS,但默认跳过的外部环境用例标为未完整覆盖,不算通过。专项显式启用的本机副本/Valkey测试已执行通过,未扩大 known-failures。
|
||||
|
||||
`.4` 测后资源申请总量约6.475/7.8核(83%)、内存11,813Mi(42%),包含实验查询Pod;低实际使用率不能替代调度申请余量。后续继续扩核须重新核对该节点及生产峰值。当前仍是限时实验Pod,保留原自动结束机制和25Gi盘;后续恢复验证按已记录UID、配置及回退步骤重建,不将该单Pod直接当生产查询库。
|
||||
|
||||
@ -74,16 +74,24 @@ func (server *server) dashboardWorkforceReport(ctx context.Context, org string,
|
||||
return workforcestats.Build(d, q), nil
|
||||
}
|
||||
|
||||
func (server *server) populateDashboardWorkforce(ctx context.Context, requestOrganizationID string, principal *authenticatedPrincipal, from time.Time, to time.Time, summary *domain.DashboardSummary) error {
|
||||
func (server *server) populateDashboardWorkforce(ctx context.Context, requestOrganizationID string, principal *authenticatedPrincipal, from time.Time, to time.Time, summary *domain.DashboardSummary, relative ...[2]bool) error {
|
||||
if summary == nil {
|
||||
return nil
|
||||
}
|
||||
q, scope := workforcePrincipalQuery(principal)
|
||||
q.From, q.To = from, to
|
||||
if len(relative) > 0 {
|
||||
q.RelativeFrom = relative[0][0]
|
||||
q.RelativeTo = relative[0][1]
|
||||
}
|
||||
report, err := server.dashboardWorkforceReport(ctx, requestOrganizationID, q)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if report.WindowFrom != "" {
|
||||
from, _ = time.Parse(time.RFC3339Nano, report.WindowFrom)
|
||||
to, _ = time.Parse(time.RFC3339Nano, report.WindowTo)
|
||||
}
|
||||
summary.AudienceScope = scope
|
||||
summary.Workforce = domain.DashboardWorkforceSummary{AsOf: report.AsOf, From: from.Format(time.RFC3339Nano), To: to.Format(time.RFC3339Nano), Totals: report.Totals, Items: report.Items}
|
||||
return nil
|
||||
|
||||
@ -17,13 +17,16 @@ import (
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"droboticflow/internal/domain"
|
||||
"droboticflow/internal/productionstats"
|
||||
"droboticflow/internal/queryserving"
|
||||
"droboticflow/internal/store"
|
||||
"droboticflow/internal/testsupport"
|
||||
valkey "github.com/valkey-io/valkey-go"
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
"go.mongodb.org/mongo-driver/event"
|
||||
"go.mongodb.org/mongo-driver/mongo"
|
||||
@ -48,6 +51,13 @@ func TestQueryReplicaIsolatedHTTPComparisonLocal(t *testing.T) {
|
||||
}
|
||||
t.Setenv("ANNOTATION_SOURCE_VALIDATION_MODE", "advisory")
|
||||
t.Setenv("QUERY_SERVING_NAMESPACE", "isolated-http-comparison")
|
||||
published := os.Getenv("QUERY_PUBLISHED_RESULTS") == "1"
|
||||
cacheURL := os.Getenv("QUERY_COMPARE_VALKEY_URL")
|
||||
if published && cacheURL == "" {
|
||||
t.Fatal("published result comparison requires explicit isolated Valkey")
|
||||
}
|
||||
t.Setenv("QUERY_SERVING_READS", "1")
|
||||
t.Setenv("QUERY_VALKEY_URL", cacheURL)
|
||||
ctx := t.Context()
|
||||
n := isolatedComparisonInt(t, "QUERY_COMPARE_TASKS", 26218, 20, 30000)
|
||||
waves := isolatedComparisonInt(t, "QUERY_COMPARE_WAVES", 2, 1, 100)
|
||||
@ -78,13 +88,17 @@ func TestQueryReplicaIsolatedHTTPComparisonLocal(t *testing.T) {
|
||||
}
|
||||
}()
|
||||
}
|
||||
var workforceAggregations atomic.Int64
|
||||
targetCommands := map[string]int64{}
|
||||
targetMaxMillis := map[string]float64{}
|
||||
targetMonitor := &event.CommandMonitor{Started: func(ctx context.Context, e *event.CommandStartedEvent) {
|
||||
name, _ := e.Command.Lookup("aggregate").StringValueOK()
|
||||
if name == "queryWorkforceFacts" {
|
||||
workforceAggregations.Add(1)
|
||||
}
|
||||
if !local {
|
||||
return
|
||||
}
|
||||
name, _ := e.Command.Lookup("aggregate").StringValueOK()
|
||||
if name != "queryWorkforceFacts" {
|
||||
return
|
||||
}
|
||||
@ -121,6 +135,16 @@ func TestQueryReplicaIsolatedHTTPComparisonLocal(t *testing.T) {
|
||||
delete(factPending, e.RequestID)
|
||||
}
|
||||
}}
|
||||
// Separate foreground report aggregation from legitimate background builds
|
||||
// and paginated detail aggregates when checking finished-result fallback.
|
||||
var apiWorkforceAggregations atomic.Int64
|
||||
apiTargetMonitor := &event.CommandMonitor{Started: func(ctx context.Context, e *event.CommandStartedEvent) {
|
||||
name, _ := e.Command.Lookup("aggregate").StringValueOK()
|
||||
if name == "queryWorkforceFacts" {
|
||||
apiWorkforceAggregations.Add(1)
|
||||
}
|
||||
targetMonitor.Started(ctx, e)
|
||||
}, Succeeded: targetMonitor.Succeeded, Failed: targetMonitor.Failed}
|
||||
monitor := &event.CommandMonitor{
|
||||
Started: func(_ context.Context, e *event.CommandStartedEvent) {
|
||||
mu.Lock()
|
||||
@ -258,7 +282,24 @@ func TestQueryReplicaIsolatedHTTPComparisonLocal(t *testing.T) {
|
||||
if err := builderTarget.VerifyQueryReplicaTarget(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
builderTarget.ConfigureQueryServing(nil, nil)
|
||||
closeBuilderCache, e := builderTarget.ConfigureQueryServingFromEnvironment()
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
t.Cleanup(closeBuilderCache)
|
||||
if published {
|
||||
cache, ok := builderTarget.SharedQueryCache().(*queryserving.ManagedValkey)
|
||||
if !ok {
|
||||
t.Fatal("missing dedicated shared cache")
|
||||
}
|
||||
until := time.Now().Add(5 * time.Second)
|
||||
for !cache.Ready() && time.Now().Before(until) {
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
}
|
||||
if !cache.Ready() {
|
||||
t.Fatal("dedicated cache not ready")
|
||||
}
|
||||
}
|
||||
if err := builderTarget.ConfigureQueryRefreshMode(os.Getenv("QUERY_REPLICA_REFRESH_MODE")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@ -333,9 +374,9 @@ func TestQueryReplicaIsolatedHTTPComparisonLocal(t *testing.T) {
|
||||
t.Setenv("QUERY_READ_MONGO_URI", uri)
|
||||
t.Setenv("QUERY_READ_MONGO_DB", db.Name())
|
||||
t.Setenv("QUERY_SERVING_READS", "1")
|
||||
t.Setenv("QUERY_VALKEY_URL", "")
|
||||
t.Setenv("QUERY_VALKEY_URL", cacheURL)
|
||||
t.Setenv("QUERY_TYPESENSE_URL", "")
|
||||
closeReplica, err := newReader.ConfigureQueryReplicaFromEnvironment(ctx, targetMonitor)
|
||||
closeReplica, err := newReader.ConfigureQueryReplicaFromEnvironment(ctx, apiTargetMonitor)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@ -354,6 +395,9 @@ func TestQueryReplicaIsolatedHTTPComparisonLocal(t *testing.T) {
|
||||
return 0, nil, err
|
||||
}
|
||||
r.AddCookie(&http.Cookie{Name: sessionCookieName, Value: fixture.tokens[actor]})
|
||||
if published {
|
||||
r.Header.Set("Accept", "application/vnd.droboticflow.prepared+json")
|
||||
}
|
||||
resp, err := client.Do(r)
|
||||
if err != nil {
|
||||
return 0, nil, err
|
||||
@ -368,6 +412,39 @@ func TestQueryReplicaIsolatedHTTPComparisonLocal(t *testing.T) {
|
||||
"/api/dashboard/workforce?from=2026-11-01T00:00:00Z&to=2026-11-13T00:00:00Z&groupBy=member",
|
||||
"/api/workforce-returns?from=2026-11-01T00:00:00Z&to=2026-11-13T00:00:00Z&limit=20&offset=0",
|
||||
}
|
||||
publicationWarmSeconds := float64(0)
|
||||
publicationPending := 0
|
||||
if published {
|
||||
warmStart := time.Now()
|
||||
deadline := materializeStarted.Add(4 * time.Minute)
|
||||
pending := map[int]bool{}
|
||||
for actor := 0; actor < maxActors; actor++ {
|
||||
pending[actor] = true
|
||||
}
|
||||
for len(pending) > 0 {
|
||||
for actor := range pending {
|
||||
if !time.Now().Before(deadline) {
|
||||
t.Fatalf("published responses did not become ready within original materialization deadline: %d", len(pending))
|
||||
}
|
||||
status, _, err := request(after.URL, paths[2], actor)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if status == 200 {
|
||||
delete(pending, actor)
|
||||
} else if status == http.StatusAccepted {
|
||||
publicationPending++
|
||||
} else {
|
||||
t.Fatalf("publication prewarm status %d", status)
|
||||
}
|
||||
}
|
||||
if len(pending) > 0 {
|
||||
time.Sleep(time.Second)
|
||||
}
|
||||
}
|
||||
publicationWarmSeconds = time.Since(warmStart).Seconds()
|
||||
t.Logf("published response startup seconds=%.3f pending_responses=%d", publicationWarmSeconds, publicationPending)
|
||||
}
|
||||
// First reads use the existing parity requests, with no extra query load.
|
||||
// Exact business JSON equality; only generated report timestamps are volatile.
|
||||
firstReads := map[string]any{}
|
||||
@ -550,6 +627,115 @@ func TestQueryReplicaIsolatedHTTPComparisonLocal(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
if published {
|
||||
// Separate list migration from count/report rollout; formal operations
|
||||
// and personal pages keep their authoritative path.
|
||||
newReader.EnableQueryReplicaPoolReads(true)
|
||||
poolCases := []any{}
|
||||
expectedPool := map[string]any{}
|
||||
for _, mode := range []string{"old", "query"} {
|
||||
url := before.URL
|
||||
if mode == "query" {
|
||||
url = after.URL
|
||||
}
|
||||
for _, pool := range []string{"available", "available&search=part0001", "available&search=absent-capture", "mine", "reviewed"} {
|
||||
principalCacheMu.Lock()
|
||||
principalCache = map[string]principalCacheEntry{}
|
||||
principalCacheMu.Unlock()
|
||||
latencies := []float64{}
|
||||
var guard sync.Mutex
|
||||
var wg sync.WaitGroup
|
||||
for actor := 61; actor < 81; actor++ {
|
||||
wg.Add(1)
|
||||
go func(actor int) {
|
||||
defer wg.Done()
|
||||
path := "/api/human-items?pluginId=annotation-qa&stageId=semantic_review&limit=20&pool=" + pool
|
||||
start := time.Now()
|
||||
status, body, err := request(url, path, actor)
|
||||
guard.Lock()
|
||||
defer guard.Unlock()
|
||||
latencies = append(latencies, float64(time.Since(start).Microseconds())/1000)
|
||||
if status != 200 || err != nil {
|
||||
t.Errorf("pool %s/%s/%d: status=%d err=%v %.200s", mode, pool, actor, status, err, body)
|
||||
return
|
||||
}
|
||||
key := fmt.Sprintf("%s/%d", pool, actor)
|
||||
value := isolatedBusinessJSON(t, body)
|
||||
if mode == "old" {
|
||||
expectedPool[key] = value
|
||||
} else if !reflect.DeepEqual(expectedPool[key], value) {
|
||||
t.Errorf("pool business mismatch %s", key)
|
||||
}
|
||||
}(actor)
|
||||
}
|
||||
wg.Wait()
|
||||
sort.Float64s(latencies)
|
||||
p95 := latencies[(len(latencies)-1)*95/100]
|
||||
poolCases = append(poolCases, map[string]any{"mode": mode, "pool": pool, "requests": len(latencies), "p95Ms": p95})
|
||||
if mode == "query" && p95 > 2000 {
|
||||
t.Errorf("%s pool p95 exceeds 2s: %f", pool, p95)
|
||||
}
|
||||
}
|
||||
}
|
||||
report["poolListCases"] = poolCases
|
||||
newReader.EnableQueryReplicaPoolReads(false)
|
||||
}
|
||||
if published {
|
||||
opts, e := valkey.ParseURL(cacheURL)
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
vc, e := valkey.NewClient(opts)
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
defer vc.Close()
|
||||
if e = vc.Do(ctx, vc.B().Flushdb().Sync().Build()).Error(); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
beforeAggregate := apiWorkforceAggregations.Load()
|
||||
fallback := map[string]float64{}
|
||||
for _, path := range paths {
|
||||
now := time.Now()
|
||||
beforeReport := apiWorkforceAggregations.Load()
|
||||
status, _, e := request(after.URL, path, 0)
|
||||
if e != nil || status != 200 {
|
||||
t.Fatalf("cache loss fallback status=%d err=%v", status, e)
|
||||
}
|
||||
if strings.HasPrefix(path, "/api/dashboard/workforce?") && apiWorkforceAggregations.Load() != beforeReport {
|
||||
t.Error("cache loss reran report aggregation")
|
||||
}
|
||||
fallback[strings.Split(path, "?")[0]] = float64(time.Since(now).Microseconds()) / 1000
|
||||
}
|
||||
report["cacheLossFallbackMs"] = fallback
|
||||
report["cacheLossDetailAndReportAggregations"] = apiWorkforceAggregations.Load() - beforeAggregate
|
||||
// A separate API client with an unreachable loopback cache must still
|
||||
// serve durable products; this does not stop any cluster service.
|
||||
t.Setenv("QUERY_VALKEY_URL", "redis://127.0.0.1:1/0")
|
||||
blocked := store.NewMongoStore(sdb)
|
||||
finish, e := blocked.ConfigureQueryReplicaFromEnvironment(ctx, apiTargetMonitor)
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
defer finish()
|
||||
unavailable := serve(blocked)
|
||||
unavailableMs := map[string]float64{}
|
||||
for _, path := range paths {
|
||||
now := time.Now()
|
||||
status, _, e := request(unavailable.URL, path, 0)
|
||||
if e != nil || status != 200 {
|
||||
t.Fatalf("unavailable cache fallback status=%d err=%v", status, e)
|
||||
}
|
||||
unavailableMs[strings.Split(path, "?")[0]] = float64(time.Since(now).Microseconds()) / 1000
|
||||
}
|
||||
report["cacheUnavailableFallbackMs"] = unavailableMs
|
||||
}
|
||||
if published {
|
||||
report["cache"] = "dedicated Valkey with majority-committed durable response fallback"
|
||||
}
|
||||
report["publishedResults"] = published
|
||||
report["publicationPrewarmSeconds"] = publicationWarmSeconds
|
||||
report["publicationPendingResponses"] = publicationPending
|
||||
report["materializationSeconds"] = materializeSeconds
|
||||
report["firstReads"] = firstReads
|
||||
report["cases"] = cases
|
||||
@ -636,7 +822,7 @@ func isolatedSourceScans(t *testing.T, db *mongo.Database) map[string]int64 {
|
||||
func isolatedSourceCPU(t *testing.T) float64 {
|
||||
t.Helper()
|
||||
if os.Getenv("QUERY_COMPARE_IN_CLUSTER") == "1" {
|
||||
if runtime.GOOS != "linux" || os.Getenv("NODE_NAME") != "10.189.16.4" || os.Getenv("POD_NAME") != "node4-query-fixture" {
|
||||
if runtime.GOOS != "linux" || (os.Getenv("NODE_NAME") != "10.189.16.4" && os.Getenv("NODE_NAME") != "10.189.16.33") || os.Getenv("POD_NAME") != "node4-query-fixture" {
|
||||
t.Fatal("isolated source process identity rejected")
|
||||
}
|
||||
raw, err := os.ReadFile("/synthetic/mongod.pid")
|
||||
|
||||
@ -1582,7 +1582,10 @@ func (server *server) handleDashboard(writer http.ResponseWriter, request *http.
|
||||
}
|
||||
summary := domain.DashboardSummary{}
|
||||
if includeWorkforce {
|
||||
if err := server.populateDashboardWorkforce(request.Context(), server.requestOrganizationID(request), principal, from, to, &summary); err != nil {
|
||||
if err := server.populateDashboardWorkforce(request.Context(), server.requestOrganizationID(request), principal, from, to, &summary, [2]bool{strings.TrimSpace(request.URL.Query().Get("from")) == "", strings.TrimSpace(request.URL.Query().Get("to")) == ""}); err != nil {
|
||||
if renderWorkforcePublicationPending(writer, request, err) {
|
||||
return
|
||||
}
|
||||
renderError(writer, http.StatusInternalServerError, err)
|
||||
return
|
||||
}
|
||||
@ -1647,7 +1650,10 @@ func (server *server) handleDashboard(writer http.ResponseWriter, request *http.
|
||||
return
|
||||
}
|
||||
if includeWorkforce {
|
||||
if err := server.populateDashboardWorkforce(request.Context(), server.requestOrganizationID(request), principal, from, to, &summary); err != nil {
|
||||
if err := server.populateDashboardWorkforce(request.Context(), server.requestOrganizationID(request), principal, from, to, &summary, [2]bool{strings.TrimSpace(request.URL.Query().Get("from")) == "", strings.TrimSpace(request.URL.Query().Get("to")) == ""}); err != nil {
|
||||
if renderWorkforcePublicationPending(writer, request, err) {
|
||||
return
|
||||
}
|
||||
renderError(writer, http.StatusInternalServerError, err)
|
||||
return
|
||||
}
|
||||
@ -1672,7 +1678,10 @@ func (server *server) handleDashboard(writer http.ResponseWriter, request *http.
|
||||
return
|
||||
}
|
||||
if includeWorkforce {
|
||||
if err := server.populateDashboardWorkforce(request.Context(), server.requestOrganizationID(request), principal, from, to, &summary); err != nil {
|
||||
if err := server.populateDashboardWorkforce(request.Context(), server.requestOrganizationID(request), principal, from, to, &summary, [2]bool{strings.TrimSpace(request.URL.Query().Get("from")) == "", strings.TrimSpace(request.URL.Query().Get("to")) == ""}); err != nil {
|
||||
if renderWorkforcePublicationPending(writer, request, err) {
|
||||
return
|
||||
}
|
||||
renderError(writer, http.StatusInternalServerError, err)
|
||||
return
|
||||
}
|
||||
|
||||
31
internal/api/workforce_publication_test.go
Normal file
31
internal/api/workforce_publication_test.go
Normal file
@ -0,0 +1,31 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"droboticflow/internal/store"
|
||||
)
|
||||
|
||||
func TestWorkforcePublicationPendingIsNotAnEmptyReport(t *testing.T) {
|
||||
r := httptest.NewRequest(http.MethodGet, "/report", nil)
|
||||
r.Header.Set("Accept", "application/vnd.droboticflow.prepared+json")
|
||||
w := httptest.NewRecorder()
|
||||
if !renderWorkforcePublicationPending(w, r, fmt.Errorf("report: %w", store.ErrWorkforcePublicationPending)) || w.Code != http.StatusAccepted || w.Header().Get("Retry-After") != "1" {
|
||||
t.Fatalf("publication readiness response: %d %s", w.Code, w.Body.String())
|
||||
}
|
||||
legacy := httptest.NewRecorder()
|
||||
if !renderWorkforcePublicationPending(legacy, httptest.NewRequest(http.MethodGet, "/report", nil), store.ErrWorkforcePublicationPending) || legacy.Code != http.StatusServiceUnavailable {
|
||||
t.Fatal("legacy client received a successful non-report body")
|
||||
}
|
||||
|
||||
for _, err := range []error{nil, store.ErrQueryReplicaNotReady, errors.New("permission denied")} {
|
||||
w = httptest.NewRecorder()
|
||||
if renderWorkforcePublicationPending(w, r, err) || w.Body.Len() != 0 {
|
||||
t.Fatal("unrelated errors must retain their existing handling")
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -2,6 +2,7 @@ package api
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strconv"
|
||||
@ -9,6 +10,7 @@ import (
|
||||
"time"
|
||||
|
||||
"droboticflow/internal/domain"
|
||||
"droboticflow/internal/store"
|
||||
"droboticflow/internal/workforcestats"
|
||||
)
|
||||
|
||||
@ -24,6 +26,8 @@ func (s *server) workforceQuery(r *http.Request, workload bool) (workforcestats.
|
||||
return q, err
|
||||
}
|
||||
q.ProjectID = strings.TrimSpace(v.Get("projectId"))
|
||||
q.RelativeFrom = strings.TrimSpace(v.Get("from")) == ""
|
||||
q.RelativeTo = strings.TrimSpace(v.Get("to")) == ""
|
||||
q.GroupTemplate = workload
|
||||
q.Mode = domain.RunMode(v.Get("mode"))
|
||||
q.Role = strings.TrimSpace(v.Get("role"))
|
||||
@ -102,9 +106,16 @@ func (s *server) handleAnnotationWorkloadSummary(w http.ResponseWriter, r *http.
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
if renderWorkforcePublicationPending(w, r, err) {
|
||||
return
|
||||
}
|
||||
renderError(w, 500, err)
|
||||
return
|
||||
}
|
||||
if report.WindowFrom != "" {
|
||||
q.From, _ = time.Parse(time.RFC3339Nano, report.WindowFrom)
|
||||
q.To, _ = time.Parse(time.RFC3339Nano, report.WindowTo)
|
||||
}
|
||||
items := make([]domain.AnnotationWorkloadAggregate, 0, len(report.Items))
|
||||
for _, m := range report.Items {
|
||||
items = append(items, domain.AnnotationWorkloadAggregate{
|
||||
@ -131,9 +142,16 @@ func (s *server) handleDashboardWorkforce(w http.ResponseWriter, r *http.Request
|
||||
q.Details = false
|
||||
report, err := s.dashboardWorkforceReport(r.Context(), s.requestOrganizationID(r), q)
|
||||
if err != nil {
|
||||
if renderWorkforcePublicationPending(w, r, err) {
|
||||
return
|
||||
}
|
||||
renderError(w, http.StatusInternalServerError, err)
|
||||
return
|
||||
}
|
||||
if report.WindowFrom != "" {
|
||||
q.From, _ = time.Parse(time.RFC3339Nano, report.WindowFrom)
|
||||
q.To, _ = time.Parse(time.RFC3339Nano, report.WindowTo)
|
||||
}
|
||||
workforcestats.ApplyRates(&report.Totals)
|
||||
for i := range report.Items {
|
||||
workforcestats.ApplyRates(&report.Items[i])
|
||||
@ -194,3 +212,17 @@ func (s *server) handleWorkforceReturns(w http.ResponseWriter, r *http.Request)
|
||||
}
|
||||
renderJSON(w, 200, map[string]any{"items": items[offset:end], "total": count, "from": q.From.Format(time.RFC3339Nano), "to": q.To.Format(time.RFC3339Nano)})
|
||||
}
|
||||
|
||||
func renderWorkforcePublicationPending(w http.ResponseWriter, r *http.Request, err error) bool {
|
||||
if !errors.Is(err, store.ErrWorkforcePublicationPending) {
|
||||
return false
|
||||
}
|
||||
w.Header().Set("Retry-After", "1")
|
||||
// Older clients must not decode a readiness envelope as an empty report.
|
||||
if !strings.Contains(r.Header.Get("Accept"), "application/vnd.droboticflow.prepared+json") {
|
||||
renderError(w, http.StatusServiceUnavailable, fmt.Errorf("看板正在准备,请稍后重试"))
|
||||
return true
|
||||
}
|
||||
renderJSON(w, http.StatusAccepted, map[string]any{"status": "building", "retryAfterMs": 1000})
|
||||
return true
|
||||
}
|
||||
|
||||
@ -861,7 +861,9 @@ func (store *MongoStore) UpsertHumanBatch(ctx context.Context, item domain.Human
|
||||
}
|
||||
|
||||
func (store *MongoStore) ListHumanItems(ctx context.Context, organizationID string, query domain.HumanItemQuery) ([]domain.HumanItemRecord, error) {
|
||||
if query.PoolCandidateProjection {
|
||||
// Personal pages must include a just-claimed/returned task before async
|
||||
// discovery catches up. These indexed actor/history scopes stay live.
|
||||
if query.PoolCandidateProjection && query.ActorUserID == "" && !query.ReviewedPool && !query.MinePoolOnly {
|
||||
if target, err := store.poolReplica(ctx); err != nil {
|
||||
return nil, err
|
||||
} else if target != nil {
|
||||
|
||||
@ -25,8 +25,12 @@ type MongoStore struct {
|
||||
queryReplicaReads *queryReplicaReads
|
||||
queryReplicaFactStore bool
|
||||
queryPeriodic bool
|
||||
queryPublishedResults bool
|
||||
workforcePublicationTouches workforcePublicationTouches
|
||||
workforcePublicationBuilds workforcePublicationBuildMemo
|
||||
queryBuildWrites *queryBuildWriteGate
|
||||
queryReplicaCounts queryCountCache
|
||||
queryReplicaPageFlights singleflight.Group
|
||||
queryReplicaBatchFacts queryBatchFacts
|
||||
queryServing *queryServingOptions
|
||||
productionReadFlights singleflight.Group
|
||||
|
||||
@ -11,6 +11,17 @@ import (
|
||||
// This is an explicit derived-data index operation, never API startup behavior.
|
||||
// Production execution requires its own reviewed migration/index authorization.
|
||||
func (s *MongoStore) EnsureQueryReadModelIndexes(ctx context.Context) error {
|
||||
if s.publishedWorkforceReads() {
|
||||
_, err := s.db.Collection(workforcePublicationData).Indexes().CreateOne(ctx, mongo.IndexModel{Keys: bson.D{{Key: "computedAt", Value: 1}}, Options: options.Index().SetName("workforce_publication_retention_v1").SetExpireAfterSeconds(172800)})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = s.db.Collection(workforcePublicationRegistry).Indexes().CreateOne(ctx, mongo.IndexModel{Keys: bson.D{{Key: "updatedAt", Value: 1}}, Options: options.Index().SetName("workforce_publication_requests_retention_v1").SetExpireAfterSeconds(172800)})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if s.queryReplicaFactStore {
|
||||
if err := s.ensureWorkforceFactIndexes(ctx); err != nil {
|
||||
return err
|
||||
|
||||
@ -475,6 +475,11 @@ func (b *QueryReadBuilder) run(ctx context.Context, started chan<- struct{}) err
|
||||
log.Print("query_model_build result=retry category=source_or_publication")
|
||||
}
|
||||
}
|
||||
for org := range scopes {
|
||||
if e := b.store.buildWorkforcePublications(ctx, l, org); e != nil {
|
||||
log.Print("workforce_publication result=retry")
|
||||
}
|
||||
}
|
||||
if time.Since(lastCleanup) > time.Minute {
|
||||
cleanup, finish := context.WithTimeout(ctx, 5*time.Second)
|
||||
_ = b.store.cleanQueryModelOrphans(cleanup)
|
||||
|
||||
@ -72,6 +72,7 @@ func (s *MongoStore) SharedQueryCache() queryserving.Cache {
|
||||
}
|
||||
|
||||
type QueryModelHead struct {
|
||||
verifiedUntil time.Time // Request-local proof; never serialized or persisted.
|
||||
ChunkBytes int `bson:"chunkBytes,omitempty" json:"chunkBytes,omitempty"`
|
||||
FactEpoch int64 `bson:"factEpoch,omitempty" json:"factEpoch,omitempty"`
|
||||
SourceRevision int64 `bson:"sourceRevision" json:"sourceRevision"`
|
||||
|
||||
79
internal/store/query_replica_page_local_test.go
Normal file
79
internal/store/query_replica_page_local_test.go
Normal file
@ -0,0 +1,79 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"droboticflow/internal/domain"
|
||||
"encoding/json"
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
"go.mongodb.org/mongo-driver/event"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestQueryReplicaPageCacheKeepsCurrentOwnershipAndPersonalReadsLocal(t *testing.T) {
|
||||
var targetFinds atomic.Int64
|
||||
source, target, p := newLocalReplica(t, nil, &event.CommandMonitor{Started: func(_ context.Context, e *event.CommandStartedEvent) {
|
||||
name, _ := e.Command.Lookup("find").StringValueOK()
|
||||
if name == "humanItems" {
|
||||
targetFinds.Add(1)
|
||||
}
|
||||
}})
|
||||
ctx := t.Context()
|
||||
doc := bson.M{"id": "i", "organizationId": "o", "projectId": "p", "status": "ready", "version": 1, "createdAt": "2026-11-01T00:00:00Z", "subject": bson.M{"payload": bson.M{"annotationDraft": "preserve"}}}
|
||||
if _, e := source.queryCollection("humanItems").InsertOne(ctx, doc); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
finishLocalReplica(t, p)
|
||||
cache := publicationTestCache{}
|
||||
target.ConfigureQueryServing(cache, nil)
|
||||
if e := source.ConfigureQueryReplicaReports(target, time.Minute); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
source.EnableQueryReplicaPoolReads(true)
|
||||
q := domain.HumanItemQuery{PoolCandidateProjection: true, Limit: 1}
|
||||
read := WithQueryReplicaPoolRead(ctx)
|
||||
targetFinds.Store(0)
|
||||
for i := 0; i < 2; i++ {
|
||||
rows, e := source.ListHumanItems(read, "o", q)
|
||||
if e != nil || len(rows) != 1 || rows[0].Version != 1 {
|
||||
t.Fatal(rows, e)
|
||||
}
|
||||
}
|
||||
if targetFinds.Load() != 1 {
|
||||
t.Fatal("warm discovery reread target tasks", targetFinds.Load())
|
||||
}
|
||||
raw, _ := json.Marshal(cache)
|
||||
if strings.Contains(string(raw), "annotationDraft") || strings.Contains(string(raw), "preserve") {
|
||||
t.Fatal("page cache copied annotation")
|
||||
}
|
||||
if _, e := source.humanItems.UpdateOne(ctx, bson.M{"id": "i"}, bson.M{"$set": bson.M{"status": "claimed", "assignedUserId": "other", "version": 2}}); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
rows, e := source.ListHumanItems(read, "o", q)
|
||||
if e != nil || len(rows) != 1 || rows[0].Version != 2 || rows[0].AssignedUserID != "other" {
|
||||
t.Fatal("cached ownership reached response", rows, e)
|
||||
}
|
||||
if _, e := source.humanItems.DeleteOne(ctx, bson.M{"id": "i"}); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
rows, e = source.ListHumanItems(read, "o", q)
|
||||
if e != nil || len(rows) != 1 || !rows[0].QueryReplicaOmit || rows[0].CreatedAt != "2026-11-01T00:00:00Z" {
|
||||
t.Fatal("deleted candidate broke pagination", rows, e)
|
||||
}
|
||||
if _, e := source.humanItems.InsertOne(ctx, bson.M{"id": "new", "organizationId": "o", "status": "claimed", "assignedUserId": "u", "version": 3}); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
mine := q
|
||||
mine.ActorUserID = "u"
|
||||
mine.MinePoolOnly = true
|
||||
rows, e = source.ListHumanItems(read, "o", mine)
|
||||
if e != nil || len(rows) != 1 || rows[0].ID != "new" {
|
||||
t.Fatal("just claimed personal task awaited replica", rows, e)
|
||||
}
|
||||
rows, e = source.ListHumanItems(read, "foreign", q)
|
||||
if e != nil || len(rows) != 0 {
|
||||
t.Fatal("page cache crossed organization", rows, e)
|
||||
}
|
||||
}
|
||||
50
internal/store/query_replica_page_test.go
Normal file
50
internal/store/query_replica_page_test.go
Normal file
@ -0,0 +1,50 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestQueryReplicaPageFlightsShareIdentityAndCancelIndependently(t *testing.T) {
|
||||
s := &MongoStore{}
|
||||
started, release := make(chan struct{}), make(chan struct{})
|
||||
var calls atomic.Int64
|
||||
read := func(ctx context.Context) (queryPoolPage, error) {
|
||||
calls.Add(1)
|
||||
deadline, ok := ctx.Deadline()
|
||||
if !ok || time.Until(deadline) > 8*time.Second {
|
||||
t.Error("page work lost original bound")
|
||||
}
|
||||
close(started)
|
||||
select {
|
||||
case <-release:
|
||||
return queryPoolPage{Key: "scope", Items: []queryPoolPageIdentity{{ID: "i", CreatedAt: "before"}}}, nil
|
||||
case <-ctx.Done():
|
||||
return queryPoolPage{}, ctx.Err()
|
||||
}
|
||||
}
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
first := make(chan error, 1)
|
||||
go func() { _, err := s.shareReplicaPoolPage(ctx, "scope", read); first <- err }()
|
||||
<-started
|
||||
second := s.queryReplicaPageFlights.DoChan("scope", func() (any, error) { t.Error("duplicate candidate query"); return queryPoolPage{}, nil })
|
||||
cancel()
|
||||
if err := <-first; !errors.Is(err, context.Canceled) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
close(release)
|
||||
result := <-second
|
||||
if result.Err != nil || result.Val.(queryPoolPage).Items[0].ID != "i" || calls.Load() != 1 {
|
||||
t.Fatal(result, calls.Load())
|
||||
}
|
||||
sentinel := errors.New("query unavailable")
|
||||
if _, err := s.shareReplicaPoolPage(t.Context(), "scope", func(context.Context) (queryPoolPage, error) { return queryPoolPage{}, sentinel }); !errors.Is(err, sentinel) {
|
||||
t.Fatal("failed query cached", err)
|
||||
}
|
||||
if _, err := s.shareReplicaPoolPage(t.Context(), "scope", func(context.Context) (queryPoolPage, error) { calls.Add(1); return queryPoolPage{}, nil }); err != nil || calls.Load() != 2 {
|
||||
t.Fatal("completed flight survived", err, calls.Load())
|
||||
}
|
||||
}
|
||||
@ -3,8 +3,10 @@ package store
|
||||
import (
|
||||
"context"
|
||||
"droboticflow/internal/domain"
|
||||
"droboticflow/internal/queryserving"
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
"go.mongodb.org/mongo-driver/mongo/options"
|
||||
"time"
|
||||
)
|
||||
|
||||
type queryReplicaPoolReadKey struct{}
|
||||
@ -46,7 +48,7 @@ func (s *MongoStore) EnableQueryReplicaCountReads(enabled bool) {
|
||||
// Missing/changed candidates keep an internal cursor placeholder so deletion
|
||||
// cannot falsely end pagination. They never enter the public response.
|
||||
func (s *MongoStore) replicaPoolCandidates(ctx context.Context, target *MongoStore, org string, q domain.HumanItemQuery) ([]domain.HumanItemRecord, error) {
|
||||
rows, e := target.ListHumanItems(ctx, org, q)
|
||||
rows, e := target.replicaPoolPageIdentities(ctx, org, q)
|
||||
if e != nil || len(rows) == 0 {
|
||||
return rows, e
|
||||
}
|
||||
@ -89,6 +91,90 @@ func (s *MongoStore) replicaPoolCandidates(ctx context.Context, target *MongoSto
|
||||
return rows, nil
|
||||
}
|
||||
|
||||
type queryPoolPageIdentity struct {
|
||||
ID string `json:"id"`
|
||||
CreatedAt string `json:"createdAt"`
|
||||
}
|
||||
type queryPoolPage struct {
|
||||
Key string `json:"key"`
|
||||
Items []queryPoolPageIdentity `json:"items"`
|
||||
}
|
||||
|
||||
// Cache only discovery identities, never payloads, permissions or canClaim.
|
||||
// The source still fetches the current page before applying business rules.
|
||||
func (s *MongoStore) replicaPoolPageIdentities(ctx context.Context, org string, q domain.HumanItemQuery) ([]domain.HumanItemRecord, error) {
|
||||
if s.SharedQueryCache() == nil || q.Limit <= 0 || q.Limit > 200 {
|
||||
return s.ListHumanItems(ctx, org, q)
|
||||
}
|
||||
epoch, err := s.countFactEpoch(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
key := queryserving.Key("pool-page-identities/v1", s.db.Name(), org, epoch, q)
|
||||
page, err := s.shareReplicaPoolPage(ctx, key, func(work context.Context) (queryPoolPage, error) {
|
||||
return s.loadReplicaPoolPage(work, key, org, q)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rows := make([]domain.HumanItemRecord, 0, len(page.Items))
|
||||
for _, item := range page.Items {
|
||||
rows = append(rows, domain.HumanItemRecord{ID: item.ID, CreatedAt: item.CreatedAt})
|
||||
}
|
||||
return rows, nil
|
||||
}
|
||||
|
||||
// Match the existing shared-name lookup contract: only in-flight data reads
|
||||
// coalesce, one canceled waiter cannot cancel another, and work stays bounded.
|
||||
func (s *MongoStore) shareReplicaPoolPage(ctx context.Context, key string, read func(context.Context) (queryPoolPage, error)) (queryPoolPage, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return queryPoolPage{}, err
|
||||
}
|
||||
flight := s.queryReplicaPageFlights.DoChan(key, func() (any, error) {
|
||||
work, cancel := context.WithTimeout(context.WithoutCancel(ctx), 8*time.Second)
|
||||
defer cancel()
|
||||
return read(work)
|
||||
})
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return queryPoolPage{}, ctx.Err()
|
||||
case result := <-flight:
|
||||
if result.Err != nil {
|
||||
return queryPoolPage{}, result.Err
|
||||
}
|
||||
page := result.Val.(queryPoolPage)
|
||||
page.Items = append([]queryPoolPageIdentity(nil), page.Items...)
|
||||
return page, nil
|
||||
}
|
||||
}
|
||||
func (s *MongoStore) loadReplicaPoolPage(ctx context.Context, key, org string, q domain.HumanItemQuery) (queryPoolPage, error) {
|
||||
var page queryPoolPage
|
||||
if queryserving.GetJSON(ctx, s.SharedQueryCache(), key, &page) && page.Key == key && len(page.Items) <= q.Limit {
|
||||
valid := true
|
||||
for _, item := range page.Items {
|
||||
if item.ID == "" {
|
||||
valid = false
|
||||
break
|
||||
}
|
||||
}
|
||||
if valid {
|
||||
return page, nil
|
||||
}
|
||||
}
|
||||
rows, err := s.ListHumanItems(ctx, org, q)
|
||||
if err != nil {
|
||||
return queryPoolPage{}, err
|
||||
}
|
||||
page = queryPoolPage{Key: key, Items: make([]queryPoolPageIdentity, 0, len(rows))}
|
||||
for _, item := range rows {
|
||||
page.Items = append(page.Items, queryPoolPageIdentity{ID: item.ID, CreatedAt: item.CreatedAt})
|
||||
}
|
||||
if len(rows) <= q.Limit {
|
||||
queryserving.PutJSON(ctx, s.SharedQueryCache(), key, page, 2*time.Second)
|
||||
}
|
||||
return page, nil
|
||||
}
|
||||
|
||||
func (s *MongoStore) replicaAdminPool(ctx context.Context, target *MongoStore, org string, q AdminAnnotationPoolQuery) ([]domain.HumanItemRecord, error) {
|
||||
rows, e := target.ListAdminAnnotationPool(ctx, org, q)
|
||||
if e != nil || len(rows) == 0 {
|
||||
|
||||
@ -11,6 +11,7 @@ import (
|
||||
// different deployments even when their database names happen to be identical.
|
||||
func (s *MongoStore) ConfigureQueryServingFromEnvironment() (func(), error) {
|
||||
close := func() {}
|
||||
s.ConfigureQueryPublishedResults(os.Getenv("QUERY_PUBLISHED_RESULTS") == "1")
|
||||
if err := s.ConfigureQueryRefreshMode(os.Getenv("QUERY_REPLICA_REFRESH_MODE")); err != nil {
|
||||
return close, err
|
||||
}
|
||||
|
||||
144
internal/store/supplier_authorization_batch_local_test.go
Normal file
144
internal/store/supplier_authorization_batch_local_test.go
Normal file
@ -0,0 +1,144 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"sort"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
|
||||
"droboticflow/internal/domain"
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
"go.mongodb.org/mongo-driver/event"
|
||||
)
|
||||
|
||||
// Keep the prior per-project algorithm as the compatibility oracle.
|
||||
func legacyAuthorizationListForTest(ctx context.Context, s *MongoStore, org, project string) ([]domain.SupplierProjectAuthorization, error) {
|
||||
projects := []domain.ProjectRecord{}
|
||||
if project != "" {
|
||||
p, e := s.GetProject(ctx, project)
|
||||
if e != nil {
|
||||
return nil, e
|
||||
}
|
||||
if p != nil && annotationOrganizationMatches(p.OrganizationID, org) {
|
||||
projects = append(projects, *p)
|
||||
}
|
||||
} else {
|
||||
var e error
|
||||
projects, e = s.ListProjectsByOrganization(ctx, org)
|
||||
if e != nil {
|
||||
return nil, e
|
||||
}
|
||||
}
|
||||
result := []domain.SupplierProjectAuthorization{}
|
||||
for _, p := range projects {
|
||||
initialized, e := s.supplierProjectAuthorizationInitialized(ctx, org, p.ID)
|
||||
if e != nil {
|
||||
return nil, e
|
||||
}
|
||||
if !initialized {
|
||||
seeds, e := s.authorizationSeeds(ctx, org, p.ID)
|
||||
if e != nil {
|
||||
return nil, e
|
||||
}
|
||||
result = append(result, seeds...)
|
||||
continue
|
||||
}
|
||||
cur, e := s.supplierProjectAuthorizations.Find(ctx, bson.M{"organizationId": normalizedOrganizationID(org), "projectId": p.ID})
|
||||
if e != nil {
|
||||
return nil, e
|
||||
}
|
||||
grants, e := decodeMany[domain.SupplierProjectAuthorization](ctx, cur)
|
||||
if e != nil {
|
||||
return nil, e
|
||||
}
|
||||
result = append(result, grants...)
|
||||
}
|
||||
sort.Slice(result, func(i, j int) bool {
|
||||
if result[i].ProjectID != result[j].ProjectID {
|
||||
return result[i].ProjectID < result[j].ProjectID
|
||||
}
|
||||
return result[i].SupplierID < result[j].SupplierID
|
||||
})
|
||||
return result, nil
|
||||
}
|
||||
func normalizedAuthorizationTestRows(rows []domain.SupplierProjectAuthorization) []domain.SupplierProjectAuthorization {
|
||||
for i := range rows {
|
||||
if rows[i].Source == "existing_member" {
|
||||
rows[i].CreatedAt = ""
|
||||
rows[i].UpdatedAt = ""
|
||||
for j := range rows[i].History {
|
||||
rows[i].History[j].At = ""
|
||||
}
|
||||
}
|
||||
}
|
||||
return rows
|
||||
}
|
||||
func TestSupplierAuthorizationBatchPreservesLegacyAndLiveRevocationLocal(t *testing.T) {
|
||||
var reads, writes atomic.Int64
|
||||
s := newQuerySyncSource(t, &event.CommandMonitor{Started: func(_ context.Context, e *event.CommandStartedEvent) {
|
||||
switch e.CommandName {
|
||||
case "find", "aggregate", "getMore":
|
||||
reads.Add(1)
|
||||
case "update", "insert", "delete", "findAndModify":
|
||||
writes.Add(1)
|
||||
}
|
||||
}})
|
||||
ctx := t.Context()
|
||||
org := domain.DefaultOrganizationID
|
||||
insert := func(name string, doc bson.M) {
|
||||
t.Helper()
|
||||
if _, e := s.db.Collection(name).InsertOne(ctx, doc); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
}
|
||||
insert("users", bson.M{"id": "active", "supplierId": "v", "role": "annotator", "status": "active"})
|
||||
insert("users", bson.M{"id": "disabled", "organizationId": org, "supplierId": "v-disabled", "role": "annotator", "status": "disabled"})
|
||||
insert("suppliers", bson.M{"id": "v", "status": "active"})
|
||||
insert("suppliers", bson.M{"id": "v-disabled", "organizationId": org, "status": "active"})
|
||||
for i := 0; i < 30; i++ {
|
||||
p := fmt.Sprintf("p-%02d", i)
|
||||
insert("projects", bson.M{"id": p, "organizationId": org, "name": p})
|
||||
insert("projectMembers", bson.M{"id": "m-" + p, "projectId": p, "userId": "active", "supplierId": "v", "createdAt": "2026-01-01T00:00:00Z"})
|
||||
insert("projectMembers", bson.M{"id": "d-" + p, "organizationId": org, "projectId": p, "userId": "disabled", "supplierId": "v-disabled"})
|
||||
}
|
||||
insert("projects", bson.M{"id": " p-space ", "organizationId": org})
|
||||
insert("projectMembers", bson.M{"id": "space", "projectId": "p-space", "userId": "active", "supplierId": "v"})
|
||||
insert("projectMembers", bson.M{"id": "raw-space", "projectId": " p-space ", "userId": "active", "supplierId": "v"})
|
||||
insert("projects", bson.M{"id": " ", "organizationId": org})
|
||||
insert("projects", bson.M{"id": "foreign", "organizationId": "other"})
|
||||
insert("supplierProjectAuthorizationInit", bson.M{"id": supplierAuthorizationInitID(org, "p-00")})
|
||||
insert("supplierProjectAuthorizations", bson.M{"id": "revoked", "organizationId": org, "projectId": "p-00", "supplierId": "v", "status": "revoked", "drainTaskIds": bson.A{"task-old"}, "version": 4})
|
||||
for _, project := range []string{"", "p-00", "p-01", " p-space ", " ", "foreign", "missing"} {
|
||||
want, e := legacyAuthorizationListForTest(ctx, s, org, project)
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
reads.Store(0)
|
||||
writes.Store(0)
|
||||
got, e := s.ListSupplierProjectAuthorizations(ctx, org, project)
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if !reflect.DeepEqual(normalizedAuthorizationTestRows(want), normalizedAuthorizationTestRows(got)) {
|
||||
t.Fatalf("scope=%q mismatch want=%+v got=%+v", project, want, got)
|
||||
}
|
||||
if writes.Load() != 0 {
|
||||
t.Fatal("authorization GET wrote business state")
|
||||
}
|
||||
if project == "" && reads.Load() > 7 {
|
||||
t.Fatal("per-project source reads remain", reads.Load())
|
||||
}
|
||||
}
|
||||
if _, e := s.db.Collection("users").UpdateOne(ctx, bson.M{"id": "active"}, bson.M{"$set": bson.M{"status": "disabled"}}); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
rows, e := s.ListSupplierProjectAuthorizations(ctx, org, "")
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if len(rows) != 1 || rows[0].Status != "revoked" {
|
||||
t.Fatal("disabled account reused a prior grant", rows)
|
||||
}
|
||||
}
|
||||
@ -357,29 +357,102 @@ func (s *MongoStore) ListSupplierProjectAuthorizations(ctx context.Context, org,
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
// Resolve initialization and legacy seed inputs once per request. No grants
|
||||
// or permission decisions survive this call; revocations remain live reads.
|
||||
if len(projects) == 0 {
|
||||
return []domain.SupplierProjectAuthorization{}, nil
|
||||
}
|
||||
initIDs := make([]string, 0, len(projects))
|
||||
for _, p := range projects {
|
||||
initIDs = append(initIDs, supplierAuthorizationInitID(org, p.ID))
|
||||
}
|
||||
cur, err := s.supplierProjectAuthorizationInit.Find(ctx, bson.M{"id": bson.M{"$in": initIDs}}, options.Find().SetProjection(bson.M{"id": 1}))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
markers, err := decodeMany[struct {
|
||||
ID string `bson:"id"`
|
||||
}](ctx, cur)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
initialized := map[string]bool{}
|
||||
for _, marker := range markers {
|
||||
initialized[marker.ID] = true
|
||||
}
|
||||
readyIDs, legacyIDs := []string{}, []string{}
|
||||
allLegacyMembers := false
|
||||
for _, p := range projects {
|
||||
if initialized[supplierAuthorizationInitID(org, p.ID)] {
|
||||
readyIDs = append(readyIDs, p.ID)
|
||||
} else {
|
||||
id := strings.TrimSpace(p.ID)
|
||||
if id == "" {
|
||||
allLegacyMembers = true
|
||||
}
|
||||
legacyIDs = append(legacyIDs, id)
|
||||
}
|
||||
}
|
||||
grantsByProject := map[string][]domain.SupplierProjectAuthorization{}
|
||||
if len(readyIDs) > 0 {
|
||||
cur, err := s.supplierProjectAuthorizations.Find(ctx, bson.M{"organizationId": normalizedOrganizationID(org), "projectId": bson.M{"$in": readyIDs}})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
grants, err := decodeMany[domain.SupplierProjectAuthorization](ctx, cur)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, grant := range grants {
|
||||
grantsByProject[grant.ProjectID] = append(grantsByProject[grant.ProjectID], grant)
|
||||
}
|
||||
}
|
||||
var members []domain.ProjectMemberRecord
|
||||
var users []domain.UserRecord
|
||||
var suppliers []domain.SupplierRecord
|
||||
if len(legacyIDs) > 0 {
|
||||
filter := orgCompatibilityFilter(org)
|
||||
if !allLegacyMembers {
|
||||
filter["projectId"] = bson.M{"$in": legacyIDs}
|
||||
}
|
||||
cur, err := s.projectMembers.Find(ctx, filter, options.Find().SetSort(bson.D{{Key: "createdAt", Value: 1}}))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
members, err = decodeMany[domain.ProjectMemberRecord](ctx, cur)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
userIDs := []string{}
|
||||
seen := map[string]bool{}
|
||||
for _, member := range members {
|
||||
if !seen[member.UserID] {
|
||||
seen[member.UserID] = true
|
||||
userIDs = append(userIDs, member.UserID)
|
||||
}
|
||||
}
|
||||
users, err = s.ListProjectMemberUsers(ctx, org, userIDs)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
suppliers, err = s.ListSuppliers(ctx, org)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
result := []domain.SupplierProjectAuthorization{}
|
||||
for _, p := range projects {
|
||||
initialized, err := s.supplierProjectAuthorizationInitialized(ctx, org, p.ID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !initialized {
|
||||
seed, err := s.authorizationSeeds(ctx, org, p.ID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
if initialized[supplierAuthorizationInitID(org, p.ID)] {
|
||||
result = append(result, grantsByProject[p.ID]...)
|
||||
} else {
|
||||
// The legacy list trims its query ID, then the seed reducer compares
|
||||
// the original ID. Preserve its empty result for malformed spaced IDs.
|
||||
seedMembers := members
|
||||
if trimmed := strings.TrimSpace(p.ID); trimmed != "" && trimmed != p.ID {
|
||||
seedMembers = nil
|
||||
}
|
||||
result = append(result, seed...)
|
||||
continue
|
||||
result = append(result, effectiveSupplierAuthorizationSeeds(org, p.ID, seedMembers, users, suppliers)...)
|
||||
}
|
||||
cursor, err := s.supplierProjectAuthorizations.Find(ctx, bson.M{"organizationId": normalizedOrganizationID(org), "projectId": p.ID})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
grants, err := decodeMany[domain.SupplierProjectAuthorization](ctx, cursor)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result = append(result, grants...)
|
||||
}
|
||||
sort.Slice(result, func(i, j int) bool {
|
||||
if result[i].ProjectID != result[j].ProjectID {
|
||||
|
||||
@ -81,6 +81,10 @@ func (s *MongoStore) workforceHead(ctx context.Context, org string) (QueryModelH
|
||||
if h.Collection != workforceFactCollection || h.FactEpoch < 1 || h.Generation != workforceFactSchema || time.Since(h.AsOf) < 0 || (!unchanged && time.Since(h.AsOf) > maxAge) {
|
||||
return h, true, ErrQueryReplicaNotReady
|
||||
}
|
||||
h.verifiedUntil = h.AsOf.Add(maxAge)
|
||||
if unchanged {
|
||||
h.verifiedUntil = time.Now().Add(maxAge)
|
||||
}
|
||||
return h, true, nil
|
||||
}
|
||||
func workforceVersionFilter(org string, epoch int64) bson.M {
|
||||
@ -549,6 +553,14 @@ func (s *MongoStore) loadWorkforceFactReport(ctx context.Context, org string, q
|
||||
if q.MatchUser && q.UserID == "" {
|
||||
return workforcestats.Report{}, false, nil
|
||||
}
|
||||
if s.publishedWorkforceReads() && !q.Details {
|
||||
raw, _ := json.Marshal(q)
|
||||
if len(raw) <= 4096 {
|
||||
if report, used, err := s.loadPublishedWorkforceReport(ctx, org, q); used || err != nil {
|
||||
return report, used, err
|
||||
}
|
||||
}
|
||||
}
|
||||
h, exists, err := s.workforceHead(ctx, org)
|
||||
if s.periodicQueryStore() && !exists && err == nil {
|
||||
return workforcestats.Report{}, true, ErrQueryReplicaNotReady
|
||||
@ -575,6 +587,14 @@ func (s *MongoStore) loadWorkforceFactReport(ctx context.Context, org string, q
|
||||
}
|
||||
}
|
||||
}
|
||||
return s.computeWorkforceFactReport(ctx, org, q, h)
|
||||
}
|
||||
|
||||
func (s *MongoStore) computeWorkforceFactReport(ctx context.Context, org string, q workforcestats.Query, h QueryModelHead) (workforcestats.Report, bool, error) {
|
||||
key := queryserving.Key(workforceFactSchema, s.db.Name(), org, h.FactEpoch, q)
|
||||
if s.periodicQueryStore() {
|
||||
key = queryserving.Key("periodic-report/v1", key)
|
||||
}
|
||||
pipeline := append(workforceFactReadStages(org, h, q), bson.D{{Key: "$project", Value: bson.M{"kind": 1, "metric": 1, "return": 1, "eventCount": 1, "userId": 1, "userName": 1, "supplierId": 1, "supplierName": 1, "role": 1, "templateVersion": 1, "day": 1}}}, bson.D{{Key: "$facet", Value: bson.M{"items": workforceMetricPipeline(q, false), "totals": workforceMetricPipeline(q, true)}}})
|
||||
cur, err := s.db.Collection(workforceFactCollection).Aggregate(ctx, pipeline, options.Aggregate().SetMaxTime(8*time.Second))
|
||||
if err != nil {
|
||||
|
||||
354
internal/store/workforce_publication.go
Normal file
354
internal/store/workforce_publication.go
Normal file
@ -0,0 +1,354 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"droboticflow/internal/queryserving"
|
||||
"droboticflow/internal/workforcestats"
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
"go.mongodb.org/mongo-driver/mongo"
|
||||
"go.mongodb.org/mongo-driver/mongo/options"
|
||||
)
|
||||
|
||||
const workforcePublicationSchema = "workforce-publication/v3"
|
||||
|
||||
var ErrWorkforcePublicationPending = fmt.Errorf("workforce publication pending: %w", ErrQueryReplicaNotReady)
|
||||
|
||||
const workforcePublicationRegistry = "queryWorkforcePublicationRequests"
|
||||
const workforcePublicationData = "queryWorkforcePublications"
|
||||
const workforcePublicationLimit = 1024
|
||||
const workforcePublicationTouchInterval = 15 * time.Minute
|
||||
const workforcePublicationMaxBytes = 12 << 20
|
||||
|
||||
type workforcePublicationRequest struct {
|
||||
Query json.RawMessage `bson:"query"`
|
||||
RequestedAt time.Time `bson:"requestedAt"`
|
||||
}
|
||||
type workforcePublication struct {
|
||||
LegacyAggregation bool `bson:"legacyAggregation,omitempty"`
|
||||
Certificate queryReplicaReadState `bson:"certificate"`
|
||||
ExpiresAt time.Time `bson:"expiresAt"`
|
||||
Key string `bson:"_id"`
|
||||
Schema string `bson:"schema"`
|
||||
Organization string `bson:"organizationId"`
|
||||
Database string `bson:"database"`
|
||||
Epoch int64 `bson:"epoch"`
|
||||
ComputedAt time.Time `bson:"computedAt"`
|
||||
AsOf time.Time `bson:"asOf"`
|
||||
Body []byte `bson:"body"`
|
||||
Digest string `bson:"digest"`
|
||||
}
|
||||
type workforcePublicationTouches struct {
|
||||
sync.Mutex
|
||||
times map[string]time.Time
|
||||
}
|
||||
|
||||
type workforcePublicationBuildStamp struct {
|
||||
Epoch int64
|
||||
CheckedAt time.Time
|
||||
}
|
||||
type workforcePublicationBuildMemo struct {
|
||||
sync.Mutex
|
||||
values map[string]workforcePublicationBuildStamp
|
||||
}
|
||||
|
||||
func (s *MongoStore) rememberWorkforcePublicationBuild(key string, epoch int64) {
|
||||
s.workforcePublicationBuilds.Lock()
|
||||
defer s.workforcePublicationBuilds.Unlock()
|
||||
if len(s.workforcePublicationBuilds.values) >= 4096 {
|
||||
clear(s.workforcePublicationBuilds.values)
|
||||
}
|
||||
if s.workforcePublicationBuilds.values == nil {
|
||||
s.workforcePublicationBuilds.values = map[string]workforcePublicationBuildStamp{}
|
||||
}
|
||||
s.workforcePublicationBuilds.values[key] = workforcePublicationBuildStamp{Epoch: epoch, CheckedAt: time.Now()}
|
||||
}
|
||||
|
||||
// Startup-only. Published reads are independent of business operation gates.
|
||||
func (s *MongoStore) ConfigureQueryPublishedResults(enabled bool) { s.queryPublishedResults = enabled }
|
||||
func (s *MongoStore) publishedWorkforceReads() bool {
|
||||
return s.periodicQueryStore() && s.queryPublishedResults
|
||||
}
|
||||
func canonicalWorkforcePublicationQuery(q workforcestats.Query) workforcestats.Query {
|
||||
if q.RelativeFrom {
|
||||
q.From = time.Time{}
|
||||
}
|
||||
if q.RelativeTo {
|
||||
q.To = time.Time{}
|
||||
}
|
||||
return q
|
||||
}
|
||||
func (s *MongoStore) workforcePublicationKey(org string, q workforcestats.Query) string {
|
||||
q = canonicalWorkforcePublicationQuery(q)
|
||||
return queryserving.Key(workforcePublicationSchema, s.db.Name(), org, q)
|
||||
}
|
||||
func (s *MongoStore) validWorkforcePublication(v workforcePublication, key, org string) bool {
|
||||
age := time.Since(v.ComputedAt)
|
||||
certificateAge := time.Since(v.Certificate.ObservedAt)
|
||||
if v.Certificate.Binding.SourceDatabase == "" || !v.Certificate.Ready || v.Certificate.Schema != queryReplicaSchema || certificateAge < 0 || certificateAge > queryPeriodicDisplayMaxAge || v.Certificate.Binding.Namespace != queryReplicaNamespace(v.Certificate.Binding.SourceDatabase) || v.ExpiresAt.After(v.Certificate.ObservedAt.Add(queryPeriodicDisplayMaxAge)) {
|
||||
return false
|
||||
}
|
||||
if v.Key != key || v.Schema != workforcePublicationSchema || v.Organization != org || v.Database != s.db.Name() || v.Epoch < 1 || age < 0 || age > queryPeriodicDisplayMaxAge || v.ExpiresAt.IsZero() || !time.Now().Before(v.ExpiresAt) || v.ExpiresAt.After(v.ComputedAt.Add(queryPeriodicDisplayMaxAge)) || v.AsOf.IsZero() || v.AsOf.After(v.ComputedAt) || (len(v.Body) == 0 && !v.LegacyAggregation) || (v.LegacyAggregation && len(v.Body) != 0) || len(v.Body) > workforcePublicationMaxBytes {
|
||||
return false
|
||||
}
|
||||
sum := sha256.Sum256(v.Body)
|
||||
return v.Digest == hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
// Only the exact server-derived scope is registered, never an authorization
|
||||
// conclusion. One bounded org document makes the admission limit atomic.
|
||||
func (s *MongoStore) requestWorkforcePublication(ctx context.Context, org string, q workforcestats.Query) error {
|
||||
raw, err := json.Marshal(canonicalWorkforcePublicationQuery(q))
|
||||
if err != nil || len(raw) > 4096 {
|
||||
return errors.New("workforce publication query exceeds budget")
|
||||
}
|
||||
key := s.workforcePublicationKey(org, q)
|
||||
now := time.Now().UTC()
|
||||
s.workforcePublicationTouches.Lock()
|
||||
if now.Sub(s.workforcePublicationTouches.times[key]) < workforcePublicationTouchInterval {
|
||||
s.workforcePublicationTouches.Unlock()
|
||||
return nil
|
||||
}
|
||||
s.workforcePublicationTouches.Unlock()
|
||||
short, cancel := context.WithTimeout(ctx, 200*time.Millisecond)
|
||||
defer cancel()
|
||||
c := s.db.Collection(workforcePublicationRegistry)
|
||||
field := "requests." + key
|
||||
filter := bson.M{"_id": org, "$or": bson.A{bson.M{field: bson.M{"$exists": true}}, bson.M{"$expr": bson.M{"$lt": bson.A{bson.M{"$size": bson.M{"$objectToArray": bson.M{"$ifNull": bson.A{"$requests", bson.M{}}}}}, workforcePublicationLimit}}}}}
|
||||
update := bson.M{"$set": bson.M{field: workforcePublicationRequest{Query: raw, RequestedAt: now}, "updatedAt": now}}
|
||||
r, err := c.UpdateOne(short, filter, update)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if r.MatchedCount == 0 {
|
||||
_, err = c.UpdateOne(short, bson.M{"_id": org}, bson.M{"$setOnInsert": bson.M{"requests": bson.M{}, "updatedAt": now}}, options.Update().SetUpsert(true))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
r, err = c.UpdateOne(short, filter, update)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if r.MatchedCount != 1 {
|
||||
return errors.New("workforce publication registry full")
|
||||
}
|
||||
s.workforcePublicationTouches.Lock()
|
||||
if len(s.workforcePublicationTouches.times) >= workforcePublicationLimit {
|
||||
clear(s.workforcePublicationTouches.times)
|
||||
}
|
||||
if s.workforcePublicationTouches.times == nil {
|
||||
s.workforcePublicationTouches.times = map[string]time.Time{}
|
||||
}
|
||||
s.workforcePublicationTouches.times[key] = now
|
||||
s.workforcePublicationTouches.Unlock()
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *MongoStore) loadPublishedWorkforceReport(ctx context.Context, org string, q workforcestats.Query) (workforcestats.Report, bool, error) {
|
||||
key := s.workforcePublicationKey(org, q)
|
||||
// A failed catalog touch must not hide an already published complete result.
|
||||
defer func() { _ = s.requestWorkforcePublication(ctx, org, q) }()
|
||||
var v workforcePublication
|
||||
if queryserving.GetJSON(ctx, s.SharedQueryCache(), key, &v) && s.validWorkforcePublication(v, key, org) {
|
||||
if v.LegacyAggregation {
|
||||
return workforcestats.Report{}, false, nil
|
||||
}
|
||||
var report workforcestats.Report
|
||||
if json.Unmarshal(v.Body, &report) == nil {
|
||||
return report, true, nil
|
||||
}
|
||||
}
|
||||
err := s.db.Collection(workforcePublicationData).FindOne(ctx, bson.M{"_id": key}).Decode(&v)
|
||||
if err != nil && !errors.Is(err, mongo.ErrNoDocuments) {
|
||||
return workforcestats.Report{}, true, err
|
||||
}
|
||||
if err != nil || !s.validWorkforcePublication(v, key, org) {
|
||||
return workforcestats.Report{}, true, ErrWorkforcePublicationPending
|
||||
}
|
||||
if v.LegacyAggregation {
|
||||
return workforcestats.Report{}, false, nil
|
||||
}
|
||||
var report workforcestats.Report
|
||||
if err = json.Unmarshal(v.Body, &report); err != nil {
|
||||
return report, true, err
|
||||
}
|
||||
queryserving.PutJSON(ctx, s.SharedQueryCache(), key, v, queryPeriodicDisplayMaxAge)
|
||||
return report, true, nil
|
||||
}
|
||||
|
||||
// This runs only under the builder's fenced lease. HTTP never calls it.
|
||||
// Failed/new computations leave the previous complete document and cache intact.
|
||||
func (s *MongoStore) buildWorkforcePublications(ctx context.Context, l queryModelLease, org string) error {
|
||||
if !s.publishedWorkforceReads() {
|
||||
return nil
|
||||
}
|
||||
var registry struct {
|
||||
Requests map[string]workforcePublicationRequest `bson:"requests"`
|
||||
}
|
||||
err := s.db.Collection(workforcePublicationRegistry).FindOne(ctx, bson.M{"_id": org}).Decode(®istry)
|
||||
if errors.Is(err, mongo.ErrNoDocuments) {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var certificate queryReplicaReadState
|
||||
if err = s.db.Collection(queryReplicaControl).FindOne(ctx, bson.M{"_id": "state"}).Decode(&certificate); err != nil {
|
||||
return err
|
||||
}
|
||||
if !certificate.Ready || certificate.Schema != queryReplicaSchema || time.Since(certificate.ObservedAt) < 0 || time.Since(certificate.ObservedAt) > queryPeriodicDisplayMaxAge {
|
||||
return ErrQueryReplicaNotReady
|
||||
}
|
||||
head, exists, err := s.workforceHead(ctx, org)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !exists {
|
||||
return ErrQueryReplicaNotReady
|
||||
}
|
||||
keys := make([]string, 0, len(registry.Requests))
|
||||
for key := range registry.Requests {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
built := 0
|
||||
passStarted := time.Now()
|
||||
for _, key := range keys {
|
||||
request := registry.Requests[key]
|
||||
if time.Since(request.RequestedAt) > 24*time.Hour {
|
||||
_, err = s.db.Collection(workforcePublicationRegistry).UpdateOne(ctx, bson.M{"_id": org, "requests." + key + ".requestedAt": request.RequestedAt}, bson.M{"$unset": bson.M{"requests." + key: ""}})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
continue
|
||||
}
|
||||
var q workforcestats.Query
|
||||
if len(request.Query) > 4096 || json.Unmarshal(request.Query, &q) != nil || key != s.workforcePublicationKey(org, q) || q.Details || (q.MatchUser && q.UserID == "") {
|
||||
continue
|
||||
}
|
||||
s.workforcePublicationBuilds.Lock()
|
||||
stamp := s.workforcePublicationBuilds.values[key]
|
||||
s.workforcePublicationBuilds.Unlock()
|
||||
if stamp.Epoch == head.FactEpoch && time.Since(stamp.CheckedAt) < time.Minute {
|
||||
continue
|
||||
}
|
||||
var current workforcePublication
|
||||
err = s.db.Collection(workforcePublicationData).FindOne(ctx, bson.M{"_id": key}).Decode(¤t)
|
||||
if err != nil && !errors.Is(err, mongo.ErrNoDocuments) {
|
||||
return err
|
||||
}
|
||||
if err == nil && s.validWorkforcePublication(current, key, org) && current.Epoch == head.FactEpoch && time.Since(current.ComputedAt) < queryPeriodicBuildInterval {
|
||||
queryserving.PutJSON(ctx, s.SharedQueryCache(), key, current, queryPeriodicDisplayMaxAge)
|
||||
s.rememberWorkforcePublicationBuild(key, head.FactEpoch)
|
||||
continue
|
||||
}
|
||||
if built >= 16 || time.Since(passStarted) > 8*time.Second {
|
||||
continue
|
||||
}
|
||||
windowNow := time.Now().UTC()
|
||||
if q.RelativeFrom {
|
||||
q.From = windowNow.Add(-30 * 24 * time.Hour)
|
||||
}
|
||||
if q.RelativeTo {
|
||||
q.To = windowNow
|
||||
}
|
||||
if (q.RelativeFrom || q.RelativeTo) && !q.From.Before(q.To) {
|
||||
continue
|
||||
}
|
||||
report, used, e := s.computeWorkforceFactReport(ctx, org, q, head)
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
if !used {
|
||||
continue
|
||||
}
|
||||
if q.RelativeFrom || q.RelativeTo {
|
||||
report.WindowFrom = q.From.Format(time.RFC3339Nano)
|
||||
report.WindowTo = q.To.Format(time.RFC3339Nano)
|
||||
}
|
||||
body, e := json.Marshal(report)
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
legacy := len(body) > workforcePublicationMaxBytes
|
||||
if legacy {
|
||||
body = nil
|
||||
}
|
||||
sum := sha256.Sum256(body)
|
||||
next := workforcePublication{LegacyAggregation: legacy, Certificate: certificate, ExpiresAt: head.verifiedUntil, Key: key, Schema: workforcePublicationSchema, Organization: org, Database: s.db.Name(), Epoch: head.FactEpoch, ComputedAt: time.Now().UTC(), AsOf: head.AsOf, Body: body, Digest: hex.EncodeToString(sum[:])}
|
||||
if next.ExpiresAt.After(next.ComputedAt.Add(queryPeriodicDisplayMaxAge)) {
|
||||
next.ExpiresAt = next.ComputedAt.Add(queryPeriodicDisplayMaxAge)
|
||||
}
|
||||
if bound := certificate.ObservedAt.Add(queryPeriodicDisplayMaxAge); bound.Before(next.ExpiresAt) {
|
||||
next.ExpiresAt = bound
|
||||
}
|
||||
if !time.Now().Before(next.ExpiresAt) {
|
||||
return ErrQueryReplicaNotReady
|
||||
}
|
||||
err = s.workforceFactTransaction(ctx, l, func(sc mongo.SessionContext) error {
|
||||
var current queryReplicaReadState
|
||||
if e := s.db.Collection(queryReplicaControl).FindOne(sc, bson.M{"_id": "state"}).Decode(¤t); e != nil {
|
||||
return e
|
||||
}
|
||||
if current.Binding != certificate.Binding || current.Schema != certificate.Schema || !current.Ready {
|
||||
return ErrQueryReplicaNotReady
|
||||
}
|
||||
h, ok, e := s.workforceHead(sc, org)
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
if !ok || h.FactEpoch != head.FactEpoch {
|
||||
return ErrQueryReplicaNotReady
|
||||
}
|
||||
if h.verifiedUntil.Before(next.ExpiresAt) {
|
||||
next.ExpiresAt = h.verifiedUntil
|
||||
}
|
||||
_, e = s.db.Collection(workforcePublicationData).ReplaceOne(sc, bson.M{"_id": key}, next, options.Replace().SetUpsert(true))
|
||||
return e
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
queryserving.PutJSON(ctx, s.SharedQueryCache(), key, next, queryPeriodicDisplayMaxAge)
|
||||
s.rememberWorkforcePublicationBuild(key, head.FactEpoch)
|
||||
built++
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// API authorization already derived q from the current principal. A complete
|
||||
// cache product carries a bounded committed sync certificate, so a hit needs no
|
||||
// query-store metadata fetch. Initial physical binding verification is retained.
|
||||
func (s *MongoStore) cachedPublishedWorkforce(ctx context.Context, org string, q workforcestats.Query) (workforcestats.Report, bool, error) {
|
||||
r := s.queryReplicaReads
|
||||
if r == nil || r.target == nil || !r.target.publishedWorkforceReads() || q.Details || (q.MatchUser && q.UserID == "") {
|
||||
return workforcestats.Report{}, false, nil
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return workforcestats.Report{}, true, err
|
||||
}
|
||||
if err := s.verifyReplicaRead(ctx, r); err != nil {
|
||||
return workforcestats.Report{}, true, err
|
||||
}
|
||||
target := r.target
|
||||
key := target.workforcePublicationKey(org, q)
|
||||
var v workforcePublication
|
||||
if !queryserving.GetJSON(ctx, target.SharedQueryCache(), key, &v) || !target.validWorkforcePublication(v, key, org) || v.Certificate.Binding.SourceDatabase != s.db.Name() || v.Certificate.Binding.Namespace != queryReplicaNamespace(s.db.Name()) {
|
||||
return workforcestats.Report{}, false, nil
|
||||
}
|
||||
var report workforcestats.Report
|
||||
if json.Unmarshal(v.Body, &report) != nil {
|
||||
return workforcestats.Report{}, false, nil
|
||||
}
|
||||
_ = target.requestWorkforcePublication(ctx, org, q)
|
||||
return report, true, nil
|
||||
}
|
||||
209
internal/store/workforce_publication_local_test.go
Normal file
209
internal/store/workforce_publication_local_test.go
Normal file
@ -0,0 +1,209 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"os"
|
||||
"reflect"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"droboticflow/internal/domain"
|
||||
"droboticflow/internal/workforcestats"
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
"go.mongodb.org/mongo-driver/event"
|
||||
)
|
||||
|
||||
type publicationTestCache map[string][]byte
|
||||
|
||||
func (c publicationTestCache) Get(_ context.Context, k string) ([]byte, bool) {
|
||||
v, ok := c[k]
|
||||
return append([]byte(nil), v...), ok
|
||||
}
|
||||
func (c publicationTestCache) Put(_ context.Context, k string, v []byte, _ time.Duration) {
|
||||
c[k] = append([]byte(nil), v...)
|
||||
}
|
||||
|
||||
func TestWorkforcePublicationDurableFallbackAndIsolationLocal(t *testing.T) {
|
||||
if os.Getenv("QUERY_REPLICA_LOCAL_TEST") != "1" {
|
||||
t.Skip("owned synthetic replicas only")
|
||||
}
|
||||
var aggregations atomic.Int32
|
||||
var targetReads atomic.Int32
|
||||
monitor := &event.CommandMonitor{Started: func(_ context.Context, e *event.CommandStartedEvent) {
|
||||
if e.CommandName == "aggregate" || e.CommandName == "find" {
|
||||
targetReads.Add(1)
|
||||
}
|
||||
if e.CommandName == "aggregate" {
|
||||
aggregations.Add(1)
|
||||
}
|
||||
}}
|
||||
source, target, p := newLocalReplica(t, nil, monitor)
|
||||
seedWorkforceStats(t, source)
|
||||
finishLocalReplica(t, p)
|
||||
target.queryPeriodic = true
|
||||
target.ConfigureQueryPublishedResults(true)
|
||||
cache := publicationTestCache{}
|
||||
target.ConfigureQueryServing(cache, nil)
|
||||
if err := target.EnsureQueryReadModelIndexes(t.Context()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
lease, ok, err := target.acquireQueryModelLease(t.Context())
|
||||
if err != nil || !ok {
|
||||
t.Fatal(ok, err)
|
||||
}
|
||||
if err = target.rebuildWorkforceFacts(t.Context(), lease, "workforce-test", 0); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
query := workforcestats.Query{Mode: domain.RunModeAll}
|
||||
snapshot, err := source.LoadWorkforceStats(t.Context(), "workforce-test", query)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := workforcestats.Build(snapshot, query)
|
||||
want.Details = nil
|
||||
aggregations.Store(0)
|
||||
if _, _, err = target.loadWorkforceFactReport(t.Context(), "workforce-test", query); !errors.Is(err, ErrQueryReplicaNotReady) {
|
||||
t.Fatal("unpublished report calculated inline", err)
|
||||
}
|
||||
if aggregations.Load() != 0 {
|
||||
t.Fatal("HTTP miss aggregated history")
|
||||
}
|
||||
if err = target.buildWorkforcePublications(t.Context(), lease, "workforce-test"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
aggregations.Store(0)
|
||||
got, _, err := target.loadWorkforceFactReport(t.Context(), "workforce-test", query)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got.AsOf = ""
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("published result differs %#v %#v", got, want)
|
||||
}
|
||||
if err = source.ConfigureQueryReplicaReports(target, time.Minute); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err = source.LoadWorkforceReportCached(t.Context(), "workforce-test", query); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
targetReads.Store(0)
|
||||
if _, err = source.LoadWorkforceReportCached(t.Context(), "workforce-test", query); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if targetReads.Load() != 0 {
|
||||
t.Fatal("warm published cache required query Mongo")
|
||||
}
|
||||
canceled, cancel := context.WithCancel(t.Context())
|
||||
cancel()
|
||||
if _, err := source.LoadWorkforceReportCached(canceled, "workforce-test", query); !errors.Is(err, context.Canceled) {
|
||||
t.Fatal("warm cache ignored cancellation", err)
|
||||
}
|
||||
|
||||
// Loss of all process/shared cache state still reads one complete durable result.
|
||||
restarted := NewMongoStore(target.db)
|
||||
restarted.queryReplicaFactStore = true
|
||||
restarted.queryPeriodic = true
|
||||
restarted.queryPublishedResults = true
|
||||
got, _, err = restarted.loadWorkforceFactReport(t.Context(), "workforce-test", query)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got.AsOf = ""
|
||||
if !reflect.DeepEqual(got, want) || aggregations.Load() != 0 {
|
||||
t.Fatal("cache loss/restart reran aggregation")
|
||||
}
|
||||
// A restricted/current scope cannot reuse the broad result.
|
||||
denied := query
|
||||
denied.AllowedProjects = map[string]bool{}
|
||||
if _, _, err = restarted.loadWorkforceFactReport(t.Context(), "workforce-test", denied); !errors.Is(err, ErrQueryReplicaNotReady) {
|
||||
t.Fatal("broader publication reused", err)
|
||||
}
|
||||
if err = target.buildWorkforcePublications(t.Context(), lease, "workforce-test"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
restricted, _, err := restarted.loadWorkforceFactReport(t.Context(), "workforce-test", denied)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(restricted.Items) != 0 {
|
||||
t.Fatal("revoked project scope exposed rows")
|
||||
}
|
||||
// An unfenced worker cannot replace durable data, including on epoch changes.
|
||||
_, err = target.queryControl().UpdateOne(t.Context(), bson.M{"_id": "global"}, bson.M{"$inc": bson.M{"epoch": 1}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
target.workforcePublicationBuilds.values = nil
|
||||
_, err = target.db.Collection(workforcePublicationData).UpdateMany(t.Context(), bson.M{}, bson.M{"$set": bson.M{"computedAt": time.Now().Add(-4 * time.Minute)}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = target.buildWorkforcePublications(t.Context(), lease, "workforce-test"); err == nil {
|
||||
t.Fatal("lost lease published")
|
||||
}
|
||||
if aggregations.Load() == 0 {
|
||||
t.Fatal("test did not reach background computation")
|
||||
}
|
||||
// Cached envelopes are bound to query/database/org, digest and time.
|
||||
key := target.workforcePublicationKey("workforce-test", query)
|
||||
var envelope workforcePublication
|
||||
if json.Unmarshal(cache[key], &envelope) != nil {
|
||||
t.Fatal("missing cached envelope")
|
||||
}
|
||||
for _, mutate := range []func(*workforcePublication){func(v *workforcePublication) { v.Organization = "other" }, func(v *workforcePublication) { v.ExpiresAt = time.Now().Add(-time.Second) }, func(v *workforcePublication) { v.ExpiresAt = v.ComputedAt.Add(16 * time.Minute) }, func(v *workforcePublication) { v.Database = "other" }, func(v *workforcePublication) { v.ComputedAt = time.Now().Add(time.Hour) }, func(v *workforcePublication) { v.ComputedAt = time.Now().Add(-16 * time.Minute) }, func(v *workforcePublication) { v.Body = []byte("changed") }} {
|
||||
bad := envelope
|
||||
mutate(&bad)
|
||||
if target.validWorkforcePublication(bad, key, "workforce-test") {
|
||||
t.Fatal("invalid cache accepted")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestWorkforcePublicationRollingWindowIsPublishedOnceLocal(t *testing.T) {
|
||||
if os.Getenv("QUERY_REPLICA_LOCAL_TEST") != "1" {
|
||||
t.Skip("owned synthetic replicas only")
|
||||
}
|
||||
source, target, p := newLocalReplica(t)
|
||||
seedWorkforceStats(t, source)
|
||||
finishLocalReplica(t, p)
|
||||
target.queryPeriodic = true
|
||||
target.queryPublishedResults = true
|
||||
lease, ok, err := target.acquireQueryModelLease(t.Context())
|
||||
if err != nil || !ok {
|
||||
t.Fatal(ok, err)
|
||||
}
|
||||
if err = target.rebuildWorkforceFacts(t.Context(), lease, "workforce-test", 0); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
q := workforcestats.Query{From: now.Add(-30 * 24 * time.Hour), To: now, RelativeFrom: true, RelativeTo: true}
|
||||
if _, _, err = target.loadWorkforceFactReport(t.Context(), "workforce-test", q); !errors.Is(err, ErrWorkforcePublicationPending) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = target.buildWorkforcePublications(t.Context(), lease, "workforce-test"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
q.From = q.From.Add(time.Second)
|
||||
q.To = q.To.Add(time.Second)
|
||||
report, _, err := target.loadWorkforceFactReport(t.Context(), "workforce-test", q)
|
||||
if err != nil {
|
||||
t.Fatal("moving defaults missed the prepared response", err)
|
||||
}
|
||||
from, e := time.Parse(time.RFC3339Nano, report.WindowFrom)
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
to, e := time.Parse(time.RFC3339Nano, report.WindowTo)
|
||||
if e != nil || to.Sub(from) != 30*24*time.Hour {
|
||||
t.Fatal("published period is not exact", e)
|
||||
}
|
||||
explicit := q
|
||||
explicit.RelativeFrom = false
|
||||
explicit.RelativeTo = false
|
||||
if target.workforcePublicationKey("workforce-test", q) == target.workforcePublicationKey("workforce-test", explicit) {
|
||||
t.Fatal("explicit query reused a rolling default")
|
||||
}
|
||||
}
|
||||
28
internal/store/workforce_publication_test.go
Normal file
28
internal/store/workforce_publication_test.go
Normal file
@ -0,0 +1,28 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"droboticflow/internal/workforcestats"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestWorkforcePublicationRelativeWindowDoesNotRoundExplicitBounds(t *testing.T) {
|
||||
a := workforcestats.Query{From: time.Date(2026, 11, 1, 0, 0, 0, 1, time.UTC), To: time.Date(2026, 11, 2, 0, 0, 0, 2, time.UTC)}
|
||||
b := a
|
||||
b.From = b.From.Add(time.Nanosecond)
|
||||
// Key recipe is independent of a Mongo connection and preserves each explicit instant.
|
||||
if canonicalWorkforcePublicationQuery(a).From.Equal(canonicalWorkforcePublicationQuery(b).From) {
|
||||
t.Fatal("explicit nanoseconds rounded")
|
||||
}
|
||||
a.RelativeFrom = true
|
||||
a.RelativeTo = true
|
||||
b.RelativeFrom = true
|
||||
b.RelativeTo = true
|
||||
ca, cb := canonicalWorkforcePublicationQuery(a), canonicalWorkforcePublicationQuery(b)
|
||||
if ca.From != cb.From || ca.To != cb.To || !ca.From.IsZero() || !ca.To.IsZero() {
|
||||
t.Fatal("rolling defaults do not have a stable recipe")
|
||||
}
|
||||
if a.From.IsZero() || b.To.IsZero() {
|
||||
t.Fatal("canonicalization mutated a caller query")
|
||||
}
|
||||
}
|
||||
@ -20,6 +20,10 @@ var workforceReportCache = &ttlResultCache[[]byte]{now: time.Now}
|
||||
// contract. Only summary callers use this path; detail readers stay uncached.
|
||||
// Cache keys contain every query scope, never an authorization decision.
|
||||
func (s *MongoStore) LoadWorkforceReportCached(ctx context.Context, org string, q workforcestats.Query) (workforcestats.Report, error) {
|
||||
if report, used, err := s.cachedPublishedWorkforce(ctx, org, q); used || err != nil {
|
||||
return report, err
|
||||
}
|
||||
|
||||
if err := ctx.Err(); err != nil {
|
||||
return workforcestats.Report{}, err
|
||||
}
|
||||
@ -106,6 +110,10 @@ func (s *MongoStore) LoadWorkforceReportCached(ctx context.Context, org string,
|
||||
// LoadWorkforceReport preserves uncached legacy summary behavior when read
|
||||
// replicas are disabled; the enabled replacement reads only published facts.
|
||||
func (s *MongoStore) LoadWorkforceReport(ctx context.Context, org string, q workforcestats.Query) (workforcestats.Report, error) {
|
||||
if report, used, err := s.cachedPublishedWorkforce(ctx, org, q); used || err != nil {
|
||||
return report, err
|
||||
}
|
||||
|
||||
target, err := s.reportReplica(ctx)
|
||||
if err != nil {
|
||||
return workforcestats.Report{}, err
|
||||
|
||||
@ -8,6 +8,8 @@ import (
|
||||
)
|
||||
|
||||
type Query struct {
|
||||
RelativeFrom bool // API supplied no lower bound: 30 days before computation.
|
||||
RelativeTo bool // API supplied no upper bound: computation time.
|
||||
AllowedProjects map[string]bool
|
||||
AllowedSuppliers map[string]bool
|
||||
MatchUser bool
|
||||
@ -41,6 +43,8 @@ type Snapshot struct {
|
||||
}
|
||||
|
||||
type Report struct {
|
||||
WindowFrom string `json:"windowFrom,omitempty"`
|
||||
WindowTo string `json:"windowTo,omitempty"`
|
||||
AsOf string `json:"asOf,omitempty"`
|
||||
Items []domain.DashboardWorkforceMetric
|
||||
Totals domain.DashboardWorkforceMetric
|
||||
|
||||
@ -105,7 +105,7 @@ class ProfileTests(unittest.TestCase):
|
||||
self.assertIn('truncate -s 26843545600', config['data']['init.sh'])
|
||||
self.assertIn('80530636800', config['data']['query25.sh'])
|
||||
self.assertIn('memory.max', config['data']['guard.sh'])
|
||||
pod['spec']['containers'][0]['resources']['limits']['memory'] = '8Gi'
|
||||
pod['spec']['containers'][0]['resources']['limits']['memory'] = '16Gi'
|
||||
with self.assertRaises(AssertionError):
|
||||
profiles.verify_resources(pod, name)
|
||||
profiles.verify_scripts(config['data'], name)
|
||||
|
||||
@ -43,8 +43,10 @@ def validate_saved(saved, pilot):
|
||||
assert saved['quota']['spec']['hard'] == profiles.quota(name)
|
||||
expected = pilot.manifests(True, name)[-2]['data']
|
||||
actual = saved['config']['data']
|
||||
prior_verifier = dict(expected)
|
||||
prior_verifier['verify25.sh'] = expected['verify25.sh'].replace('Number(db.serverStatus().wiredTiger.cache["maximum bytes configured"])', 'db.serverStatus().wiredTiger.cache["maximum bytes configured"]')
|
||||
legacy = name == 'baseline' and hashlib.sha256(json.dumps(actual, sort_keys=True).encode()).hexdigest() == 'e5eebd9dacae29e817d90909fa97dd31e648ef8eaf4aa81f6bcb31ac58ee8eed'
|
||||
assert actual == expected or legacy, 'rollback scripts are not an approved profile'
|
||||
assert actual == expected or actual == prior_verifier or legacy, 'rollback scripts are not an approved profile'
|
||||
|
||||
|
||||
def main():
|
||||
|
||||
@ -23,7 +23,7 @@ ROOT = Path(__file__).resolve().parents[2]
|
||||
HARD = {"pods": "2", "requests.cpu": "2", "limits.cpu": "5", "requests.memory": "3Gi", "limits.memory": "5Gi", "services": "0", "persistentvolumeclaims": "0"}
|
||||
|
||||
|
||||
def manifest():
|
||||
def manifest(source_node=NODE, cache=None):
|
||||
env = [{"name": name, "valueFrom": {"fieldRef": {"fieldPath": field}}} for name, field in (("POD_UID", "metadata.uid"), ("POD_NAME", "metadata.name"), ("NODE_NAME", "spec.nodeName"))]
|
||||
env += [{"name": "HOME", "value": "/tmp"}, {"name": "MONGOSH_DISABLE_TELEMETRY", "value": "1"}, {"name": "GOMAXPROCS", "value": "2"}, {"name": "GOMEMLIMIT", "value": "1536MiB"}]
|
||||
command = """set -euo pipefail
|
||||
@ -36,10 +36,15 @@ trap 'exit 0' TERM INT
|
||||
while kill -0 "$pid" 2>/dev/null; do sleep 10 & wait $!; done
|
||||
wait "$pid"
|
||||
"""
|
||||
pod = {"apiVersion": "v1", "kind": "Pod", "metadata": {"name": "node4-query-fixture", "namespace": NS, "labels": {"app": "node4-query-fixture", "purpose": "mongo-cpu-synthetic"}}, "spec": {"nodeName": NODE, "automountServiceAccountToken": False, "enableServiceLinks": False, "restartPolicy": "Never", "activeDeadlineSeconds": 7200, "terminationGracePeriodSeconds": 45, "securityContext": {"runAsUser": 999, "runAsGroup": 999, "runAsNonRoot": True, "fsGroup": 999}, "containers": [{"name": "lab", "image": IMAGE, "imagePullPolicy": "Never", "command": ["bash", "-c", command], "env": env, "resources": {"requests": {"cpu": "1", "memory": "2Gi", "ephemeral-storage": "32Mi"}, "limits": {"cpu": "4", "memory": "4Gi", "ephemeral-storage": "128Mi"}}, "securityContext": {"allowPrivilegeEscalation": False, "readOnlyRootFilesystem": True, "capabilities": {"drop": ["ALL"]}, "seccompProfile": {"type": "RuntimeDefault"}}, "readinessProbe": {"exec": {"command": ["mongosh", "--quiet", "--host", "127.0.0.1", "--port", "27119", "--eval", "quit(db.adminCommand({ping:1}).ok?0:1)"]}, "periodSeconds": 15, "timeoutSeconds": 5}, "volumeMounts": [{"name": "synthetic", "mountPath": "/synthetic"}, {"name": "tmp", "mountPath": "/tmp"}]}], "volumes": [{"name": "synthetic", "emptyDir": {"medium": "Memory", "sizeLimit": "2Gi"}}, {"name": "tmp", "emptyDir": {"medium": "Memory", "sizeLimit": "64Mi"}}]}}
|
||||
pod = {"apiVersion": "v1", "kind": "Pod", "metadata": {"name": "node4-query-fixture", "namespace": NS, "labels": {"app": "node4-query-fixture", "purpose": "mongo-cpu-synthetic"}}, "spec": {"nodeName": source_node, "automountServiceAccountToken": False, "enableServiceLinks": False, "restartPolicy": "Never", "activeDeadlineSeconds": 7200, "terminationGracePeriodSeconds": 45, "securityContext": {"runAsUser": 999, "runAsGroup": 999, "runAsNonRoot": True, "fsGroup": 999}, "containers": [{"name": "lab", "image": IMAGE, "imagePullPolicy": "Never", "command": ["bash", "-c", command], "env": env, "resources": {"requests": {"cpu": "1", "memory": "2Gi", "ephemeral-storage": "32Mi"}, "limits": {"cpu": "4", "memory": "4Gi", "ephemeral-storage": "128Mi"}}, "securityContext": {"allowPrivilegeEscalation": False, "readOnlyRootFilesystem": True, "capabilities": {"drop": ["ALL"]}, "seccompProfile": {"type": "RuntimeDefault"}}, "readinessProbe": {"exec": {"command": ["mongosh", "--quiet", "--host", "127.0.0.1", "--port", "27119", "--eval", "quit(db.adminCommand({ping:1}).ok?0:1)"]}, "periodSeconds": 15, "timeoutSeconds": 5}, "volumeMounts": [{"name": "synthetic", "mountPath": "/synthetic"}, {"name": "tmp", "mountPath": "/tmp"}]}], "volumes": [{"name": "synthetic", "emptyDir": {"medium": "Memory", "sizeLimit": "2Gi"}}, {"name": "tmp", "emptyDir": {"medium": "Memory", "sizeLimit": "64Mi"}}]}}
|
||||
ingress = {"apiVersion": "networking.k8s.io/v1", "kind": "NetworkPolicy", "metadata": {"name": "query-fixture-ingress", "namespace": NS}, "spec": {"podSelector": {"matchLabels": {"app": "node4-query25"}}, "policyTypes": ["Ingress"], "ingress": [{"from": [{"podSelector": {"matchLabels": {"app": "node4-query-fixture"}}}], "ports": [{"protocol": "TCP", "port": 27130}]}]}}
|
||||
egress = {"apiVersion": "networking.k8s.io/v1", "kind": "NetworkPolicy", "metadata": {"name": "query-fixture-egress", "namespace": NS}, "spec": {"podSelector": {"matchLabels": {"app": "node4-query-fixture"}}, "policyTypes": ["Egress"], "egress": [{"to": [{"podSelector": {"matchLabels": {"app": "node4-query25"}}}], "ports": [{"protocol": "TCP", "port": 27130}]}]}}
|
||||
return [ingress, egress, pod]
|
||||
if source_node != NODE:
|
||||
pod["spec"]["containers"][0]["imagePullPolicy"] = "IfNotPresent"
|
||||
policies = [ingress, egress]
|
||||
if cache:
|
||||
policies.append({"apiVersion":"networking.k8s.io/v1","kind":"NetworkPolicy","metadata":{"name":"query-fixture-cache-egress","namespace":NS},"spec":{"podSelector":{"matchLabels":{"app":"node4-query-fixture"}},"policyTypes":["Egress"],"egress":[{"to":[{"namespaceSelector":{"matchLabels":{"kubernetes.io/metadata.name":cache["namespace"]}},"podSelector":{"matchLabels":{"app":cache["pod"]}}}],"ports":[{"protocol":"TCP","port":6379}]}]}})
|
||||
return policies + [pod]
|
||||
|
||||
|
||||
def verify_query_pod(query, expected_uid, query_profile="baseline"):
|
||||
@ -77,6 +82,8 @@ def main():
|
||||
ap.add_argument("--confirm-plan")
|
||||
ap.add_argument("--query-pod-uid", default=QUERY_UID, help="exact inspected UID after an authorized synthetic Pod rebuild")
|
||||
ap.add_argument("--query-profile", choices=query_lab_profiles.PROFILES, default="baseline")
|
||||
ap.add_argument("--source-node", choices=(NODE,"10.189.16.33"), default=NODE)
|
||||
ap.add_argument("--valkey-owned")
|
||||
args = ap.parse_args()
|
||||
global HARD
|
||||
HARD = query_lab_profiles.quota(args.query_profile, fixture=True)
|
||||
@ -94,6 +101,21 @@ def main():
|
||||
def get(*argv):
|
||||
return json.loads(call(*argv, "-o", "json"))
|
||||
|
||||
cache = None
|
||||
if args.valkey_owned:
|
||||
cache = json.loads(Path(args.valkey_owned).read_text())
|
||||
assert cache["namespace"] == "droboticflow-query-cache-lab" and cache["pod"] == "query-cache-synthetic"
|
||||
assert get("get","namespace",cache["namespace"])["metadata"]["uid"] == cache["namespaceUID"]
|
||||
cp = get("-n",cache["namespace"],"get","pod",cache["pod"])
|
||||
assert cp["metadata"]["uid"] == cache["podUID"] and cp["spec"]["nodeName"] == "10.189.16.33" and cp["status"]["phase"] == "Running"
|
||||
assert cp["spec"]["containers"][0]["image"] == cache["image"]
|
||||
assert cache["url"] == "redis://"+cp["status"]["podIP"]+":6379/0"
|
||||
if args.source_node != NODE:
|
||||
other = get("get","node",args.source_node)
|
||||
assert other["metadata"]["uid"] == "77747f1c-42d8-40e8-a8d0-91f0e56ba3bd"
|
||||
assert all(c["status"]=="False" for c in other["status"]["conditions"] if c["type"] in ("DiskPressure","MemoryPressure","PIDPressure"))
|
||||
usage=call("top","node",args.source_node,"--no-headers").split()
|
||||
assert float(usage[2].rstrip("%"))<25 and float(usage[4].rstrip("%"))<60
|
||||
node = get("get", "node", NODE)
|
||||
assert node["metadata"]["uid"] == NODE_UID
|
||||
assert all(c["status"] == "False" for c in node["status"]["conditions"] if c["type"] in ("DiskPressure", "MemoryPressure", "PIDPressure"))
|
||||
@ -116,10 +138,10 @@ def main():
|
||||
assert recorded["planSHA256"] == previous_digest
|
||||
assert quota["spec"]["hard"] == HARD
|
||||
existing = get("-n", NS, "get", "pod", "node4-query-fixture")
|
||||
assert existing["metadata"]["uid"] == recorded["fixtureUID"] and existing["spec"]["nodeName"] == NODE
|
||||
assert existing["metadata"]["uid"] == recorded["fixtureUID"] and existing["spec"]["nodeName"] == args.source_node
|
||||
assert existing["spec"]["containers"][0]["resources"]["limits"]["cpu"] == "4"
|
||||
assert existing["spec"]["containers"][0]["resources"]["limits"]["memory"] == "4Gi"
|
||||
for obj in manifest()[:2]:
|
||||
for obj in manifest(args.source_node, cache)[:-1]:
|
||||
assert get("-n", NS, "get", "networkpolicy", obj["metadata"]["name"])["spec"] == obj["spec"]
|
||||
else:
|
||||
assert quota["spec"]["hard"] == original_quota
|
||||
@ -138,8 +160,8 @@ def main():
|
||||
assert hashes == previous["binarySHA256"], "resume must use the recorded binaries"
|
||||
else:
|
||||
call("-n", NS, "exec", "node4-query-fixture", "-c", "lab", "--", "sh", "-c", 'for p in /proc/[0-9]*/exe; do x=$(readlink "$p" 2>/dev/null || true); case "$x" in /synthetic/query-http*.test) exit 1;; esac; done')
|
||||
objects = manifest()
|
||||
plan = {"kind": "synthetic-in-cluster-comparison", "node": NODE, "namespaceUID": NS_UID, "queryUID": query_uid, "queryIP": ip, "quotaVersion": quota["metadata"]["resourceVersion"], "originalQuota": original_quota, "temporaryQuota": HARD, "objects": objects, "binarySHA256": hashes, "customerData": "none", "newDisk": "none; source is bounded tmpfs", "productionChanges": "none", "queryBridgeStorage": "bounded /tmp memory volume; database noexec retained", "stop": "three degraded readiness/IO/CPU samples; delete only owned fixture", "rollback": "stop owned bridges, delete fixture and its two policies, CAS restore original quota; keep 25Gi data"}
|
||||
objects = manifest(args.source_node, cache)
|
||||
plan = {"kind": "synthetic-in-cluster-comparison", "sourceNode": args.source_node, "valkey": cache, "node": NODE, "namespaceUID": NS_UID, "queryUID": query_uid, "queryIP": ip, "quotaVersion": quota["metadata"]["resourceVersion"], "originalQuota": original_quota, "temporaryQuota": HARD, "objects": objects, "binarySHA256": hashes, "customerData": "none", "newDisk": "none; source is bounded tmpfs", "productionChanges": "none", "queryBridgeStorage": "bounded /tmp memory volume; database noexec retained", "stop": "three degraded readiness/IO/CPU samples; delete only owned fixture", "rollback": "stop owned bridges, delete fixture and its two policies, CAS restore original quota; keep 25Gi data"}
|
||||
if recorded:
|
||||
plan["resumeFrom"] = recorded
|
||||
raw = json.dumps(plan, sort_keys=True).encode()
|
||||
@ -163,7 +185,7 @@ def main():
|
||||
(out / "created.json").write_text(json.dumps({"fixtureUID": created["metadata"]["uid"], "planSHA256": digest}))
|
||||
call("-n", NS, "wait", "--for=condition=Ready", "pod/node4-query-fixture", "--timeout=180s", timeout=190)
|
||||
fixture = get("-n", NS, "get", "pod", "node4-query-fixture")
|
||||
assert fixture["spec"]["nodeName"] == NODE
|
||||
assert fixture["spec"]["nodeName"] == args.source_node
|
||||
fuid = fixture["metadata"]["uid"]
|
||||
assert str(uuid.UUID(fuid)) == fuid
|
||||
assert get("-n", NS, "get", "pod", "node4-query25")["metadata"]["uid"] == query_uid
|
||||
@ -192,7 +214,7 @@ def main():
|
||||
fcommand = "set -eu; if test -s /synthetic/bridge.pid; then p=$(cat /synthetic/bridge.pid); if kill -0 \"$p\" 2>/dev/null; then test \"$(readlink /proc/$p/exe)\" = /synthetic/bridge; exit 0; fi; fi; nohup /synthetic/bridge -listen 127.0.0.1:27229 -upstream " + ip + ":27130 -pod-uid " + fuid + " >/synthetic/bridge.log 2>&1 </dev/null & echo $! >/synthetic/bridge.pid"
|
||||
call("-n", NS, "exec", "node4-query-fixture", "-c", "lab", "--", "sh", "-c", fcommand)
|
||||
call("-n", NS, "exec", "node4-query-fixture", "-c", "lab", "--", "mongosh", "--quiet", "mongodb://127.0.0.1:27229/?directConnection=true&replicaSet=query-lab-rs", "--eval", 'const m=db.getSiblingDB("query25_synthetic").marker.findOne({_id:"query25-confirmed"});quit(m&&m.value==="preserve-query25-v1"?0:2)')
|
||||
owned = {"fixtureUID": fuid, "queryUID": query_uid, "queryBridgeDirectory": qdir, "apiBinaryPath": api_path, "policies": {obj["metadata"]["name"]: get("-n", NS, "get", "networkpolicy", obj["metadata"]["name"])["metadata"]["uid"] for obj in objects[:2]}, "binarySHA256": hashes, "planSHA256": digest}
|
||||
owned = {"fixtureUID": fuid, "queryUID": query_uid, "sourceNode": args.source_node, "valkey": cache, "queryBridgeDirectory": qdir, "apiBinaryPath": api_path, "policies": {obj["metadata"]["name"]: get("-n", NS, "get", "networkpolicy", obj["metadata"]["name"])["metadata"]["uid"] for obj in objects[:-1]}, "binarySHA256": hashes, "planSHA256": digest}
|
||||
(out / "owned.json").write_text(json.dumps(owned, indent=2))
|
||||
print("ISOLATED_FIXTURE_READY", flush=True)
|
||||
|
||||
|
||||
@ -136,6 +136,7 @@ def main():
|
||||
ap.add_argument("--periodic-refresh", action="store_true", help="explicit periodic display policy; same load and safety limits")
|
||||
ap.add_argument("--cold-pace-kib", type=int, choices=(0, 128), default=0)
|
||||
ap.add_argument("--query-profile", choices=query_lab_profiles.PROFILES, default="baseline")
|
||||
ap.add_argument("--published-results", action="store_true")
|
||||
args = ap.parse_args()
|
||||
assert args.cold_write_probe or args.cold_pace_kib == 0
|
||||
assert not args.cold_write_probe or args.fixture_plan, "cold diagnostic requires the owned in-cluster fixture"
|
||||
@ -160,12 +161,17 @@ def main():
|
||||
run(k + ["-n", "droboticflow-query-lab", "exec", "node4-query25", "-c", "probe", "--", "bash", "/scripts/verify25.sh"])
|
||||
fixture = None
|
||||
owned = None
|
||||
source_node = NODE
|
||||
cache_owned = None
|
||||
if args.fixture_plan:
|
||||
owned = json.loads(Path(args.fixture_plan).read_text())
|
||||
source_node = owned.get("sourceNode",NODE)
|
||||
assert source_node in (NODE,"10.189.16.33")
|
||||
cache_owned = owned.get("valkey")
|
||||
fixture = get("-n", "droboticflow-query-lab", "get", "pod", "node4-query-fixture")
|
||||
assert fixture["metadata"]["uid"] == owned["fixtureUID"]
|
||||
assert query["metadata"]["uid"] == owned["queryUID"]
|
||||
assert fixture["status"]["phase"] == "Running" and fixture["spec"]["nodeName"] == NODE
|
||||
assert fixture["status"]["phase"] == "Running" and fixture["spec"]["nodeName"] == source_node
|
||||
budget = fixture["spec"]["containers"][0]["resources"]["limits"]
|
||||
assert budget["cpu"] == "4" and budget["memory"] == "4Gi"
|
||||
api_binary = owned.get("apiBinaryPath", "/synthetic/query-http.test")
|
||||
@ -196,7 +202,7 @@ def main():
|
||||
script += "echo PROTECTED " + p["metadata"]["uid"] + "; cat " + cgroup(p) + "/io.pressure; cat " + cgroup(p) + "/io.stat; "
|
||||
for filename in ("cpu.stat", "memory.current", "memory.events", "io.stat", "io.max"):
|
||||
script += "echo QUERY " + filename + "; cat " + cgroup(query) + "/" + filename + "; "
|
||||
if fixture:
|
||||
if fixture and source_node == NODE:
|
||||
for filename in ("cpu.stat", "memory.current", "memory.events", "io.stat"):
|
||||
script += "echo SOURCE " + filename + "; cat " + cgroup(fixture) + "/" + filename + "; "
|
||||
previous = None
|
||||
@ -213,7 +219,18 @@ def main():
|
||||
health = get("get", "node", NODE)
|
||||
assert all(c["status"] == "False" for c in health["status"]["conditions"] if c["type"] in ("DiskPressure", "MemoryPressure", "PIDPressure"))
|
||||
host = run(k + ["-n", "kube-system", "exec", "node-problem-detector-6v59v", "-c", "node-problem-detector", "--", "sh", "-c", script])
|
||||
local = "included in host SOURCE fields" if fixture else run(["docker", "exec", CONTAINER, "sh", "-c", "cat /sys/fs/cgroup/cpu.stat; cat /sys/fs/cgroup/memory.current; cat /sys/fs/cgroup/io.stat"])
|
||||
if fixture and source_node != NODE:
|
||||
local = run(k+["-n","droboticflow-query-lab","exec","node4-query-fixture","-c","lab","--","sh","-c","cat /sys/fs/cgroup/cpu.stat; cat /sys/fs/cgroup/memory.current; cat /sys/fs/cgroup/memory.events; cat /sys/fs/cgroup/io.stat"])
|
||||
health_other=get("get","node",source_node)
|
||||
assert health_other["metadata"]["uid"]=="77747f1c-42d8-40e8-a8d0-91f0e56ba3bd"
|
||||
assert all(c["status"]=="False" for c in health_other["status"]["conditions"] if c["type"] in ("DiskPressure","MemoryPressure","PIDPressure"))
|
||||
other_top=run(k+["top","node",source_node,"--no-headers"]).split()
|
||||
assert float(other_top[2].rstrip("%"))<=60 and float(other_top[4].rstrip("%"))<=70, "source/cache node overloaded"
|
||||
else:
|
||||
local = "included in host SOURCE fields" if fixture else run(["docker", "exec", CONTAINER, "sh", "-c", "cat /sys/fs/cgroup/cpu.stat; cat /sys/fs/cgroup/memory.current; cat /sys/fs/cgroup/io.stat"])
|
||||
if cache_owned:
|
||||
cp=get("-n",cache_owned["namespace"],"get","pod",cache_owned["pod"])
|
||||
assert cp["metadata"]["uid"]==cache_owned["podUID"] and cp["status"]["phase"]=="Running", "owned cache replaced/stopped"
|
||||
top = run(k + ["top", "node", NODE, "--no-headers"]).split()
|
||||
cpu_pct = float(top[2].rstrip("%"))
|
||||
if fixture:
|
||||
@ -251,6 +268,9 @@ def main():
|
||||
baseline = collect_baseline(sample)
|
||||
limits = {"ready": max(1, max(s["readySeconds"] for s in baseline) * 2), "pressure": max(0.5, max(s["protectedFullAvg10"] for s in baseline) + 0.5), "await": max(10, max(s["awaitMs"] for s in baseline) * 3)}
|
||||
manifest = {"sourceCommit": run(["git", "-C", str(ROOT), "rev-parse", "HEAD"]), "sourceFingerprint": source_fingerprint(), "sourceContainer": source["Id"], "sourceImage": source["Image"], "sourceCPU": 4, "sourceMemory": 4_294_967_296, "queryPodUID": query["metadata"]["uid"], "queryImage": query["spec"]["containers"][0]["image"], "queryCPU": int(query_lab_profiles.profile(args.query_profile)["cpu"]), "queryMemory": query_lab_profiles.resources(args.query_profile)["memory"], "queryProfile": query_lab_profiles.profile(args.query_profile), "queryScriptSHA256": hashlib.sha256(json.dumps(query_scripts, sort_keys=True).encode()).hexdigest(), "tasks": args.tasks, "waves": args.waves, "maxActors": args.max_actors, "wireCompressionBothVariants": os.getenv("QUERY_COMPARE_WIRE_COMPRESSION", ""), "minimumPoolBothVariants": os.getenv("QUERY_COMPARE_MIN_POOL", "0"), "limits": limits, "limitations": ["source Docker VM shares CPU with existing local containers", "target loop and backing device use the recorded finite queryProfile IO caps", "short comparison; not 30-minute three-round production capacity acceptance", "health probes do not establish customer workflow latency"]}
|
||||
manifest["sourceNode"] = source_node
|
||||
manifest["publishedResults"] = args.published_results
|
||||
manifest["valkeyOwned"] = cache_owned
|
||||
manifest["baselineSamples"] = len(baseline)
|
||||
manifest["refreshMode"] = "periodic" if args.periodic_refresh else "continuous"
|
||||
manifest["baselineSeconds"] = baseline[-1]["time"] - baseline[0]["time"]
|
||||
@ -269,6 +289,9 @@ def main():
|
||||
if args.cold_write_probe:
|
||||
settings += ["QUERY_REPLICA_COLD_WRITE_PROBE=1", "QUERY_COLD_PACE_KIB=" + str(args.cold_pace_kib)]
|
||||
settings += ["QUERY_REPLICA_REFRESH_MODE=" + manifest["refreshMode"]]
|
||||
if args.published_results:
|
||||
assert cache_owned, "published cluster comparison requires its owned dedicated cache"
|
||||
settings += ["QUERY_PUBLISHED_RESULTS=1","QUERY_COMPARE_VALKEY_URL="+cache_owned["url"]]
|
||||
argv = k + ["-n", "droboticflow-query-lab", "exec", "node4-query-fixture", "-c", "lab", "--", "env"] + settings + [api_binary, "-test.run=^" + test_name + "$", "-test.count=1", "-test.timeout=45m", "-test.v"]
|
||||
with (out / "test.log").open("w") as log:
|
||||
child = subprocess.Popen(argv, cwd=ROOT, env=env, stdout=log, stderr=subprocess.STDOUT, start_new_session=True)
|
||||
|
||||
60
scripts/local/query-valkey-lab.py
Normal file
60
scripts/local/query-valkey-lab.py
Normal file
@ -0,0 +1,60 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Bounded derived-cache-only Valkey on node33; no Service, PVC or customer data."""
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import urllib.request
|
||||
|
||||
NODE='10.189.16.33'
|
||||
NODE_UID='77747f1c-42d8-40e8-a8d0-91f0e56ba3bd'
|
||||
NS='droboticflow-query-cache-lab'
|
||||
POD='query-cache-synthetic'
|
||||
IMAGE='docker.io/valkey/valkey@sha256:e51a82741b780e4bf315db10753edf07eea6496d405fa7df2a8677a18f5e7464'
|
||||
SOURCE_NS='droboticflow-query-lab'
|
||||
|
||||
|
||||
def manifests():
|
||||
return [
|
||||
{'apiVersion':'v1','kind':'Namespace','metadata':{'name':NS,'labels':{'purpose':'mongo-cpu-synthetic-cache'}}},
|
||||
{'apiVersion':'v1','kind':'ResourceQuota','metadata':{'name':'cache-budget','namespace':NS},'spec':{'hard':{'pods':'1','requests.cpu':'1','limits.cpu':'2','requests.memory':'2Gi','limits.memory':'4Gi','services':'0','persistentvolumeclaims':'0'}}},
|
||||
{'apiVersion':'networking.k8s.io/v1','kind':'NetworkPolicy','metadata':{'name':'cache-isolation','namespace':NS},'spec':{'podSelector':{},'policyTypes':['Ingress','Egress'],'egress':[],'ingress':[{'from':[{'namespaceSelector':{'matchLabels':{'kubernetes.io/metadata.name':SOURCE_NS}},'podSelector':{'matchLabels':{'app':'node4-query-fixture'}}}],'ports':[{'protocol':'TCP','port':6379}]}]}},
|
||||
{'apiVersion':'v1','kind':'Pod','metadata':{'name':POD,'namespace':NS,'labels':{'app':POD,'purpose':'mongo-cpu-synthetic-cache'}},'spec':{'nodeName':NODE,'automountServiceAccountToken':False,'enableServiceLinks':False,'restartPolicy':'Never','activeDeadlineSeconds':7200,'terminationGracePeriodSeconds':30,'securityContext':{'runAsUser':1000,'runAsGroup':1000,'runAsNonRoot':True,'fsGroup':1000},'containers':[{'name':'cache','image':IMAGE,'imagePullPolicy':'IfNotPresent','command':['valkey-server','--bind','0.0.0.0','--protected-mode','no','--port','6379','--maxmemory','2gb','--maxmemory-policy','allkeys-lru','--save','','--appendonly','no','--dir','/cache','--loglevel','warning'],'resources':{'requests':{'cpu':'1','memory':'2Gi'},'limits':{'cpu':'2','memory':'4Gi'}},'securityContext':{'allowPrivilegeEscalation':False,'readOnlyRootFilesystem':True,'capabilities':{'drop':['ALL']}},'volumeMounts':[{'name':'cache','mountPath':'/cache'}],'readinessProbe':{'exec':{'command':['valkey-cli','ping']},'periodSeconds':5,'timeoutSeconds':3}}],'volumes':[{'name':'cache','emptyDir':{'medium':'Memory','sizeLimit':'64Mi'}}]}}
|
||||
]
|
||||
|
||||
|
||||
def main():
|
||||
ap=argparse.ArgumentParser(description=__doc__)
|
||||
ap.add_argument('--kubeconfig',required=True);ap.add_argument('--output',required=True)
|
||||
ap.add_argument('--execute',action='store_true');ap.add_argument('--confirm-plan')
|
||||
a=ap.parse_args();out=Path(a.output).resolve();out.mkdir(parents=True,exist_ok=True)
|
||||
k=['kubectl','--kubeconfig',a.kubeconfig]
|
||||
def call(*cmd,data=None,timeout=30):
|
||||
p=subprocess.run(k+list(cmd),input=data,text=True,capture_output=True,timeout=timeout)
|
||||
if p.returncode:raise RuntimeError(p.stderr[:1500])
|
||||
return p.stdout
|
||||
def get(*cmd):return json.loads(call(*cmd,'-o','json'))
|
||||
node=get('get','node',NODE);assert node['metadata']['uid']==NODE_UID
|
||||
assert all(c['status']=='False' for c in node['status']['conditions'] if c['type'] in ['DiskPressure','MemoryPressure','PIDPressure'])
|
||||
top=call('top','node',NODE,'--no-headers').split();assert float(top[2].rstrip('%'))<25 and float(top[4].rstrip('%'))<60
|
||||
assert not call('get','namespace',NS,'--ignore-not-found','-o','name').strip(),'existing namespace must be inspected, not overwritten'
|
||||
with urllib.request.urlopen('http://120.48.180.80/api/ready',timeout=5) as r:assert json.load(r)['ready']
|
||||
objects=manifests();plan={'nodeUID':NODE_UID,'objects':objects,'productionWrites':False,'durability':'Valkey is disposable derived cache; complete responses remain majority+j in query Mongo','rollback':'UID-delete only these owned resources; never delete a reused namespace'}
|
||||
sha=hashlib.sha256(json.dumps(plan,sort_keys=True).encode()).hexdigest()
|
||||
(out/'plan.json').write_text(json.dumps(plan,indent=2));(out/'objects.json').write_text(json.dumps({'apiVersion':'v1','kind':'List','items':objects},indent=2))
|
||||
print('PLAN',sha,flush=True)
|
||||
if not a.execute:return
|
||||
assert a.confirm_plan==sha
|
||||
call('create','-f',str(out/'objects.json'))
|
||||
ns=get('get','namespace',NS);pod=get('-n',NS,'get','pod',POD)
|
||||
(out/'created.json').write_text(json.dumps({'namespaceUID':ns['metadata']['uid'],'podUID':pod['metadata']['uid'],'node':NODE,'image':IMAGE}))
|
||||
call('-n',NS,'wait','--for=condition=Ready','pod/'+POD,'--timeout=180s',timeout=185)
|
||||
pod=get('-n',NS,'get','pod',POD);assert pod['spec']['nodeName']==NODE
|
||||
info=call('-n',NS,'exec',POD,'-c','cache','--','valkey-cli','INFO','memory')
|
||||
assert 'maxmemory:2147483648' in info
|
||||
owned={'namespace':NS,'namespaceUID':ns['metadata']['uid'],'pod':POD,'podUID':pod['metadata']['uid'],'node':NODE,'nodeUID':NODE_UID,'image':IMAGE,'url':'redis://'+pod['status']['podIP']+':6379/0','planSHA256':sha}
|
||||
(out/'owned.json').write_text(json.dumps(owned,indent=2));(out/'memory.log').write_text(info)
|
||||
print('CACHE_READY',pod['metadata']['uid'],flush=True)
|
||||
|
||||
if __name__=='__main__':main()
|
||||
@ -6,6 +6,8 @@ PROFILES = {
|
||||
"compute": {"cpu": "2", "memoryGi": 4, "cacheGi": "1", "rbps": 2097152, "wbps": 1048576, "riops": 64, "wiops": 32},
|
||||
"balanced": {"cpu": "2", "memoryGi": 4, "cacheGi": "1", "rbps": 4194304, "wbps": 2097152, "riops": 128, "wiops": 64},
|
||||
"read-headroom": {"cpu": "2", "memoryGi": 4, "cacheGi": "1", "rbps": 8388608, "wbps": 2097152, "riops": 256, "wiops": 64},
|
||||
"query4-write4": {"cpu": "4", "memoryGi": 8, "cacheGi": "2", "rbps": 4194304, "wbps": 4194304, "riops": 128, "wiops": 128},
|
||||
"query4-write8": {"cpu": "4", "memoryGi": 8, "cacheGi": "2", "rbps": 4194304, "wbps": 8388608, "riops": 128, "wiops": 256},
|
||||
}
|
||||
|
||||
|
||||
@ -42,7 +44,7 @@ def render(objects, name):
|
||||
config["data"]["guard.sh"] += f'\n[[ $(cat "$group/cpu.max") == "{int(p["cpu"]) * 100000} 100000" ]]\n[[ $(cat "$group/memory.max") == {p["memoryGi"] * 1024 ** 3} ]]\n'
|
||||
config["data"]["query25.sh"] = config["data"]["query25.sh"].replace('--wiredTigerCacheSizeGB 0.25', '--wiredTigerCacheSizeGB ' + p['cacheGi'])
|
||||
# Verify actual server settings, not only command text or admission requests.
|
||||
config["data"]["verify25.sh"] = config["data"]["verify25.sh"].replace('const d=db.getSiblingDB', f'if(db.serverStatus().wiredTiger.cache["maximum bytes configured"]!=={int(float(p["cacheGi"]) * 1024 ** 3)})throw Error("WT cache drift");const d=db.getSiblingDB')
|
||||
config["data"]["verify25.sh"] = config["data"]["verify25.sh"].replace('const d=db.getSiblingDB', f'if(Number(db.serverStatus().wiredTiger.cache["maximum bytes configured"])!=={int(float(p["cacheGi"]) * 1024 ** 3)})throw Error("WT cache drift");const d=db.getSiblingDB')
|
||||
config["data"]["profile.json"] = __import__('json').dumps({"name": name, **p}, sort_keys=True)
|
||||
pod["metadata"].setdefault("annotations", {})["query-lab/profile"] = name
|
||||
for container in pod["spec"]["containers"] + pod["spec"]["initContainers"]:
|
||||
|
||||
@ -11,8 +11,30 @@ compare_spec = importlib.util.spec_from_file_location("isolated_compare", Path(_
|
||||
compare = importlib.util.module_from_spec(compare_spec)
|
||||
compare_spec.loader.exec_module(compare)
|
||||
|
||||
valkey_spec = importlib.util.spec_from_file_location("valkey_lab", Path(__file__).with_name("query-valkey-lab.py"))
|
||||
valkey_lab = importlib.util.module_from_spec(valkey_spec)
|
||||
valkey_spec.loader.exec_module(valkey_lab)
|
||||
|
||||
|
||||
class FixtureSafety(unittest.TestCase):
|
||||
def test_dedicated_valkey_is_disposable_bounded_and_network_isolated(self):
|
||||
objects = valkey_lab.manifests()
|
||||
self.assertEqual([o["kind"] for o in objects], ["Namespace", "ResourceQuota", "NetworkPolicy", "Pod"])
|
||||
policy, pod = objects[-2:]
|
||||
self.assertEqual(policy["spec"]["egress"], [])
|
||||
source = policy["spec"]["ingress"][0]["from"][0]
|
||||
self.assertEqual(source["namespaceSelector"]["matchLabels"]["kubernetes.io/metadata.name"], "droboticflow-query-lab")
|
||||
self.assertEqual(source["podSelector"]["matchLabels"]["app"], "node4-query-fixture")
|
||||
self.assertEqual(pod["spec"]["nodeName"], "10.189.16.33")
|
||||
self.assertFalse(pod["spec"]["automountServiceAccountToken"])
|
||||
self.assertTrue(all("hostPath" not in v and "persistentVolumeClaim" not in v for v in pod["spec"]["volumes"]))
|
||||
container = pod["spec"]["containers"][0]
|
||||
self.assertEqual(container["resources"]["limits"], {"cpu": "2", "memory": "4Gi"})
|
||||
self.assertIn("--maxmemory", container["command"])
|
||||
self.assertIn("2gb", container["command"])
|
||||
self.assertIn("--appendonly", container["command"])
|
||||
self.assertTrue(container["securityContext"]["readOnlyRootFilesystem"])
|
||||
|
||||
def test_rebuilt_query_uid_requires_exact_storage_and_resources(self):
|
||||
import copy
|
||||
uid = "e1210ce7-16ee-4bda-9f8f-e3a02efefa01"
|
||||
|
||||
@ -23,6 +23,9 @@ CASES = ('internal/store/query_replica_local_test.go',
|
||||
'internal/store/query_replica_valkey_local_test.go',
|
||||
'internal/store/workforce_report_cache_local_test.go',
|
||||
'internal/store/workforce_facts_local_test.go',
|
||||
'internal/store/workforce_publication_local_test.go',
|
||||
'internal/store/query_replica_page_local_test.go',
|
||||
'internal/store/supplier_authorization_batch_local_test.go',
|
||||
'internal/store/query_cold_write_local_test.go',
|
||||
'internal/store/query_periodic_local_test.go',
|
||||
'internal/store/query_pool_buckets_local_test.go')
|
||||
|
||||
@ -522,6 +522,7 @@
|
||||
"argv": ["npm", "run", "test:browser:dashboard"],
|
||||
"tests": true
|
||||
},
|
||||
{ "argv": ["node", "tests/browser/dashboard-publication.mjs"], "tests": true },
|
||||
{
|
||||
"argv": ["python3", "scripts/test_dashboard_stability.py"]
|
||||
},
|
||||
@ -975,7 +976,7 @@
|
||||
"name": "node4-query-pilot-guardrails",
|
||||
"paths": [
|
||||
"^scripts/local/node4-query-pilot/",
|
||||
"^scripts/local/(query-lab-resource-stage|query_lab_profiles)\\.py$",
|
||||
"^scripts/local/(query-lab-resource-stage|query-valkey-lab|query_lab_profiles)\\.py$",
|
||||
"^docs/operations/query-replica-node4-(isolation-plan|smoke-results)\\.md$",
|
||||
"^scripts/local/(query-replica-(cluster-fixture|isolated-compare)|test_query_replica_cluster_fixture)\\.py$",
|
||||
"^docs/operations/query-replica-query25-comparison\\.md$"
|
||||
@ -1019,10 +1020,11 @@
|
||||
"paths": [
|
||||
"^internal/store/query_(replica|sync|tracked_collection|collection_interface).*\\.go$",
|
||||
"^internal/api/query_replica.*\\.go$",
|
||||
"^internal/store/workforce_(report_cache|heap_diagnostic|facts|return_page).*\\.go$",
|
||||
"^internal/store/workforce_(report_cache|heap_diagnostic|facts|return_page|publication).*\\.go$",
|
||||
"^internal/store/query_pool_buckets.*\\.go$",
|
||||
"^internal/store/query_cold_write.*\\.go$",
|
||||
"^internal/store/query_periodic.*\\.go$",
|
||||
"^internal/store/supplier_authorization_batch_local_test\\.go$",
|
||||
"^internal/api/query_periodic.*\\.go$",
|
||||
"^internal/api/query_fact_routes_test\\.go$",
|
||||
"^internal/api/workforce_compact_report_test\\.go$",
|
||||
|
||||
@ -158,10 +158,11 @@ function DailyDetails({ query }: { query: WorkforceReportQuery }) {
|
||||
const identity = JSON.stringify({ ...query, groupBy: "day" });
|
||||
useEffect(() => {
|
||||
let active = true;
|
||||
const controller = new AbortController();
|
||||
setItems(null);
|
||||
setError("");
|
||||
workforceApi
|
||||
.report(JSON.parse(identity))
|
||||
.report(JSON.parse(identity), controller.signal)
|
||||
.then((result) => {
|
||||
if (active) setItems(result.items);
|
||||
})
|
||||
@ -170,6 +171,7 @@ function DailyDetails({ query }: { query: WorkforceReportQuery }) {
|
||||
});
|
||||
return () => {
|
||||
active = false;
|
||||
controller.abort();
|
||||
};
|
||||
}, [identity]);
|
||||
if (error) return <p role="alert">{userErrorMessage(error)}</p>;
|
||||
@ -258,12 +260,13 @@ export function WorkforceStatisticsPanel({
|
||||
useEffect(() => {
|
||||
if (collapsed || !deferLoad) return;
|
||||
let active = true;
|
||||
const controller = new AbortController();
|
||||
setLoading(true);
|
||||
setRemoteItems(null);
|
||||
setRemoteTotals(undefined);
|
||||
setError("");
|
||||
workforceApi
|
||||
.report({ ...apiFilters, role, groupBy: "member" })
|
||||
.report({ ...apiFilters, role, groupBy: "member" }, controller.signal)
|
||||
.then((result) => {
|
||||
if (!active) return;
|
||||
setRemoteItems(result.items);
|
||||
@ -277,6 +280,7 @@ export function WorkforceStatisticsPanel({
|
||||
});
|
||||
return () => {
|
||||
active = false;
|
||||
controller.abort();
|
||||
};
|
||||
}, [collapsed, deferLoad, role, filterKey]);
|
||||
const visibleItems = deferLoad ? remoteItems || [] : items;
|
||||
@ -461,7 +465,7 @@ export function WorkforceStatisticsPanel({
|
||||
})}
|
||||
</tbody>
|
||||
</table>
|
||||
{!filtered.length && (
|
||||
{!loading && !error && !filtered.length && (
|
||||
<p className="p-8 text-center text-gray-500">
|
||||
{visibleItems.length ? "没有匹配的成员记录" : "当前区间暂无工作记录"}
|
||||
</p>
|
||||
|
||||
@ -15,7 +15,7 @@ import type {
|
||||
SliceSegment,
|
||||
} from "../../types";
|
||||
import { compactAnnotationPayload } from "../../lib/annotation-operation-history.js";
|
||||
import { apiRequest } from "./client";
|
||||
import { apiRequest, preparedAPIRequest } from "./client";
|
||||
import { decodeAnnotationTimeline } from "../../lib/annotation-timeline-wire.js";
|
||||
import { normalizeAnnotationWorkspace, normalizeAnnotationTaskDetail } from "../../lib/annotation-workspace.js";
|
||||
|
||||
@ -495,8 +495,8 @@ export const annotationOperationsApi = {
|
||||
payload,
|
||||
),
|
||||
workloadSummary: (projectId?: string, options: { signal?: AbortSignal } = {}) =>
|
||||
apiRequest<AnnotationWorkloadSummary>(
|
||||
preparedAPIRequest<AnnotationWorkloadSummary>(
|
||||
`/api/plugins/annotation-qa/workload-summary${query({ projectId })}`,
|
||||
options,
|
||||
options.signal,
|
||||
),
|
||||
};
|
||||
|
||||
@ -189,3 +189,37 @@ export async function apiRequest<T>(
|
||||
|
||||
throw lastError || new Error("Request failed");
|
||||
}
|
||||
|
||||
/** Wait for an optional derived report without extending the normal read budget. */
|
||||
export async function preparedAPIRequest<T>(path: string, signal?: AbortSignal): Promise<T> {
|
||||
const deadline = Date.now() + DEFAULT_REQUEST_TIMEOUT_MS;
|
||||
let failures = 0;
|
||||
while (true) {
|
||||
const remaining = deadline - Date.now();
|
||||
if (remaining <= 0) throw new APITimeoutError("看板正在准备,请稍后重试");
|
||||
let response: Response;
|
||||
try {
|
||||
response = await apiFetch(
|
||||
path,
|
||||
{ signal, headers: { Accept: "application/vnd.droboticflow.prepared+json" } },
|
||||
{ timeoutMs: remaining },
|
||||
);
|
||||
} catch (error) {
|
||||
if (isAbortError(error) || error instanceof APITimeoutError || ++failures >= 4) throw error;
|
||||
await abortableDelay(Math.min(failures * 500, Math.max(0, deadline - Date.now())), signal);
|
||||
continue;
|
||||
}
|
||||
if (!response.ok) {
|
||||
if (RETRIABLE_STATUS.has(response.status) && ++failures < 4) {
|
||||
await abortableDelay(Math.min(failures * 500, Math.max(0, deadline - Date.now())), signal);
|
||||
continue;
|
||||
}
|
||||
throw await responseError(response);
|
||||
}
|
||||
const result = await response.json();
|
||||
if (response.status !== 202) return result as T;
|
||||
if (result?.status !== "building") throw new APIError("看板暂时无法加载,请重试", 202);
|
||||
const delay = Math.min(2000, Math.max(500, Number(result.retryAfterMs) || 1000));
|
||||
await abortableDelay(Math.min(delay, Math.max(0, deadline - Date.now())), signal);
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,10 +1,10 @@
|
||||
import type { DashboardSummary } from "../../types";
|
||||
import { apiRequest } from "./client";
|
||||
import { preparedAPIRequest } from "./client";
|
||||
|
||||
export const dashboardApi = {
|
||||
getSummary: (options: { includeWorkforce?: boolean } = {}) => {
|
||||
const suffix =
|
||||
options.includeWorkforce === undefined ? "" : `?includeWorkforce=${options.includeWorkforce ? "true" : "false"}`;
|
||||
return apiRequest<DashboardSummary>(`/api/dashboard${suffix}`);
|
||||
return preparedAPIRequest<DashboardSummary>(`/api/dashboard${suffix}`);
|
||||
},
|
||||
};
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
import type { WorkforceReturnDetail } from "../../types";
|
||||
import { apiRequest } from "./client";
|
||||
import { apiRequest, preparedAPIRequest } from "./client";
|
||||
|
||||
export type WorkforceDetailQuery = {
|
||||
userId: string;
|
||||
@ -22,19 +22,19 @@ export type WorkforceReportQuery = {
|
||||
};
|
||||
|
||||
export const workforceApi = {
|
||||
report: (query: WorkforceReportQuery = {}) => {
|
||||
report: (query: WorkforceReportQuery = {}, signal?: AbortSignal) => {
|
||||
const search = new URLSearchParams();
|
||||
Object.entries(query).forEach(([key, value]) => {
|
||||
if (value !== undefined && value !== "") search.set(key, String(value));
|
||||
});
|
||||
return apiRequest<{
|
||||
return preparedAPIRequest<{
|
||||
from: string;
|
||||
to: string;
|
||||
groupBy: "member" | "day";
|
||||
totals: import("../../types").DashboardWorkforceMetric;
|
||||
items: import("../../types").DashboardWorkforceMetric[];
|
||||
eventCount: number;
|
||||
}>(`/api/dashboard/workforce?${search}`);
|
||||
}>(`/api/dashboard/workforce?${search}`, signal);
|
||||
},
|
||||
returns: (query: WorkforceDetailQuery, offset = 0) => {
|
||||
const search = new URLSearchParams({ limit: "20", offset: String(offset) });
|
||||
|
||||
90
tests/browser/dashboard-publication.mjs
Normal file
90
tests/browser/dashboard-publication.mjs
Normal file
@ -0,0 +1,90 @@
|
||||
import { createServer } from "vite";
|
||||
import { chromium } from "playwright";
|
||||
import assert from "node:assert/strict";
|
||||
import fs from "node:fs/promises";
|
||||
import { browserExecutableOptions } from "../../scripts/lib/browser-executable.mjs";
|
||||
const out = process.env.DASHBOARD_PUBLICATION_OUTPUT || "local-data/test-results/dashboard-publication";
|
||||
await fs.mkdir(out, { recursive: true });
|
||||
const fixture = `import React from 'react'; import {createRoot} from 'react-dom/client'; import {UserProvider} from '/src/context/UserContext'; import {TimeZoneProvider} from '/src/context/TimeZoneContext'; import {WorkforceStatisticsPanel} from '/src/components/WorkforceStatisticsPanel'; import '/src/index.css'; createRoot(document.getElementById('root')!).render(<UserProvider><TimeZoneProvider><WorkforceStatisticsPanel items={[]} deferLoad={true} collapsedByDefault={false}/></TimeZoneProvider></UserProvider>);`;
|
||||
const server = await createServer({
|
||||
configFile: "tests/browser/vite.platform-feedback.config.ts",
|
||||
cacheDir: out + "/vite-cache",
|
||||
server: { host: "127.0.0.1", port: 0 },
|
||||
plugins: [
|
||||
{
|
||||
name: "publication-fixture",
|
||||
enforce: "pre",
|
||||
transform(code, id) {
|
||||
if (id.endsWith("/tests/browser/fixtures/acceptance-playback-rate.tsx")) return fixture;
|
||||
},
|
||||
},
|
||||
],
|
||||
});
|
||||
let browser;
|
||||
const report = { checks: [], errors: [] };
|
||||
try {
|
||||
await server.listen();
|
||||
const origin = server.resolvedUrls.local[0].replace(/\/$/, "");
|
||||
browser = await chromium.launch({ ...browserExecutableOptions(), headless: true, args: ["--no-proxy-server"] });
|
||||
const page = await browser.newPage({ viewport: { width: 1440, height: 1000 } });
|
||||
page.on("pageerror", (e) => report.errors.push(e.message));
|
||||
page.setDefaultTimeout(15000);
|
||||
let ready = false,
|
||||
denied = false,
|
||||
calls = 0;
|
||||
await page.route("**/api/**", (route) => {
|
||||
const path = new URL(route.request().url()).pathname;
|
||||
if (!path.startsWith("/api/")) return route.continue();
|
||||
if (path === "/api/auth/me")
|
||||
return route.fulfill({
|
||||
json: {
|
||||
user: { id: "synthetic", name: "Synthetic", role: "admin", status: "active" },
|
||||
organization: { id: "synthetic", name: "Synthetic" },
|
||||
},
|
||||
});
|
||||
if (path === "/api/dashboard/workforce") {
|
||||
calls++;
|
||||
if (denied) return route.fulfill({ status: 403, json: { error: "无权访问" } });
|
||||
if (!ready) return route.fulfill({ status: 202, json: { status: "building", retryAfterMs: 500 } });
|
||||
return route.fulfill({ json: { items: [], totals: {}, from: "2026-11-01", to: "2026-11-13" } });
|
||||
}
|
||||
return route.fulfill({ json: { items: [] } });
|
||||
});
|
||||
await page.route("**/publication-test", (route) =>
|
||||
route.fulfill({
|
||||
contentType: "text/html",
|
||||
body: `<html><body><div id='root'></div><script type='module'>import RefreshRuntime from '/@react-refresh';RefreshRuntime.injectIntoGlobalHook(window);window.$RefreshReg$=()=>{};window.$RefreshSig$=()=>type=>type;window.__vite_plugin_react_preamble_installed__=true;</script><script type='module' src='/tests/browser/fixtures/acceptance-playback-rate.tsx'></script></body></html>`,
|
||||
}),
|
||||
);
|
||||
await page.goto(origin + "/publication-test");
|
||||
await page.getByText("正在加载工作记录…", { exact: true }).waitFor();
|
||||
await page.waitForTimeout(1100);
|
||||
assert(calls >= 2);
|
||||
assert.equal(await page.getByText("当前区间暂无工作记录", { exact: true }).count(), 0);
|
||||
await page.screenshot({ path: out + "/building.png" });
|
||||
report.checks.push("202 retains loading state rather than false empty");
|
||||
ready = true;
|
||||
await page.getByText("当前区间暂无工作记录", { exact: true }).waitFor();
|
||||
await page.screenshot({ path: out + "/complete-empty.png" });
|
||||
report.checks.push("completed empty report replaces loading");
|
||||
ready = false;
|
||||
await page.reload();
|
||||
await page.getByText("正在加载工作记录…", { exact: true }).waitFor();
|
||||
await page.goto("about:blank");
|
||||
const before = calls;
|
||||
await page.waitForTimeout(1200);
|
||||
assert.equal(calls, before);
|
||||
report.checks.push("unmount cancels future report polls");
|
||||
denied = true;
|
||||
await page.goto(origin + "/publication-test");
|
||||
await page.getByRole("alert").waitFor();
|
||||
await page.screenshot({ path: out + "/denied.png" });
|
||||
report.checks.push("permission failure shows error");
|
||||
assert.deepEqual(report.errors, []);
|
||||
report.passed = true;
|
||||
} finally {
|
||||
await fs.writeFile(out + "/report.json", JSON.stringify(report, null, 2));
|
||||
await browser?.close();
|
||||
await server.close();
|
||||
}
|
||||
console.log(JSON.stringify(report));
|
||||
90
tests/frontend/prepared-report.test.mjs
Normal file
90
tests/frontend/prepared-report.test.mjs
Normal file
@ -0,0 +1,90 @@
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import fs from "node:fs";
|
||||
import ts from "typescript";
|
||||
import { buildAPIRequestHeaders } from "../../src/services/api/http.js";
|
||||
|
||||
const code = ts.transpileModule(fs.readFileSync(new URL("../../src/services/api/client.ts", import.meta.url), "utf8"), {
|
||||
compilerOptions: { module: ts.ModuleKind.CommonJS, target: ts.ScriptTarget.ES2022 },
|
||||
}).outputText;
|
||||
function client(fetcher, onDelay = () => {}) {
|
||||
let now = 0;
|
||||
const exports = {};
|
||||
const window = {
|
||||
setTimeout(fn, ms) {
|
||||
return setImmediate(() => {
|
||||
now += ms;
|
||||
onDelay();
|
||||
fn();
|
||||
});
|
||||
},
|
||||
clearTimeout(id) {
|
||||
clearImmediate(id);
|
||||
},
|
||||
};
|
||||
new Function("exports", "require", "fetch", "Date", "window", code)(
|
||||
exports,
|
||||
(path) => (path === "./http.js" ? { buildAPIRequestHeaders } : { buildAPIURL: (p) => p }),
|
||||
fetcher,
|
||||
{ now: () => now },
|
||||
window,
|
||||
);
|
||||
return exports;
|
||||
}
|
||||
const response = (body, status = 200) => new Response(JSON.stringify(body), { status });
|
||||
test("prepared reports wait for publication and return only a completed report", async () => {
|
||||
let calls = 0;
|
||||
const c = client(async (_url, options) => {
|
||||
assert.equal(options.credentials, "include");
|
||||
assert.equal(new Headers(options.headers).get("Accept"), "application/vnd.droboticflow.prepared+json");
|
||||
assert.equal(options.body, undefined);
|
||||
return ++calls === 1 ? response({ status: "building", retryAfterMs: 1000 }, 202) : response({ rows: [1] });
|
||||
});
|
||||
assert.deepEqual(await c.preparedAPIRequest("/api/dashboard/workforce"), { rows: [1] });
|
||||
assert.equal(calls, 2);
|
||||
});
|
||||
test("publication polling retains the original total 30 second budget", async () => {
|
||||
let calls = 0;
|
||||
const c = client(async () => {
|
||||
calls++;
|
||||
return response({ status: "building", retryAfterMs: 2000 }, 202);
|
||||
});
|
||||
await assert.rejects(c.preparedAPIRequest("/report"), { name: "APITimeoutError" });
|
||||
assert.equal(calls, 15);
|
||||
});
|
||||
test("leaving the report cancels publication polling before another read", async () => {
|
||||
const abort = new AbortController();
|
||||
let calls = 0;
|
||||
const c = client(
|
||||
async () => {
|
||||
calls++;
|
||||
return response({ status: "building" }, 202);
|
||||
},
|
||||
() => abort.abort(),
|
||||
);
|
||||
await assert.rejects(c.preparedAPIRequest("/report", abort.signal), { name: "AbortError" });
|
||||
assert.equal(calls, 1);
|
||||
});
|
||||
test("permission errors and unrelated 202 responses are never treated as finished data", async () => {
|
||||
const denied = client(async () => response({ error: "forbidden" }, 403));
|
||||
await assert.rejects(denied.preparedAPIRequest("/report"), { status: 403 });
|
||||
const unknown = client(async () => response({ rows: [] }, 202));
|
||||
await assert.rejects(unknown.preparedAPIRequest("/report"), { status: 202 });
|
||||
});
|
||||
|
||||
test("prepared reads retain bounded transport retry and never retry cancellation", async () => {
|
||||
let calls = 0;
|
||||
const c = client(async () => {
|
||||
if (++calls === 1) throw new TypeError("network");
|
||||
return response({ items: [] });
|
||||
});
|
||||
assert.deepEqual(await c.preparedAPIRequest("/report"), { items: [] });
|
||||
assert.equal(calls, 2);
|
||||
calls = 0;
|
||||
const aborted = client(async () => {
|
||||
calls++;
|
||||
throw new DOMException("canceled", "AbortError");
|
||||
});
|
||||
await assert.rejects(aborted.preparedAPIRequest("/report"), { name: "AbortError" });
|
||||
assert.equal(calls, 1);
|
||||
});
|
||||
Loading…
x
Reference in New Issue
Block a user