⚡ Publish query reports and coalesce task pool reads
Some checks are pending
web-ci / checks (push) Waiting to run
web-ci / pool-search-mongo (push) Waiting to run
web-ci / work-availability (push) Blocked by required conditions
web-ci / query-serving (push) Blocked by required conditions

This commit is contained in:
d-robotics 2026-11-13 22:31:44 -08:00
parent 3e59fa19a9
commit fc7786190f
42 changed files with 1816 additions and 61 deletions

View File

@ -17,7 +17,7 @@ import (
)
func allowedFlow(listen, upstream, node, actualUID, expectedUID string) bool {
if node != "10.189.16.4" || actualUID == "" || actualUID != expectedUID {
if (node != "10.189.16.4" && node != "10.189.16.33") || actualUID == "" || actualUID != expectedUID {
return false
}
l, le := netip.ParseAddrPort(listen)
@ -27,7 +27,7 @@ func allowedFlow(listen, upstream, node, actualUID, expectedUID string) bool {
}
private := netip.MustParsePrefix("10.189.16.0/24")
return l == netip.MustParseAddrPort("127.0.0.1:27229") && private.Contains(u.Addr()) && u.Port() == 27130 ||
private.Contains(l.Addr()) && l.Port() == 27130 && u == netip.MustParseAddrPort("127.0.0.1:27129")
node == "10.189.16.4" && private.Contains(l.Addr()) && l.Port() == 27130 && u == netip.MustParseAddrPort("127.0.0.1:27129")
}
// Do not embed TCPConn: promoted ReadFrom/WriteTo would bypass the deadlines

View File

@ -9,6 +9,8 @@ func TestIsolatedBridgeRejectsOtherNodesIdentitiesAndDestinations(t *testing.T)
}{
{"127.0.0.1:27229", "10.189.16.46:27130", "10.189.16.4", "owned", "owned", true},
{"10.189.16.46:27130", "127.0.0.1:27129", "10.189.16.4", "owned", "owned", true},
{"127.0.0.1:27229", "10.189.16.46:27130", "10.189.16.33", "owned", "owned", true},
{"10.189.16.46:27130", "127.0.0.1:27129", "10.189.16.33", "owned", "owned", false},
{"0.0.0.0:27229", "10.189.16.46:27130", "10.189.16.4", "owned", "owned", false},
{"127.0.0.1:27229", "8.8.8.8:27130", "10.189.16.4", "owned", "owned", false},
{"127.0.0.1:27229", "10.189.16.46:27017", "10.189.16.4", "owned", "owned", false},

View File

@ -52,3 +52,17 @@
计数展示不再读取原库脏任务/删除意图做即时补偿,因此数量是查询库已提交事实的结果,不保证包含刚刚发生的操作。批次权限/策略、账号、成员、供应商权限继续实时校验;列表维持本页当前归属/版本读取与原服务端可见性/`canClaim`,领取、保存、送检、质检、验收和后台调度均沿原权威路径。此变更不允许旧缓存绕过权限,也不允许以旧任务列表重复领取。
本次只在合成隔离环境验证,不切换生产、不修改客户数据。固定原请求超时、租约和多数提交要求;延迟展示是独立业务契约,不是延长请求超时或降低持久性来通过测试。验证必须单列本机与25Gi共享盘结果,以及冷构建、故障恢复和持续变化期间的覆盖限制。
## 最终展示结果后台发布(2026-11-14,隔离实施)
在周期模式外再显式启用 `QUERY_PUBLISHED_RESULTS=1`,人员看板和作业记录汇总把原归约器的完整结果发布到查询库,并在提交后写入专用 Valkey。HTTP 不因普通缓存未命中重新聚合历史。Valkey 丢失或不可达时读取同一查询库中的完整成品;缺少成品返回 HTTP 202 `building`,客户端保留加载状态并在既有30秒总预算内等待,离开页面取消请求。权限失败和其他错误不伪装成准备状态。
成品键包含查询库、组织、当前服务端权限范围和精确筛选;每次请求仍实时建立权限范围,缓存不保存授权结论。默认滚动日期保存为后台计算配方,发布时记录实际区间;显式日期保持纳秒边界,避免每次请求的当前时间产生不同键。事实版本、构建租约和绑定在事务提交时重新核验;旧完整版本保留到新版本成功,失效或错误不集中回源原库。
缓存信封带已提交事实证书、摘要与到期时间,热读无需额外查查询库水位;首次读取仍验证物理绑定。最长展示年龄仍为15分钟,不能通过重新包装旧成品延长。成品生成受每轮16项/8秒调度预算约束;单组织登记最多1024种活跃查询,24小时后清理闲置登记。超过查询描述或12MiB成品边界的历史特殊结果保留查询库原兼容聚合,不截断结果,不回扫权威库。
该机制只接管派生展示。领取、标注、切片、送检、正式质检及验收不等待成品生成,也不读取缓存决定权限、任务归属或版本。独立 Valkey 是可丢弃缓存,持久结果继续由查询 Mongo 的原多数提交与日志持久性保证;尚未切换生产入口。
准备状态通过显式媒体类型协商;旧客户端收到未就绪错误而非成功的非报表JSON。活跃配方的登记触碰合并到15分钟一次,后台统计刷新周期不变。任务池只共享候选身份,个人任务/审查历史维持当前权威查询;权限目录的逐项目重复读合并为请求内批量读,继续复用原授权种子规则,不形成共享权限缓存。
候选身份的冷缓存查询合并仅作用于相同数据库、组织、事实代次和完整筛选,沿用既有共享名称读取的8秒工作上限及独立取消规则;完成后的长效成品仍只在共享缓存/查询库中。每个请求各自取当前任务与运行服务端权限校验,合并结果不带授权结论、正文或可领取标志。

View File

@ -153,3 +153,43 @@ QUERY_REPLICA_LOCAL_TEST=1 QUERY_REPLICA_SCALE_LOCAL=1 go test ./internal/store
- 最终快速检查 `verify-1794589491130862000/report.json` 所选检查通过;最终完整副本race/cache记录 `check-5b8d9540efff4b01825a878e91549000.log` 无缺失/跳过/源码变化。新增串行调度保护回归保留,失败的并行实验日志保留。前述原始任务检查的交付夹具跳过、原始API既有失败仍如实保留,不将最终状态写成整体验收通过。
本批没有推送、PR、生产应用或数据变更;独立查询库周期读取开关仍默认关闭。下一步先稳定复现冷态统计超时,并减少旧产出快照和统计的重复读/持久化确认轮次;达到原配置下的完整正确性、速度与资源对照后再准备上线。
## 成品发布与专用缓存隔离验证(2026-11-14 UTC)
本批基线 `3e59fa19`,仅在 `mongo-cpu` 工作树和已授权合成命名空间实施。没有切换生产查询入口、修改客户数据或购买磁盘。
- `QUERY_PUBLISHED_RESULTS=1` 与周期模式组合启用。常规人员/工作记录汇总由后台登记、归约、事务发布,再放入专用 Valkey;前台优先读取完整成品。缓存丢失、连接失败或API进程重启后只读查询库成品,不触发权威库历史聚合。明细仍在查询库分页后补当前页正文。
- 首次未准备好时,新客户端通过 `Accept: application/vnd.droboticflow.prepared+json` 协商 HTTP 202;旧客户端得到503,避免将准备信封当成空报告。完成后的响应保持旧格式。等待、重试、切换页面取消均沿原30秒预算,不延长接口或聚合超时。
- 已登记范围的活跃时间最多每15分钟触碰一次,减少面板访问引发的查询库写入;这不改变3分钟的后台统计刷新。每组织最多1024个活跃配方、24小时闲置清理,尚未证明500人任意筛选组合的容量。
- Valkey 位于 `.33` 的独立实验命名空间,限2核/4Gi、maxmemory 2Gi,仅允许指定合成夹具访问。它是可丢弃派生缓存;持久成品仍由查询Mongo保证。该单Pod、无持久化缓存配置不是生产高可用方案。
- 查询Mongo在 `.4` 保留25Gi盘,使用4核/8Gi、WiredTiger 2Gi;读限4MiB/128 IOPS,写依次试验4MiB/128 IOPS、8MiB/256 IOPS。初始化器、运行守护、loop与底层设备限额同步更新,Pod重建后核实实际值。多数提交、日志确认、原有超时和磁盘边界均未降低。
- 任务池共享缓存仅保存本页候选ID和游标身份;返回前按ID取当前归属、状态、版本,服务端继续执行可见性与 `canClaim`。“我的任务”和个人审查历史维持权威读取,避免刚领取/提交后因异步发现滞后而消失。权限材料改为请求内批量读取,不跨请求缓存授权结论;默认组织、旧授权、撤销、禁用账号、异常历史项目ID均与原算法对照。
证据根目录:`local-data/diagnostics/mongo-cpu/published-results-20261114/`(本工作树忽略目录)。`write4-products-pool-v4`、`write8-products-pool-v4` 是同一二进制、相同26,218任务/81人/两轮的IO对照,均结果等价、无新业务错误。该窗口源库CPU工作量约下降90%、文档扫描约下降93%;它是合成源进程的CPU秒数对照,不是生产Mongo即时CPU百分比。首次复制约55秒→27秒,统计物化约118秒→112秒,说明增加写IO对首次复制帮助更明显,不能将所有查询收益归于扩资源。
`query4-write8` 保存前一4MiB配置及UID回退步骤。Pod重建保留原验证标记,并将一份合成结果以majority+j写入后验证逐字节和SHA恢复;这不代替完整生产备份恢复演练。两轮采样中受保护Pod的IO full PSI为0,生产就绪、Pod身份/重启集合正常;只是该窗口未观察到影响。
首次测试失败记录保留:旧节点夹具检查拒绝`.33`、旧汇总计量混入合法分页聚合、2Gi缓存的BSON Long比较错误,分别修正后重测。真实浏览器发现加载时仍同时展示空表提示,已修复;准备、空结果、权限错误和退出取消均验证。任务门禁原600秒整体预算耗尽后按原单项超时续跑;浏览器清理超时单项原样重跑通过。完整API包失败通过官方Go基线门禁核验,无新增失败;未扩大已知失败名单。具体来源、环境、命令和各次失败保留在 `verify-task-console.log`、`verify-continuation.log` 和其引用的JSON报告中。
v5的广范围搜索及冷暖读取对照记录于 `final-products-pool-v5`;后续v6修复了该轮暴露的冷缓存并发重复查询。本批不是30分钟三轮、200/500人、持续写入积压或节点故障容量验收,未准备生产切读。上线前仍须明确生产HA/备份、所有写者标记覆盖、源索引与限速回填范围,并重新核对当前 `dev` 和实际线上版本。
最终 `final-products-pool-v6` 使用相同26,218任务/81人两轮主读取及20名审查账号的五类任务池读取(含广范围命中、无结果搜索)。主读取162次、任务池200次新旧结果一致;查询库/cache故障回退验证通过。大量命中搜索在v5为1.52秒,定位同条件冷缓存并发重复查询后增加有界在途合并;独立取消与本页当前版本读取保留,v6降至约664毫秒。不是通过少返回任务、放宽权限或增加超时取得结果。
| 读取 | 旧路径 p95 | 最终查询路径 p95 |
| --- | ---: | ---: |
| 人员看板 | 17,524ms | 75ms |
| 作业记录分页 | 17,618ms | 407ms |
| 任务池计数 | 18,394ms | 986ms |
| 可领取列表 | 370ms | 384ms |
| 大量命中搜索 | 781ms | 664ms |
| 无结果搜索 | 5,670ms | 297ms |
普通列表未观察到速度提升,不宣称所有读取均变快。“我的任务/已审”仍为权威路径,约4~5毫秒。生产总览仅2次暖读,不能据此承诺该角色的持续容量。源库处理同批主读取的CPU耗时23.74→2.40秒,扫描9,887,630→914,716文档;此对照包含测量窗口的后台读写,但一次回填和首次物化另列,不能替代30分钟持续同步负载。
首次复制约30秒、查询库初始物化约107秒,登记的81个成品另需约8秒准备;后台未完成前没有“瞬间冷启动”承诺。暖读与缓存故障均有独立记录。最终窗口受保护业务IO full PSI为0、健康最大约160毫秒、物理盘平均等待采样最大约4.55毫秒、节点CPU最高6%,无生产Pod身份/重启变化。数据库结果及标注/版本摘要保持一致。
临时合成源Pod和桥接在测后按UID清理,查询库配额恢复。实验Pod继续保留原自动到期机制,25Gi数据盘不因到期删除。专用Valkey曾正常到期,已按原参数重建并在最后一轮从空缓存重测;没有取消到期机制或扩大生产入口。下一阶段须补充真实验收聚合包形态、持续变更和200/500人验证,再制定生产影子读取与切换计划。
最终集成报告:`local-data/test-results/verify-1794636683705497000/report.json`,源码未在运行中变化、无未映射文件。41项中39项通过,包含隔离副本/缓存/Mongo专项、源码校验不入队、标注/切片浏览器、搜索及面板、生产前端构建和静态启动。API整包原始检查仍报告已有失败,最终官方Go基线门禁确认68项均在70项既有基线中,无新增;store整包PASS,但默认跳过的外部环境用例标为未完整覆盖,不算通过。专项显式启用的本机副本/Valkey测试已执行通过,未扩大 known-failures。
`.4` 测后资源申请总量约6.475/7.8核(83%)、内存11,813Mi(42%),包含实验查询Pod;低实际使用率不能替代调度申请余量。后续继续扩核须重新核对该节点及生产峰值。当前仍是限时实验Pod,保留原自动结束机制和25Gi盘;后续恢复验证按已记录UID、配置及回退步骤重建,不将该单Pod直接当生产查询库。

View File

@ -74,16 +74,24 @@ func (server *server) dashboardWorkforceReport(ctx context.Context, org string,
return workforcestats.Build(d, q), nil
}
func (server *server) populateDashboardWorkforce(ctx context.Context, requestOrganizationID string, principal *authenticatedPrincipal, from time.Time, to time.Time, summary *domain.DashboardSummary) error {
func (server *server) populateDashboardWorkforce(ctx context.Context, requestOrganizationID string, principal *authenticatedPrincipal, from time.Time, to time.Time, summary *domain.DashboardSummary, relative ...[2]bool) error {
if summary == nil {
return nil
}
q, scope := workforcePrincipalQuery(principal)
q.From, q.To = from, to
if len(relative) > 0 {
q.RelativeFrom = relative[0][0]
q.RelativeTo = relative[0][1]
}
report, err := server.dashboardWorkforceReport(ctx, requestOrganizationID, q)
if err != nil {
return err
}
if report.WindowFrom != "" {
from, _ = time.Parse(time.RFC3339Nano, report.WindowFrom)
to, _ = time.Parse(time.RFC3339Nano, report.WindowTo)
}
summary.AudienceScope = scope
summary.Workforce = domain.DashboardWorkforceSummary{AsOf: report.AsOf, From: from.Format(time.RFC3339Nano), To: to.Format(time.RFC3339Nano), Totals: report.Totals, Items: report.Items}
return nil

View File

@ -17,13 +17,16 @@ import (
"strconv"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
"droboticflow/internal/domain"
"droboticflow/internal/productionstats"
"droboticflow/internal/queryserving"
"droboticflow/internal/store"
"droboticflow/internal/testsupport"
valkey "github.com/valkey-io/valkey-go"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/event"
"go.mongodb.org/mongo-driver/mongo"
@ -48,6 +51,13 @@ func TestQueryReplicaIsolatedHTTPComparisonLocal(t *testing.T) {
}
t.Setenv("ANNOTATION_SOURCE_VALIDATION_MODE", "advisory")
t.Setenv("QUERY_SERVING_NAMESPACE", "isolated-http-comparison")
published := os.Getenv("QUERY_PUBLISHED_RESULTS") == "1"
cacheURL := os.Getenv("QUERY_COMPARE_VALKEY_URL")
if published && cacheURL == "" {
t.Fatal("published result comparison requires explicit isolated Valkey")
}
t.Setenv("QUERY_SERVING_READS", "1")
t.Setenv("QUERY_VALKEY_URL", cacheURL)
ctx := t.Context()
n := isolatedComparisonInt(t, "QUERY_COMPARE_TASKS", 26218, 20, 30000)
waves := isolatedComparisonInt(t, "QUERY_COMPARE_WAVES", 2, 1, 100)
@ -78,13 +88,17 @@ func TestQueryReplicaIsolatedHTTPComparisonLocal(t *testing.T) {
}
}()
}
var workforceAggregations atomic.Int64
targetCommands := map[string]int64{}
targetMaxMillis := map[string]float64{}
targetMonitor := &event.CommandMonitor{Started: func(ctx context.Context, e *event.CommandStartedEvent) {
name, _ := e.Command.Lookup("aggregate").StringValueOK()
if name == "queryWorkforceFacts" {
workforceAggregations.Add(1)
}
if !local {
return
}
name, _ := e.Command.Lookup("aggregate").StringValueOK()
if name != "queryWorkforceFacts" {
return
}
@ -121,6 +135,16 @@ func TestQueryReplicaIsolatedHTTPComparisonLocal(t *testing.T) {
delete(factPending, e.RequestID)
}
}}
// Separate foreground report aggregation from legitimate background builds
// and paginated detail aggregates when checking finished-result fallback.
var apiWorkforceAggregations atomic.Int64
apiTargetMonitor := &event.CommandMonitor{Started: func(ctx context.Context, e *event.CommandStartedEvent) {
name, _ := e.Command.Lookup("aggregate").StringValueOK()
if name == "queryWorkforceFacts" {
apiWorkforceAggregations.Add(1)
}
targetMonitor.Started(ctx, e)
}, Succeeded: targetMonitor.Succeeded, Failed: targetMonitor.Failed}
monitor := &event.CommandMonitor{
Started: func(_ context.Context, e *event.CommandStartedEvent) {
mu.Lock()
@ -258,7 +282,24 @@ func TestQueryReplicaIsolatedHTTPComparisonLocal(t *testing.T) {
if err := builderTarget.VerifyQueryReplicaTarget(ctx); err != nil {
t.Fatal(err)
}
builderTarget.ConfigureQueryServing(nil, nil)
closeBuilderCache, e := builderTarget.ConfigureQueryServingFromEnvironment()
if e != nil {
t.Fatal(e)
}
t.Cleanup(closeBuilderCache)
if published {
cache, ok := builderTarget.SharedQueryCache().(*queryserving.ManagedValkey)
if !ok {
t.Fatal("missing dedicated shared cache")
}
until := time.Now().Add(5 * time.Second)
for !cache.Ready() && time.Now().Before(until) {
time.Sleep(20 * time.Millisecond)
}
if !cache.Ready() {
t.Fatal("dedicated cache not ready")
}
}
if err := builderTarget.ConfigureQueryRefreshMode(os.Getenv("QUERY_REPLICA_REFRESH_MODE")); err != nil {
t.Fatal(err)
}
@ -333,9 +374,9 @@ func TestQueryReplicaIsolatedHTTPComparisonLocal(t *testing.T) {
t.Setenv("QUERY_READ_MONGO_URI", uri)
t.Setenv("QUERY_READ_MONGO_DB", db.Name())
t.Setenv("QUERY_SERVING_READS", "1")
t.Setenv("QUERY_VALKEY_URL", "")
t.Setenv("QUERY_VALKEY_URL", cacheURL)
t.Setenv("QUERY_TYPESENSE_URL", "")
closeReplica, err := newReader.ConfigureQueryReplicaFromEnvironment(ctx, targetMonitor)
closeReplica, err := newReader.ConfigureQueryReplicaFromEnvironment(ctx, apiTargetMonitor)
if err != nil {
t.Fatal(err)
}
@ -354,6 +395,9 @@ func TestQueryReplicaIsolatedHTTPComparisonLocal(t *testing.T) {
return 0, nil, err
}
r.AddCookie(&http.Cookie{Name: sessionCookieName, Value: fixture.tokens[actor]})
if published {
r.Header.Set("Accept", "application/vnd.droboticflow.prepared+json")
}
resp, err := client.Do(r)
if err != nil {
return 0, nil, err
@ -368,6 +412,39 @@ func TestQueryReplicaIsolatedHTTPComparisonLocal(t *testing.T) {
"/api/dashboard/workforce?from=2026-11-01T00:00:00Z&to=2026-11-13T00:00:00Z&groupBy=member",
"/api/workforce-returns?from=2026-11-01T00:00:00Z&to=2026-11-13T00:00:00Z&limit=20&offset=0",
}
publicationWarmSeconds := float64(0)
publicationPending := 0
if published {
warmStart := time.Now()
deadline := materializeStarted.Add(4 * time.Minute)
pending := map[int]bool{}
for actor := 0; actor < maxActors; actor++ {
pending[actor] = true
}
for len(pending) > 0 {
for actor := range pending {
if !time.Now().Before(deadline) {
t.Fatalf("published responses did not become ready within original materialization deadline: %d", len(pending))
}
status, _, err := request(after.URL, paths[2], actor)
if err != nil {
t.Fatal(err)
}
if status == 200 {
delete(pending, actor)
} else if status == http.StatusAccepted {
publicationPending++
} else {
t.Fatalf("publication prewarm status %d", status)
}
}
if len(pending) > 0 {
time.Sleep(time.Second)
}
}
publicationWarmSeconds = time.Since(warmStart).Seconds()
t.Logf("published response startup seconds=%.3f pending_responses=%d", publicationWarmSeconds, publicationPending)
}
// First reads use the existing parity requests, with no extra query load.
// Exact business JSON equality; only generated report timestamps are volatile.
firstReads := map[string]any{}
@ -550,6 +627,115 @@ func TestQueryReplicaIsolatedHTTPComparisonLocal(t *testing.T) {
}
}
}
if published {
// Separate list migration from count/report rollout; formal operations
// and personal pages keep their authoritative path.
newReader.EnableQueryReplicaPoolReads(true)
poolCases := []any{}
expectedPool := map[string]any{}
for _, mode := range []string{"old", "query"} {
url := before.URL
if mode == "query" {
url = after.URL
}
for _, pool := range []string{"available", "available&search=part0001", "available&search=absent-capture", "mine", "reviewed"} {
principalCacheMu.Lock()
principalCache = map[string]principalCacheEntry{}
principalCacheMu.Unlock()
latencies := []float64{}
var guard sync.Mutex
var wg sync.WaitGroup
for actor := 61; actor < 81; actor++ {
wg.Add(1)
go func(actor int) {
defer wg.Done()
path := "/api/human-items?pluginId=annotation-qa&stageId=semantic_review&limit=20&pool=" + pool
start := time.Now()
status, body, err := request(url, path, actor)
guard.Lock()
defer guard.Unlock()
latencies = append(latencies, float64(time.Since(start).Microseconds())/1000)
if status != 200 || err != nil {
t.Errorf("pool %s/%s/%d: status=%d err=%v %.200s", mode, pool, actor, status, err, body)
return
}
key := fmt.Sprintf("%s/%d", pool, actor)
value := isolatedBusinessJSON(t, body)
if mode == "old" {
expectedPool[key] = value
} else if !reflect.DeepEqual(expectedPool[key], value) {
t.Errorf("pool business mismatch %s", key)
}
}(actor)
}
wg.Wait()
sort.Float64s(latencies)
p95 := latencies[(len(latencies)-1)*95/100]
poolCases = append(poolCases, map[string]any{"mode": mode, "pool": pool, "requests": len(latencies), "p95Ms": p95})
if mode == "query" && p95 > 2000 {
t.Errorf("%s pool p95 exceeds 2s: %f", pool, p95)
}
}
}
report["poolListCases"] = poolCases
newReader.EnableQueryReplicaPoolReads(false)
}
if published {
opts, e := valkey.ParseURL(cacheURL)
if e != nil {
t.Fatal(e)
}
vc, e := valkey.NewClient(opts)
if e != nil {
t.Fatal(e)
}
defer vc.Close()
if e = vc.Do(ctx, vc.B().Flushdb().Sync().Build()).Error(); e != nil {
t.Fatal(e)
}
beforeAggregate := apiWorkforceAggregations.Load()
fallback := map[string]float64{}
for _, path := range paths {
now := time.Now()
beforeReport := apiWorkforceAggregations.Load()
status, _, e := request(after.URL, path, 0)
if e != nil || status != 200 {
t.Fatalf("cache loss fallback status=%d err=%v", status, e)
}
if strings.HasPrefix(path, "/api/dashboard/workforce?") && apiWorkforceAggregations.Load() != beforeReport {
t.Error("cache loss reran report aggregation")
}
fallback[strings.Split(path, "?")[0]] = float64(time.Since(now).Microseconds()) / 1000
}
report["cacheLossFallbackMs"] = fallback
report["cacheLossDetailAndReportAggregations"] = apiWorkforceAggregations.Load() - beforeAggregate
// A separate API client with an unreachable loopback cache must still
// serve durable products; this does not stop any cluster service.
t.Setenv("QUERY_VALKEY_URL", "redis://127.0.0.1:1/0")
blocked := store.NewMongoStore(sdb)
finish, e := blocked.ConfigureQueryReplicaFromEnvironment(ctx, apiTargetMonitor)
if e != nil {
t.Fatal(e)
}
defer finish()
unavailable := serve(blocked)
unavailableMs := map[string]float64{}
for _, path := range paths {
now := time.Now()
status, _, e := request(unavailable.URL, path, 0)
if e != nil || status != 200 {
t.Fatalf("unavailable cache fallback status=%d err=%v", status, e)
}
unavailableMs[strings.Split(path, "?")[0]] = float64(time.Since(now).Microseconds()) / 1000
}
report["cacheUnavailableFallbackMs"] = unavailableMs
}
if published {
report["cache"] = "dedicated Valkey with majority-committed durable response fallback"
}
report["publishedResults"] = published
report["publicationPrewarmSeconds"] = publicationWarmSeconds
report["publicationPendingResponses"] = publicationPending
report["materializationSeconds"] = materializeSeconds
report["firstReads"] = firstReads
report["cases"] = cases
@ -636,7 +822,7 @@ func isolatedSourceScans(t *testing.T, db *mongo.Database) map[string]int64 {
func isolatedSourceCPU(t *testing.T) float64 {
t.Helper()
if os.Getenv("QUERY_COMPARE_IN_CLUSTER") == "1" {
if runtime.GOOS != "linux" || os.Getenv("NODE_NAME") != "10.189.16.4" || os.Getenv("POD_NAME") != "node4-query-fixture" {
if runtime.GOOS != "linux" || (os.Getenv("NODE_NAME") != "10.189.16.4" && os.Getenv("NODE_NAME") != "10.189.16.33") || os.Getenv("POD_NAME") != "node4-query-fixture" {
t.Fatal("isolated source process identity rejected")
}
raw, err := os.ReadFile("/synthetic/mongod.pid")

View File

@ -1582,7 +1582,10 @@ func (server *server) handleDashboard(writer http.ResponseWriter, request *http.
}
summary := domain.DashboardSummary{}
if includeWorkforce {
if err := server.populateDashboardWorkforce(request.Context(), server.requestOrganizationID(request), principal, from, to, &summary); err != nil {
if err := server.populateDashboardWorkforce(request.Context(), server.requestOrganizationID(request), principal, from, to, &summary, [2]bool{strings.TrimSpace(request.URL.Query().Get("from")) == "", strings.TrimSpace(request.URL.Query().Get("to")) == ""}); err != nil {
if renderWorkforcePublicationPending(writer, request, err) {
return
}
renderError(writer, http.StatusInternalServerError, err)
return
}
@ -1647,7 +1650,10 @@ func (server *server) handleDashboard(writer http.ResponseWriter, request *http.
return
}
if includeWorkforce {
if err := server.populateDashboardWorkforce(request.Context(), server.requestOrganizationID(request), principal, from, to, &summary); err != nil {
if err := server.populateDashboardWorkforce(request.Context(), server.requestOrganizationID(request), principal, from, to, &summary, [2]bool{strings.TrimSpace(request.URL.Query().Get("from")) == "", strings.TrimSpace(request.URL.Query().Get("to")) == ""}); err != nil {
if renderWorkforcePublicationPending(writer, request, err) {
return
}
renderError(writer, http.StatusInternalServerError, err)
return
}
@ -1672,7 +1678,10 @@ func (server *server) handleDashboard(writer http.ResponseWriter, request *http.
return
}
if includeWorkforce {
if err := server.populateDashboardWorkforce(request.Context(), server.requestOrganizationID(request), principal, from, to, &summary); err != nil {
if err := server.populateDashboardWorkforce(request.Context(), server.requestOrganizationID(request), principal, from, to, &summary, [2]bool{strings.TrimSpace(request.URL.Query().Get("from")) == "", strings.TrimSpace(request.URL.Query().Get("to")) == ""}); err != nil {
if renderWorkforcePublicationPending(writer, request, err) {
return
}
renderError(writer, http.StatusInternalServerError, err)
return
}

View File

@ -0,0 +1,31 @@
package api
import (
"errors"
"fmt"
"net/http"
"net/http/httptest"
"testing"
"droboticflow/internal/store"
)
func TestWorkforcePublicationPendingIsNotAnEmptyReport(t *testing.T) {
r := httptest.NewRequest(http.MethodGet, "/report", nil)
r.Header.Set("Accept", "application/vnd.droboticflow.prepared+json")
w := httptest.NewRecorder()
if !renderWorkforcePublicationPending(w, r, fmt.Errorf("report: %w", store.ErrWorkforcePublicationPending)) || w.Code != http.StatusAccepted || w.Header().Get("Retry-After") != "1" {
t.Fatalf("publication readiness response: %d %s", w.Code, w.Body.String())
}
legacy := httptest.NewRecorder()
if !renderWorkforcePublicationPending(legacy, httptest.NewRequest(http.MethodGet, "/report", nil), store.ErrWorkforcePublicationPending) || legacy.Code != http.StatusServiceUnavailable {
t.Fatal("legacy client received a successful non-report body")
}
for _, err := range []error{nil, store.ErrQueryReplicaNotReady, errors.New("permission denied")} {
w = httptest.NewRecorder()
if renderWorkforcePublicationPending(w, r, err) || w.Body.Len() != 0 {
t.Fatal("unrelated errors must retain their existing handling")
}
}
}

View File

@ -2,6 +2,7 @@ package api
import (
"context"
"errors"
"fmt"
"net/http"
"strconv"
@ -9,6 +10,7 @@ import (
"time"
"droboticflow/internal/domain"
"droboticflow/internal/store"
"droboticflow/internal/workforcestats"
)
@ -24,6 +26,8 @@ func (s *server) workforceQuery(r *http.Request, workload bool) (workforcestats.
return q, err
}
q.ProjectID = strings.TrimSpace(v.Get("projectId"))
q.RelativeFrom = strings.TrimSpace(v.Get("from")) == ""
q.RelativeTo = strings.TrimSpace(v.Get("to")) == ""
q.GroupTemplate = workload
q.Mode = domain.RunMode(v.Get("mode"))
q.Role = strings.TrimSpace(v.Get("role"))
@ -102,9 +106,16 @@ func (s *server) handleAnnotationWorkloadSummary(w http.ResponseWriter, r *http.
}
}
if err != nil {
if renderWorkforcePublicationPending(w, r, err) {
return
}
renderError(w, 500, err)
return
}
if report.WindowFrom != "" {
q.From, _ = time.Parse(time.RFC3339Nano, report.WindowFrom)
q.To, _ = time.Parse(time.RFC3339Nano, report.WindowTo)
}
items := make([]domain.AnnotationWorkloadAggregate, 0, len(report.Items))
for _, m := range report.Items {
items = append(items, domain.AnnotationWorkloadAggregate{
@ -131,9 +142,16 @@ func (s *server) handleDashboardWorkforce(w http.ResponseWriter, r *http.Request
q.Details = false
report, err := s.dashboardWorkforceReport(r.Context(), s.requestOrganizationID(r), q)
if err != nil {
if renderWorkforcePublicationPending(w, r, err) {
return
}
renderError(w, http.StatusInternalServerError, err)
return
}
if report.WindowFrom != "" {
q.From, _ = time.Parse(time.RFC3339Nano, report.WindowFrom)
q.To, _ = time.Parse(time.RFC3339Nano, report.WindowTo)
}
workforcestats.ApplyRates(&report.Totals)
for i := range report.Items {
workforcestats.ApplyRates(&report.Items[i])
@ -194,3 +212,17 @@ func (s *server) handleWorkforceReturns(w http.ResponseWriter, r *http.Request)
}
renderJSON(w, 200, map[string]any{"items": items[offset:end], "total": count, "from": q.From.Format(time.RFC3339Nano), "to": q.To.Format(time.RFC3339Nano)})
}
func renderWorkforcePublicationPending(w http.ResponseWriter, r *http.Request, err error) bool {
if !errors.Is(err, store.ErrWorkforcePublicationPending) {
return false
}
w.Header().Set("Retry-After", "1")
// Older clients must not decode a readiness envelope as an empty report.
if !strings.Contains(r.Header.Get("Accept"), "application/vnd.droboticflow.prepared+json") {
renderError(w, http.StatusServiceUnavailable, fmt.Errorf("看板正在准备,请稍后重试"))
return true
}
renderJSON(w, http.StatusAccepted, map[string]any{"status": "building", "retryAfterMs": 1000})
return true
}

View File

@ -861,7 +861,9 @@ func (store *MongoStore) UpsertHumanBatch(ctx context.Context, item domain.Human
}
func (store *MongoStore) ListHumanItems(ctx context.Context, organizationID string, query domain.HumanItemQuery) ([]domain.HumanItemRecord, error) {
if query.PoolCandidateProjection {
// Personal pages must include a just-claimed/returned task before async
// discovery catches up. These indexed actor/history scopes stay live.
if query.PoolCandidateProjection && query.ActorUserID == "" && !query.ReviewedPool && !query.MinePoolOnly {
if target, err := store.poolReplica(ctx); err != nil {
return nil, err
} else if target != nil {

View File

@ -25,8 +25,12 @@ type MongoStore struct {
queryReplicaReads *queryReplicaReads
queryReplicaFactStore bool
queryPeriodic bool
queryPublishedResults bool
workforcePublicationTouches workforcePublicationTouches
workforcePublicationBuilds workforcePublicationBuildMemo
queryBuildWrites *queryBuildWriteGate
queryReplicaCounts queryCountCache
queryReplicaPageFlights singleflight.Group
queryReplicaBatchFacts queryBatchFacts
queryServing *queryServingOptions
productionReadFlights singleflight.Group

View File

@ -11,6 +11,17 @@ import (
// This is an explicit derived-data index operation, never API startup behavior.
// Production execution requires its own reviewed migration/index authorization.
func (s *MongoStore) EnsureQueryReadModelIndexes(ctx context.Context) error {
if s.publishedWorkforceReads() {
_, err := s.db.Collection(workforcePublicationData).Indexes().CreateOne(ctx, mongo.IndexModel{Keys: bson.D{{Key: "computedAt", Value: 1}}, Options: options.Index().SetName("workforce_publication_retention_v1").SetExpireAfterSeconds(172800)})
if err != nil {
return err
}
_, err = s.db.Collection(workforcePublicationRegistry).Indexes().CreateOne(ctx, mongo.IndexModel{Keys: bson.D{{Key: "updatedAt", Value: 1}}, Options: options.Index().SetName("workforce_publication_requests_retention_v1").SetExpireAfterSeconds(172800)})
if err != nil {
return err
}
}
if s.queryReplicaFactStore {
if err := s.ensureWorkforceFactIndexes(ctx); err != nil {
return err

View File

@ -475,6 +475,11 @@ func (b *QueryReadBuilder) run(ctx context.Context, started chan<- struct{}) err
log.Print("query_model_build result=retry category=source_or_publication")
}
}
for org := range scopes {
if e := b.store.buildWorkforcePublications(ctx, l, org); e != nil {
log.Print("workforce_publication result=retry")
}
}
if time.Since(lastCleanup) > time.Minute {
cleanup, finish := context.WithTimeout(ctx, 5*time.Second)
_ = b.store.cleanQueryModelOrphans(cleanup)

View File

@ -72,6 +72,7 @@ func (s *MongoStore) SharedQueryCache() queryserving.Cache {
}
type QueryModelHead struct {
verifiedUntil time.Time // Request-local proof; never serialized or persisted.
ChunkBytes int `bson:"chunkBytes,omitempty" json:"chunkBytes,omitempty"`
FactEpoch int64 `bson:"factEpoch,omitempty" json:"factEpoch,omitempty"`
SourceRevision int64 `bson:"sourceRevision" json:"sourceRevision"`

View File

@ -0,0 +1,79 @@
package store
import (
"context"
"droboticflow/internal/domain"
"encoding/json"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/event"
"strings"
"sync/atomic"
"testing"
"time"
)
func TestQueryReplicaPageCacheKeepsCurrentOwnershipAndPersonalReadsLocal(t *testing.T) {
var targetFinds atomic.Int64
source, target, p := newLocalReplica(t, nil, &event.CommandMonitor{Started: func(_ context.Context, e *event.CommandStartedEvent) {
name, _ := e.Command.Lookup("find").StringValueOK()
if name == "humanItems" {
targetFinds.Add(1)
}
}})
ctx := t.Context()
doc := bson.M{"id": "i", "organizationId": "o", "projectId": "p", "status": "ready", "version": 1, "createdAt": "2026-11-01T00:00:00Z", "subject": bson.M{"payload": bson.M{"annotationDraft": "preserve"}}}
if _, e := source.queryCollection("humanItems").InsertOne(ctx, doc); e != nil {
t.Fatal(e)
}
finishLocalReplica(t, p)
cache := publicationTestCache{}
target.ConfigureQueryServing(cache, nil)
if e := source.ConfigureQueryReplicaReports(target, time.Minute); e != nil {
t.Fatal(e)
}
source.EnableQueryReplicaPoolReads(true)
q := domain.HumanItemQuery{PoolCandidateProjection: true, Limit: 1}
read := WithQueryReplicaPoolRead(ctx)
targetFinds.Store(0)
for i := 0; i < 2; i++ {
rows, e := source.ListHumanItems(read, "o", q)
if e != nil || len(rows) != 1 || rows[0].Version != 1 {
t.Fatal(rows, e)
}
}
if targetFinds.Load() != 1 {
t.Fatal("warm discovery reread target tasks", targetFinds.Load())
}
raw, _ := json.Marshal(cache)
if strings.Contains(string(raw), "annotationDraft") || strings.Contains(string(raw), "preserve") {
t.Fatal("page cache copied annotation")
}
if _, e := source.humanItems.UpdateOne(ctx, bson.M{"id": "i"}, bson.M{"$set": bson.M{"status": "claimed", "assignedUserId": "other", "version": 2}}); e != nil {
t.Fatal(e)
}
rows, e := source.ListHumanItems(read, "o", q)
if e != nil || len(rows) != 1 || rows[0].Version != 2 || rows[0].AssignedUserID != "other" {
t.Fatal("cached ownership reached response", rows, e)
}
if _, e := source.humanItems.DeleteOne(ctx, bson.M{"id": "i"}); e != nil {
t.Fatal(e)
}
rows, e = source.ListHumanItems(read, "o", q)
if e != nil || len(rows) != 1 || !rows[0].QueryReplicaOmit || rows[0].CreatedAt != "2026-11-01T00:00:00Z" {
t.Fatal("deleted candidate broke pagination", rows, e)
}
if _, e := source.humanItems.InsertOne(ctx, bson.M{"id": "new", "organizationId": "o", "status": "claimed", "assignedUserId": "u", "version": 3}); e != nil {
t.Fatal(e)
}
mine := q
mine.ActorUserID = "u"
mine.MinePoolOnly = true
rows, e = source.ListHumanItems(read, "o", mine)
if e != nil || len(rows) != 1 || rows[0].ID != "new" {
t.Fatal("just claimed personal task awaited replica", rows, e)
}
rows, e = source.ListHumanItems(read, "foreign", q)
if e != nil || len(rows) != 0 {
t.Fatal("page cache crossed organization", rows, e)
}
}

View File

@ -0,0 +1,50 @@
package store
import (
"context"
"errors"
"sync/atomic"
"testing"
"time"
)
func TestQueryReplicaPageFlightsShareIdentityAndCancelIndependently(t *testing.T) {
s := &MongoStore{}
started, release := make(chan struct{}), make(chan struct{})
var calls atomic.Int64
read := func(ctx context.Context) (queryPoolPage, error) {
calls.Add(1)
deadline, ok := ctx.Deadline()
if !ok || time.Until(deadline) > 8*time.Second {
t.Error("page work lost original bound")
}
close(started)
select {
case <-release:
return queryPoolPage{Key: "scope", Items: []queryPoolPageIdentity{{ID: "i", CreatedAt: "before"}}}, nil
case <-ctx.Done():
return queryPoolPage{}, ctx.Err()
}
}
ctx, cancel := context.WithCancel(t.Context())
first := make(chan error, 1)
go func() { _, err := s.shareReplicaPoolPage(ctx, "scope", read); first <- err }()
<-started
second := s.queryReplicaPageFlights.DoChan("scope", func() (any, error) { t.Error("duplicate candidate query"); return queryPoolPage{}, nil })
cancel()
if err := <-first; !errors.Is(err, context.Canceled) {
t.Fatal(err)
}
close(release)
result := <-second
if result.Err != nil || result.Val.(queryPoolPage).Items[0].ID != "i" || calls.Load() != 1 {
t.Fatal(result, calls.Load())
}
sentinel := errors.New("query unavailable")
if _, err := s.shareReplicaPoolPage(t.Context(), "scope", func(context.Context) (queryPoolPage, error) { return queryPoolPage{}, sentinel }); !errors.Is(err, sentinel) {
t.Fatal("failed query cached", err)
}
if _, err := s.shareReplicaPoolPage(t.Context(), "scope", func(context.Context) (queryPoolPage, error) { calls.Add(1); return queryPoolPage{}, nil }); err != nil || calls.Load() != 2 {
t.Fatal("completed flight survived", err, calls.Load())
}
}

View File

@ -3,8 +3,10 @@ package store
import (
"context"
"droboticflow/internal/domain"
"droboticflow/internal/queryserving"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo/options"
"time"
)
type queryReplicaPoolReadKey struct{}
@ -46,7 +48,7 @@ func (s *MongoStore) EnableQueryReplicaCountReads(enabled bool) {
// Missing/changed candidates keep an internal cursor placeholder so deletion
// cannot falsely end pagination. They never enter the public response.
func (s *MongoStore) replicaPoolCandidates(ctx context.Context, target *MongoStore, org string, q domain.HumanItemQuery) ([]domain.HumanItemRecord, error) {
rows, e := target.ListHumanItems(ctx, org, q)
rows, e := target.replicaPoolPageIdentities(ctx, org, q)
if e != nil || len(rows) == 0 {
return rows, e
}
@ -89,6 +91,90 @@ func (s *MongoStore) replicaPoolCandidates(ctx context.Context, target *MongoSto
return rows, nil
}
type queryPoolPageIdentity struct {
ID string `json:"id"`
CreatedAt string `json:"createdAt"`
}
type queryPoolPage struct {
Key string `json:"key"`
Items []queryPoolPageIdentity `json:"items"`
}
// Cache only discovery identities, never payloads, permissions or canClaim.
// The source still fetches the current page before applying business rules.
func (s *MongoStore) replicaPoolPageIdentities(ctx context.Context, org string, q domain.HumanItemQuery) ([]domain.HumanItemRecord, error) {
if s.SharedQueryCache() == nil || q.Limit <= 0 || q.Limit > 200 {
return s.ListHumanItems(ctx, org, q)
}
epoch, err := s.countFactEpoch(ctx)
if err != nil {
return nil, err
}
key := queryserving.Key("pool-page-identities/v1", s.db.Name(), org, epoch, q)
page, err := s.shareReplicaPoolPage(ctx, key, func(work context.Context) (queryPoolPage, error) {
return s.loadReplicaPoolPage(work, key, org, q)
})
if err != nil {
return nil, err
}
rows := make([]domain.HumanItemRecord, 0, len(page.Items))
for _, item := range page.Items {
rows = append(rows, domain.HumanItemRecord{ID: item.ID, CreatedAt: item.CreatedAt})
}
return rows, nil
}
// Match the existing shared-name lookup contract: only in-flight data reads
// coalesce, one canceled waiter cannot cancel another, and work stays bounded.
func (s *MongoStore) shareReplicaPoolPage(ctx context.Context, key string, read func(context.Context) (queryPoolPage, error)) (queryPoolPage, error) {
if err := ctx.Err(); err != nil {
return queryPoolPage{}, err
}
flight := s.queryReplicaPageFlights.DoChan(key, func() (any, error) {
work, cancel := context.WithTimeout(context.WithoutCancel(ctx), 8*time.Second)
defer cancel()
return read(work)
})
select {
case <-ctx.Done():
return queryPoolPage{}, ctx.Err()
case result := <-flight:
if result.Err != nil {
return queryPoolPage{}, result.Err
}
page := result.Val.(queryPoolPage)
page.Items = append([]queryPoolPageIdentity(nil), page.Items...)
return page, nil
}
}
func (s *MongoStore) loadReplicaPoolPage(ctx context.Context, key, org string, q domain.HumanItemQuery) (queryPoolPage, error) {
var page queryPoolPage
if queryserving.GetJSON(ctx, s.SharedQueryCache(), key, &page) && page.Key == key && len(page.Items) <= q.Limit {
valid := true
for _, item := range page.Items {
if item.ID == "" {
valid = false
break
}
}
if valid {
return page, nil
}
}
rows, err := s.ListHumanItems(ctx, org, q)
if err != nil {
return queryPoolPage{}, err
}
page = queryPoolPage{Key: key, Items: make([]queryPoolPageIdentity, 0, len(rows))}
for _, item := range rows {
page.Items = append(page.Items, queryPoolPageIdentity{ID: item.ID, CreatedAt: item.CreatedAt})
}
if len(rows) <= q.Limit {
queryserving.PutJSON(ctx, s.SharedQueryCache(), key, page, 2*time.Second)
}
return page, nil
}
func (s *MongoStore) replicaAdminPool(ctx context.Context, target *MongoStore, org string, q AdminAnnotationPoolQuery) ([]domain.HumanItemRecord, error) {
rows, e := target.ListAdminAnnotationPool(ctx, org, q)
if e != nil || len(rows) == 0 {

View File

@ -11,6 +11,7 @@ import (
// different deployments even when their database names happen to be identical.
func (s *MongoStore) ConfigureQueryServingFromEnvironment() (func(), error) {
close := func() {}
s.ConfigureQueryPublishedResults(os.Getenv("QUERY_PUBLISHED_RESULTS") == "1")
if err := s.ConfigureQueryRefreshMode(os.Getenv("QUERY_REPLICA_REFRESH_MODE")); err != nil {
return close, err
}

View File

@ -0,0 +1,144 @@
package store
import (
"context"
"fmt"
"reflect"
"sort"
"sync/atomic"
"testing"
"droboticflow/internal/domain"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/event"
)
// Keep the prior per-project algorithm as the compatibility oracle.
func legacyAuthorizationListForTest(ctx context.Context, s *MongoStore, org, project string) ([]domain.SupplierProjectAuthorization, error) {
projects := []domain.ProjectRecord{}
if project != "" {
p, e := s.GetProject(ctx, project)
if e != nil {
return nil, e
}
if p != nil && annotationOrganizationMatches(p.OrganizationID, org) {
projects = append(projects, *p)
}
} else {
var e error
projects, e = s.ListProjectsByOrganization(ctx, org)
if e != nil {
return nil, e
}
}
result := []domain.SupplierProjectAuthorization{}
for _, p := range projects {
initialized, e := s.supplierProjectAuthorizationInitialized(ctx, org, p.ID)
if e != nil {
return nil, e
}
if !initialized {
seeds, e := s.authorizationSeeds(ctx, org, p.ID)
if e != nil {
return nil, e
}
result = append(result, seeds...)
continue
}
cur, e := s.supplierProjectAuthorizations.Find(ctx, bson.M{"organizationId": normalizedOrganizationID(org), "projectId": p.ID})
if e != nil {
return nil, e
}
grants, e := decodeMany[domain.SupplierProjectAuthorization](ctx, cur)
if e != nil {
return nil, e
}
result = append(result, grants...)
}
sort.Slice(result, func(i, j int) bool {
if result[i].ProjectID != result[j].ProjectID {
return result[i].ProjectID < result[j].ProjectID
}
return result[i].SupplierID < result[j].SupplierID
})
return result, nil
}
func normalizedAuthorizationTestRows(rows []domain.SupplierProjectAuthorization) []domain.SupplierProjectAuthorization {
for i := range rows {
if rows[i].Source == "existing_member" {
rows[i].CreatedAt = ""
rows[i].UpdatedAt = ""
for j := range rows[i].History {
rows[i].History[j].At = ""
}
}
}
return rows
}
func TestSupplierAuthorizationBatchPreservesLegacyAndLiveRevocationLocal(t *testing.T) {
var reads, writes atomic.Int64
s := newQuerySyncSource(t, &event.CommandMonitor{Started: func(_ context.Context, e *event.CommandStartedEvent) {
switch e.CommandName {
case "find", "aggregate", "getMore":
reads.Add(1)
case "update", "insert", "delete", "findAndModify":
writes.Add(1)
}
}})
ctx := t.Context()
org := domain.DefaultOrganizationID
insert := func(name string, doc bson.M) {
t.Helper()
if _, e := s.db.Collection(name).InsertOne(ctx, doc); e != nil {
t.Fatal(e)
}
}
insert("users", bson.M{"id": "active", "supplierId": "v", "role": "annotator", "status": "active"})
insert("users", bson.M{"id": "disabled", "organizationId": org, "supplierId": "v-disabled", "role": "annotator", "status": "disabled"})
insert("suppliers", bson.M{"id": "v", "status": "active"})
insert("suppliers", bson.M{"id": "v-disabled", "organizationId": org, "status": "active"})
for i := 0; i < 30; i++ {
p := fmt.Sprintf("p-%02d", i)
insert("projects", bson.M{"id": p, "organizationId": org, "name": p})
insert("projectMembers", bson.M{"id": "m-" + p, "projectId": p, "userId": "active", "supplierId": "v", "createdAt": "2026-01-01T00:00:00Z"})
insert("projectMembers", bson.M{"id": "d-" + p, "organizationId": org, "projectId": p, "userId": "disabled", "supplierId": "v-disabled"})
}
insert("projects", bson.M{"id": " p-space ", "organizationId": org})
insert("projectMembers", bson.M{"id": "space", "projectId": "p-space", "userId": "active", "supplierId": "v"})
insert("projectMembers", bson.M{"id": "raw-space", "projectId": " p-space ", "userId": "active", "supplierId": "v"})
insert("projects", bson.M{"id": " ", "organizationId": org})
insert("projects", bson.M{"id": "foreign", "organizationId": "other"})
insert("supplierProjectAuthorizationInit", bson.M{"id": supplierAuthorizationInitID(org, "p-00")})
insert("supplierProjectAuthorizations", bson.M{"id": "revoked", "organizationId": org, "projectId": "p-00", "supplierId": "v", "status": "revoked", "drainTaskIds": bson.A{"task-old"}, "version": 4})
for _, project := range []string{"", "p-00", "p-01", " p-space ", " ", "foreign", "missing"} {
want, e := legacyAuthorizationListForTest(ctx, s, org, project)
if e != nil {
t.Fatal(e)
}
reads.Store(0)
writes.Store(0)
got, e := s.ListSupplierProjectAuthorizations(ctx, org, project)
if e != nil {
t.Fatal(e)
}
if !reflect.DeepEqual(normalizedAuthorizationTestRows(want), normalizedAuthorizationTestRows(got)) {
t.Fatalf("scope=%q mismatch want=%+v got=%+v", project, want, got)
}
if writes.Load() != 0 {
t.Fatal("authorization GET wrote business state")
}
if project == "" && reads.Load() > 7 {
t.Fatal("per-project source reads remain", reads.Load())
}
}
if _, e := s.db.Collection("users").UpdateOne(ctx, bson.M{"id": "active"}, bson.M{"$set": bson.M{"status": "disabled"}}); e != nil {
t.Fatal(e)
}
rows, e := s.ListSupplierProjectAuthorizations(ctx, org, "")
if e != nil {
t.Fatal(e)
}
if len(rows) != 1 || rows[0].Status != "revoked" {
t.Fatal("disabled account reused a prior grant", rows)
}
}

View File

@ -357,29 +357,102 @@ func (s *MongoStore) ListSupplierProjectAuthorizations(ctx context.Context, org,
return nil, err
}
}
// Resolve initialization and legacy seed inputs once per request. No grants
// or permission decisions survive this call; revocations remain live reads.
if len(projects) == 0 {
return []domain.SupplierProjectAuthorization{}, nil
}
initIDs := make([]string, 0, len(projects))
for _, p := range projects {
initIDs = append(initIDs, supplierAuthorizationInitID(org, p.ID))
}
cur, err := s.supplierProjectAuthorizationInit.Find(ctx, bson.M{"id": bson.M{"$in": initIDs}}, options.Find().SetProjection(bson.M{"id": 1}))
if err != nil {
return nil, err
}
markers, err := decodeMany[struct {
ID string `bson:"id"`
}](ctx, cur)
if err != nil {
return nil, err
}
initialized := map[string]bool{}
for _, marker := range markers {
initialized[marker.ID] = true
}
readyIDs, legacyIDs := []string{}, []string{}
allLegacyMembers := false
for _, p := range projects {
if initialized[supplierAuthorizationInitID(org, p.ID)] {
readyIDs = append(readyIDs, p.ID)
} else {
id := strings.TrimSpace(p.ID)
if id == "" {
allLegacyMembers = true
}
legacyIDs = append(legacyIDs, id)
}
}
grantsByProject := map[string][]domain.SupplierProjectAuthorization{}
if len(readyIDs) > 0 {
cur, err := s.supplierProjectAuthorizations.Find(ctx, bson.M{"organizationId": normalizedOrganizationID(org), "projectId": bson.M{"$in": readyIDs}})
if err != nil {
return nil, err
}
grants, err := decodeMany[domain.SupplierProjectAuthorization](ctx, cur)
if err != nil {
return nil, err
}
for _, grant := range grants {
grantsByProject[grant.ProjectID] = append(grantsByProject[grant.ProjectID], grant)
}
}
var members []domain.ProjectMemberRecord
var users []domain.UserRecord
var suppliers []domain.SupplierRecord
if len(legacyIDs) > 0 {
filter := orgCompatibilityFilter(org)
if !allLegacyMembers {
filter["projectId"] = bson.M{"$in": legacyIDs}
}
cur, err := s.projectMembers.Find(ctx, filter, options.Find().SetSort(bson.D{{Key: "createdAt", Value: 1}}))
if err != nil {
return nil, err
}
members, err = decodeMany[domain.ProjectMemberRecord](ctx, cur)
if err != nil {
return nil, err
}
userIDs := []string{}
seen := map[string]bool{}
for _, member := range members {
if !seen[member.UserID] {
seen[member.UserID] = true
userIDs = append(userIDs, member.UserID)
}
}
users, err = s.ListProjectMemberUsers(ctx, org, userIDs)
if err != nil {
return nil, err
}
suppliers, err = s.ListSuppliers(ctx, org)
if err != nil {
return nil, err
}
}
result := []domain.SupplierProjectAuthorization{}
for _, p := range projects {
initialized, err := s.supplierProjectAuthorizationInitialized(ctx, org, p.ID)
if err != nil {
return nil, err
}
if !initialized {
seed, err := s.authorizationSeeds(ctx, org, p.ID)
if err != nil {
return nil, err
if initialized[supplierAuthorizationInitID(org, p.ID)] {
result = append(result, grantsByProject[p.ID]...)
} else {
// The legacy list trims its query ID, then the seed reducer compares
// the original ID. Preserve its empty result for malformed spaced IDs.
seedMembers := members
if trimmed := strings.TrimSpace(p.ID); trimmed != "" && trimmed != p.ID {
seedMembers = nil
}
result = append(result, seed...)
continue
result = append(result, effectiveSupplierAuthorizationSeeds(org, p.ID, seedMembers, users, suppliers)...)
}
cursor, err := s.supplierProjectAuthorizations.Find(ctx, bson.M{"organizationId": normalizedOrganizationID(org), "projectId": p.ID})
if err != nil {
return nil, err
}
grants, err := decodeMany[domain.SupplierProjectAuthorization](ctx, cursor)
if err != nil {
return nil, err
}
result = append(result, grants...)
}
sort.Slice(result, func(i, j int) bool {
if result[i].ProjectID != result[j].ProjectID {

View File

@ -81,6 +81,10 @@ func (s *MongoStore) workforceHead(ctx context.Context, org string) (QueryModelH
if h.Collection != workforceFactCollection || h.FactEpoch < 1 || h.Generation != workforceFactSchema || time.Since(h.AsOf) < 0 || (!unchanged && time.Since(h.AsOf) > maxAge) {
return h, true, ErrQueryReplicaNotReady
}
h.verifiedUntil = h.AsOf.Add(maxAge)
if unchanged {
h.verifiedUntil = time.Now().Add(maxAge)
}
return h, true, nil
}
func workforceVersionFilter(org string, epoch int64) bson.M {
@ -549,6 +553,14 @@ func (s *MongoStore) loadWorkforceFactReport(ctx context.Context, org string, q
if q.MatchUser && q.UserID == "" {
return workforcestats.Report{}, false, nil
}
if s.publishedWorkforceReads() && !q.Details {
raw, _ := json.Marshal(q)
if len(raw) <= 4096 {
if report, used, err := s.loadPublishedWorkforceReport(ctx, org, q); used || err != nil {
return report, used, err
}
}
}
h, exists, err := s.workforceHead(ctx, org)
if s.periodicQueryStore() && !exists && err == nil {
return workforcestats.Report{}, true, ErrQueryReplicaNotReady
@ -575,6 +587,14 @@ func (s *MongoStore) loadWorkforceFactReport(ctx context.Context, org string, q
}
}
}
return s.computeWorkforceFactReport(ctx, org, q, h)
}
func (s *MongoStore) computeWorkforceFactReport(ctx context.Context, org string, q workforcestats.Query, h QueryModelHead) (workforcestats.Report, bool, error) {
key := queryserving.Key(workforceFactSchema, s.db.Name(), org, h.FactEpoch, q)
if s.periodicQueryStore() {
key = queryserving.Key("periodic-report/v1", key)
}
pipeline := append(workforceFactReadStages(org, h, q), bson.D{{Key: "$project", Value: bson.M{"kind": 1, "metric": 1, "return": 1, "eventCount": 1, "userId": 1, "userName": 1, "supplierId": 1, "supplierName": 1, "role": 1, "templateVersion": 1, "day": 1}}}, bson.D{{Key: "$facet", Value: bson.M{"items": workforceMetricPipeline(q, false), "totals": workforceMetricPipeline(q, true)}}})
cur, err := s.db.Collection(workforceFactCollection).Aggregate(ctx, pipeline, options.Aggregate().SetMaxTime(8*time.Second))
if err != nil {

View File

@ -0,0 +1,354 @@
package store
import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"sort"
"sync"
"time"
"droboticflow/internal/queryserving"
"droboticflow/internal/workforcestats"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
)
const workforcePublicationSchema = "workforce-publication/v3"
var ErrWorkforcePublicationPending = fmt.Errorf("workforce publication pending: %w", ErrQueryReplicaNotReady)
const workforcePublicationRegistry = "queryWorkforcePublicationRequests"
const workforcePublicationData = "queryWorkforcePublications"
const workforcePublicationLimit = 1024
const workforcePublicationTouchInterval = 15 * time.Minute
const workforcePublicationMaxBytes = 12 << 20
type workforcePublicationRequest struct {
Query json.RawMessage `bson:"query"`
RequestedAt time.Time `bson:"requestedAt"`
}
type workforcePublication struct {
LegacyAggregation bool `bson:"legacyAggregation,omitempty"`
Certificate queryReplicaReadState `bson:"certificate"`
ExpiresAt time.Time `bson:"expiresAt"`
Key string `bson:"_id"`
Schema string `bson:"schema"`
Organization string `bson:"organizationId"`
Database string `bson:"database"`
Epoch int64 `bson:"epoch"`
ComputedAt time.Time `bson:"computedAt"`
AsOf time.Time `bson:"asOf"`
Body []byte `bson:"body"`
Digest string `bson:"digest"`
}
type workforcePublicationTouches struct {
sync.Mutex
times map[string]time.Time
}
type workforcePublicationBuildStamp struct {
Epoch int64
CheckedAt time.Time
}
type workforcePublicationBuildMemo struct {
sync.Mutex
values map[string]workforcePublicationBuildStamp
}
func (s *MongoStore) rememberWorkforcePublicationBuild(key string, epoch int64) {
s.workforcePublicationBuilds.Lock()
defer s.workforcePublicationBuilds.Unlock()
if len(s.workforcePublicationBuilds.values) >= 4096 {
clear(s.workforcePublicationBuilds.values)
}
if s.workforcePublicationBuilds.values == nil {
s.workforcePublicationBuilds.values = map[string]workforcePublicationBuildStamp{}
}
s.workforcePublicationBuilds.values[key] = workforcePublicationBuildStamp{Epoch: epoch, CheckedAt: time.Now()}
}
// Startup-only. Published reads are independent of business operation gates.
func (s *MongoStore) ConfigureQueryPublishedResults(enabled bool) { s.queryPublishedResults = enabled }
func (s *MongoStore) publishedWorkforceReads() bool {
return s.periodicQueryStore() && s.queryPublishedResults
}
func canonicalWorkforcePublicationQuery(q workforcestats.Query) workforcestats.Query {
if q.RelativeFrom {
q.From = time.Time{}
}
if q.RelativeTo {
q.To = time.Time{}
}
return q
}
func (s *MongoStore) workforcePublicationKey(org string, q workforcestats.Query) string {
q = canonicalWorkforcePublicationQuery(q)
return queryserving.Key(workforcePublicationSchema, s.db.Name(), org, q)
}
func (s *MongoStore) validWorkforcePublication(v workforcePublication, key, org string) bool {
age := time.Since(v.ComputedAt)
certificateAge := time.Since(v.Certificate.ObservedAt)
if v.Certificate.Binding.SourceDatabase == "" || !v.Certificate.Ready || v.Certificate.Schema != queryReplicaSchema || certificateAge < 0 || certificateAge > queryPeriodicDisplayMaxAge || v.Certificate.Binding.Namespace != queryReplicaNamespace(v.Certificate.Binding.SourceDatabase) || v.ExpiresAt.After(v.Certificate.ObservedAt.Add(queryPeriodicDisplayMaxAge)) {
return false
}
if v.Key != key || v.Schema != workforcePublicationSchema || v.Organization != org || v.Database != s.db.Name() || v.Epoch < 1 || age < 0 || age > queryPeriodicDisplayMaxAge || v.ExpiresAt.IsZero() || !time.Now().Before(v.ExpiresAt) || v.ExpiresAt.After(v.ComputedAt.Add(queryPeriodicDisplayMaxAge)) || v.AsOf.IsZero() || v.AsOf.After(v.ComputedAt) || (len(v.Body) == 0 && !v.LegacyAggregation) || (v.LegacyAggregation && len(v.Body) != 0) || len(v.Body) > workforcePublicationMaxBytes {
return false
}
sum := sha256.Sum256(v.Body)
return v.Digest == hex.EncodeToString(sum[:])
}
// Only the exact server-derived scope is registered, never an authorization
// conclusion. One bounded org document makes the admission limit atomic.
func (s *MongoStore) requestWorkforcePublication(ctx context.Context, org string, q workforcestats.Query) error {
raw, err := json.Marshal(canonicalWorkforcePublicationQuery(q))
if err != nil || len(raw) > 4096 {
return errors.New("workforce publication query exceeds budget")
}
key := s.workforcePublicationKey(org, q)
now := time.Now().UTC()
s.workforcePublicationTouches.Lock()
if now.Sub(s.workforcePublicationTouches.times[key]) < workforcePublicationTouchInterval {
s.workforcePublicationTouches.Unlock()
return nil
}
s.workforcePublicationTouches.Unlock()
short, cancel := context.WithTimeout(ctx, 200*time.Millisecond)
defer cancel()
c := s.db.Collection(workforcePublicationRegistry)
field := "requests." + key
filter := bson.M{"_id": org, "$or": bson.A{bson.M{field: bson.M{"$exists": true}}, bson.M{"$expr": bson.M{"$lt": bson.A{bson.M{"$size": bson.M{"$objectToArray": bson.M{"$ifNull": bson.A{"$requests", bson.M{}}}}}, workforcePublicationLimit}}}}}
update := bson.M{"$set": bson.M{field: workforcePublicationRequest{Query: raw, RequestedAt: now}, "updatedAt": now}}
r, err := c.UpdateOne(short, filter, update)
if err != nil {
return err
}
if r.MatchedCount == 0 {
_, err = c.UpdateOne(short, bson.M{"_id": org}, bson.M{"$setOnInsert": bson.M{"requests": bson.M{}, "updatedAt": now}}, options.Update().SetUpsert(true))
if err != nil {
return err
}
r, err = c.UpdateOne(short, filter, update)
if err != nil {
return err
}
}
if r.MatchedCount != 1 {
return errors.New("workforce publication registry full")
}
s.workforcePublicationTouches.Lock()
if len(s.workforcePublicationTouches.times) >= workforcePublicationLimit {
clear(s.workforcePublicationTouches.times)
}
if s.workforcePublicationTouches.times == nil {
s.workforcePublicationTouches.times = map[string]time.Time{}
}
s.workforcePublicationTouches.times[key] = now
s.workforcePublicationTouches.Unlock()
return nil
}
func (s *MongoStore) loadPublishedWorkforceReport(ctx context.Context, org string, q workforcestats.Query) (workforcestats.Report, bool, error) {
key := s.workforcePublicationKey(org, q)
// A failed catalog touch must not hide an already published complete result.
defer func() { _ = s.requestWorkforcePublication(ctx, org, q) }()
var v workforcePublication
if queryserving.GetJSON(ctx, s.SharedQueryCache(), key, &v) && s.validWorkforcePublication(v, key, org) {
if v.LegacyAggregation {
return workforcestats.Report{}, false, nil
}
var report workforcestats.Report
if json.Unmarshal(v.Body, &report) == nil {
return report, true, nil
}
}
err := s.db.Collection(workforcePublicationData).FindOne(ctx, bson.M{"_id": key}).Decode(&v)
if err != nil && !errors.Is(err, mongo.ErrNoDocuments) {
return workforcestats.Report{}, true, err
}
if err != nil || !s.validWorkforcePublication(v, key, org) {
return workforcestats.Report{}, true, ErrWorkforcePublicationPending
}
if v.LegacyAggregation {
return workforcestats.Report{}, false, nil
}
var report workforcestats.Report
if err = json.Unmarshal(v.Body, &report); err != nil {
return report, true, err
}
queryserving.PutJSON(ctx, s.SharedQueryCache(), key, v, queryPeriodicDisplayMaxAge)
return report, true, nil
}
// This runs only under the builder's fenced lease. HTTP never calls it.
// Failed/new computations leave the previous complete document and cache intact.
func (s *MongoStore) buildWorkforcePublications(ctx context.Context, l queryModelLease, org string) error {
if !s.publishedWorkforceReads() {
return nil
}
var registry struct {
Requests map[string]workforcePublicationRequest `bson:"requests"`
}
err := s.db.Collection(workforcePublicationRegistry).FindOne(ctx, bson.M{"_id": org}).Decode(&registry)
if errors.Is(err, mongo.ErrNoDocuments) {
return nil
}
if err != nil {
return err
}
var certificate queryReplicaReadState
if err = s.db.Collection(queryReplicaControl).FindOne(ctx, bson.M{"_id": "state"}).Decode(&certificate); err != nil {
return err
}
if !certificate.Ready || certificate.Schema != queryReplicaSchema || time.Since(certificate.ObservedAt) < 0 || time.Since(certificate.ObservedAt) > queryPeriodicDisplayMaxAge {
return ErrQueryReplicaNotReady
}
head, exists, err := s.workforceHead(ctx, org)
if err != nil {
return err
}
if !exists {
return ErrQueryReplicaNotReady
}
keys := make([]string, 0, len(registry.Requests))
for key := range registry.Requests {
keys = append(keys, key)
}
sort.Strings(keys)
built := 0
passStarted := time.Now()
for _, key := range keys {
request := registry.Requests[key]
if time.Since(request.RequestedAt) > 24*time.Hour {
_, err = s.db.Collection(workforcePublicationRegistry).UpdateOne(ctx, bson.M{"_id": org, "requests." + key + ".requestedAt": request.RequestedAt}, bson.M{"$unset": bson.M{"requests." + key: ""}})
if err != nil {
return err
}
continue
}
var q workforcestats.Query
if len(request.Query) > 4096 || json.Unmarshal(request.Query, &q) != nil || key != s.workforcePublicationKey(org, q) || q.Details || (q.MatchUser && q.UserID == "") {
continue
}
s.workforcePublicationBuilds.Lock()
stamp := s.workforcePublicationBuilds.values[key]
s.workforcePublicationBuilds.Unlock()
if stamp.Epoch == head.FactEpoch && time.Since(stamp.CheckedAt) < time.Minute {
continue
}
var current workforcePublication
err = s.db.Collection(workforcePublicationData).FindOne(ctx, bson.M{"_id": key}).Decode(&current)
if err != nil && !errors.Is(err, mongo.ErrNoDocuments) {
return err
}
if err == nil && s.validWorkforcePublication(current, key, org) && current.Epoch == head.FactEpoch && time.Since(current.ComputedAt) < queryPeriodicBuildInterval {
queryserving.PutJSON(ctx, s.SharedQueryCache(), key, current, queryPeriodicDisplayMaxAge)
s.rememberWorkforcePublicationBuild(key, head.FactEpoch)
continue
}
if built >= 16 || time.Since(passStarted) > 8*time.Second {
continue
}
windowNow := time.Now().UTC()
if q.RelativeFrom {
q.From = windowNow.Add(-30 * 24 * time.Hour)
}
if q.RelativeTo {
q.To = windowNow
}
if (q.RelativeFrom || q.RelativeTo) && !q.From.Before(q.To) {
continue
}
report, used, e := s.computeWorkforceFactReport(ctx, org, q, head)
if e != nil {
return e
}
if !used {
continue
}
if q.RelativeFrom || q.RelativeTo {
report.WindowFrom = q.From.Format(time.RFC3339Nano)
report.WindowTo = q.To.Format(time.RFC3339Nano)
}
body, e := json.Marshal(report)
if e != nil {
return e
}
legacy := len(body) > workforcePublicationMaxBytes
if legacy {
body = nil
}
sum := sha256.Sum256(body)
next := workforcePublication{LegacyAggregation: legacy, Certificate: certificate, ExpiresAt: head.verifiedUntil, Key: key, Schema: workforcePublicationSchema, Organization: org, Database: s.db.Name(), Epoch: head.FactEpoch, ComputedAt: time.Now().UTC(), AsOf: head.AsOf, Body: body, Digest: hex.EncodeToString(sum[:])}
if next.ExpiresAt.After(next.ComputedAt.Add(queryPeriodicDisplayMaxAge)) {
next.ExpiresAt = next.ComputedAt.Add(queryPeriodicDisplayMaxAge)
}
if bound := certificate.ObservedAt.Add(queryPeriodicDisplayMaxAge); bound.Before(next.ExpiresAt) {
next.ExpiresAt = bound
}
if !time.Now().Before(next.ExpiresAt) {
return ErrQueryReplicaNotReady
}
err = s.workforceFactTransaction(ctx, l, func(sc mongo.SessionContext) error {
var current queryReplicaReadState
if e := s.db.Collection(queryReplicaControl).FindOne(sc, bson.M{"_id": "state"}).Decode(&current); e != nil {
return e
}
if current.Binding != certificate.Binding || current.Schema != certificate.Schema || !current.Ready {
return ErrQueryReplicaNotReady
}
h, ok, e := s.workforceHead(sc, org)
if e != nil {
return e
}
if !ok || h.FactEpoch != head.FactEpoch {
return ErrQueryReplicaNotReady
}
if h.verifiedUntil.Before(next.ExpiresAt) {
next.ExpiresAt = h.verifiedUntil
}
_, e = s.db.Collection(workforcePublicationData).ReplaceOne(sc, bson.M{"_id": key}, next, options.Replace().SetUpsert(true))
return e
})
if err != nil {
return err
}
queryserving.PutJSON(ctx, s.SharedQueryCache(), key, next, queryPeriodicDisplayMaxAge)
s.rememberWorkforcePublicationBuild(key, head.FactEpoch)
built++
}
return nil
}
// API authorization already derived q from the current principal. A complete
// cache product carries a bounded committed sync certificate, so a hit needs no
// query-store metadata fetch. Initial physical binding verification is retained.
func (s *MongoStore) cachedPublishedWorkforce(ctx context.Context, org string, q workforcestats.Query) (workforcestats.Report, bool, error) {
r := s.queryReplicaReads
if r == nil || r.target == nil || !r.target.publishedWorkforceReads() || q.Details || (q.MatchUser && q.UserID == "") {
return workforcestats.Report{}, false, nil
}
if err := ctx.Err(); err != nil {
return workforcestats.Report{}, true, err
}
if err := s.verifyReplicaRead(ctx, r); err != nil {
return workforcestats.Report{}, true, err
}
target := r.target
key := target.workforcePublicationKey(org, q)
var v workforcePublication
if !queryserving.GetJSON(ctx, target.SharedQueryCache(), key, &v) || !target.validWorkforcePublication(v, key, org) || v.Certificate.Binding.SourceDatabase != s.db.Name() || v.Certificate.Binding.Namespace != queryReplicaNamespace(s.db.Name()) {
return workforcestats.Report{}, false, nil
}
var report workforcestats.Report
if json.Unmarshal(v.Body, &report) != nil {
return workforcestats.Report{}, false, nil
}
_ = target.requestWorkforcePublication(ctx, org, q)
return report, true, nil
}

View File

@ -0,0 +1,209 @@
package store
import (
"context"
"encoding/json"
"errors"
"os"
"reflect"
"sync/atomic"
"testing"
"time"
"droboticflow/internal/domain"
"droboticflow/internal/workforcestats"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/event"
)
type publicationTestCache map[string][]byte
func (c publicationTestCache) Get(_ context.Context, k string) ([]byte, bool) {
v, ok := c[k]
return append([]byte(nil), v...), ok
}
func (c publicationTestCache) Put(_ context.Context, k string, v []byte, _ time.Duration) {
c[k] = append([]byte(nil), v...)
}
func TestWorkforcePublicationDurableFallbackAndIsolationLocal(t *testing.T) {
if os.Getenv("QUERY_REPLICA_LOCAL_TEST") != "1" {
t.Skip("owned synthetic replicas only")
}
var aggregations atomic.Int32
var targetReads atomic.Int32
monitor := &event.CommandMonitor{Started: func(_ context.Context, e *event.CommandStartedEvent) {
if e.CommandName == "aggregate" || e.CommandName == "find" {
targetReads.Add(1)
}
if e.CommandName == "aggregate" {
aggregations.Add(1)
}
}}
source, target, p := newLocalReplica(t, nil, monitor)
seedWorkforceStats(t, source)
finishLocalReplica(t, p)
target.queryPeriodic = true
target.ConfigureQueryPublishedResults(true)
cache := publicationTestCache{}
target.ConfigureQueryServing(cache, nil)
if err := target.EnsureQueryReadModelIndexes(t.Context()); err != nil {
t.Fatal(err)
}
lease, ok, err := target.acquireQueryModelLease(t.Context())
if err != nil || !ok {
t.Fatal(ok, err)
}
if err = target.rebuildWorkforceFacts(t.Context(), lease, "workforce-test", 0); err != nil {
t.Fatal(err)
}
query := workforcestats.Query{Mode: domain.RunModeAll}
snapshot, err := source.LoadWorkforceStats(t.Context(), "workforce-test", query)
if err != nil {
t.Fatal(err)
}
want := workforcestats.Build(snapshot, query)
want.Details = nil
aggregations.Store(0)
if _, _, err = target.loadWorkforceFactReport(t.Context(), "workforce-test", query); !errors.Is(err, ErrQueryReplicaNotReady) {
t.Fatal("unpublished report calculated inline", err)
}
if aggregations.Load() != 0 {
t.Fatal("HTTP miss aggregated history")
}
if err = target.buildWorkforcePublications(t.Context(), lease, "workforce-test"); err != nil {
t.Fatal(err)
}
aggregations.Store(0)
got, _, err := target.loadWorkforceFactReport(t.Context(), "workforce-test", query)
if err != nil {
t.Fatal(err)
}
got.AsOf = ""
if !reflect.DeepEqual(got, want) {
t.Fatalf("published result differs %#v %#v", got, want)
}
if err = source.ConfigureQueryReplicaReports(target, time.Minute); err != nil {
t.Fatal(err)
}
if _, err = source.LoadWorkforceReportCached(t.Context(), "workforce-test", query); err != nil {
t.Fatal(err)
}
targetReads.Store(0)
if _, err = source.LoadWorkforceReportCached(t.Context(), "workforce-test", query); err != nil {
t.Fatal(err)
}
if targetReads.Load() != 0 {
t.Fatal("warm published cache required query Mongo")
}
canceled, cancel := context.WithCancel(t.Context())
cancel()
if _, err := source.LoadWorkforceReportCached(canceled, "workforce-test", query); !errors.Is(err, context.Canceled) {
t.Fatal("warm cache ignored cancellation", err)
}
// Loss of all process/shared cache state still reads one complete durable result.
restarted := NewMongoStore(target.db)
restarted.queryReplicaFactStore = true
restarted.queryPeriodic = true
restarted.queryPublishedResults = true
got, _, err = restarted.loadWorkforceFactReport(t.Context(), "workforce-test", query)
if err != nil {
t.Fatal(err)
}
got.AsOf = ""
if !reflect.DeepEqual(got, want) || aggregations.Load() != 0 {
t.Fatal("cache loss/restart reran aggregation")
}
// A restricted/current scope cannot reuse the broad result.
denied := query
denied.AllowedProjects = map[string]bool{}
if _, _, err = restarted.loadWorkforceFactReport(t.Context(), "workforce-test", denied); !errors.Is(err, ErrQueryReplicaNotReady) {
t.Fatal("broader publication reused", err)
}
if err = target.buildWorkforcePublications(t.Context(), lease, "workforce-test"); err != nil {
t.Fatal(err)
}
restricted, _, err := restarted.loadWorkforceFactReport(t.Context(), "workforce-test", denied)
if err != nil {
t.Fatal(err)
}
if len(restricted.Items) != 0 {
t.Fatal("revoked project scope exposed rows")
}
// An unfenced worker cannot replace durable data, including on epoch changes.
_, err = target.queryControl().UpdateOne(t.Context(), bson.M{"_id": "global"}, bson.M{"$inc": bson.M{"epoch": 1}})
if err != nil {
t.Fatal(err)
}
target.workforcePublicationBuilds.values = nil
_, err = target.db.Collection(workforcePublicationData).UpdateMany(t.Context(), bson.M{}, bson.M{"$set": bson.M{"computedAt": time.Now().Add(-4 * time.Minute)}})
if err != nil {
t.Fatal(err)
}
if err = target.buildWorkforcePublications(t.Context(), lease, "workforce-test"); err == nil {
t.Fatal("lost lease published")
}
if aggregations.Load() == 0 {
t.Fatal("test did not reach background computation")
}
// Cached envelopes are bound to query/database/org, digest and time.
key := target.workforcePublicationKey("workforce-test", query)
var envelope workforcePublication
if json.Unmarshal(cache[key], &envelope) != nil {
t.Fatal("missing cached envelope")
}
for _, mutate := range []func(*workforcePublication){func(v *workforcePublication) { v.Organization = "other" }, func(v *workforcePublication) { v.ExpiresAt = time.Now().Add(-time.Second) }, func(v *workforcePublication) { v.ExpiresAt = v.ComputedAt.Add(16 * time.Minute) }, func(v *workforcePublication) { v.Database = "other" }, func(v *workforcePublication) { v.ComputedAt = time.Now().Add(time.Hour) }, func(v *workforcePublication) { v.ComputedAt = time.Now().Add(-16 * time.Minute) }, func(v *workforcePublication) { v.Body = []byte("changed") }} {
bad := envelope
mutate(&bad)
if target.validWorkforcePublication(bad, key, "workforce-test") {
t.Fatal("invalid cache accepted")
}
}
}
func TestWorkforcePublicationRollingWindowIsPublishedOnceLocal(t *testing.T) {
if os.Getenv("QUERY_REPLICA_LOCAL_TEST") != "1" {
t.Skip("owned synthetic replicas only")
}
source, target, p := newLocalReplica(t)
seedWorkforceStats(t, source)
finishLocalReplica(t, p)
target.queryPeriodic = true
target.queryPublishedResults = true
lease, ok, err := target.acquireQueryModelLease(t.Context())
if err != nil || !ok {
t.Fatal(ok, err)
}
if err = target.rebuildWorkforceFacts(t.Context(), lease, "workforce-test", 0); err != nil {
t.Fatal(err)
}
now := time.Now().UTC()
q := workforcestats.Query{From: now.Add(-30 * 24 * time.Hour), To: now, RelativeFrom: true, RelativeTo: true}
if _, _, err = target.loadWorkforceFactReport(t.Context(), "workforce-test", q); !errors.Is(err, ErrWorkforcePublicationPending) {
t.Fatal(err)
}
if err = target.buildWorkforcePublications(t.Context(), lease, "workforce-test"); err != nil {
t.Fatal(err)
}
q.From = q.From.Add(time.Second)
q.To = q.To.Add(time.Second)
report, _, err := target.loadWorkforceFactReport(t.Context(), "workforce-test", q)
if err != nil {
t.Fatal("moving defaults missed the prepared response", err)
}
from, e := time.Parse(time.RFC3339Nano, report.WindowFrom)
if e != nil {
t.Fatal(e)
}
to, e := time.Parse(time.RFC3339Nano, report.WindowTo)
if e != nil || to.Sub(from) != 30*24*time.Hour {
t.Fatal("published period is not exact", e)
}
explicit := q
explicit.RelativeFrom = false
explicit.RelativeTo = false
if target.workforcePublicationKey("workforce-test", q) == target.workforcePublicationKey("workforce-test", explicit) {
t.Fatal("explicit query reused a rolling default")
}
}

View File

@ -0,0 +1,28 @@
package store
import (
"droboticflow/internal/workforcestats"
"testing"
"time"
)
func TestWorkforcePublicationRelativeWindowDoesNotRoundExplicitBounds(t *testing.T) {
a := workforcestats.Query{From: time.Date(2026, 11, 1, 0, 0, 0, 1, time.UTC), To: time.Date(2026, 11, 2, 0, 0, 0, 2, time.UTC)}
b := a
b.From = b.From.Add(time.Nanosecond)
// Key recipe is independent of a Mongo connection and preserves each explicit instant.
if canonicalWorkforcePublicationQuery(a).From.Equal(canonicalWorkforcePublicationQuery(b).From) {
t.Fatal("explicit nanoseconds rounded")
}
a.RelativeFrom = true
a.RelativeTo = true
b.RelativeFrom = true
b.RelativeTo = true
ca, cb := canonicalWorkforcePublicationQuery(a), canonicalWorkforcePublicationQuery(b)
if ca.From != cb.From || ca.To != cb.To || !ca.From.IsZero() || !ca.To.IsZero() {
t.Fatal("rolling defaults do not have a stable recipe")
}
if a.From.IsZero() || b.To.IsZero() {
t.Fatal("canonicalization mutated a caller query")
}
}

View File

@ -20,6 +20,10 @@ var workforceReportCache = &ttlResultCache[[]byte]{now: time.Now}
// contract. Only summary callers use this path; detail readers stay uncached.
// Cache keys contain every query scope, never an authorization decision.
func (s *MongoStore) LoadWorkforceReportCached(ctx context.Context, org string, q workforcestats.Query) (workforcestats.Report, error) {
if report, used, err := s.cachedPublishedWorkforce(ctx, org, q); used || err != nil {
return report, err
}
if err := ctx.Err(); err != nil {
return workforcestats.Report{}, err
}
@ -106,6 +110,10 @@ func (s *MongoStore) LoadWorkforceReportCached(ctx context.Context, org string,
// LoadWorkforceReport preserves uncached legacy summary behavior when read
// replicas are disabled; the enabled replacement reads only published facts.
func (s *MongoStore) LoadWorkforceReport(ctx context.Context, org string, q workforcestats.Query) (workforcestats.Report, error) {
if report, used, err := s.cachedPublishedWorkforce(ctx, org, q); used || err != nil {
return report, err
}
target, err := s.reportReplica(ctx)
if err != nil {
return workforcestats.Report{}, err

View File

@ -8,6 +8,8 @@ import (
)
type Query struct {
RelativeFrom bool // API supplied no lower bound: 30 days before computation.
RelativeTo bool // API supplied no upper bound: computation time.
AllowedProjects map[string]bool
AllowedSuppliers map[string]bool
MatchUser bool
@ -41,6 +43,8 @@ type Snapshot struct {
}
type Report struct {
WindowFrom string `json:"windowFrom,omitempty"`
WindowTo string `json:"windowTo,omitempty"`
AsOf string `json:"asOf,omitempty"`
Items []domain.DashboardWorkforceMetric
Totals domain.DashboardWorkforceMetric

View File

@ -105,7 +105,7 @@ class ProfileTests(unittest.TestCase):
self.assertIn('truncate -s 26843545600', config['data']['init.sh'])
self.assertIn('80530636800', config['data']['query25.sh'])
self.assertIn('memory.max', config['data']['guard.sh'])
pod['spec']['containers'][0]['resources']['limits']['memory'] = '8Gi'
pod['spec']['containers'][0]['resources']['limits']['memory'] = '16Gi'
with self.assertRaises(AssertionError):
profiles.verify_resources(pod, name)
profiles.verify_scripts(config['data'], name)

View File

@ -43,8 +43,10 @@ def validate_saved(saved, pilot):
assert saved['quota']['spec']['hard'] == profiles.quota(name)
expected = pilot.manifests(True, name)[-2]['data']
actual = saved['config']['data']
prior_verifier = dict(expected)
prior_verifier['verify25.sh'] = expected['verify25.sh'].replace('Number(db.serverStatus().wiredTiger.cache["maximum bytes configured"])', 'db.serverStatus().wiredTiger.cache["maximum bytes configured"]')
legacy = name == 'baseline' and hashlib.sha256(json.dumps(actual, sort_keys=True).encode()).hexdigest() == 'e5eebd9dacae29e817d90909fa97dd31e648ef8eaf4aa81f6bcb31ac58ee8eed'
assert actual == expected or legacy, 'rollback scripts are not an approved profile'
assert actual == expected or actual == prior_verifier or legacy, 'rollback scripts are not an approved profile'
def main():

View File

@ -23,7 +23,7 @@ ROOT = Path(__file__).resolve().parents[2]
HARD = {"pods": "2", "requests.cpu": "2", "limits.cpu": "5", "requests.memory": "3Gi", "limits.memory": "5Gi", "services": "0", "persistentvolumeclaims": "0"}
def manifest():
def manifest(source_node=NODE, cache=None):
env = [{"name": name, "valueFrom": {"fieldRef": {"fieldPath": field}}} for name, field in (("POD_UID", "metadata.uid"), ("POD_NAME", "metadata.name"), ("NODE_NAME", "spec.nodeName"))]
env += [{"name": "HOME", "value": "/tmp"}, {"name": "MONGOSH_DISABLE_TELEMETRY", "value": "1"}, {"name": "GOMAXPROCS", "value": "2"}, {"name": "GOMEMLIMIT", "value": "1536MiB"}]
command = """set -euo pipefail
@ -36,10 +36,15 @@ trap 'exit 0' TERM INT
while kill -0 "$pid" 2>/dev/null; do sleep 10 & wait $!; done
wait "$pid"
"""
pod = {"apiVersion": "v1", "kind": "Pod", "metadata": {"name": "node4-query-fixture", "namespace": NS, "labels": {"app": "node4-query-fixture", "purpose": "mongo-cpu-synthetic"}}, "spec": {"nodeName": NODE, "automountServiceAccountToken": False, "enableServiceLinks": False, "restartPolicy": "Never", "activeDeadlineSeconds": 7200, "terminationGracePeriodSeconds": 45, "securityContext": {"runAsUser": 999, "runAsGroup": 999, "runAsNonRoot": True, "fsGroup": 999}, "containers": [{"name": "lab", "image": IMAGE, "imagePullPolicy": "Never", "command": ["bash", "-c", command], "env": env, "resources": {"requests": {"cpu": "1", "memory": "2Gi", "ephemeral-storage": "32Mi"}, "limits": {"cpu": "4", "memory": "4Gi", "ephemeral-storage": "128Mi"}}, "securityContext": {"allowPrivilegeEscalation": False, "readOnlyRootFilesystem": True, "capabilities": {"drop": ["ALL"]}, "seccompProfile": {"type": "RuntimeDefault"}}, "readinessProbe": {"exec": {"command": ["mongosh", "--quiet", "--host", "127.0.0.1", "--port", "27119", "--eval", "quit(db.adminCommand({ping:1}).ok?0:1)"]}, "periodSeconds": 15, "timeoutSeconds": 5}, "volumeMounts": [{"name": "synthetic", "mountPath": "/synthetic"}, {"name": "tmp", "mountPath": "/tmp"}]}], "volumes": [{"name": "synthetic", "emptyDir": {"medium": "Memory", "sizeLimit": "2Gi"}}, {"name": "tmp", "emptyDir": {"medium": "Memory", "sizeLimit": "64Mi"}}]}}
pod = {"apiVersion": "v1", "kind": "Pod", "metadata": {"name": "node4-query-fixture", "namespace": NS, "labels": {"app": "node4-query-fixture", "purpose": "mongo-cpu-synthetic"}}, "spec": {"nodeName": source_node, "automountServiceAccountToken": False, "enableServiceLinks": False, "restartPolicy": "Never", "activeDeadlineSeconds": 7200, "terminationGracePeriodSeconds": 45, "securityContext": {"runAsUser": 999, "runAsGroup": 999, "runAsNonRoot": True, "fsGroup": 999}, "containers": [{"name": "lab", "image": IMAGE, "imagePullPolicy": "Never", "command": ["bash", "-c", command], "env": env, "resources": {"requests": {"cpu": "1", "memory": "2Gi", "ephemeral-storage": "32Mi"}, "limits": {"cpu": "4", "memory": "4Gi", "ephemeral-storage": "128Mi"}}, "securityContext": {"allowPrivilegeEscalation": False, "readOnlyRootFilesystem": True, "capabilities": {"drop": ["ALL"]}, "seccompProfile": {"type": "RuntimeDefault"}}, "readinessProbe": {"exec": {"command": ["mongosh", "--quiet", "--host", "127.0.0.1", "--port", "27119", "--eval", "quit(db.adminCommand({ping:1}).ok?0:1)"]}, "periodSeconds": 15, "timeoutSeconds": 5}, "volumeMounts": [{"name": "synthetic", "mountPath": "/synthetic"}, {"name": "tmp", "mountPath": "/tmp"}]}], "volumes": [{"name": "synthetic", "emptyDir": {"medium": "Memory", "sizeLimit": "2Gi"}}, {"name": "tmp", "emptyDir": {"medium": "Memory", "sizeLimit": "64Mi"}}]}}
ingress = {"apiVersion": "networking.k8s.io/v1", "kind": "NetworkPolicy", "metadata": {"name": "query-fixture-ingress", "namespace": NS}, "spec": {"podSelector": {"matchLabels": {"app": "node4-query25"}}, "policyTypes": ["Ingress"], "ingress": [{"from": [{"podSelector": {"matchLabels": {"app": "node4-query-fixture"}}}], "ports": [{"protocol": "TCP", "port": 27130}]}]}}
egress = {"apiVersion": "networking.k8s.io/v1", "kind": "NetworkPolicy", "metadata": {"name": "query-fixture-egress", "namespace": NS}, "spec": {"podSelector": {"matchLabels": {"app": "node4-query-fixture"}}, "policyTypes": ["Egress"], "egress": [{"to": [{"podSelector": {"matchLabels": {"app": "node4-query25"}}}], "ports": [{"protocol": "TCP", "port": 27130}]}]}}
return [ingress, egress, pod]
if source_node != NODE:
pod["spec"]["containers"][0]["imagePullPolicy"] = "IfNotPresent"
policies = [ingress, egress]
if cache:
policies.append({"apiVersion":"networking.k8s.io/v1","kind":"NetworkPolicy","metadata":{"name":"query-fixture-cache-egress","namespace":NS},"spec":{"podSelector":{"matchLabels":{"app":"node4-query-fixture"}},"policyTypes":["Egress"],"egress":[{"to":[{"namespaceSelector":{"matchLabels":{"kubernetes.io/metadata.name":cache["namespace"]}},"podSelector":{"matchLabels":{"app":cache["pod"]}}}],"ports":[{"protocol":"TCP","port":6379}]}]}})
return policies + [pod]
def verify_query_pod(query, expected_uid, query_profile="baseline"):
@ -77,6 +82,8 @@ def main():
ap.add_argument("--confirm-plan")
ap.add_argument("--query-pod-uid", default=QUERY_UID, help="exact inspected UID after an authorized synthetic Pod rebuild")
ap.add_argument("--query-profile", choices=query_lab_profiles.PROFILES, default="baseline")
ap.add_argument("--source-node", choices=(NODE,"10.189.16.33"), default=NODE)
ap.add_argument("--valkey-owned")
args = ap.parse_args()
global HARD
HARD = query_lab_profiles.quota(args.query_profile, fixture=True)
@ -94,6 +101,21 @@ def main():
def get(*argv):
return json.loads(call(*argv, "-o", "json"))
cache = None
if args.valkey_owned:
cache = json.loads(Path(args.valkey_owned).read_text())
assert cache["namespace"] == "droboticflow-query-cache-lab" and cache["pod"] == "query-cache-synthetic"
assert get("get","namespace",cache["namespace"])["metadata"]["uid"] == cache["namespaceUID"]
cp = get("-n",cache["namespace"],"get","pod",cache["pod"])
assert cp["metadata"]["uid"] == cache["podUID"] and cp["spec"]["nodeName"] == "10.189.16.33" and cp["status"]["phase"] == "Running"
assert cp["spec"]["containers"][0]["image"] == cache["image"]
assert cache["url"] == "redis://"+cp["status"]["podIP"]+":6379/0"
if args.source_node != NODE:
other = get("get","node",args.source_node)
assert other["metadata"]["uid"] == "77747f1c-42d8-40e8-a8d0-91f0e56ba3bd"
assert all(c["status"]=="False" for c in other["status"]["conditions"] if c["type"] in ("DiskPressure","MemoryPressure","PIDPressure"))
usage=call("top","node",args.source_node,"--no-headers").split()
assert float(usage[2].rstrip("%"))<25 and float(usage[4].rstrip("%"))<60
node = get("get", "node", NODE)
assert node["metadata"]["uid"] == NODE_UID
assert all(c["status"] == "False" for c in node["status"]["conditions"] if c["type"] in ("DiskPressure", "MemoryPressure", "PIDPressure"))
@ -116,10 +138,10 @@ def main():
assert recorded["planSHA256"] == previous_digest
assert quota["spec"]["hard"] == HARD
existing = get("-n", NS, "get", "pod", "node4-query-fixture")
assert existing["metadata"]["uid"] == recorded["fixtureUID"] and existing["spec"]["nodeName"] == NODE
assert existing["metadata"]["uid"] == recorded["fixtureUID"] and existing["spec"]["nodeName"] == args.source_node
assert existing["spec"]["containers"][0]["resources"]["limits"]["cpu"] == "4"
assert existing["spec"]["containers"][0]["resources"]["limits"]["memory"] == "4Gi"
for obj in manifest()[:2]:
for obj in manifest(args.source_node, cache)[:-1]:
assert get("-n", NS, "get", "networkpolicy", obj["metadata"]["name"])["spec"] == obj["spec"]
else:
assert quota["spec"]["hard"] == original_quota
@ -138,8 +160,8 @@ def main():
assert hashes == previous["binarySHA256"], "resume must use the recorded binaries"
else:
call("-n", NS, "exec", "node4-query-fixture", "-c", "lab", "--", "sh", "-c", 'for p in /proc/[0-9]*/exe; do x=$(readlink "$p" 2>/dev/null || true); case "$x" in /synthetic/query-http*.test) exit 1;; esac; done')
objects = manifest()
plan = {"kind": "synthetic-in-cluster-comparison", "node": NODE, "namespaceUID": NS_UID, "queryUID": query_uid, "queryIP": ip, "quotaVersion": quota["metadata"]["resourceVersion"], "originalQuota": original_quota, "temporaryQuota": HARD, "objects": objects, "binarySHA256": hashes, "customerData": "none", "newDisk": "none; source is bounded tmpfs", "productionChanges": "none", "queryBridgeStorage": "bounded /tmp memory volume; database noexec retained", "stop": "three degraded readiness/IO/CPU samples; delete only owned fixture", "rollback": "stop owned bridges, delete fixture and its two policies, CAS restore original quota; keep 25Gi data"}
objects = manifest(args.source_node, cache)
plan = {"kind": "synthetic-in-cluster-comparison", "sourceNode": args.source_node, "valkey": cache, "node": NODE, "namespaceUID": NS_UID, "queryUID": query_uid, "queryIP": ip, "quotaVersion": quota["metadata"]["resourceVersion"], "originalQuota": original_quota, "temporaryQuota": HARD, "objects": objects, "binarySHA256": hashes, "customerData": "none", "newDisk": "none; source is bounded tmpfs", "productionChanges": "none", "queryBridgeStorage": "bounded /tmp memory volume; database noexec retained", "stop": "three degraded readiness/IO/CPU samples; delete only owned fixture", "rollback": "stop owned bridges, delete fixture and its two policies, CAS restore original quota; keep 25Gi data"}
if recorded:
plan["resumeFrom"] = recorded
raw = json.dumps(plan, sort_keys=True).encode()
@ -163,7 +185,7 @@ def main():
(out / "created.json").write_text(json.dumps({"fixtureUID": created["metadata"]["uid"], "planSHA256": digest}))
call("-n", NS, "wait", "--for=condition=Ready", "pod/node4-query-fixture", "--timeout=180s", timeout=190)
fixture = get("-n", NS, "get", "pod", "node4-query-fixture")
assert fixture["spec"]["nodeName"] == NODE
assert fixture["spec"]["nodeName"] == args.source_node
fuid = fixture["metadata"]["uid"]
assert str(uuid.UUID(fuid)) == fuid
assert get("-n", NS, "get", "pod", "node4-query25")["metadata"]["uid"] == query_uid
@ -192,7 +214,7 @@ def main():
fcommand = "set -eu; if test -s /synthetic/bridge.pid; then p=$(cat /synthetic/bridge.pid); if kill -0 \"$p\" 2>/dev/null; then test \"$(readlink /proc/$p/exe)\" = /synthetic/bridge; exit 0; fi; fi; nohup /synthetic/bridge -listen 127.0.0.1:27229 -upstream " + ip + ":27130 -pod-uid " + fuid + " >/synthetic/bridge.log 2>&1 </dev/null & echo $! >/synthetic/bridge.pid"
call("-n", NS, "exec", "node4-query-fixture", "-c", "lab", "--", "sh", "-c", fcommand)
call("-n", NS, "exec", "node4-query-fixture", "-c", "lab", "--", "mongosh", "--quiet", "mongodb://127.0.0.1:27229/?directConnection=true&replicaSet=query-lab-rs", "--eval", 'const m=db.getSiblingDB("query25_synthetic").marker.findOne({_id:"query25-confirmed"});quit(m&&m.value==="preserve-query25-v1"?0:2)')
owned = {"fixtureUID": fuid, "queryUID": query_uid, "queryBridgeDirectory": qdir, "apiBinaryPath": api_path, "policies": {obj["metadata"]["name"]: get("-n", NS, "get", "networkpolicy", obj["metadata"]["name"])["metadata"]["uid"] for obj in objects[:2]}, "binarySHA256": hashes, "planSHA256": digest}
owned = {"fixtureUID": fuid, "queryUID": query_uid, "sourceNode": args.source_node, "valkey": cache, "queryBridgeDirectory": qdir, "apiBinaryPath": api_path, "policies": {obj["metadata"]["name"]: get("-n", NS, "get", "networkpolicy", obj["metadata"]["name"])["metadata"]["uid"] for obj in objects[:-1]}, "binarySHA256": hashes, "planSHA256": digest}
(out / "owned.json").write_text(json.dumps(owned, indent=2))
print("ISOLATED_FIXTURE_READY", flush=True)

View File

@ -136,6 +136,7 @@ def main():
ap.add_argument("--periodic-refresh", action="store_true", help="explicit periodic display policy; same load and safety limits")
ap.add_argument("--cold-pace-kib", type=int, choices=(0, 128), default=0)
ap.add_argument("--query-profile", choices=query_lab_profiles.PROFILES, default="baseline")
ap.add_argument("--published-results", action="store_true")
args = ap.parse_args()
assert args.cold_write_probe or args.cold_pace_kib == 0
assert not args.cold_write_probe or args.fixture_plan, "cold diagnostic requires the owned in-cluster fixture"
@ -160,12 +161,17 @@ def main():
run(k + ["-n", "droboticflow-query-lab", "exec", "node4-query25", "-c", "probe", "--", "bash", "/scripts/verify25.sh"])
fixture = None
owned = None
source_node = NODE
cache_owned = None
if args.fixture_plan:
owned = json.loads(Path(args.fixture_plan).read_text())
source_node = owned.get("sourceNode",NODE)
assert source_node in (NODE,"10.189.16.33")
cache_owned = owned.get("valkey")
fixture = get("-n", "droboticflow-query-lab", "get", "pod", "node4-query-fixture")
assert fixture["metadata"]["uid"] == owned["fixtureUID"]
assert query["metadata"]["uid"] == owned["queryUID"]
assert fixture["status"]["phase"] == "Running" and fixture["spec"]["nodeName"] == NODE
assert fixture["status"]["phase"] == "Running" and fixture["spec"]["nodeName"] == source_node
budget = fixture["spec"]["containers"][0]["resources"]["limits"]
assert budget["cpu"] == "4" and budget["memory"] == "4Gi"
api_binary = owned.get("apiBinaryPath", "/synthetic/query-http.test")
@ -196,7 +202,7 @@ def main():
script += "echo PROTECTED " + p["metadata"]["uid"] + "; cat " + cgroup(p) + "/io.pressure; cat " + cgroup(p) + "/io.stat; "
for filename in ("cpu.stat", "memory.current", "memory.events", "io.stat", "io.max"):
script += "echo QUERY " + filename + "; cat " + cgroup(query) + "/" + filename + "; "
if fixture:
if fixture and source_node == NODE:
for filename in ("cpu.stat", "memory.current", "memory.events", "io.stat"):
script += "echo SOURCE " + filename + "; cat " + cgroup(fixture) + "/" + filename + "; "
previous = None
@ -213,7 +219,18 @@ def main():
health = get("get", "node", NODE)
assert all(c["status"] == "False" for c in health["status"]["conditions"] if c["type"] in ("DiskPressure", "MemoryPressure", "PIDPressure"))
host = run(k + ["-n", "kube-system", "exec", "node-problem-detector-6v59v", "-c", "node-problem-detector", "--", "sh", "-c", script])
local = "included in host SOURCE fields" if fixture else run(["docker", "exec", CONTAINER, "sh", "-c", "cat /sys/fs/cgroup/cpu.stat; cat /sys/fs/cgroup/memory.current; cat /sys/fs/cgroup/io.stat"])
if fixture and source_node != NODE:
local = run(k+["-n","droboticflow-query-lab","exec","node4-query-fixture","-c","lab","--","sh","-c","cat /sys/fs/cgroup/cpu.stat; cat /sys/fs/cgroup/memory.current; cat /sys/fs/cgroup/memory.events; cat /sys/fs/cgroup/io.stat"])
health_other=get("get","node",source_node)
assert health_other["metadata"]["uid"]=="77747f1c-42d8-40e8-a8d0-91f0e56ba3bd"
assert all(c["status"]=="False" for c in health_other["status"]["conditions"] if c["type"] in ("DiskPressure","MemoryPressure","PIDPressure"))
other_top=run(k+["top","node",source_node,"--no-headers"]).split()
assert float(other_top[2].rstrip("%"))<=60 and float(other_top[4].rstrip("%"))<=70, "source/cache node overloaded"
else:
local = "included in host SOURCE fields" if fixture else run(["docker", "exec", CONTAINER, "sh", "-c", "cat /sys/fs/cgroup/cpu.stat; cat /sys/fs/cgroup/memory.current; cat /sys/fs/cgroup/io.stat"])
if cache_owned:
cp=get("-n",cache_owned["namespace"],"get","pod",cache_owned["pod"])
assert cp["metadata"]["uid"]==cache_owned["podUID"] and cp["status"]["phase"]=="Running", "owned cache replaced/stopped"
top = run(k + ["top", "node", NODE, "--no-headers"]).split()
cpu_pct = float(top[2].rstrip("%"))
if fixture:
@ -251,6 +268,9 @@ def main():
baseline = collect_baseline(sample)
limits = {"ready": max(1, max(s["readySeconds"] for s in baseline) * 2), "pressure": max(0.5, max(s["protectedFullAvg10"] for s in baseline) + 0.5), "await": max(10, max(s["awaitMs"] for s in baseline) * 3)}
manifest = {"sourceCommit": run(["git", "-C", str(ROOT), "rev-parse", "HEAD"]), "sourceFingerprint": source_fingerprint(), "sourceContainer": source["Id"], "sourceImage": source["Image"], "sourceCPU": 4, "sourceMemory": 4_294_967_296, "queryPodUID": query["metadata"]["uid"], "queryImage": query["spec"]["containers"][0]["image"], "queryCPU": int(query_lab_profiles.profile(args.query_profile)["cpu"]), "queryMemory": query_lab_profiles.resources(args.query_profile)["memory"], "queryProfile": query_lab_profiles.profile(args.query_profile), "queryScriptSHA256": hashlib.sha256(json.dumps(query_scripts, sort_keys=True).encode()).hexdigest(), "tasks": args.tasks, "waves": args.waves, "maxActors": args.max_actors, "wireCompressionBothVariants": os.getenv("QUERY_COMPARE_WIRE_COMPRESSION", ""), "minimumPoolBothVariants": os.getenv("QUERY_COMPARE_MIN_POOL", "0"), "limits": limits, "limitations": ["source Docker VM shares CPU with existing local containers", "target loop and backing device use the recorded finite queryProfile IO caps", "short comparison; not 30-minute three-round production capacity acceptance", "health probes do not establish customer workflow latency"]}
manifest["sourceNode"] = source_node
manifest["publishedResults"] = args.published_results
manifest["valkeyOwned"] = cache_owned
manifest["baselineSamples"] = len(baseline)
manifest["refreshMode"] = "periodic" if args.periodic_refresh else "continuous"
manifest["baselineSeconds"] = baseline[-1]["time"] - baseline[0]["time"]
@ -269,6 +289,9 @@ def main():
if args.cold_write_probe:
settings += ["QUERY_REPLICA_COLD_WRITE_PROBE=1", "QUERY_COLD_PACE_KIB=" + str(args.cold_pace_kib)]
settings += ["QUERY_REPLICA_REFRESH_MODE=" + manifest["refreshMode"]]
if args.published_results:
assert cache_owned, "published cluster comparison requires its owned dedicated cache"
settings += ["QUERY_PUBLISHED_RESULTS=1","QUERY_COMPARE_VALKEY_URL="+cache_owned["url"]]
argv = k + ["-n", "droboticflow-query-lab", "exec", "node4-query-fixture", "-c", "lab", "--", "env"] + settings + [api_binary, "-test.run=^" + test_name + "$", "-test.count=1", "-test.timeout=45m", "-test.v"]
with (out / "test.log").open("w") as log:
child = subprocess.Popen(argv, cwd=ROOT, env=env, stdout=log, stderr=subprocess.STDOUT, start_new_session=True)

View File

@ -0,0 +1,60 @@
#!/usr/bin/env python3
"""Bounded derived-cache-only Valkey on node33; no Service, PVC or customer data."""
import argparse
import hashlib
import json
from pathlib import Path
import subprocess
import urllib.request
NODE='10.189.16.33'
NODE_UID='77747f1c-42d8-40e8-a8d0-91f0e56ba3bd'
NS='droboticflow-query-cache-lab'
POD='query-cache-synthetic'
IMAGE='docker.io/valkey/valkey@sha256:e51a82741b780e4bf315db10753edf07eea6496d405fa7df2a8677a18f5e7464'
SOURCE_NS='droboticflow-query-lab'
def manifests():
return [
{'apiVersion':'v1','kind':'Namespace','metadata':{'name':NS,'labels':{'purpose':'mongo-cpu-synthetic-cache'}}},
{'apiVersion':'v1','kind':'ResourceQuota','metadata':{'name':'cache-budget','namespace':NS},'spec':{'hard':{'pods':'1','requests.cpu':'1','limits.cpu':'2','requests.memory':'2Gi','limits.memory':'4Gi','services':'0','persistentvolumeclaims':'0'}}},
{'apiVersion':'networking.k8s.io/v1','kind':'NetworkPolicy','metadata':{'name':'cache-isolation','namespace':NS},'spec':{'podSelector':{},'policyTypes':['Ingress','Egress'],'egress':[],'ingress':[{'from':[{'namespaceSelector':{'matchLabels':{'kubernetes.io/metadata.name':SOURCE_NS}},'podSelector':{'matchLabels':{'app':'node4-query-fixture'}}}],'ports':[{'protocol':'TCP','port':6379}]}]}},
{'apiVersion':'v1','kind':'Pod','metadata':{'name':POD,'namespace':NS,'labels':{'app':POD,'purpose':'mongo-cpu-synthetic-cache'}},'spec':{'nodeName':NODE,'automountServiceAccountToken':False,'enableServiceLinks':False,'restartPolicy':'Never','activeDeadlineSeconds':7200,'terminationGracePeriodSeconds':30,'securityContext':{'runAsUser':1000,'runAsGroup':1000,'runAsNonRoot':True,'fsGroup':1000},'containers':[{'name':'cache','image':IMAGE,'imagePullPolicy':'IfNotPresent','command':['valkey-server','--bind','0.0.0.0','--protected-mode','no','--port','6379','--maxmemory','2gb','--maxmemory-policy','allkeys-lru','--save','','--appendonly','no','--dir','/cache','--loglevel','warning'],'resources':{'requests':{'cpu':'1','memory':'2Gi'},'limits':{'cpu':'2','memory':'4Gi'}},'securityContext':{'allowPrivilegeEscalation':False,'readOnlyRootFilesystem':True,'capabilities':{'drop':['ALL']}},'volumeMounts':[{'name':'cache','mountPath':'/cache'}],'readinessProbe':{'exec':{'command':['valkey-cli','ping']},'periodSeconds':5,'timeoutSeconds':3}}],'volumes':[{'name':'cache','emptyDir':{'medium':'Memory','sizeLimit':'64Mi'}}]}}
]
def main():
ap=argparse.ArgumentParser(description=__doc__)
ap.add_argument('--kubeconfig',required=True);ap.add_argument('--output',required=True)
ap.add_argument('--execute',action='store_true');ap.add_argument('--confirm-plan')
a=ap.parse_args();out=Path(a.output).resolve();out.mkdir(parents=True,exist_ok=True)
k=['kubectl','--kubeconfig',a.kubeconfig]
def call(*cmd,data=None,timeout=30):
p=subprocess.run(k+list(cmd),input=data,text=True,capture_output=True,timeout=timeout)
if p.returncode:raise RuntimeError(p.stderr[:1500])
return p.stdout
def get(*cmd):return json.loads(call(*cmd,'-o','json'))
node=get('get','node',NODE);assert node['metadata']['uid']==NODE_UID
assert all(c['status']=='False' for c in node['status']['conditions'] if c['type'] in ['DiskPressure','MemoryPressure','PIDPressure'])
top=call('top','node',NODE,'--no-headers').split();assert float(top[2].rstrip('%'))<25 and float(top[4].rstrip('%'))<60
assert not call('get','namespace',NS,'--ignore-not-found','-o','name').strip(),'existing namespace must be inspected, not overwritten'
with urllib.request.urlopen('http://120.48.180.80/api/ready',timeout=5) as r:assert json.load(r)['ready']
objects=manifests();plan={'nodeUID':NODE_UID,'objects':objects,'productionWrites':False,'durability':'Valkey is disposable derived cache; complete responses remain majority+j in query Mongo','rollback':'UID-delete only these owned resources; never delete a reused namespace'}
sha=hashlib.sha256(json.dumps(plan,sort_keys=True).encode()).hexdigest()
(out/'plan.json').write_text(json.dumps(plan,indent=2));(out/'objects.json').write_text(json.dumps({'apiVersion':'v1','kind':'List','items':objects},indent=2))
print('PLAN',sha,flush=True)
if not a.execute:return
assert a.confirm_plan==sha
call('create','-f',str(out/'objects.json'))
ns=get('get','namespace',NS);pod=get('-n',NS,'get','pod',POD)
(out/'created.json').write_text(json.dumps({'namespaceUID':ns['metadata']['uid'],'podUID':pod['metadata']['uid'],'node':NODE,'image':IMAGE}))
call('-n',NS,'wait','--for=condition=Ready','pod/'+POD,'--timeout=180s',timeout=185)
pod=get('-n',NS,'get','pod',POD);assert pod['spec']['nodeName']==NODE
info=call('-n',NS,'exec',POD,'-c','cache','--','valkey-cli','INFO','memory')
assert 'maxmemory:2147483648' in info
owned={'namespace':NS,'namespaceUID':ns['metadata']['uid'],'pod':POD,'podUID':pod['metadata']['uid'],'node':NODE,'nodeUID':NODE_UID,'image':IMAGE,'url':'redis://'+pod['status']['podIP']+':6379/0','planSHA256':sha}
(out/'owned.json').write_text(json.dumps(owned,indent=2));(out/'memory.log').write_text(info)
print('CACHE_READY',pod['metadata']['uid'],flush=True)
if __name__=='__main__':main()

View File

@ -6,6 +6,8 @@ PROFILES = {
"compute": {"cpu": "2", "memoryGi": 4, "cacheGi": "1", "rbps": 2097152, "wbps": 1048576, "riops": 64, "wiops": 32},
"balanced": {"cpu": "2", "memoryGi": 4, "cacheGi": "1", "rbps": 4194304, "wbps": 2097152, "riops": 128, "wiops": 64},
"read-headroom": {"cpu": "2", "memoryGi": 4, "cacheGi": "1", "rbps": 8388608, "wbps": 2097152, "riops": 256, "wiops": 64},
"query4-write4": {"cpu": "4", "memoryGi": 8, "cacheGi": "2", "rbps": 4194304, "wbps": 4194304, "riops": 128, "wiops": 128},
"query4-write8": {"cpu": "4", "memoryGi": 8, "cacheGi": "2", "rbps": 4194304, "wbps": 8388608, "riops": 128, "wiops": 256},
}
@ -42,7 +44,7 @@ def render(objects, name):
config["data"]["guard.sh"] += f'\n[[ $(cat "$group/cpu.max") == "{int(p["cpu"]) * 100000} 100000" ]]\n[[ $(cat "$group/memory.max") == {p["memoryGi"] * 1024 ** 3} ]]\n'
config["data"]["query25.sh"] = config["data"]["query25.sh"].replace('--wiredTigerCacheSizeGB 0.25', '--wiredTigerCacheSizeGB ' + p['cacheGi'])
# Verify actual server settings, not only command text or admission requests.
config["data"]["verify25.sh"] = config["data"]["verify25.sh"].replace('const d=db.getSiblingDB', f'if(db.serverStatus().wiredTiger.cache["maximum bytes configured"]!=={int(float(p["cacheGi"]) * 1024 ** 3)})throw Error("WT cache drift");const d=db.getSiblingDB')
config["data"]["verify25.sh"] = config["data"]["verify25.sh"].replace('const d=db.getSiblingDB', f'if(Number(db.serverStatus().wiredTiger.cache["maximum bytes configured"])!=={int(float(p["cacheGi"]) * 1024 ** 3)})throw Error("WT cache drift");const d=db.getSiblingDB')
config["data"]["profile.json"] = __import__('json').dumps({"name": name, **p}, sort_keys=True)
pod["metadata"].setdefault("annotations", {})["query-lab/profile"] = name
for container in pod["spec"]["containers"] + pod["spec"]["initContainers"]:

View File

@ -11,8 +11,30 @@ compare_spec = importlib.util.spec_from_file_location("isolated_compare", Path(_
compare = importlib.util.module_from_spec(compare_spec)
compare_spec.loader.exec_module(compare)
valkey_spec = importlib.util.spec_from_file_location("valkey_lab", Path(__file__).with_name("query-valkey-lab.py"))
valkey_lab = importlib.util.module_from_spec(valkey_spec)
valkey_spec.loader.exec_module(valkey_lab)
class FixtureSafety(unittest.TestCase):
def test_dedicated_valkey_is_disposable_bounded_and_network_isolated(self):
objects = valkey_lab.manifests()
self.assertEqual([o["kind"] for o in objects], ["Namespace", "ResourceQuota", "NetworkPolicy", "Pod"])
policy, pod = objects[-2:]
self.assertEqual(policy["spec"]["egress"], [])
source = policy["spec"]["ingress"][0]["from"][0]
self.assertEqual(source["namespaceSelector"]["matchLabels"]["kubernetes.io/metadata.name"], "droboticflow-query-lab")
self.assertEqual(source["podSelector"]["matchLabels"]["app"], "node4-query-fixture")
self.assertEqual(pod["spec"]["nodeName"], "10.189.16.33")
self.assertFalse(pod["spec"]["automountServiceAccountToken"])
self.assertTrue(all("hostPath" not in v and "persistentVolumeClaim" not in v for v in pod["spec"]["volumes"]))
container = pod["spec"]["containers"][0]
self.assertEqual(container["resources"]["limits"], {"cpu": "2", "memory": "4Gi"})
self.assertIn("--maxmemory", container["command"])
self.assertIn("2gb", container["command"])
self.assertIn("--appendonly", container["command"])
self.assertTrue(container["securityContext"]["readOnlyRootFilesystem"])
def test_rebuilt_query_uid_requires_exact_storage_and_resources(self):
import copy
uid = "e1210ce7-16ee-4bda-9f8f-e3a02efefa01"

View File

@ -23,6 +23,9 @@ CASES = ('internal/store/query_replica_local_test.go',
'internal/store/query_replica_valkey_local_test.go',
'internal/store/workforce_report_cache_local_test.go',
'internal/store/workforce_facts_local_test.go',
'internal/store/workforce_publication_local_test.go',
'internal/store/query_replica_page_local_test.go',
'internal/store/supplier_authorization_batch_local_test.go',
'internal/store/query_cold_write_local_test.go',
'internal/store/query_periodic_local_test.go',
'internal/store/query_pool_buckets_local_test.go')

View File

@ -522,6 +522,7 @@
"argv": ["npm", "run", "test:browser:dashboard"],
"tests": true
},
{ "argv": ["node", "tests/browser/dashboard-publication.mjs"], "tests": true },
{
"argv": ["python3", "scripts/test_dashboard_stability.py"]
},
@ -975,7 +976,7 @@
"name": "node4-query-pilot-guardrails",
"paths": [
"^scripts/local/node4-query-pilot/",
"^scripts/local/(query-lab-resource-stage|query_lab_profiles)\\.py$",
"^scripts/local/(query-lab-resource-stage|query-valkey-lab|query_lab_profiles)\\.py$",
"^docs/operations/query-replica-node4-(isolation-plan|smoke-results)\\.md$",
"^scripts/local/(query-replica-(cluster-fixture|isolated-compare)|test_query_replica_cluster_fixture)\\.py$",
"^docs/operations/query-replica-query25-comparison\\.md$"
@ -1019,10 +1020,11 @@
"paths": [
"^internal/store/query_(replica|sync|tracked_collection|collection_interface).*\\.go$",
"^internal/api/query_replica.*\\.go$",
"^internal/store/workforce_(report_cache|heap_diagnostic|facts|return_page).*\\.go$",
"^internal/store/workforce_(report_cache|heap_diagnostic|facts|return_page|publication).*\\.go$",
"^internal/store/query_pool_buckets.*\\.go$",
"^internal/store/query_cold_write.*\\.go$",
"^internal/store/query_periodic.*\\.go$",
"^internal/store/supplier_authorization_batch_local_test\\.go$",
"^internal/api/query_periodic.*\\.go$",
"^internal/api/query_fact_routes_test\\.go$",
"^internal/api/workforce_compact_report_test\\.go$",

View File

@ -158,10 +158,11 @@ function DailyDetails({ query }: { query: WorkforceReportQuery }) {
const identity = JSON.stringify({ ...query, groupBy: "day" });
useEffect(() => {
let active = true;
const controller = new AbortController();
setItems(null);
setError("");
workforceApi
.report(JSON.parse(identity))
.report(JSON.parse(identity), controller.signal)
.then((result) => {
if (active) setItems(result.items);
})
@ -170,6 +171,7 @@ function DailyDetails({ query }: { query: WorkforceReportQuery }) {
});
return () => {
active = false;
controller.abort();
};
}, [identity]);
if (error) return <p role="alert">{userErrorMessage(error)}</p>;
@ -258,12 +260,13 @@ export function WorkforceStatisticsPanel({
useEffect(() => {
if (collapsed || !deferLoad) return;
let active = true;
const controller = new AbortController();
setLoading(true);
setRemoteItems(null);
setRemoteTotals(undefined);
setError("");
workforceApi
.report({ ...apiFilters, role, groupBy: "member" })
.report({ ...apiFilters, role, groupBy: "member" }, controller.signal)
.then((result) => {
if (!active) return;
setRemoteItems(result.items);
@ -277,6 +280,7 @@ export function WorkforceStatisticsPanel({
});
return () => {
active = false;
controller.abort();
};
}, [collapsed, deferLoad, role, filterKey]);
const visibleItems = deferLoad ? remoteItems || [] : items;
@ -461,7 +465,7 @@ export function WorkforceStatisticsPanel({
})}
</tbody>
</table>
{!filtered.length && (
{!loading && !error && !filtered.length && (
<p className="p-8 text-center text-gray-500">
{visibleItems.length ? "没有匹配的成员记录" : "当前区间暂无工作记录"}
</p>

View File

@ -15,7 +15,7 @@ import type {
SliceSegment,
} from "../../types";
import { compactAnnotationPayload } from "../../lib/annotation-operation-history.js";
import { apiRequest } from "./client";
import { apiRequest, preparedAPIRequest } from "./client";
import { decodeAnnotationTimeline } from "../../lib/annotation-timeline-wire.js";
import { normalizeAnnotationWorkspace, normalizeAnnotationTaskDetail } from "../../lib/annotation-workspace.js";
@ -495,8 +495,8 @@ export const annotationOperationsApi = {
payload,
),
workloadSummary: (projectId?: string, options: { signal?: AbortSignal } = {}) =>
apiRequest<AnnotationWorkloadSummary>(
preparedAPIRequest<AnnotationWorkloadSummary>(
`/api/plugins/annotation-qa/workload-summary${query({ projectId })}`,
options,
options.signal,
),
};

View File

@ -189,3 +189,37 @@ export async function apiRequest<T>(
throw lastError || new Error("Request failed");
}
/** Wait for an optional derived report without extending the normal read budget. */
export async function preparedAPIRequest<T>(path: string, signal?: AbortSignal): Promise<T> {
const deadline = Date.now() + DEFAULT_REQUEST_TIMEOUT_MS;
let failures = 0;
while (true) {
const remaining = deadline - Date.now();
if (remaining <= 0) throw new APITimeoutError("看板正在准备,请稍后重试");
let response: Response;
try {
response = await apiFetch(
path,
{ signal, headers: { Accept: "application/vnd.droboticflow.prepared+json" } },
{ timeoutMs: remaining },
);
} catch (error) {
if (isAbortError(error) || error instanceof APITimeoutError || ++failures >= 4) throw error;
await abortableDelay(Math.min(failures * 500, Math.max(0, deadline - Date.now())), signal);
continue;
}
if (!response.ok) {
if (RETRIABLE_STATUS.has(response.status) && ++failures < 4) {
await abortableDelay(Math.min(failures * 500, Math.max(0, deadline - Date.now())), signal);
continue;
}
throw await responseError(response);
}
const result = await response.json();
if (response.status !== 202) return result as T;
if (result?.status !== "building") throw new APIError("看板暂时无法加载,请重试", 202);
const delay = Math.min(2000, Math.max(500, Number(result.retryAfterMs) || 1000));
await abortableDelay(Math.min(delay, Math.max(0, deadline - Date.now())), signal);
}
}

View File

@ -1,10 +1,10 @@
import type { DashboardSummary } from "../../types";
import { apiRequest } from "./client";
import { preparedAPIRequest } from "./client";
export const dashboardApi = {
getSummary: (options: { includeWorkforce?: boolean } = {}) => {
const suffix =
options.includeWorkforce === undefined ? "" : `?includeWorkforce=${options.includeWorkforce ? "true" : "false"}`;
return apiRequest<DashboardSummary>(`/api/dashboard${suffix}`);
return preparedAPIRequest<DashboardSummary>(`/api/dashboard${suffix}`);
},
};

View File

@ -1,5 +1,5 @@
import type { WorkforceReturnDetail } from "../../types";
import { apiRequest } from "./client";
import { apiRequest, preparedAPIRequest } from "./client";
export type WorkforceDetailQuery = {
userId: string;
@ -22,19 +22,19 @@ export type WorkforceReportQuery = {
};
export const workforceApi = {
report: (query: WorkforceReportQuery = {}) => {
report: (query: WorkforceReportQuery = {}, signal?: AbortSignal) => {
const search = new URLSearchParams();
Object.entries(query).forEach(([key, value]) => {
if (value !== undefined && value !== "") search.set(key, String(value));
});
return apiRequest<{
return preparedAPIRequest<{
from: string;
to: string;
groupBy: "member" | "day";
totals: import("../../types").DashboardWorkforceMetric;
items: import("../../types").DashboardWorkforceMetric[];
eventCount: number;
}>(`/api/dashboard/workforce?${search}`);
}>(`/api/dashboard/workforce?${search}`, signal);
},
returns: (query: WorkforceDetailQuery, offset = 0) => {
const search = new URLSearchParams({ limit: "20", offset: String(offset) });

View File

@ -0,0 +1,90 @@
import { createServer } from "vite";
import { chromium } from "playwright";
import assert from "node:assert/strict";
import fs from "node:fs/promises";
import { browserExecutableOptions } from "../../scripts/lib/browser-executable.mjs";
const out = process.env.DASHBOARD_PUBLICATION_OUTPUT || "local-data/test-results/dashboard-publication";
await fs.mkdir(out, { recursive: true });
const fixture = `import React from 'react'; import {createRoot} from 'react-dom/client'; import {UserProvider} from '/src/context/UserContext'; import {TimeZoneProvider} from '/src/context/TimeZoneContext'; import {WorkforceStatisticsPanel} from '/src/components/WorkforceStatisticsPanel'; import '/src/index.css'; createRoot(document.getElementById('root')!).render(<UserProvider><TimeZoneProvider><WorkforceStatisticsPanel items={[]} deferLoad={true} collapsedByDefault={false}/></TimeZoneProvider></UserProvider>);`;
const server = await createServer({
configFile: "tests/browser/vite.platform-feedback.config.ts",
cacheDir: out + "/vite-cache",
server: { host: "127.0.0.1", port: 0 },
plugins: [
{
name: "publication-fixture",
enforce: "pre",
transform(code, id) {
if (id.endsWith("/tests/browser/fixtures/acceptance-playback-rate.tsx")) return fixture;
},
},
],
});
let browser;
const report = { checks: [], errors: [] };
try {
await server.listen();
const origin = server.resolvedUrls.local[0].replace(/\/$/, "");
browser = await chromium.launch({ ...browserExecutableOptions(), headless: true, args: ["--no-proxy-server"] });
const page = await browser.newPage({ viewport: { width: 1440, height: 1000 } });
page.on("pageerror", (e) => report.errors.push(e.message));
page.setDefaultTimeout(15000);
let ready = false,
denied = false,
calls = 0;
await page.route("**/api/**", (route) => {
const path = new URL(route.request().url()).pathname;
if (!path.startsWith("/api/")) return route.continue();
if (path === "/api/auth/me")
return route.fulfill({
json: {
user: { id: "synthetic", name: "Synthetic", role: "admin", status: "active" },
organization: { id: "synthetic", name: "Synthetic" },
},
});
if (path === "/api/dashboard/workforce") {
calls++;
if (denied) return route.fulfill({ status: 403, json: { error: "无权访问" } });
if (!ready) return route.fulfill({ status: 202, json: { status: "building", retryAfterMs: 500 } });
return route.fulfill({ json: { items: [], totals: {}, from: "2026-11-01", to: "2026-11-13" } });
}
return route.fulfill({ json: { items: [] } });
});
await page.route("**/publication-test", (route) =>
route.fulfill({
contentType: "text/html",
body: `<html><body><div id='root'></div><script type='module'>import RefreshRuntime from '/@react-refresh';RefreshRuntime.injectIntoGlobalHook(window);window.$RefreshReg$=()=>{};window.$RefreshSig$=()=>type=>type;window.__vite_plugin_react_preamble_installed__=true;</script><script type='module' src='/tests/browser/fixtures/acceptance-playback-rate.tsx'></script></body></html>`,
}),
);
await page.goto(origin + "/publication-test");
await page.getByText("正在加载工作记录…", { exact: true }).waitFor();
await page.waitForTimeout(1100);
assert(calls >= 2);
assert.equal(await page.getByText("当前区间暂无工作记录", { exact: true }).count(), 0);
await page.screenshot({ path: out + "/building.png" });
report.checks.push("202 retains loading state rather than false empty");
ready = true;
await page.getByText("当前区间暂无工作记录", { exact: true }).waitFor();
await page.screenshot({ path: out + "/complete-empty.png" });
report.checks.push("completed empty report replaces loading");
ready = false;
await page.reload();
await page.getByText("正在加载工作记录…", { exact: true }).waitFor();
await page.goto("about:blank");
const before = calls;
await page.waitForTimeout(1200);
assert.equal(calls, before);
report.checks.push("unmount cancels future report polls");
denied = true;
await page.goto(origin + "/publication-test");
await page.getByRole("alert").waitFor();
await page.screenshot({ path: out + "/denied.png" });
report.checks.push("permission failure shows error");
assert.deepEqual(report.errors, []);
report.passed = true;
} finally {
await fs.writeFile(out + "/report.json", JSON.stringify(report, null, 2));
await browser?.close();
await server.close();
}
console.log(JSON.stringify(report));

View File

@ -0,0 +1,90 @@
import test from "node:test";
import assert from "node:assert/strict";
import fs from "node:fs";
import ts from "typescript";
import { buildAPIRequestHeaders } from "../../src/services/api/http.js";
const code = ts.transpileModule(fs.readFileSync(new URL("../../src/services/api/client.ts", import.meta.url), "utf8"), {
compilerOptions: { module: ts.ModuleKind.CommonJS, target: ts.ScriptTarget.ES2022 },
}).outputText;
function client(fetcher, onDelay = () => {}) {
let now = 0;
const exports = {};
const window = {
setTimeout(fn, ms) {
return setImmediate(() => {
now += ms;
onDelay();
fn();
});
},
clearTimeout(id) {
clearImmediate(id);
},
};
new Function("exports", "require", "fetch", "Date", "window", code)(
exports,
(path) => (path === "./http.js" ? { buildAPIRequestHeaders } : { buildAPIURL: (p) => p }),
fetcher,
{ now: () => now },
window,
);
return exports;
}
const response = (body, status = 200) => new Response(JSON.stringify(body), { status });
test("prepared reports wait for publication and return only a completed report", async () => {
let calls = 0;
const c = client(async (_url, options) => {
assert.equal(options.credentials, "include");
assert.equal(new Headers(options.headers).get("Accept"), "application/vnd.droboticflow.prepared+json");
assert.equal(options.body, undefined);
return ++calls === 1 ? response({ status: "building", retryAfterMs: 1000 }, 202) : response({ rows: [1] });
});
assert.deepEqual(await c.preparedAPIRequest("/api/dashboard/workforce"), { rows: [1] });
assert.equal(calls, 2);
});
test("publication polling retains the original total 30 second budget", async () => {
let calls = 0;
const c = client(async () => {
calls++;
return response({ status: "building", retryAfterMs: 2000 }, 202);
});
await assert.rejects(c.preparedAPIRequest("/report"), { name: "APITimeoutError" });
assert.equal(calls, 15);
});
test("leaving the report cancels publication polling before another read", async () => {
const abort = new AbortController();
let calls = 0;
const c = client(
async () => {
calls++;
return response({ status: "building" }, 202);
},
() => abort.abort(),
);
await assert.rejects(c.preparedAPIRequest("/report", abort.signal), { name: "AbortError" });
assert.equal(calls, 1);
});
test("permission errors and unrelated 202 responses are never treated as finished data", async () => {
const denied = client(async () => response({ error: "forbidden" }, 403));
await assert.rejects(denied.preparedAPIRequest("/report"), { status: 403 });
const unknown = client(async () => response({ rows: [] }, 202));
await assert.rejects(unknown.preparedAPIRequest("/report"), { status: 202 });
});
test("prepared reads retain bounded transport retry and never retry cancellation", async () => {
let calls = 0;
const c = client(async () => {
if (++calls === 1) throw new TypeError("network");
return response({ items: [] });
});
assert.deepEqual(await c.preparedAPIRequest("/report"), { items: [] });
assert.equal(calls, 2);
calls = 0;
const aborted = client(async () => {
calls++;
throw new DOMException("canceled", "AbortError");
});
await assert.rejects(aborted.preparedAPIRequest("/report"), { name: "AbortError" });
assert.equal(calls, 1);
});