✨ Add optional background query models, shared cache and search

This commit is contained in:
Codex 2026-11-12 01:18:17 -08:00
parent 9232bace60
commit 1050f48286
39 changed files with 3180 additions and 48 deletions

View File

@ -105,3 +105,25 @@ jobs:
path: local-data/
if-no-files-found: error
retention-days: 14
query-serving:
runs-on: ubuntu-24.04
needs: pool-search-mongo
if: ${{ !cancelled() }}
steps:
- uses: actions/checkout@v4
- uses: actions/setup-go@v5
with:
go-version-file: go.mod
cache: true
- run: docker compose version
- run: python3 scripts/test_query_serving.py --start --race
- uses: actions/upload-artifact@v4
if: always()
with:
name: query-serving-logs
path: local-data/query-serving-local/logs/
if-no-files-found: error
retention-days: 14
- run: python3 scripts/test_query_serving.py --down
if: always()

View File

@ -0,0 +1,22 @@
# 后台物化读表、Valkey 与 Typesense 实施
Status: claimed
Type: task
Base: 9232bace
用户批准依次实施物化读取与共享缓存,再接搜索;正确性与性能达标后才准备发布。本次仅本地实现、隔离验证和提交,不推送、不部署、不写生产。
- 后台独立进程构建完整代次,复用已有统计口径与旧记录解析;读取不触发新组件的全量构建。后台并发有界、分布式租约、原子发布,失败保留上一完整代次,支持重启恢复和删除对账。
- Valkey 共享不可变统计/查询结果,包含数据库、组织、项目范围、代次与查询参数;缓存故障、丢失与未准备就绪退回现有读取,不能阻断作业。复用既有快照发布协议,不重写标注/审核写路径。
- Typesense 仅提供搜索候选;保留精确子串、Capture/part、特殊字符、旧记录、排序、完整分页与统计语义。实时权限、可领取、自审禁止、归属、状态、版本与写锁仍由现有服务端代码判定。
- 新能力默认关闭,运行地址/凭证从环境或现有私有配置读取。新增集合只存可重建派生数据,后台和查询组件的生产安装、索引及回填另列计划。
- 使用隔离 Mongo、真实 Valkey/Typesense 和合成数据,对照旧路径、撤权/归属变化、删除/重启、组件故障、并发与查询资源;接近线上规模,搜索 p95≤2秒,面板暖态 p95≤500ms、预热首访≤2秒。未实测或未达标保持待验收。
- 保留原工作树两个 admin pool 测试及旧发布笔记,阶段性本地提交;不修改既有标注和正式审查记录。
## 本地进展
后台读表、Valkey、Typesense 已实现,默认关闭。真实三组件12项(含竞态、范围、失联、重启、旧数据和缓存损坏)通过;既有五阶段作业/恢复、搜索与统计浏览器回归通过。官方 Go 基线为68项已有失败、无新增;前端基线与构建/生产启动检查通过。范围总报告保留了原生产诊断跳过、原测试债务和最初三组件瞬时回退断言失败,不将其写成全绿;修正的专项已重跑通过。
81人、约2.6万任务/34万事件的首轮短测:候选零错误,347组账号/接口业务结果一致;管理员全部项目搜索p95 2592→799ms、生产总览3516→162ms;Mongo内部返回字节199215338→152507258,Go CPU +3.3%。这是短测,不是容量/生产验收。1GiB Mongo 对照OOM已作废,两版统一固定4GiB重新测得上述结果。
独立后台镜像构建和启动帮助检查已通过。待完成:冻结源码后的三轮持续对照和资源审查。未达标不准备发布;生产拓扑、新组件及派生索引/构建范围仍需具体计划与单独授权。现有两个受保护文件的哈希保持不变。

View File

@ -0,0 +1,17 @@
FROM golang:1.24-bookworm@sha256:1a6d4452c65dea36aac2e2d606b01b4a029ec90cc1ae53890540ce6173ea77ac AS build
WORKDIR /src
ARG GOPROXY=https://proxy.golang.org,direct
COPY go.mod go.sum ./
RUN GOPROXY="${GOPROXY}" go mod download
COPY cmd/query-read-builder ./cmd/query-read-builder
COPY internal ./internal
RUN CGO_ENABLED=0 go build -trimpath -o /out/query-read-builder ./cmd/query-read-builder
# A static binary only needs trusted CA roots; no media/worker tools or app code.
FROM scratch
COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
COPY --from=build /out/query-read-builder /query-read-builder
ARG DROBOTICFLOW_BUILD_COMMIT=unknown
LABEL org.opencontainers.image.revision=${DROBOTICFLOW_BUILD_COMMIT}
USER 10001:10001
ENTRYPOINT ["/query-read-builder"]

View File

@ -173,6 +173,14 @@ func main() {
}()
dataStore := store.NewMongoStore(client.Database(cfg.MongoDB))
if os.Getenv("ANNOTATION_PROCESSOR_MODE") != "processor" {
closeServing, servingErr := dataStore.ConfigureQueryServingFromEnvironment()
if servingErr != nil {
fmt.Println("[api] query accelerator configuration unavailable; using existing reads")
} else {
defer closeServing()
}
}
if os.Getenv("ANNOTATION_PROCESSOR_MODE") == "processor" {
checkIndexes := dataStore.EnsureIndexes
mode := os.Getenv("DROBOTICFLOW_DATABASE_BOOTSTRAP")

View File

@ -0,0 +1,65 @@
// query-read-builder only writes derived read models. It never processes work.
package main
import (
"context"
"droboticflow/internal/config"
"droboticflow/internal/store"
"flag"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
"log"
"os"
"os/signal"
"syscall"
"time"
)
func main() {
once := flag.Bool("once", false, "build once; streaming freshness expires after exit")
ensure := flag.Bool("ensure-indexes", false, "explicitly create derived-data indexes before building")
flag.Parse()
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
defer stop()
cfg, err := config.LoadValidated()
if err != nil {
log.Fatal("invalid builder configuration")
}
if os.Getenv("QUERY_SERVING_READS") != "1" {
log.Fatal("explicit QUERY_SERVING_READS=1 is required")
}
client, err := mongo.Connect(ctx, options.Client().ApplyURI(cfg.MongoURI))
if err != nil {
log.Fatal("builder database unavailable")
}
defer client.Disconnect(context.Background())
s := store.NewMongoStore(client.Database(cfg.MongoDB))
closeServing, err := s.ConfigureQueryServingFromEnvironment()
if err != nil {
log.Fatal("builder accelerator configuration invalid")
}
defer closeServing()
if *ensure {
if s.EnsureQueryReadModelIndexes(ctx) != nil {
log.Fatal("derived index creation failed")
}
}
b := store.NewQueryReadBuilder(s)
defer b.Close()
if *once {
if b.RunOnce(ctx) != nil {
log.Fatal("initial materialization incomplete")
}
return
}
for ctx.Err() == nil {
if b.Run(ctx) != nil && ctx.Err() == nil {
log.Print("query builder paused; retrying with full reconciliation")
}
select {
case <-ctx.Done():
return
case <-time.After(5 * time.Second):
}
}
}

View File

@ -0,0 +1,38 @@
# 后台查询读表与可选加速组件
状态:本地实现;生产保持关闭。速度与资源结论见同任务证据,未通过的门槛不能据本说明视为通过。
## 责任与兼容
Mongo 仍是任务、权限、归属、版本与正式审查记录的权威存储。独立 `query-read-builder` 只写 `queryReadModelControl`、`queryReadModelData` 及既有 `annotationOutputSnapshot*` 派生集合,不领取、退回、重置、补验或改写客户任务。读取新集合不触发构建。
后台使用租约代次隔离写者,先完整构建再发布清单。单个失败不会发布半份结果。进程重启先打开变更流,再核对源身份和重建;不把遗留索引视为最新。副本集/分片集群变更流是持续搜索加速的前提,普通独立 Mongo 不支持时后台暂停,API 继续旧路径。
- 生产总览复用原事实读取与 `productionstats.Build`,保留原日期、供应商、人员和首次送检批次口径。当前配置项目集合生成一份报告;其他未预建项目组合走原路径。来源无变化且观察器新鲜时复用同一代次;有变化按原事实缓存时限停止使用旧结果。
- 人员与供应商看板复用既有分片快照和聚合器,后台无人访问也预热。Valkey 共享不可变分片及按完整范围、用户、供应商、日期、时区、代次隔离的结果;本进程已算出的结果优先,避免每次重复访问缓存服务器。
- Typesense 只存任务 ID 与名称搜索窗口,返回保守候选 ID。用 Unicode 窗口交集保持任意子串、Capture/part 和标点语义;最终匹配、排序、分页、实时可领取及权限仍走原服务端规则。候选过大、超时、索引过期或不完整时退回原查询,不能截断候选凑性能指标。
- 派生集合对通用数据库浏览入口隐藏,防止绕过组织和个人权限。Valkey、Typesense 只允许服务端私网访问,不开放浏览器直连。
## 配置与本地验证
API、后台分别显式设置 `QUERY_SERVING_READS=1` 与唯一 `QUERY_SERVING_NAMESPACE`,默认关闭。可选 `QUERY_VALKEY_URL`、`QUERY_TYPESENSE_URL`、`QUERY_TYPESENSE_API_KEY` 均通过现有私有环境配置提供;凭证不提交。不同集群即便数据库同名也必须使用不同 namespace。缓存连接异步建立,启动不可用不会阻止 API;缓存请求有短时限和退避。
本机合成依赖:`python3 scripts/test_query_serving.py --start`。仅暴露 loopback 27019/16379/18108;随机测试密钥在忽略目录,Mongo 使用独立 query-rs 和独立数据卷。`--down` 只停这一项目,保留卷;不停止共享快照库和其他项目。Mongo 文件句柄上限显式设为65536,避免多轮隔离测试建索引触发默认低限制。
功能与故障回归使用真实本机三组件,`--race` 检查并发。规模对照入口为 `python3 scripts/test_query_serving_performance.py --production-shape`,使用同一合成数据、索引、资源及固定压缩设置;可通过 `--database` 复用脚本生成的专用数据库。完整稳态验收另设 `--rounds 3 --warmup 300 --duration 1800`;短测通过不代表线上容量验收。
本机规模夹具统一使用 Mongo 2 CPU/4GiB(WiredTiger 0.5GiB)、Valkey 1 CPU/256MiB、Typesense 2 CPU/1GiB;这是隔离对照的固定配置,不是生产扩容建议。1GiB Mongo 的首轮对照因旧路径 OOM 作废,不能计为优化收益。
## 运行与清理
后台单进程生成、跨副本租约;Valkey 不持久化,清空后仅损失速度。Typesense 索引可重建。清理只删除当前数据库登记、已不可达且超过保留期的派生代次,当前清单和其他部署不在清理范围。固定构建预算和载荷预算超限时保留旧路径,不改任务准入。
显式 `query-read-builder --ensure-indexes` 只为派生行创建 `query_model_rows_v1`;API 启动不建索引。生产新增组件、该索引及派生数据构建会产生数据库读取和索引维护开销,需另列具体部署、资源和授权计划。不上线时不向公司集群安装或写入这些数据。
修改生产统计或名称匹配的语义时必须递增 `queryModelSchema`,防止新代码复用旧口径的派生代次。API 与后台应发布同一已核验源码版本。
## 发布与回退条件
先核对 Mongo 拓扑与权限、私网连接及组件资源,再在隔离环境验收完整任务链和查询结果。比较搜索 p95、冷/暖看板延迟、错误、结果/权限等价、后台业务写入为零、Mongo 读取字节与命令、Go/三组件 CPU 和内存。未达到搜索p95≤2秒、暖面板p95≤500ms、预热首访≤2秒,或三轮持续资源恶化未解释时保持待验收。
上线采用独立开关,先预热后逐步启用读取。回退关闭 API 的 `QUERY_SERVING_READS` 并停后台,恢复原读取;不改写客户记录,不要求删除新派生集合。组件失联仅是性能降级,不改变标注、切片、送检、质检和验收准入,也不恢复严格源校验。

1
go.mod
View File

@ -6,6 +6,7 @@ require (
github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible
github.com/baidubce/bce-sdk-go v0.9.264
github.com/go-chi/chi/v5 v5.2.3
github.com/valkey-io/valkey-go v1.0.60
go.etcd.io/bbolt v1.4.3
go.mongodb.org/mongo-driver v1.17.6
golang.org/x/crypto v0.36.0

4
go.sum
View File

@ -76,6 +76,7 @@ github.com/onsi/ginkgo/v2 v2.21.0 h1:7rg/4f3rB88pb5obDgNZrNHrQ4e6WpjonchcpuBRnZM
github.com/onsi/ginkgo/v2 v2.21.0/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo=
github.com/onsi/gomega v1.35.1 h1:Cwbd75ZBPxFSuZ6T+rN/WCb/gOc6YgFBXLlZLhC7Ds4=
github.com/onsi/gomega v1.35.1/go.mod h1:PvZbdDc8J6XJEpDK4HCuRBm8a6Fzp9/DmhC9C7yFlog=
github.com/onsi/gomega v1.36.2 h1:koNYke6TVk6ZmnyHrCXba/T/MoLBXFjeC1PtvYgw0A8=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
@ -95,6 +96,8 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
github.com/valkey-io/valkey-go v1.0.60 h1:idh959D20H5n7D/kwEdTKNaMn5+4HpZTn7bLXnAhQIw=
github.com/valkey-io/valkey-go v1.0.60/go.mod h1:bHmwjIEOrGq/ubOJfh5uMRs7Xj6mV3mQ/ZXUbmqpjqY=
github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM=
github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg=
github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
@ -169,6 +172,7 @@ golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4f
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
golang.org/x/tools v0.26.0 h1:v/60pFQmzmT9ExmjDv2gGIfi3OqfKoEP6I5+umXlbnQ=
golang.org/x/tools v0.26.0/go.mod h1:TPVVj70c7JJ3WCazhD8OdXcZg/og+b9+tH/KxylGwH0=
golang.org/x/tools v0.31.0 h1:0EedkvKDbh+qistFTd0Bcwe/YLh4vHwWEkiI0toFIBU=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

View File

@ -10,7 +10,6 @@ import (
"time"
"droboticflow/internal/domain"
"droboticflow/internal/outputsnapshot"
"droboticflow/internal/store"
"github.com/go-chi/chi/v5"
@ -148,7 +147,7 @@ func normalizeDatabaseBrowserTarget(raw string) databaseBrowserTarget {
func validateDatabaseCollectionName(field string, value string) *validationIssue {
name := strings.TrimSpace(value)
if outputsnapshot.IsPrivateCollection(name) {
if store.IsPrivateDerivedCollection(name) {
return &validationIssue{Field: field, Message: "internal statistics collection is not accessible"}
}
label := "collection name"

View File

@ -26,6 +26,8 @@ import (
"time"
"droboticflow/internal/domain"
"droboticflow/internal/productionstats"
"droboticflow/internal/queryserving"
"droboticflow/internal/store"
"go.mongodb.org/mongo-driver/event"
"go.mongodb.org/mongo-driver/mongo"
@ -96,6 +98,8 @@ func TestPollPoolReadMixLocal(t *testing.T) {
pageShape := os.Getenv("POLL_POOL_PAGE_SHAPE") == "1"
allowBaselineFailure := os.Getenv("POLL_POOL_ALLOW_BASELINE_FAILURE") == "1"
readModels := os.Getenv("POLL_POOL_READMODELS") == "1"
queryServing := os.Getenv("POLL_POOL_QUERY_SERVING") == "1"
queryPanels := os.Getenv("POLL_POOL_QUERY_PANELS") == "1"
mode := os.Getenv("POLL_POOL_MODE")
if mode == "" {
@ -108,6 +112,7 @@ func TestPollPoolReadMixLocal(t *testing.T) {
defer cancel()
var measured atomic.Bool
var commands, mongoBytes, mongoNS, writes atomic.Int64
var derivedWrites atomic.Int64
var probeMu sync.Mutex
pending := map[int64]struct {
key string
@ -154,10 +159,12 @@ func TestPollPoolReadMixLocal(t *testing.T) {
}
},
Succeeded: func(_ context.Context, e *event.CommandSucceededEvent) {
derived := false
probeMu.Lock()
if entry, ok := pending[e.RequestID]; ok {
key := entry.key
row := observations[key]
derived = strings.HasPrefix(row.Collection, "queryReadModel") || strings.HasPrefix(row.Collection, "annotationOutputSnapshot")
row.Count++
row.Bytes += int64(len(e.Reply))
row.Nanoseconds += e.Duration.Nanoseconds()
@ -170,7 +177,11 @@ func TestPollPoolReadMixLocal(t *testing.T) {
mongoNS.Add(e.Duration.Nanoseconds())
switch e.CommandName {
case "insert", "update", "delete", "findAndModify":
writes.Add(1)
if queryServing && derived {
derivedWrites.Add(1)
} else {
writes.Add(1)
}
}
}
}}
@ -235,6 +246,12 @@ func TestPollPoolReadMixLocal(t *testing.T) {
}
const org = "load-org"
projects := []string{"load-project-a", "load-project-b"}
if os.Getenv("POLL_POOL_QUERY_SERVING_PREPARE") == "1" && !reuseFixture {
must(ds.UpsertOrganization(ctx, domain.OrganizationRecord{ID: org, Status: domain.OrganizationStatusActive}))
_, err := ds.SaveProductionStatsScope(ctx, org, productionstats.Scope{ProjectIDs: projects, Revision: 1}, 0)
must(err)
must(ds.EnsureQueryReadModelIndexes(ctx))
}
if !reuseFixture {
for _, p := range projects {
must(ds.UpsertProject(ctx, domain.ProjectRecord{ID: p, OrganizationID: org, Name: p}))
@ -487,11 +504,13 @@ func TestPollPoolReadMixLocal(t *testing.T) {
elapsed := time.Since(start)
completedRequests.Add(1)
var body struct {
Count *int `json:"count"`
Counts map[string]int `json:"counts"`
Code string `json:"code"`
Error string `json:"error"`
Items []struct {
Summary *productionstats.Summary `json:"summary"`
Totals json.RawMessage `json:"totals"`
Count *int `json:"count"`
Counts map[string]int `json:"counts"`
Code string `json:"code"`
Error string `json:"error"`
Items []struct {
ID string `json:"id"`
CanClaim *bool `json:"canClaim"`
TraceSummary *domain.HumanTraceSummary `json:"traceSummary"`
@ -534,6 +553,15 @@ func TestPollPoolReadMixLocal(t *testing.T) {
completedErrors.Add(1)
}
rowCount := len(body.Items) + len(body.Summaries)
if body.Summary != nil {
rowCount = len(body.Summary.Rows)
raw, _ := json.Marshal(body.Summary)
ids = append(ids, string(raw))
}
if strings.HasSuffix(key, ".panel") && len(body.Totals) > 0 {
rowCount = 1
ids = append(ids, string(body.Totals))
}
if body.Count != nil {
rowCount = *body.Count
ids = append(ids, fmt.Sprintf("count:%d", *body.Count))
@ -704,8 +732,23 @@ func TestPollPoolReadMixLocal(t *testing.T) {
}
return result
}
if queryPanels {
original := paths
paths = func(a pollLoadActor, all, countCycle bool, scenario int) map[string]string {
out := original(a, all, countCycle, scenario)
out[a.Role+".panel"] = "/api/dashboard/production-quality?from=2026-10-01&to=2026-11-13"
if a.Role == "admin" {
out["admin.production"] = "/api/dashboard/production?from=2026-10-01&to=2026-11-13"
}
return out
}
}
fixtureSpec := fmt.Sprintf("isolated-preparation-v5|%d|%d|%d|%d|%d|%d|%d|%t|%t|%t|%s|%s", annotators, quality, acceptance, availablePerStage, runPayloadKiB, taskPayloadKiB, authorHistory, productionShape, readModels, claimCounts, authorizationState, os.Getenv("POLL_POOL_SEARCH_INDEXES"))
fixtureSpec += "|admin-tasks=" + os.Getenv("POLL_POOL_ADMIN_TASKS")
if os.Getenv("POLL_POOL_QUERY_SERVING_PREPARE") == "1" {
fixtureSpec += "|query-serving-preparation-v1"
}
fixtureDigest := fmt.Sprintf("%x", sha256.Sum256([]byte(fixtureSpec)))
if reuseFixture {
var manifest struct {
@ -742,6 +785,30 @@ func TestPollPoolReadMixLocal(t *testing.T) {
if os.Getenv("POLL_POOL_FIXTURE_ONLY") == "1" {
return
}
var preparationCPU float64
if queryServing {
startCPU, _ := pollLoadProcessCPU()
if os.Getenv("QUERY_SERVING_LOCAL_TEST") != "1" {
t.Fatal("query acceleration requires isolated local dependencies")
}
cache, err := queryserving.NewManagedValkey("valkey://127.0.0.1:16379", db.Name())
must(err)
defer cache.Close()
search, err := queryserving.NewTypesense("http://127.0.0.1:18108", os.Getenv("QUERY_TEST_TYPESENSE_API_KEY"))
must(err)
ds.ConfigureQueryServing(cache, search)
builder := store.NewQueryReadBuilder(ds)
buildCtx, stop := context.WithCancel(ctx)
buildDone := make(chan error, 1)
go func() { buildDone <- builder.Run(buildCtx) }()
defer func() { stop(); <-buildDone; builder.Close() }()
prepare, finish := context.WithTimeout(ctx, 4*time.Minute)
err = builder.WaitReady(prepare, org, projects)
finish()
must(err)
endCPU, _ := pollLoadProcessCPU()
preparationCPU = endCPU - startCPU
}
// Fail before timing if permissions or fixtures accidentally produce empty work.
for _, a := range actors {
for key, path := range paths(a, false, true, 0) {
@ -752,7 +819,11 @@ func TestPollPoolReadMixLocal(t *testing.T) {
}
t.Fatalf("preflight %s status=%d", key, result.Code)
}
if (a.Role == "annotation" || a.Role == "admin" && os.Getenv("POLL_POOL_ADMIN_TASKS") != "1" || strings.HasSuffix(key, ".available") || strings.HasSuffix(key, ".mine") && !strings.Contains(key, ".acceptance.")) && result.Rows == 0 {
// This workload seeds assigned annotation work, but no unassigned
// annotation pool unless claim-count coverage was requested. An empty
// admin available pool is then the correct fixture result, not failure.
requireRows := a.Role == "annotation" || a.Role == "admin" && os.Getenv("POLL_POOL_ADMIN_TASKS") != "1" || a.Role != "admin" && (strings.HasSuffix(key, ".available") || strings.HasSuffix(key, ".mine") && !strings.Contains(key, ".acceptance.")) || a.Role == "admin" && claimCounts && strings.HasSuffix(key, ".available")
if requireRows && result.Rows == 0 {
t.Fatalf("preflight %s unexpectedly empty for %s", key, a.User.ID)
}
}
@ -922,7 +993,7 @@ func TestPollPoolReadMixLocal(t *testing.T) {
}
}
}
report := map[string]any{"reused_prepared_fixture": reuseFixture, "fixture_digest": fixtureDigest, "cold_samples": coldSamples, "slow_search_endpoints": slowSearch, "baseline_failure_collection": allowBaselineFailure, "production_shape": productionShape, "page_shape": pageShape, "fixture_counts": fixtureCounts, "search_indexes": os.Getenv("POLL_POOL_SEARCH_INDEXES") == "1", "read_models": readModels, "search_queries": os.Getenv("POLL_POOL_SEARCH") == "1", "schema": 2, "resource_samples": resourceRows, "process_cpu_seconds": cpuAfter - cpuBefore, "authorization_state": authorizationState, "fixture": "read-mix-v5-isolated-preparation", "available_per_stage": availablePerStage, "task_payload_kib": taskPayloadKiB, "run_payload_kib": runPayloadKiB, "author_history_per_reviewer": authorHistory, "mode": mode, "revision": os.Getenv("POLL_POOL_REVISION"), "actors": totalActors, "actor_counts": map[string]int{"annotation": annotators, "quality": quality, "acceptance": acceptance, "admin": 1}, "claim_counts": claimCounts, "warmup_seconds": warmup, "scheduled_seconds": duration, "actual_seconds": actual.Seconds(), "errors": failures, "failure_samples": failureSamples, "metrics": metrics, "mongo_breakdown": observations, "mongo_commands": commands.Load(), "mongo_reply_bytes": mongoBytes.Load(), "mongo_command_ms": float64(mongoNS.Load()) / 1e6, "mongo_write_commands": writes.Load(), "go_heap_before": memBefore.HeapAlloc, "go_heap_after": memAfter.HeapAlloc, "go_heap_after_gc": memAfterGC.HeapAlloc, "go_gc_cycles": memAfter.NumGC - memBefore.NumGC, "go_allocated_bytes": memAfter.TotalAlloc - memBefore.TotalAlloc, "limitations": []string{"handler timing excludes ingress/network/authentication", "gzip computed separately with fixed Go default compression", "no continuation renew, media or workflow writes", "synthetic fixture; not capacity acceptance", "CPU covers handler and in-process load harness, not Mongo/Pod throttling", "resource heap includes retained benchmark request samples; not standalone service RSS"}, "samples": samples}
report := map[string]any{"query_serving": queryServing, "query_preparation_cpu_seconds": preparationCPU, "query_serving_statistics": ds.QueryServingStatistics(), "query_panels": queryPanels, "derived_write_commands": derivedWrites.Load(), "reused_prepared_fixture": reuseFixture, "fixture_digest": fixtureDigest, "cold_samples": coldSamples, "slow_search_endpoints": slowSearch, "baseline_failure_collection": allowBaselineFailure, "production_shape": productionShape, "page_shape": pageShape, "fixture_counts": fixtureCounts, "search_indexes": os.Getenv("POLL_POOL_SEARCH_INDEXES") == "1", "read_models": readModels, "search_queries": os.Getenv("POLL_POOL_SEARCH") == "1", "schema": 2, "resource_samples": resourceRows, "process_cpu_seconds": cpuAfter - cpuBefore, "authorization_state": authorizationState, "fixture": "read-mix-v5-isolated-preparation", "available_per_stage": availablePerStage, "task_payload_kib": taskPayloadKiB, "run_payload_kib": runPayloadKiB, "author_history_per_reviewer": authorHistory, "mode": mode, "revision": os.Getenv("POLL_POOL_REVISION"), "actors": totalActors, "actor_counts": map[string]int{"annotation": annotators, "quality": quality, "acceptance": acceptance, "admin": 1}, "claim_counts": claimCounts, "warmup_seconds": warmup, "scheduled_seconds": duration, "actual_seconds": actual.Seconds(), "errors": failures, "failure_samples": failureSamples, "metrics": metrics, "mongo_breakdown": observations, "mongo_commands": commands.Load(), "mongo_reply_bytes": mongoBytes.Load(), "mongo_command_ms": float64(mongoNS.Load()) / 1e6, "mongo_write_commands": writes.Load(), "go_heap_before": memBefore.HeapAlloc, "go_heap_after": memAfter.HeapAlloc, "go_heap_after_gc": memAfterGC.HeapAlloc, "go_gc_cycles": memAfter.NumGC - memBefore.NumGC, "go_allocated_bytes": memAfter.TotalAlloc - memBefore.TotalAlloc, "limitations": []string{"handler timing excludes ingress/network/authentication", "gzip computed separately with fixed Go default compression", "no continuation renew, media or workflow writes", "synthetic fixture; not capacity acceptance", "CPU covers handler and in-process load harness, not Mongo/Pod throttling", "resource heap includes retained benchmark request samples; not standalone service RSS"}, "samples": samples}
encoded, e := json.MarshalIndent(report, "", " ")
must(e)
must(os.WriteFile(out, encoded, 0600))

View File

@ -5,6 +5,7 @@ import (
"droboticflow/internal/domain"
"droboticflow/internal/productionstats"
"encoding/json"
"fmt"
"net/http"
"sort"
"strconv"
@ -138,19 +139,42 @@ func (s *server) handleProductionDashboard(w http.ResponseWriter, r *http.Reques
return
}
}
var snapshot productionstats.Snapshot
if cached, ok := s.store.(interface {
LoadProductionStatsCached(context.Context, string, []string) (productionstats.Snapshot, error)
responseIdentity := fmt.Sprint(scope.Revision) + ":" + q.From.Format(time.RFC3339) + ":" + q.To.Format(time.RFC3339) + ":" + params.Encode()
if prepared, ok := s.store.(interface {
PreparedProductionResponse(context.Context, string, []string, string) ([]byte, bool)
}); ok {
snapshot, err = cached.LoadProductionStatsCached(ctx, org, ids)
if data, hit := prepared.PreparedProductionResponse(ctx, org, ids, responseIdentity); hit {
w.Header().Set("Content-Type", "application/json")
w.Header().Set("Cache-Control", "no-store")
w.WriteHeader(200)
_, _ = w.Write(data)
return
}
}
var report productionstats.Report
var asOf string
if prepared, ok := s.store.(interface {
LoadProductionReport(context.Context, string, []string) (productionstats.Report, string, error)
}); ok {
report, asOf, err = prepared.LoadProductionReport(ctx, org, ids)
} else {
snapshot, err = s.store.LoadProductionStats(ctx, org, ids)
var snapshot productionstats.Snapshot
if cached, ok := s.store.(interface {
LoadProductionStatsCached(context.Context, string, []string) (productionstats.Snapshot, error)
}); ok {
snapshot, err = cached.LoadProductionStatsCached(ctx, org, ids)
} else {
snapshot, err = s.store.LoadProductionStats(ctx, org, ids)
}
if err == nil {
report = productionstats.Build(snapshot)
asOf = snapshot.AsOf
}
}
if err != nil {
renderError(w, 500, err)
return
}
report := productionstats.Build(snapshot)
summary := productionstats.Summarize(report, q)
suppliers, err := s.store.ListSuppliers(ctx, org)
if err != nil {
@ -196,7 +220,6 @@ func (s *server) handleProductionDashboard(w http.ResponseWriter, r *http.Reques
}
}
}
asOf := snapshot.AsOf
if asOf == "" {
asOf = started.Format(time.RFC3339Nano)
}
@ -223,5 +246,10 @@ func (s *server) handleProductionDashboard(w http.ResponseWriter, r *http.Reques
response["total"] = len(items)
response["page"] = page
}
if prepared, ok := s.store.(interface {
RememberProductionResponse(context.Context, string, []string, string, string, any)
}); ok {
prepared.RememberProductionResponse(ctx, org, ids, responseIdentity, asOf, response)
}
renderJSON(w, 200, response)
}

View File

@ -0,0 +1,76 @@
package api
import (
"context"
"droboticflow/internal/domain"
"droboticflow/internal/productionstats"
"droboticflow/internal/store"
"fmt"
"strings"
"sync/atomic"
"testing"
)
type preparedProductionStore struct {
store.Store
calls atomic.Int64
allowedIdentity string
}
func (s *preparedProductionStore) PreparedProductionResponse(_ context.Context, org string, projects []string, query string) ([]byte, bool) {
s.calls.Add(1)
key := fmt.Sprint(org, projects, query)
if s.allowedIdentity == "" {
s.allowedIdentity = key
}
if key != s.allowedIdentity {
return nil, false
}
return []byte(`{"cachedFixture":true}`), true
}
func TestQueryServingProductionResponseChecksCurrentPermissionBeforeCache(t *testing.T) {
f := newCombinedDraftFixture(t)
_, err := f.store.SaveProductionStatsScope(f.ctx, domain.DefaultOrganizationID, productionstats.Scope{ProjectIDs: []string{f.task.ProjectID}, Revision: 1}, 0)
if err != nil {
t.Fatal(err)
}
s := &preparedProductionStore{Store: f.store}
f.handler = NewHandler(s, nil)
path := "/api/dashboard/production?from=2026-11-01&to=2026-11-12"
r := f.request("GET", path, f.admin, nil)
if r.Code != 200 || !strings.Contains(r.Body.String(), "cachedFixture") {
t.Fatalf("cache not used %d %s", r.Code, r.Body.String())
}
before := s.calls.Load()
for _, user := range []domain.UserRecord{f.annotator, f.reviewer, {ID: "supplier-manager", OrganizationID: domain.DefaultOrganizationID, SupplierID: f.annotator.SupplierID, Role: domain.UserRoleSupplierManager, Status: domain.UserStatusActive}} {
if r = f.request("GET", path, user, nil); r.Code != 403 {
t.Fatalf("unauthorized cache read role=%s status=%d", user.Role, r.Code)
}
}
if s.calls.Load() != before {
t.Fatal("unauthorized request touched prepared response")
}
// Revoking a configured project changes identity before any cached body is
// served, even when its old generation and cache entry still exist.
_, err = f.store.SaveProductionStatsScope(f.ctx, domain.DefaultOrganizationID, productionstats.Scope{ProjectIDs: []string{}, Revision: 2}, 1)
if err != nil {
t.Fatal(err)
}
f.handler = NewHandler(s, nil)
r = f.request("GET", path, f.admin, nil)
if r.Code != 200 || strings.Contains(r.Body.String(), "cachedFixture") {
t.Fatal("revoked project reused old cached body", r.Code)
}
}
func TestQueryServingPrivateMaterializationsRejectDatabaseBrowser(t *testing.T) {
f := newCombinedDraftFixture(t)
for _, name := range []string{"queryReadModelControl", "queryReadModelData"} {
for _, target := range []string{"platform", "business"} {
if r := f.request("GET", "/api/database/collections/"+name+"/documents?target="+target, f.admin, nil); r.Code < 400 {
t.Fatalf("private read model leaked: %s %s", name, target)
}
}
}
}

View File

@ -0,0 +1,86 @@
package outputsnapshot
import (
"context"
"droboticflow/internal/queryserving"
"droboticflow/internal/workforcestats"
"sync"
"sync/atomic"
"testing"
"time"
)
type queryTestCache struct {
mu sync.Mutex
data map[string][]byte
}
func (c *queryTestCache) Get(_ context.Context, k string) ([]byte, bool) {
c.mu.Lock()
defer c.mu.Unlock()
v, ok := c.data[k]
return append([]byte(nil), v...), ok
}
func (c *queryTestCache) Put(_ context.Context, k string, v []byte, _ time.Duration) {
c.mu.Lock()
defer c.mu.Unlock()
if c.data == nil {
c.data = map[string][]byte{}
}
c.data[k] = append([]byte(nil), v...)
}
type queryCachedBackend struct {
MemoryBackend
cache queryTestCache
blobs atomic.Int64
}
func (b *queryCachedBackend) SharedQueryCache() queryserving.Cache { return &b.cache }
func (b *queryCachedBackend) OutputSnapshotBlob(ctx context.Context, g Generation) ([]byte, error) {
b.blobs.Add(1)
return b.MemoryBackend.OutputSnapshotBlob(ctx, g)
}
func TestQueryServingSharedOutputCacheKeepsScopeLocationAndGeneration(t *testing.T) {
now := time.Now().UTC()
backend := &queryCachedBackend{}
source := func(context.Context, string, []string) (workforcestats.OutputSnapshot, error) {
return testData(now), nil
}
one := NewReader(backend, source)
defer one.Close()
if _, err := one.Prewarm(t.Context(), "org", []string{"p"}); err != nil {
t.Fatal(err)
}
_, info := ready(t, one, "")
q := workforcestats.OutputQuery{From: now.Add(-24 * time.Hour), To: now.Add(time.Hour), Location: time.UTC, MatchUser: true, ScopeUserID: "u"}
r, _, err := one.Query(t.Context(), "org", []string{"p"}, info.ID, q)
if err != nil || r.Totals.Submitted.Count != 12 {
t.Fatal("initial report", err)
}
calls := backend.blobs.Load()
two := NewReader(backend, source)
defer two.Close()
q.Location = time.FixedZone("UTC", 0)
r, _, err = two.Query(t.Context(), "org", []string{"p"}, info.ID, q)
if err != nil || r.Totals.Submitted.Count != 12 || backend.blobs.Load() != calls {
t.Fatal("same immutable scope failed cross-reader reuse", err)
}
r.Tasks = nil
r, _, err = two.Query(t.Context(), "org", []string{"p"}, info.ID, q)
if err != nil || len(r.Tasks) == 0 {
t.Fatal("shared response mutated by another caller")
}
q.ScopeUserID = "not-authorized-user"
r, _, err = two.Query(t.Context(), "org", []string{"p"}, info.ID, q)
if err != nil || r.Totals.Submitted.Count != 0 {
t.Fatal("personal cache leaked another user's data", err)
}
if _, _, err = two.Query(t.Context(), "other-org", []string{"p"}, info.ID, q); err != ErrExpired {
t.Fatal("foreign organization borrowed generation", err)
}
if _, _, err = two.Query(t.Context(), "org", []string{"not-visible"}, info.ID, q); err != ErrExpired {
t.Fatal("changed project scope borrowed generation", err)
}
}

View File

@ -5,6 +5,7 @@ import (
"compress/gzip"
"context"
"crypto/sha256"
"droboticflow/internal/queryserving"
"droboticflow/internal/workforcestats"
"encoding/hex"
"encoding/json"
@ -43,6 +44,13 @@ func NewReader(b Backend, source Source) *Reader {
return &Reader{queryMemo: map[string][]byte{}, querySlots: make(chan struct{}, 2), backend: b, source: source, now: time.Now, ctx: ctx, cancel: cancel, attempts: map[string]time.Time{}, cache: map[string][]byte{}, reads: make(chan struct{}, 2)}
}
func (r *Reader) Close() { r.mu.Lock(); r.closed = true; r.cancel(); r.mu.Unlock(); r.wg.Wait() }
// Prewarm uses the existing fenced publication protocol, without a customer
// request or mutation. Repeated workers cannot duplicate a successful refresh.
func (r *Reader) Prewarm(ctx context.Context, org string, projects []string) (Info, error) {
_, info, err := r.resolve(ctx, org, projects, "")
return info, err
}
func scopeKey(org string, projects []string) string {
ids := append([]string(nil), projects...)
sort.Strings(ids)
@ -256,7 +264,7 @@ func (r *Reader) build(key, org string, projects []string) {
return
}
hash := sha256.Sum256(payload)
gen := Generation{Metrics: &BuildMetrics{DurationMS: time.Since(started).Milliseconds(), DecodedBytes: decodedBytes, Parts: len(data.Parts), Source: raw.SourceStats}, ID: fmt.Sprintf("of1-%s-%d", lease.Owner, lease.Epoch), Slot: lease.Slot, Epoch: lease.Epoch, AsOf: asOf, SHA256: hex.EncodeToString(hash[:]), Bytes: len(payload)}
gen := Generation{Metrics: &BuildMetrics{DurationMS: time.Since(started).Milliseconds(), DecodedBytes: decodedBytes, Parts: len(data.Parts), Source: raw.SourceStats}, ID: GenerationID(lease), Slot: lease.Slot, Epoch: lease.Epoch, AsOf: asOf, SHA256: hex.EncodeToString(hash[:]), Bytes: len(payload)}
stage = "publish"
published, buildErr = r.backend.PublishOutputSnapshot(ctx, lease, gen, payload, r.now())
if published {
@ -349,6 +357,17 @@ func (r *Reader) Query(ctx context.Context, org string, projects []string, pin s
// generation and location identity participate in the key, including personal
// and supplier isolation. No permissions or mutable datasets are cached.
q.Now = gen.AsOf
var shared queryserving.Cache
if provider, ok := r.backend.(interface{ SharedQueryCache() queryserving.Cache }); ok {
shared = provider.SharedQueryCache()
}
location := ""
if q.Location != nil {
_, fromOffset := q.From.In(q.Location).Zone()
_, toOffset := q.To.In(q.Location).Zone()
location = fmt.Sprintf("%s:%d:%d", q.Location.String(), fromOffset, toOffset)
}
sharedKey := queryserving.Key("output-query/v1", scopeKey(org, projects), gen.ID, q, location)
keyData, _ := json.Marshal([]any{gen.ID, q, fmt.Sprintf("%p", q.Location)})
key := string(keyData)
select {
@ -385,6 +404,11 @@ func (r *Reader) Query(ctx context.Context, org string, projects []string, pin s
if report := r.memoReport(key); report != nil {
return answer{report, info}, nil
}
var sharedReport workforcestats.OutputReport
if queryserving.GetJSON(buildCtx, shared, sharedKey, &sharedReport) {
r.rememberReport(key, &sharedReport)
return answer{&sharedReport, info}, nil
}
report, actual, err := r.queryResolved(buildCtx, gen, info, pin, q)
if err != nil || report == nil {
return nil, err
@ -392,6 +416,7 @@ func (r *Reader) Query(ctx context.Context, org string, projects []string, pin s
// Corrupt-current fallback must not poison the current generation's key.
if actual.ID == gen.ID {
r.rememberReport(key, report)
queryserving.PutJSON(buildCtx, shared, sharedKey, report, MaxAge)
}
return answer{report, actual}, nil
})

View File

@ -91,7 +91,7 @@ func (r *Reader) buildShards(ctx context.Context, lease Lease, org string, proje
}
metrics.Source = source
metrics.DurationMS = time.Since(started).Milliseconds()
g := Generation{ID: fmt.Sprintf("of1-%s-%d", lease.Owner, lease.Epoch), Slot: lease.Slot, Epoch: lease.Epoch, AsOf: asOf, SHA256: digest(payload), Bytes: total, Shards: len(manifest.Shards), ManifestBytes: len(payload), Metrics: metrics}
g := Generation{ID: GenerationID(lease), Slot: lease.Slot, Epoch: lease.Epoch, AsOf: asOf, SHA256: digest(payload), Bytes: total, Shards: len(manifest.Shards), ManifestBytes: len(payload), Metrics: metrics}
return g, payload, nil
}
func decodeShard(payload []byte, g Generation) (*workforcestats.OutputDataset, error) {

View File

@ -6,9 +6,12 @@ import (
"context"
"droboticflow/internal/workforcestats"
"errors"
"fmt"
"time"
)
func GenerationID(l Lease) string { return fmt.Sprintf("of1-%s-%d", l.Owner, l.Epoch) }
const (
Schema = "annotation-output-facts/v5"
RefreshInterval = 120 * time.Second

View File

@ -0,0 +1,199 @@
// Package queryserving contains optional derived-data accelerators. None owns
// business state, permissions or task mutations; every failure is a cache miss.
package queryserving
import (
"bytes"
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"sync"
"sync/atomic"
"time"
valkey "github.com/valkey-io/valkey-go"
)
const MaxValueBytes = 4 << 20
type Cache interface {
Get(context.Context, string) ([]byte, bool)
Put(context.Context, string, []byte, time.Duration)
}
// Key includes the schema/database namespace supplied by the caller and only
// hashes its remaining identity; no customer names or query text become keys.
func Key(namespace string, identity ...any) string {
b, _ := json.Marshal(identity)
sum := sha256.Sum256(b)
return namespace + ":" + hex.EncodeToString(sum[:])
}
type Valkey struct {
client valkey.Client
prefix string
until atomic.Int64
Hits, Misses, Errors atomic.Uint64
}
func NewValkey(address, namespace string) (*Valkey, error) {
opt, err := valkey.ParseURL(address)
if err != nil {
return nil, errors.New("invalid shared cache configuration")
}
opt.DisableCache = true // no implicit client-side cache or permission reuse
opt.Dialer.Timeout = 200 * time.Millisecond
opt.ConnWriteTimeout = 200 * time.Millisecond
opt.DisableRetry = true
opt.PipelineMultiplex = -1
opt.BlockingPoolSize = 4
opt.ReadBufferEachConn = 64 << 10
opt.WriteBufferEachConn = 64 << 10
client, err := valkey.NewClient(opt)
if err != nil {
return nil, errors.New("shared cache unavailable")
}
return &Valkey{client: client, prefix: Key("droboticflow-query/v1", namespace)}, nil
}
// ManagedValkey connects asynchronously so an absent cache at startup cannot
// delay or fail API availability. Connections are retried without request load.
type ManagedValkey struct {
mu sync.RWMutex
value *Valkey
cancel context.CancelFunc
done chan struct{}
}
func NewManagedValkey(address, namespace string) (*ManagedValkey, error) {
if _, err := valkey.ParseURL(address); err != nil {
return nil, errors.New("invalid shared cache configuration")
}
ctx, cancel := context.WithCancel(context.Background())
m := &ManagedValkey{cancel: cancel, done: make(chan struct{})}
go func() {
defer close(m.done)
for {
v, err := NewValkey(address, namespace)
if err == nil {
m.mu.Lock()
if ctx.Err() == nil {
m.value = v
m.mu.Unlock()
<-ctx.Done()
v.Close()
return
}
m.mu.Unlock()
v.Close()
return
}
select {
case <-ctx.Done():
return
case <-time.After(time.Second):
}
}
}()
return m, nil
}
func (m *ManagedValkey) Get(ctx context.Context, key string) ([]byte, bool) {
m.mu.RLock()
v := m.value
m.mu.RUnlock()
if v == nil {
return nil, false
}
return v.Get(ctx, key)
}
func (m *ManagedValkey) Put(ctx context.Context, key string, b []byte, ttl time.Duration) {
m.mu.RLock()
v := m.value
m.mu.RUnlock()
if v != nil {
v.Put(ctx, key, b, ttl)
}
}
func (m *ManagedValkey) Close() { m.cancel(); <-m.done }
func (m *ManagedValkey) Ready() bool { m.mu.RLock(); defer m.mu.RUnlock(); return m.value != nil }
func (v *Valkey) Close() { v.client.Close() }
func (v *Valkey) failed() { v.Errors.Add(1); v.until.Store(time.Now().Add(time.Second).UnixNano()) }
func (v *Valkey) Get(ctx context.Context, key string) ([]byte, bool) {
parent := ctx
if parent.Err() != nil {
return nil, false
}
if time.Now().UnixNano() < v.until.Load() {
v.Misses.Add(1)
return nil, false
}
ctx, cancel := context.WithTimeout(ctx, 40*time.Millisecond)
defer cancel()
b, err := v.client.Do(ctx, v.client.B().Get().Key(v.prefix+":"+key).Build()).AsBytes()
if err != nil {
if !valkey.IsValkeyNil(err) && parent.Err() == nil {
v.failed()
}
v.Misses.Add(1)
return nil, false
}
if len(b) < 36 || len(b) > MaxValueBytes+36 || string(b[:4]) != "DFQ1" {
v.Misses.Add(1)
return nil, false
}
sum := sha256.Sum256(append([]byte(key+"\x00"), b[36:]...))
if !bytes.Equal(sum[:], b[4:36]) {
v.Misses.Add(1)
return nil, false
}
v.Hits.Add(1)
return append([]byte(nil), b[36:]...), true
}
func (v *Valkey) Put(ctx context.Context, key string, b []byte, ttl time.Duration) {
parent := ctx
if parent.Err() != nil {
return
}
if len(b) > MaxValueBytes || ttl <= 0 || time.Now().UnixNano() < v.until.Load() {
return
}
ctx, cancel := context.WithTimeout(ctx, 40*time.Millisecond)
defer cancel()
sum := sha256.Sum256(append([]byte(key+"\x00"), b...))
encoded := append(append([]byte("DFQ1"), sum[:]...), b...)
if err := v.client.Do(ctx, v.client.B().Set().Key(v.prefix+":"+key).Value(string(encoded)).Px(ttl).Build()).Error(); err != nil {
if parent.Err() == nil {
v.failed()
}
}
}
func GetJSON(ctx context.Context, c Cache, key string, value any) bool {
if c == nil {
return false
}
b, ok := c.Get(ctx, key)
return ok && json.Unmarshal(b, value) == nil
}
func PutJSON(ctx context.Context, c Cache, key string, value any, ttl time.Duration) {
if c == nil {
return
}
b, err := json.Marshal(value)
if err == nil {
c.Put(ctx, key, b, ttl)
}
}
func (m *ManagedValkey) Statistics() map[string]uint64 {
m.mu.RLock()
v := m.value
m.mu.RUnlock()
if v == nil {
return map[string]uint64{}
}
return map[string]uint64{"hits": v.Hits.Load(), "misses": v.Misses.Load(), "errors": v.Errors.Load()}
}

View File

@ -0,0 +1,138 @@
package queryserving
import (
"context"
"encoding/json"
"net/http"
"net/http/httptest"
"os"
"reflect"
"strings"
"testing"
"time"
)
func TestQueryServingGramsPreserveSubstringCandidates(t *testing.T) {
for _, text := range []string{"FYLX_B07_AZER764008K_20260927_074440/part0002", "İ中文😀_左右/part", "a.b+?x[1]", "abc def ghi"} {
r := []rune(text)
indexed := map[string]bool{}
for _, g := range IndexGrams(text) {
indexed[g] = true
}
for a := range r {
for z := a + 1; z <= len(r); z++ {
for _, g := range Grams(string(r[a:z])) {
if !indexed[g] {
t.Fatalf("substring lost %q in %q", r[a:z], text)
}
}
}
}
}
if reflect.DeepEqual(Key("namespace-a", "org", []string{"p"}), Key("namespace-b", "org", []string{"p"})) {
t.Fatal("namespace collision")
}
}
func TestQueryServingTypesenseIncompleteImportAndExportFailClosed(t *testing.T) {
mode := "partial"
secret := "synthetic-secret-only"
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Header.Get("X-TYPESENSE-API-KEY") != secret {
t.Error("missing backend authentication")
}
switch mode {
case "partial":
_, _ = w.Write([]byte("{\"success\":true}\n{\"success\":false,\"error\":\"" + secret + "\"}\n"))
case "short":
_, _ = w.Write([]byte("{\"success\":true}\n"))
case "malformed":
_, _ = w.Write([]byte("{\"item_id\":\"one\"}\n{bad"))
case "failure":
w.WriteHeader(503)
_, _ = w.Write([]byte(secret))
case "ok":
_, _ = w.Write([]byte("{\"item_id\":\"one\"}\n{\"item_id\":\"two\"}\n"))
}
}))
defer server.Close()
client, _ := NewTypesense(server.URL, secret)
docs := []SearchDocument{{ID: "one"}, {ID: "two"}}
for _, m := range []string{"partial", "short"} {
mode = m
if err := client.Import(t.Context(), "dfq_test", docs); err == nil || strings.Contains(err.Error(), secret) {
t.Fatal("partial import accepted or secret leaked", err)
}
}
for _, m := range []string{"malformed", "failure"} {
mode = m
if ids, err := client.Candidates(t.Context(), "dfq_test", "_x/"); err == nil || ids != nil || strings.Contains(err.Error(), secret) {
t.Fatal("incomplete export accepted or secret leaked", ids, err)
}
}
mode = "ok"
if ids, err := client.Candidates(t.Context(), "dfq_test", "part"); err != nil || !reflect.DeepEqual(ids, []string{"one", "two"}) {
t.Fatal(ids, err)
}
}
func TestQueryServingValkeyRealIsolationCorruptionAndColdRecoveryLocal(t *testing.T) {
if os.Getenv("QUERY_SERVING_LOCAL_TEST") != "1" {
t.Skip("isolated loopback query dependencies")
}
namespace := t.Name() + time.Now().Format(time.RFC3339Nano)
a, err := NewValkey("valkey://127.0.0.1:16379", namespace)
if err != nil {
t.Fatal(err)
}
defer a.Close()
b, err := NewValkey("valkey://127.0.0.1:16379", namespace)
if err != nil {
t.Fatal(err)
}
defer b.Close()
foreign, err := NewValkey("valkey://127.0.0.1:16379", namespace+"-foreign")
if err != nil {
t.Fatal(err)
}
defer foreign.Close()
a.Put(t.Context(), "generation-one", []byte(`{"value":42}`), time.Second)
if data, ok := b.Get(t.Context(), "generation-one"); !ok || string(data) != `{"value":42}` {
t.Fatal("cross-instance cache missed")
}
if _, ok := foreign.Get(t.Context(), "generation-one"); ok {
t.Fatal("foreign namespace read cache")
}
if err = a.client.Do(t.Context(), a.client.B().Set().Key(a.prefix+":generation-one").Value(`{}`).Build()).Error(); err != nil {
t.Fatal(err)
}
if _, ok := b.Get(t.Context(), "generation-one"); ok {
t.Fatal("corrupt valid JSON treated as a valid cache value")
}
managed, err := NewManagedValkey("valkey://127.0.0.1:16379", namespace)
if err != nil {
t.Fatal(err)
}
defer managed.Close()
deadline := time.Now().Add(2 * time.Second)
for !managed.Ready() && time.Now().Before(deadline) {
time.Sleep(10 * time.Millisecond)
}
if !managed.Ready() {
t.Fatal("managed cache did not connect")
}
ctx, cancel := context.WithCancel(t.Context())
cancel()
if _, ok := managed.Get(ctx, "generation-one"); ok {
t.Fatal("cancelled cache read succeeded")
}
managed.Put(t.Context(), "different-query", []byte("[]"), time.Second)
data, ok := managed.Get(t.Context(), "different-query")
if !ok {
t.Fatal("managed cache did not warm")
}
var value []string
if json.Unmarshal(data, &value) != nil {
t.Fatal("cache payload changed")
}
}

View File

@ -0,0 +1,190 @@
package queryserving
import (
"bufio"
"bytes"
"context"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"sort"
"strings"
"time"
)
type SearchDocument struct {
ID string `json:"id"`
ItemID string `json:"item_id"`
Grams []string `json:"grams"`
}
// Grams are hex-encoded Unicode windows, independent of Typesense's tokenizer,
// punctuation/typo/prefix rules. Intersection is only a conservative prefilter;
// the existing Go matcher remains the final authority. No query is truncated.
func Grams(text string) []string {
r := []rune(strings.ToLower(text))
width := min(3, len(r))
if width == 0 {
return nil
}
set := map[string]bool{}
for i := 0; i+width <= len(r); i++ {
set[hex.EncodeToString([]byte(string(r[i:i+width])))] = true
}
out := make([]string, 0, len(set))
for s := range set {
out = append(out, s)
}
sort.Strings(out)
return out
}
func IndexGrams(text string) []string {
r := []rune(strings.ToLower(text))
set := map[string]bool{}
for width := 1; width <= 3; width++ {
for i := 0; i+width <= len(r); i++ {
set[hex.EncodeToString([]byte(string(r[i:i+width])))] = true
}
}
out := make([]string, 0, len(set))
for s := range set {
out = append(out, s)
}
sort.Strings(out)
return out
}
type Typesense struct {
base, key string
client *http.Client
}
type HTTPError struct{ Status int }
func (e *HTTPError) Error() string { return fmt.Sprintf("search backend status %d", e.Status) }
func NewTypesense(address, key string) (*Typesense, error) {
u, err := url.Parse(address)
if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" || u.User != nil || u.RawQuery != "" || u.Fragment != "" || (u.Path != "" && u.Path != "/") || key == "" {
return nil, errors.New("invalid search configuration")
}
return &Typesense{base: strings.TrimRight(address, "/"), key: key, client: &http.Client{Timeout: 30 * time.Second, CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }}}, nil
}
func (t *Typesense) request(ctx context.Context, method, path string, body []byte) ([]byte, error) {
r, err := http.NewRequestWithContext(ctx, method, t.base+path, bytes.NewReader(body))
if err != nil {
return nil, errors.New("search request invalid")
}
r.Header.Set("X-TYPESENSE-API-KEY", t.key)
r.Header.Set("Content-Type", "application/json")
res, err := t.client.Do(r)
if err != nil {
return nil, errors.New("search backend unavailable")
}
defer res.Body.Close()
if res.StatusCode < 200 || res.StatusCode >= 300 {
return nil, &HTTPError{Status: res.StatusCode}
}
b, err := io.ReadAll(io.LimitReader(res.Body, 32<<20+1))
if err != nil || len(b) > 32<<20 {
return nil, errors.New("search response incomplete")
}
return b, nil
}
func validCollection(c string) bool {
if !strings.HasPrefix(c, "dfq_") || len(c) > 120 {
return false
}
for _, r := range c {
if r != '_' && (r < 'a' || r > 'z') && (r < '0' || r > '9') {
return false
}
}
return true
}
func (t *Typesense) Create(ctx context.Context, c string) error {
if !validCollection(c) {
return errors.New("invalid search collection")
}
b, _ := json.Marshal(map[string]any{"name": c, "fields": []map[string]any{{"name": "item_id", "type": "string", "index": false}, {"name": "grams", "type": "string[]", "facet": true}}})
_, err := t.request(ctx, "POST", "/collections", b)
return err
}
func (t *Typesense) Import(ctx context.Context, c string, docs []SearchDocument) error {
if !validCollection(c) {
return errors.New("invalid search collection")
}
var data bytes.Buffer
for _, d := range docs {
if err := json.NewEncoder(&data).Encode(d); err != nil {
return err
}
}
b, err := t.request(ctx, "POST", "/collections/"+c+"/documents/import?action=create", data.Bytes())
if err != nil {
return err
}
s := bufio.NewScanner(bytes.NewReader(b))
s.Buffer(make([]byte, 4096), 1<<20)
count := 0
for s.Scan() {
var result struct {
Success bool `json:"success"`
}
if json.Unmarshal(s.Bytes(), &result) != nil || !result.Success {
return errors.New("search import incomplete")
}
count++
}
if s.Err() != nil || count != len(docs) {
return errors.New("search import response incomplete")
}
return nil
}
func (t *Typesense) Candidates(ctx context.Context, c string, terms ...string) ([]string, error) {
if !validCollection(c) {
return nil, errors.New("invalid search collection")
}
filters := []string{}
for _, term := range terms {
for _, g := range Grams(term) {
filters = append(filters, "grams:=["+g+"]")
}
}
params := url.Values{"include_fields": {"item_id"}}
if len(filters) > 0 {
params.Set("filter_by", strings.Join(filters, " && "))
}
b, err := t.request(ctx, "GET", "/collections/"+c+"/documents/export?"+params.Encode(), nil)
if err != nil {
return nil, err
}
s := bufio.NewScanner(bytes.NewReader(b))
s.Buffer(make([]byte, 4096), 1<<20)
ids := []string{}
for s.Scan() {
var d SearchDocument
if json.Unmarshal(s.Bytes(), &d) != nil || d.ItemID == "" {
return nil, errors.New("search candidates incomplete")
}
ids = append(ids, d.ItemID)
}
if s.Err() != nil {
return nil, errors.New("search candidates incomplete")
}
return ids, nil
}
func (t *Typesense) Delete(ctx context.Context, c string) error {
if !validCollection(c) {
return errors.New("invalid search collection")
}
_, err := t.request(ctx, "DELETE", "/collections/"+c, nil)
var status *HTTPError
if errors.As(err, &status) && status.Status == 404 {
return nil
}
return err
}

View File

@ -157,6 +157,29 @@ func adminPoolRowMatches(row adminPoolRow, q AdminAnnotationPoolQuery) bool {
if q.Search == "" && q.Capture == "" {
return true
}
values := adminPoolCaptureValues(row)
if q.Capture != "" {
found := false
for _, v := range values {
if strings.EqualFold(strings.TrimSpace(v), q.Capture) {
found = true
}
}
if !found {
return false
}
}
if q.Search != "" && !strings.Contains(adminPoolSearchText(row), strings.ToLower(q.Search)) {
return false
}
return true
}
func adminPoolCaptureValues(row adminPoolRow) []string {
var task domain.AnnotationTaskRecord
if len(row.Tasks) > 0 {
task = row.Tasks[0]
}
values := []string{}
if len(row.Captures) > 0 {
c := row.Captures[0]
@ -180,29 +203,18 @@ func adminPoolRowMatches(row adminPoolRow, q AdminAnnotationPoolQuery) bool {
if len(row.Runs) > 0 {
values = append(values, domain.CaptureDirectoriesFromRun(row.Runs[0])...)
}
if q.Capture != "" {
found := false
for _, v := range values {
if strings.EqualFold(strings.TrimSpace(v), q.Capture) {
found = true
}
}
if !found {
return false
}
}
if q.Search != "" {
if len(row.Projects) > 0 {
values = append(values, row.Projects[0].Name)
} else if len(row.Runs) > 0 {
values = append(values, row.Runs[0].ProjectName)
}
if !strings.Contains(strings.ToLower(strings.Join(values, " ")), strings.ToLower(q.Search)) {
return false
}
}
return true
return values
}
func adminPoolSearchText(row adminPoolRow) string {
values := adminPoolCaptureValues(row)
if len(row.Projects) > 0 {
values = append(values, row.Projects[0].Name)
} else if len(row.Runs) > 0 {
values = append(values, row.Runs[0].ProjectName)
}
return strings.ToLower(strings.Join(values, " "))
}
func (s *MemoryStore) adminPoolRowsLocked(org string, q AdminAnnotationPoolQuery) []adminPoolRow {
rows := []adminPoolRow{}
for _, i := range s.humanItems {
@ -288,7 +300,14 @@ func (s *MongoStore) scanAdminAnnotationPool(ctx context.Context, org string, q
}
ctx, cancel := context.WithTimeout(ctx, 8*time.Second)
defer cancel()
cur, err := s.humanItems.Aggregate(ctx, adminPoolPagePipeline(org, q), options.Aggregate().SetAllowDiskUse(false).SetMaxTime(8*time.Second).SetBatchSize(200))
pipeline := adminPoolPagePipeline(org, q)
if ids, ready := s.querySearchCandidates(ctx, org, q.Search, q.Capture); ready {
if len(ids) == 0 {
return nil
}
pipeline[0] = bson.D{{Key: "$match", Value: bson.M{"$and": bson.A{pipeline[0][0].Value, bson.M{"id": bson.M{"$in": ids}}}}}}
}
cur, err := s.humanItems.Aggregate(ctx, pipeline, options.Aggregate().SetAllowDiskUse(false).SetMaxTime(8*time.Second).SetBatchSize(200))
if err != nil {
return err
}

View File

@ -4,6 +4,7 @@ import (
"context"
"droboticflow/internal/domain"
"droboticflow/internal/outputsnapshot"
"droboticflow/internal/queryserving"
"fmt"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo"
@ -113,6 +114,12 @@ func (s *MongoStore) FailOutputSnapshot(ctx context.Context, l outputsnapshot.Le
return err
}
func (s *MongoStore) OutputSnapshotBlob(ctx context.Context, g outputsnapshot.Generation) ([]byte, error) {
key := queryserving.Key("output-blob/v1", s.db.Name(), g.ID, g.Slot, g.Epoch, g.SHA256)
if cache := s.SharedQueryCache(); cache != nil {
if b, hit := cache.Get(ctx, key); hit {
return b, nil
}
}
var doc struct {
Data []byte `bson:"data"`
}
@ -120,6 +127,11 @@ func (s *MongoStore) OutputSnapshotBlob(ctx context.Context, g outputsnapshot.Ge
if err == mongo.ErrNoDocuments {
err = outputsnapshot.ErrExpired
}
if err == nil {
if cache := s.SharedQueryCache(); cache != nil {
cache.Put(ctx, key, doc.Data, outputsnapshot.MaxAge)
}
}
return doc.Data, err
}
@ -143,9 +155,20 @@ func (s *MongoStore) StageOutputSnapshotShard(ctx context.Context, l outputsnaps
// Immutable: never overwrite a staged shard.
id := outputShardID(l.Slot, l.Epoch, index)
_, err = s.outputBlobs().InsertOne(ctx, bson.M{"_id": id, "data": data})
if err == nil {
if cache := s.SharedQueryCache(); cache != nil {
cache.Put(ctx, queryserving.Key("output-shard/v1", s.db.Name(), outputsnapshot.GenerationID(l), l.Slot, l.Epoch, index), data, outputsnapshot.MaxAge)
}
}
return err == nil, err
}
func (s *MongoStore) OutputSnapshotShard(ctx context.Context, g outputsnapshot.Generation, index int) ([]byte, error) {
key := queryserving.Key("output-shard/v1", s.db.Name(), g.ID, g.Slot, g.Epoch, index)
if cache := s.SharedQueryCache(); cache != nil {
if b, hit := cache.Get(ctx, key); hit {
return b, nil
}
}
var doc struct {
Data []byte `bson:"data"`
}
@ -153,6 +176,11 @@ func (s *MongoStore) OutputSnapshotShard(ctx context.Context, g outputsnapshot.G
if err == mongo.ErrNoDocuments {
err = outputsnapshot.ErrExpired
}
if err == nil {
if cache := s.SharedQueryCache(); cache != nil {
cache.Put(ctx, key, doc.Data, outputsnapshot.MaxAge)
}
}
return doc.Data, err
}
func (s *MemoryStore) StageOutputSnapshotShard(ctx context.Context, l outputsnapshot.Lease, i int, b []byte, t time.Time) (bool, error) {

View File

@ -46,6 +46,9 @@ func humanPoolSearchRowMatches(row adminPoolRow, q domain.HumanItemQuery) bool {
if adminPoolRowMatches(row, AdminAnnotationPoolQuery{Search: q.Search}) {
return true
}
return strings.Contains(humanPoolFallbackSearchText(row), strings.ToLower(q.Search))
}
func humanPoolFallbackSearchText(row adminPoolRow) string {
values := []string{row.ID, row.LegacyTaskID, row.Subject.ItemKey, row.Subject.ArtifactID}
for _, t := range row.Tasks {
values = append(values, t.ArtifactID, t.SourceVideo)
@ -76,7 +79,7 @@ func humanPoolSearchRowMatches(row adminPoolRow, q domain.HumanItemQuery) bool {
values = append(values, capture+"/"+match[1])
}
}
return strings.Contains(strings.ToLower(strings.Join(values, " ")), strings.ToLower(q.Search))
return strings.ToLower(strings.Join(values, " "))
}
// Stream metadata through the existing search rules before applying the page
@ -121,6 +124,12 @@ func (s *MongoStore) prepareHumanPoolSearchFilter(ctx context.Context, org strin
}
filter = bson.M{"$and": bson.A{filter, visibility}}
}
if ids, ready := s.querySearchCandidates(ctx, org, q.Search, q.Capture); ready {
if len(ids) == 0 {
return nil, true, nil
}
return bson.M{"$and": bson.A{filter, bson.M{"id": bson.M{"$in": ids}}}}, false, nil
}
candidates, err := s.humanPoolSearchCandidates(ctx, org, q, filter)
if err != nil {
return nil, false, err

View File

@ -3266,7 +3266,7 @@ func (store *MemoryStore) GetDatabaseOverview(context.Context) (domain.DatabaseO
Collections: make([]domain.DatabaseCollectionSummary, 0, len(names)),
}
for _, name := range names {
if outputsnapshot.IsPrivateCollection(name) {
if IsPrivateDerivedCollection(name) {
continue
}
documents := collectionsMap[name]
@ -3283,7 +3283,7 @@ func (store *MemoryStore) GetDatabaseOverview(context.Context) (domain.DatabaseO
}
func (store *MemoryStore) QueryDatabaseCollection(_ context.Context, input domain.DatabaseQueryInput) (domain.DatabaseQueryResult, error) {
if outputsnapshot.IsPrivateCollection(input.Collection) {
if IsPrivateDerivedCollection(input.Collection) {
return domain.DatabaseQueryResult{}, fmt.Errorf("collection %s not found", input.Collection)
}
store.mu.Lock()

View File

@ -12,7 +12,6 @@ import (
"time"
"droboticflow/internal/domain"
"droboticflow/internal/outputsnapshot"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/bson/primitive"
@ -21,6 +20,7 @@ import (
)
type MongoStore struct {
queryServing *queryServingOptions
productionReadFlights singleflight.Group
claimReadFlights singleflight.Group
poolSearchReadFlights singleflight.Group
@ -2808,7 +2808,7 @@ func (store *MongoStore) GetDatabaseOverview(ctx context.Context) (domain.Databa
Collections: make([]domain.DatabaseCollectionSummary, 0, len(names)),
}
for _, name := range names {
if outputsnapshot.IsPrivateCollection(name) {
if IsPrivateDerivedCollection(name) {
continue
}
summary, err := mongoBrowserCollectionSummary(ctx, store.db.Collection(name), name)
@ -2825,7 +2825,7 @@ func (store *MongoStore) QueryDatabaseCollection(ctx context.Context, input doma
if collectionName == "" {
return domain.DatabaseQueryResult{}, fmt.Errorf("collection is required")
}
if outputsnapshot.IsPrivateCollection(collectionName) {
if IsPrivateDerivedCollection(collectionName) {
return domain.DatabaseQueryResult{}, fmt.Errorf("collection %s not found", collectionName)
}
limit := input.Limit

View File

@ -0,0 +1,9 @@
package store
import "droboticflow/internal/outputsnapshot"
// Derived generations contain facts from multiple authorization scopes and
// must only be served through their permission-aware reporting/search readers.
func IsPrivateDerivedCollection(name string) bool {
return outputsnapshot.IsPrivateCollection(name) || name == "queryReadModelControl" || name == "queryReadModelData"
}

View File

@ -0,0 +1,84 @@
package store
import (
"context"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
"time"
)
// This is an explicit derived-data index operation, never API startup behavior.
// Production execution requires its own reviewed migration/index authorization.
func (s *MongoStore) EnsureQueryReadModelIndexes(ctx context.Context) error {
_, err := s.queryData().Indexes().CreateOne(ctx, mongo.IndexModel{Keys: bson.D{{Key: "generation", Value: 1}, {Key: "organizationId", Value: 1}, {Key: "itemId", Value: 1}}, Options: options.Index().SetName("query_model_rows_v1")})
return err
}
func (s *MongoStore) cleanQueryModelOrphans(ctx context.Context) error {
var state struct {
Heads map[string]QueryModelHead `bson:"heads"`
}
if err := s.queryControl().FindOne(ctx, bson.M{"_id": "global"}).Decode(&state); err != nil {
return err
}
keep := []string{}
for _, h := range state.Heads {
keep = append(keep, h.Generation)
}
filter := bson.M{"generation": bson.M{"$nin": keep}, "createdAt": bson.M{"$lt": time.Now().UTC().Add(-15 * time.Minute)}}
// Only this database's registered, unreachable search generations are
// eligible; other deployments and current generations remain untouched.
cur, err := s.queryData().Find(ctx, bson.M{"$and": bson.A{filter, bson.M{"kind": "search_collection"}}}, options.Find().SetProjection(bson.M{"_id": 1, "collection": 1}))
if err != nil {
return err
}
defer cur.Close(ctx)
for cur.Next(ctx) {
var row struct {
ID string `bson:"_id"`
Collection string `bson:"collection"`
}
if err = cur.Decode(&row); err != nil {
return err
}
if s.queryServing != nil && s.queryServing.search != nil {
if err = s.queryServing.search.Delete(ctx, row.Collection); err != nil {
return err
}
} else {
return nil
}
}
if err = cur.Err(); err != nil {
return err
}
_, err = s.queryData().DeleteMany(ctx, filter)
return err
}
// Obsolete configured scopes cannot pin an unbounded history of generations.
// Only the current builder may change this derived manifest directory.
func (s *MongoStore) pruneQueryModelScopes(ctx context.Context, l queryModelLease, scopes map[string][]string) error {
var st struct {
Heads map[string]QueryModelHead `bson:"heads"`
}
if err := s.queryControl().FindOne(ctx, queryLeaseFilter(l)).Decode(&st); err != nil {
return err
}
keep := map[string]bool{}
for org, projects := range scopes {
keep[queryModelKey("production", org, projects)] = true
keep[queryModelKey("search", org, nil)] = true
}
unset := bson.M{}
for key := range st.Heads {
if !keep[key] {
unset["heads."+key] = ""
}
}
if len(unset) == 0 {
return nil
}
_, err := s.queryControl().UpdateOne(ctx, queryLeaseFilter(l), bson.M{"$unset": unset})
return err
}

View File

@ -0,0 +1,384 @@
package store
import (
"context"
"errors"
"fmt"
"log"
"strings"
"sync"
"time"
"droboticflow/internal/outputsnapshot"
"droboticflow/internal/productionstats"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/bson/primitive"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
)
var querySourceCollections = []string{"organizations", "projects", "runs", "captureRecords", "humanItems", "humanBatches", "annotationTasks", "annotationWorklogEvents", "humanEvents", "deliveryPackages", "annotationRevisions", "sliceRevisions", "annotationReviews", "users", "suppliers", "productionStatsScopes"}
// QueryReadBuilder runs only in a dedicated, explicitly configured process.
// It uses the existing source loaders and publication protocol, never claims or
// mutates a business task. Restart starts a fresh stream BEFORE the full rebuild.
type QueryReadBuilder struct {
store *MongoStore
mu sync.Mutex
dirty map[string]bool
all bool
reader *outputsnapshot.Reader
searchDirty map[string]bool
allSearch bool
sourceDigests map[string]querySourceDigest
}
func NewQueryReadBuilder(s *MongoStore) *QueryReadBuilder {
r := outputsnapshot.NewStreamReader(s, s.StreamAnnotationOutput)
return &QueryReadBuilder{store: s, dirty: map[string]bool{}, all: true, reader: r, searchDirty: map[string]bool{}, allSearch: true, sourceDigests: map[string]querySourceDigest{}}
}
func (b *QueryReadBuilder) Close() { b.reader.Close() }
// WaitReady is for explicit prewarming and isolated acceptance, not HTTP reads.
func (b *QueryReadBuilder) WaitReady(ctx context.Context, org string, projects []string) error {
tick := time.NewTicker(100 * time.Millisecond)
defer tick.Stop()
for {
_, production := b.store.readQueryModelHead(ctx, queryModelKey("production", org, projects), 5*time.Minute)
st, err := b.store.querySearchState(ctx)
h, exists := st.Heads[queryModelKey("search", org, nil)]
search := err == nil && exists && h.Generation != "" && h.Revision == st.Revisions[queryModelKey("search", org, nil)] && h.GlobalRevision == st.Global && time.Since(st.ObservedAt) >= 0 && time.Since(st.ObservedAt) < 5*time.Second
output := len(projects) == 0
if !output {
health, e := b.reader.Inspect(ctx, org, projects)
output = e == nil && health.Available
}
if production && search && output {
return nil
}
select {
case <-ctx.Done():
return ctx.Err()
case <-tick.C:
}
}
}
func (b *QueryReadBuilder) scopes(ctx context.Context) (map[string][]string, error) {
// Public ListOrganizations also bootstraps a default business organization;
// the read builder must enumerate without that write side effect.
cur, err := b.store.organizations.Find(ctx, bson.M{}, options.Find().SetProjection(bson.M{"id": 1, "_id": 0}))
if err != nil {
return nil, err
}
defer cur.Close(ctx)
var orgs []struct {
ID string `bson:"id"`
}
if err = cur.All(ctx, &orgs); err != nil {
return nil, err
}
out := map[string][]string{}
for _, org := range orgs {
scope, err := b.store.GetProductionStatsScope(ctx, org.ID)
if err != nil {
return nil, err
}
projects, err := b.store.ListProjectsByOrganization(ctx, org.ID)
if err != nil {
return nil, err
}
selected := map[string]bool{}
for _, id := range scope.ProjectIDs {
selected[id] = true
}
ids := []string{}
for _, p := range projects {
if selected[p.ID] {
ids = append(ids, p.ID)
}
}
out[org.ID] = ids
}
return out, nil
}
func (b *QueryReadBuilder) rebuild(ctx context.Context, l queryModelLease, org string, projects []string) error {
started := time.Now()
var source struct {
Revision int64 `bson:"productionRevision"`
}
if err := b.store.queryControl().FindOne(ctx, bson.M{"_id": "global"}, options.FindOne().SetProjection(bson.M{"productionRevision": 1})).Decode(&source); err != nil {
return err
}
buildCtx, finish := context.WithTimeout(ctx, 5*time.Minute)
defer finish()
snapshot, err := b.store.loadProductionStats(buildCtx, org, projects)
if err != nil {
return err
}
asOf, err := time.Parse(time.RFC3339Nano, snapshot.AsOf)
if err != nil {
return err
}
key := queryModelKey("production", org, projects)
head, err := b.store.stageQueryModel(ctx, key, productionstats.Build(snapshot), asOf)
if err != nil {
return err
}
head.SourceRevision = source.Revision
if err = b.store.publishQueryModelHead(ctx, l, key, head); err != nil {
return err
}
if len(projects) > 0 {
if _, err = b.reader.Prewarm(ctx, org, projects); err != nil {
return err
}
}
log.Printf("query_model_build result=complete duration_ms=%d", time.Since(started).Milliseconds())
return nil
}
// RunOnce is for bounded initial prewarming and isolated tests, not a freshness
// substitute for Run. After the heartbeat expires search falls back to old reads.
func (b *QueryReadBuilder) RunOnce(ctx context.Context) error {
l, ok, err := b.store.acquireQueryModelLease(ctx)
if err != nil {
return err
}
if !ok {
return errors.New("query builder already active")
}
return b.withLease(ctx, l, func(ctx context.Context) error {
scopes, err := b.scopes(ctx)
if err != nil {
return err
}
for org, ids := range scopes {
if err = b.rebuild(ctx, l, org, ids); err != nil {
return err
}
if err = b.store.buildQuerySearchModel(ctx, l, org); err != nil {
return err
}
}
return nil
})
}
func (b *QueryReadBuilder) withLease(ctx context.Context, l queryModelLease, run func(context.Context) error) error {
ctx, cancel := context.WithCancel(ctx)
defer cancel()
done := make(chan struct{})
go func() {
defer close(done)
tick := time.NewTicker(10 * time.Second)
defer tick.Stop()
for {
select {
case <-ctx.Done():
return
case <-tick.C:
if b.store.renewQueryModelLease(ctx, l) != nil {
cancel()
return
}
}
}
}()
err := run(ctx)
cancel()
<-done
cleanup, stop := context.WithTimeout(context.Background(), time.Second)
defer stop()
_, _ = b.store.queryControl().UpdateOne(cleanup, queryLeaseFilter(l), bson.M{"$set": bson.M{"owner": "", "until": time.Now().UTC()}})
return err
}
func (b *QueryReadBuilder) Run(ctx context.Context) error {
b.mu.Lock()
b.all, b.allSearch = true, true
b.sourceDigests = map[string]querySourceDigest{}
b.mu.Unlock()
l, ok, err := b.store.acquireQueryModelLease(ctx)
if err != nil {
return err
}
if !ok {
return errors.New("query builder already active")
}
return b.withLease(ctx, l, func(ctx context.Context) error {
// No fullDocument lookup: projecting only event identity avoids copying
// annotation bodies, credentials or full video payloads on each save.
pipeline := mongo.Pipeline{{{Key: "$match", Value: bson.M{"ns.coll": bson.M{"$in": querySourceCollections}}}}, {{Key: "$project", Value: bson.M{"_id": 1, "ns": 1, "documentKey": 1, "operationType": 1, "clusterTime": 1}}}}
stream, err := b.store.db.Watch(ctx, pipeline, options.ChangeStream().SetMaxAwaitTime(time.Second))
if err != nil {
return fmt.Errorf("query builder needs available Mongo change streams: %w", err)
}
defer stream.Close(context.Background())
// Seed before consuming: events already captured by the open stream
// close the initial snapshot/change gap without retaining customer data.
seedCtx, seedCancel := context.WithTimeout(ctx, 2*time.Minute)
err = b.seedSourceDigests(seedCtx)
seedCancel()
if err != nil {
return err
}
watchCtx, stop := context.WithCancel(ctx)
defer stop()
watchDone := make(chan error, 1)
watchExited := make(chan struct{})
go func() {
defer close(watchExited)
lastHeartbeat := time.Time{}
pendingChange := false
for watchCtx.Err() == nil {
observed := time.Now().UTC()
if stream.TryNext(watchCtx) {
var event struct {
NS struct {
Collection string `bson:"coll"`
} `bson:"ns"`
Operation string `bson:"operationType"`
Key bson.M `bson:"documentKey"`
ClusterTime primitive.Timestamp `bson:"clusterTime"`
}
if e := stream.Decode(&event); e != nil {
watchDone <- e
return
}
pendingChange = true
if event.ClusterTime.T > 0 {
observed = time.Unix(int64(event.ClusterTime.T), 0).UTC()
}
if e := b.observeQuerySourceEvent(watchCtx, l, event.NS.Collection, event.Operation, event.Key); e != nil {
watchDone <- e
return
}
} else if e := stream.Err(); e != nil {
watchDone <- e
return
}
if time.Since(lastHeartbeat) < time.Second {
continue
}
if e := b.recordObserved(watchCtx, l, observed, pendingChange); e != nil {
watchDone <- e
return
}
pendingChange = false
lastHeartbeat = time.Now()
}
watchDone <- watchCtx.Err()
}()
defer func() { stop(); <-watchExited }()
tick := time.NewTicker(time.Second)
defer tick.Stop()
pendingStats, pendingSearch := map[string]bool{}, map[string]bool{}
lastStats := map[string]time.Time{}
var scopes map[string][]string
refreshScopes := true
lastScopeRead := time.Time{}
lastCleanup := time.Now()
for {
b.mu.Lock()
all, allSearch := b.all, b.allSearch
for org := range b.dirty {
pendingStats[org] = true
}
for org := range b.searchDirty {
pendingSearch[org] = true
}
b.dirty, b.searchDirty = map[string]bool{}, map[string]bool{}
b.all, b.allSearch = false, false
b.mu.Unlock()
if refreshScopes || all || time.Since(lastScopeRead) > 30*time.Second {
var e error
scopes, e = b.scopes(ctx)
if e != nil {
return e
}
if e = b.store.pruneQueryModelScopes(ctx, l, scopes); e != nil {
return e
}
refreshScopes = false
lastScopeRead = time.Now()
}
for org := range scopes {
if all {
pendingStats[org] = true
}
if allSearch {
pendingSearch[org] = true
}
}
for org := range pendingSearch {
if _, exists := scopes[org]; !exists {
delete(pendingSearch, org)
continue
}
if e := b.store.buildQuerySearchModel(ctx, l, org); e == nil {
delete(pendingSearch, org)
} else {
log.Print("query_search_build result=retry category=source_or_publication")
}
}
for org := range pendingStats {
ids, exists := scopes[org]
if !exists {
delete(pendingStats, org)
continue
}
if time.Since(lastStats[org]) < 30*time.Second {
continue
}
if e := b.rebuild(ctx, l, org, ids); e == nil {
delete(pendingStats, org)
lastStats[org] = time.Now()
} else {
log.Print("query_model_build result=retry category=source_or_publication")
}
}
if time.Since(lastCleanup) > time.Minute {
cleanup, finish := context.WithTimeout(ctx, 5*time.Second)
_ = b.store.cleanQueryModelOrphans(cleanup)
finish()
lastCleanup = time.Now()
}
// Prewarm due generations even when no browser is connected.
for org, ids := range scopes {
if len(ids) > 0 {
_, _ = b.reader.Prewarm(ctx, org, ids)
}
}
select {
case <-ctx.Done():
stop()
<-watchDone
return ctx.Err()
case e := <-watchDone:
return e
case <-tick.C:
}
}
})
}
func (s *MongoStore) markQuerySearchDirty(ctx context.Context, l queryModelLease, org string) error {
if org != "" {
_, err := s.queryControl().UpdateOne(ctx, queryLeaseFilter(l), bson.M{"$inc": bson.M{"searchRevisions." + queryModelKey("search", org, nil): int64(1)}})
return err
}
_, err := s.queryControl().UpdateOne(ctx, queryLeaseFilter(l), bson.M{"$inc": bson.M{"searchGlobalRevision": int64(1)}})
return err
}
// The builder's source collection list excludes all derived collections so its
// own publications cannot recursively schedule rebuilds.
func querySourceCollection(name string) bool {
for _, c := range querySourceCollections {
if strings.EqualFold(c, name) {
return true
}
}
return false
}

View File

@ -0,0 +1,278 @@
package store
import (
"bytes"
"compress/gzip"
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"sort"
"sync/atomic"
"time"
"droboticflow/internal/domain"
"droboticflow/internal/productionstats"
"droboticflow/internal/queryserving"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
)
const queryModelSchema = "query-model/v1"
const queryModelChunkBytes = 2 << 20
const queryModelMaxBytes = 64 << 20
type queryServingOptions struct {
cache queryserving.Cache
search *queryserving.Typesense
stats queryServingCounters
}
type queryServingCounters struct{ reportHits, responseHits, searchHits, typesenseReads, mongoSearchReads atomic.Uint64 }
// QueryServingStatistics exposes counters only, never names or query contents.
func (s *MongoStore) QueryServingStatistics() map[string]uint64 {
if s.queryServing == nil {
return map[string]uint64{}
}
c := &s.queryServing.stats
out := map[string]uint64{"report_hits": c.reportHits.Load(), "response_hits": c.responseHits.Load(), "search_hits": c.searchHits.Load(), "typesense_reads": c.typesenseReads.Load(), "materialized_search_reads": c.mongoSearchReads.Load()}
if cache, ok := s.SharedQueryCache().(interface{ Statistics() map[string]uint64 }); ok {
for k, v := range cache.Statistics() {
out["cache_"+k] = v
}
}
return out
}
// ConfigureQueryServing is startup-only. Optional accelerators are never
// created by a business read or mutation. Nil dependencies preserve old reads.
func (s *MongoStore) ConfigureQueryServing(cache queryserving.Cache, search *queryserving.Typesense) {
s.queryServing = &queryServingOptions{cache: cache, search: search}
}
func (s *MongoStore) SharedQueryCache() queryserving.Cache {
if s.queryServing == nil {
return nil
}
return s.queryServing.cache
}
type QueryModelHead struct {
SourceRevision int64 `bson:"sourceRevision" json:"sourceRevision"`
Revision int64 `bson:"revision" json:"revision"`
GlobalRevision int64 `bson:"globalRevision" json:"globalRevision"`
Generation string `bson:"generation" json:"generation"`
AsOf time.Time `bson:"asOf" json:"asOf"`
Chunks int `bson:"chunks" json:"chunks"`
Bytes int `bson:"bytes" json:"bytes"`
Digest string `bson:"digest" json:"digest"`
Collection string `bson:"searchCollection,omitempty" json:"searchCollection,omitempty"`
Rows int `bson:"rows" json:"rows"`
}
type queryModelLease struct {
Owner string
Epoch int64
}
func queryModelKey(kind, org string, projects []string) string {
return queryserving.Key(queryModelSchema, kind, normalizedOrganizationID(org), normalizedIDs(projects))
}
func normalizedIDs(ids []string) []string {
out := append([]string{}, ids...)
sort.Strings(out)
return out
}
func (s *MongoStore) queryControl() *mongo.Collection {
return s.db.Collection("queryReadModelControl")
}
func (s *MongoStore) queryData() *mongo.Collection { return s.db.Collection("queryReadModelData") }
func (s *MongoStore) acquireQueryModelLease(ctx context.Context) (queryModelLease, bool, error) {
now := time.Now().UTC()
owner := domain.NewID("query-builder")
var st struct {
Owner string `bson:"owner"`
Epoch int64 `bson:"epoch"`
}
err := s.queryControl().FindOneAndUpdate(ctx, bson.M{"_id": "global", "$or": bson.A{bson.M{"owner": ""}, bson.M{"until": bson.M{"$lte": now}}, bson.M{"owner": bson.M{"$exists": false}}}}, bson.M{"$set": bson.M{"owner": owner, "until": now.Add(60 * time.Second)}, "$inc": bson.M{"epoch": int64(1)}}, options.FindOneAndUpdate().SetUpsert(true).SetReturnDocument(options.After)).Decode(&st)
if mongo.IsDuplicateKeyError(err) {
return queryModelLease{}, false, nil
}
return queryModelLease{Owner: st.Owner, Epoch: st.Epoch}, err == nil, err
}
func queryLeaseFilter(l queryModelLease) bson.M {
return bson.M{"_id": "global", "owner": l.Owner, "epoch": l.Epoch, "until": bson.M{"$gt": time.Now().UTC()}}
}
func (s *MongoStore) renewQueryModelLease(ctx context.Context, l queryModelLease) error {
r, err := s.queryControl().UpdateOne(ctx, queryLeaseFilter(l), bson.M{"$set": bson.M{"until": time.Now().UTC().Add(60 * time.Second)}})
if err != nil {
return err
}
if r.MatchedCount != 1 {
return errors.New("query builder lease lost")
}
return nil
}
func (s *MongoStore) readQueryModelHead(ctx context.Context, key string, maxAge time.Duration) (QueryModelHead, bool) {
var st struct {
Heads map[string]QueryModelHead `bson:"heads"`
ObservedAt time.Time `bson:"observedAt"`
SourceRevision int64 `bson:"productionRevision"`
}
short, cancel := context.WithTimeout(ctx, 200*time.Millisecond)
defer cancel()
if s.queryControl().FindOne(short, bson.M{"_id": "global"}, options.FindOne().SetProjection(bson.M{"heads." + key: 1, "observedAt": 1, "productionRevision": 1})).Decode(&st) != nil {
return QueryModelHead{}, false
}
h, ok := st.Heads[key]
age := time.Since(h.AsOf)
verifiedUnchanged := st.SourceRevision == h.SourceRevision && time.Since(st.ObservedAt) >= 0 && time.Since(st.ObservedAt) <= 5*time.Second
return h, ok && h.Generation != "" && age >= 0 && (age <= maxAge || verifiedUnchanged)
}
func (s *MongoStore) publishQueryModelHead(ctx context.Context, l queryModelLease, key string, h QueryModelHead) error {
r, err := s.queryControl().UpdateOne(ctx, queryLeaseFilter(l), bson.M{"$set": bson.M{"heads." + key: h}})
if err != nil {
return err
}
if r.MatchedCount != 1 {
return errors.New("query publication fenced")
}
return nil
}
func queryModelBlobKey(key, generation string, index int) string {
return fmt.Sprintf("%s:%s:%09d", key, generation, index)
}
func (s *MongoStore) stageQueryModel(ctx context.Context, key string, value any, asOf time.Time) (QueryModelHead, error) {
var data bytes.Buffer
zw := gzip.NewWriter(&data)
budget := &queryModelBudgetWriter{writer: zw}
if err := json.NewEncoder(budget).Encode(value); err != nil {
return QueryModelHead{}, err
}
if err := zw.Close(); err != nil {
return QueryModelHead{}, err
}
if data.Len() > queryModelMaxBytes {
return QueryModelHead{}, errors.New("query materialization exceeds budget")
}
h := QueryModelHead{Generation: domain.NewID("query-generation"), AsOf: asOf, Bytes: data.Len(), Chunks: (data.Len() + queryModelChunkBytes - 1) / queryModelChunkBytes}
sum := sha256.Sum256(data.Bytes())
h.Digest = hex.EncodeToString(sum[:])
for i := 0; i < h.Chunks; i++ {
b := data.Bytes()[i*queryModelChunkBytes : min((i+1)*queryModelChunkBytes, data.Len())]
_, err := s.queryData().InsertOne(ctx, bson.M{"_id": queryModelBlobKey(key, h.Generation, i), "generation": h.Generation, "data": b, "createdAt": time.Now().UTC()})
if err != nil {
return QueryModelHead{}, err
}
if cache := s.SharedQueryCache(); cache != nil {
cache.Put(ctx, queryModelBlobKey(key, h.Generation, i), b, 15*time.Minute)
}
}
return h, nil
}
func (s *MongoStore) loadQueryModel(ctx context.Context, key string, h QueryModelHead, out any) error {
if h.Chunks < 1 || h.Bytes < 1 || h.Bytes > queryModelMaxBytes || h.Chunks != (h.Bytes+queryModelChunkBytes-1)/queryModelChunkBytes {
return errors.New("invalid query manifest")
}
var data bytes.Buffer
for i := 0; i < h.Chunks; i++ {
blobKey := queryModelBlobKey(key, h.Generation, i)
var b []byte
cache := s.SharedQueryCache()
if cache != nil {
b, _ = cache.Get(ctx, blobKey)
}
if b == nil {
var doc struct {
Data []byte `bson:"data"`
}
if err := s.queryData().FindOne(ctx, bson.M{"_id": blobKey}).Decode(&doc); err != nil {
return err
}
b = doc.Data
if cache != nil {
cache.Put(ctx, blobKey, b, 15*time.Minute)
}
}
if len(b) > queryModelChunkBytes {
return errors.New("invalid query chunk")
}
data.Write(b)
}
sum := sha256.Sum256(data.Bytes())
if data.Len() != h.Bytes || hex.EncodeToString(sum[:]) != h.Digest {
return errors.New("query materialization corrupt")
}
zr, err := gzip.NewReader(bytes.NewReader(data.Bytes()))
if err != nil {
return err
}
defer zr.Close()
b, err := io.ReadAll(io.LimitReader(zr, 128<<20+1))
if err != nil || len(b) > 128<<20 {
return errors.New("query decode exceeds budget")
}
return json.Unmarshal(b, out)
}
// LoadProductionReport serves a derived, scope-qualified report if available.
// Legacy reads remain the compatibility/failure path. No refresh is triggered.
func (s *MongoStore) LoadProductionReport(ctx context.Context, org string, projects []string) (productionstats.Report, string, error) {
if s.queryServing != nil {
key := queryModelKey("production", org, projects)
if head, ok := s.readQueryModelHead(ctx, key, dashboardStatsCacheTTL); ok {
var report productionstats.Report
if err := s.loadQueryModel(ctx, key, head, &report); err == nil {
s.queryServing.stats.reportHits.Add(1)
return report, head.AsOf.Format(time.RFC3339Nano), nil
}
}
}
snapshot, err := s.LoadProductionStatsCached(ctx, org, projects)
if err != nil {
return productionstats.Report{}, "", err
}
return productionstats.Build(snapshot), snapshot.AsOf, nil
}
func (s *MongoStore) PreparedProductionResponse(ctx context.Context, org string, projects []string, query string) ([]byte, bool) {
if s.queryServing == nil {
return nil, false
}
head, ok := s.readQueryModelHead(ctx, queryModelKey("production", org, projects), dashboardStatsCacheTTL)
if !ok || s.SharedQueryCache() == nil {
return nil, false
}
b, hit := s.SharedQueryCache().Get(ctx, queryserving.Key("production-response/v1", s.db.Name(), org, normalizedIDs(projects), head.Generation, query))
if hit {
s.queryServing.stats.responseHits.Add(1)
}
return b, hit
}
func (s *MongoStore) RememberProductionResponse(ctx context.Context, org string, projects []string, query, asOf string, value any) {
if s.queryServing == nil || s.SharedQueryCache() == nil {
return
}
head, ok := s.readQueryModelHead(ctx, queryModelKey("production", org, projects), dashboardStatsCacheTTL)
if !ok || head.AsOf.Format(time.RFC3339Nano) != asOf {
return
}
queryserving.PutJSON(ctx, s.SharedQueryCache(), queryserving.Key("production-response/v1", s.db.Name(), org, normalizedIDs(projects), head.Generation, query), value, 5*time.Minute)
}
type queryModelBudgetWriter struct {
writer io.Writer
bytes int
}
func (w *queryModelBudgetWriter) Write(p []byte) (int, error) {
if len(p) > (128<<20)-w.bytes {
return 0, errors.New("query decode exceeds budget")
}
n, e := w.writer.Write(p)
w.bytes += n
return n, e
}

View File

@ -0,0 +1,311 @@
package store
import (
"context"
"encoding/json"
"fmt"
"os"
"reflect"
"strings"
"testing"
"time"
"droboticflow/internal/domain"
"droboticflow/internal/productionstats"
"droboticflow/internal/queryserving"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
)
type queryModelFixture struct {
s *MongoStore
cache *queryserving.ManagedValkey
search *queryserving.Typesense
org string
}
func newQueryModelFixture(t *testing.T, n int) queryModelFixture {
t.Helper()
if os.Getenv("QUERY_SERVING_LOCAL_TEST") != "1" {
t.Skip("isolated query-serving dependencies")
}
ctx := t.Context()
client, err := mongo.Connect(ctx, options.Client().ApplyURI("mongodb://127.0.0.1:27019/?replicaSet=query-rs&directConnection=true").SetServerSelectionTimeout(3*time.Second))
if err != nil {
t.Fatal(err)
}
db := client.Database(fmt.Sprintf("query_serving_test_%d", time.Now().UnixNano()))
s := NewMongoStore(db)
search, err := queryserving.NewTypesense("http://127.0.0.1:18108", os.Getenv("QUERY_TEST_TYPESENSE_API_KEY"))
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() {
cleanup, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
var control struct {
Heads map[string]QueryModelHead `bson:"heads"`
}
if s.queryControl().FindOne(cleanup, bson.M{"_id": "global"}).Decode(&control) == nil {
for _, h := range control.Heads {
if h.Collection != "" {
_ = search.Delete(cleanup, h.Collection)
}
}
}
_ = db.Drop(cleanup)
_ = client.Disconnect(cleanup)
})
cache, err := queryserving.NewManagedValkey("valkey://127.0.0.1:16379", db.Name())
if err != nil {
t.Fatal(err)
}
t.Cleanup(cache.Close)
deadline := time.Now().Add(3 * time.Second)
for !cache.Ready() && time.Now().Before(deadline) {
time.Sleep(10 * time.Millisecond)
}
if !cache.Ready() {
t.Fatal("isolated cache unavailable")
}
s.ConfigureQueryServing(cache, search)
must := func(err error) {
t.Helper()
if err != nil {
t.Fatal(err)
}
}
for _, c := range []string{"humanItems", "runs", "captureRecords", "annotationTasks", "projects", "humanBatches"} {
_, err := db.Collection(c).Indexes().CreateOne(ctx, mongo.IndexModel{Keys: bson.D{{Key: "id", Value: 1}}, Options: options.Index().SetUnique(true)})
must(err)
}
must(s.EnsureQueryReadModelIndexes(ctx))
org := "query-org"
must(s.UpsertOrganization(ctx, domain.OrganizationRecord{ID: org, Status: domain.OrganizationStatusActive}))
must(s.UpsertProject(ctx, domain.ProjectRecord{ID: "p", OrganizationID: org, Name: "历史项目"}))
must(s.UpsertHumanBatch(ctx, domain.HumanBatchRecord{ID: "batch", OrganizationID: org, ProjectID: "p", PluginID: "annotation-qa", Stages: domain.DefaultFanyuV58HumanStages(), Status: domain.HumanBatchActive}))
_, err = s.SaveProductionStatsScope(ctx, org, productionstats.Scope{ProjectIDs: []string{"p"}, Revision: 1}, 0)
must(err)
docs := map[string][]any{}
push := func(c string, value any, legacy bool) {
raw, e := bson.Marshal(value)
must(e)
var d bson.M
must(bson.Unmarshal(raw, &d))
if !legacy {
stampPoolSearchMetadata(c, d)
}
docs[c] = append(docs[c], d)
}
flush := func() {
for c, rows := range docs {
if len(rows) > 0 {
_, e := db.Collection(c).InsertMany(ctx, rows)
must(e)
}
}
docs = map[string][]any{}
}
for i := 0; i < n; i++ {
id := fmt.Sprintf("item-%06d", i)
capture := fmt.Sprintf("FYLX_TEST_%06d", i)
if i == 0 {
capture = "FYLX_B07_AZER764008K_20260927_074440"
}
if i == 1 {
capture = "İ中文😀"
}
run := domain.RunRecord{ID: "run-" + id, OrganizationID: org, ProjectID: "p", TaskID: "capture-" + id, CaptureDirectories: []string{capture}, CreatedAt: "2026-11-10T00:00:00Z", Status: domain.RunStatusWaitingForHuman}
task := domain.AnnotationTaskRecord{ID: "task-" + id, OrganizationID: org, ProjectID: "p", RunID: run.ID, ArtifactID: capture + "/part0002/camera_left.mp4", SourceVideo: capture + "/part0002/camera_left.mp4", ClipEnd: 10}
item := domain.HumanItemRecord{ID: id, OrganizationID: org, ProjectID: "p", BatchID: "batch", PluginID: "annotation-qa", RunID: run.ID, LegacyTaskID: task.ID, CurrentStageID: "quality_review", Status: domain.HumanItemReady, SupplierID: fmt.Sprintf("supplier-%d", i%2), Version: 1, CreatedAt: time.Date(2026, 11, 10, 0, 0, i, 0, time.UTC).Format(time.RFC3339), Subject: domain.HumanSubjectRef{ItemKey: task.ArtifactID, Payload: map[string]any{"captureDirectory": capture, "annotationDraft": map[string]any{"description": "synthetic annotation must remain unchanged"}}}}
if i < 3 {
item.Subject.Payload["unusedBody"] = strings.Repeat("x", 1<<20)
}
push("runs", run, i%3 == 0)
push("annotationTasks", task, i%3 == 0)
push("captureRecords", domain.CaptureRecord{ID: run.TaskID, OrganizationID: org, ProjectID: "p", CaptureID: capture, RunID: run.ID}, i%3 == 0)
push("humanItems", item, i%3 == 0)
if (i+1)%1000 == 0 {
flush()
}
}
flush()
return queryModelFixture{s: s, cache: cache, search: search, org: org}
}
func queryIDs(items []domain.HumanItemRecord) []string {
out := []string{}
for _, i := range items {
out = append(out, i.ID)
}
return out
}
func TestQueryReadModelsMongoEquivalenceFencingAndFailure(t *testing.T) {
f := newQueryModelFixture(t, 241)
ctx := t.Context()
s := f.s
before, err := s.GetHumanItem(ctx, "item-000000")
if err != nil {
t.Fatal(err)
}
old := s.queryServing
s.queryServing = nil
queries := []string{"FYLX_B07_AZER764008K_20260927_074440/part0002", "TEST_0000", "中文", "İ", "item-000001", "历史项目", "[missing]", "part0002", "camera_left", "000000/part"}
expected := map[string][]string{}
for _, q := range queries {
items, e := s.ListHumanItems(ctx, f.org, domain.HumanItemQuery{PluginID: "annotation-qa", Search: q, Limit: 500})
if e != nil {
t.Fatal(e)
}
expected[q] = queryIDs(items)
}
base, e := s.LoadProductionStats(ctx, f.org, []string{"p"})
if e != nil {
t.Fatal(e)
}
wantReport := productionstats.Build(base)
s.queryServing = old
b := NewQueryReadBuilder(s)
defer b.Close()
if err = b.RunOnce(ctx); err != nil {
t.Fatal(err)
}
for _, q := range queries {
candidateIDs, ready := s.querySearchCandidates(ctx, f.org, q, "")
if !ready {
t.Logf("accelerator unavailable for %q; verifying compatibility result", q)
}
set := map[string]bool{}
for _, id := range candidateIDs {
set[id] = true
}
for _, id := range expected[q] {
if ready && !set[id] {
t.Fatalf("candidate false negative query=%q id=%s", q, id)
}
}
items, e := s.ListHumanItems(ctx, f.org, domain.HumanItemQuery{PluginID: "annotation-qa", Search: q, Limit: 500})
if e != nil || !reflect.DeepEqual(queryIDs(items), expected[q]) {
t.Fatalf("result mismatch query=%q got=%v expected=%v err=%v", q, queryIDs(items), expected[q], e)
}
}
if stats := s.QueryServingStatistics(); stats["search_hits"] == 0 || stats["typesense_reads"] == 0 {
t.Fatal("new search path never exercised", stats)
}
report, asOf, err := s.LoadProductionReport(ctx, f.org, []string{"p"})
if err != nil || asOf == "" || !reflect.DeepEqual(report, wantReport) {
t.Fatal("materialized production report differs", err)
}
after, err := s.GetHumanItem(ctx, before.ID)
if err != nil || !reflect.DeepEqual(before, after) {
t.Fatal("read model changed annotation, ownership or version")
}
// A late writer can stage data but cannot replace a newer owner's head.
l1, ok, err := s.acquireQueryModelLease(ctx)
if err != nil || !ok {
t.Fatal(err)
}
_, err = s.queryControl().UpdateOne(ctx, bson.M{"_id": "global"}, bson.M{"$set": bson.M{"until": time.Now().Add(-time.Second)}})
if err != nil {
t.Fatal(err)
}
l2, ok, err := s.acquireQueryModelLease(ctx)
if err != nil || !ok {
t.Fatal(err)
}
key := queryModelKey("production", f.org, []string{"p"})
head, ok := s.readQueryModelHead(ctx, key, time.Hour)
if !ok {
t.Fatal("missing production head")
}
if s.publishQueryModelHead(ctx, l1, key, head) == nil {
t.Fatal("expired writer published")
}
if err = s.publishQueryModelHead(ctx, l2, key, head); err != nil {
t.Fatal(err)
}
// Incomplete/corrupt derived storage never becomes zero business data.
_, err = s.queryData().UpdateOne(ctx, bson.M{"_id": queryModelBlobKey(key, head.Generation, 0)}, bson.M{"$set": bson.M{"data": []byte("corrupt")}})
if err != nil {
t.Fatal(err)
}
s.queryServing.cache = nil
got, _, err := s.LoadProductionReport(ctx, f.org, []string{"p"})
if err != nil || !reflect.DeepEqual(got, wantReport) {
t.Fatal("corrupt materialization failed to retain compatibility path", err)
}
}
func TestQuerySearchChangeObservationIgnoresDraftsAndRecoversNamesLocal(t *testing.T) {
f := newQueryModelFixture(t, 12)
s := f.s
ctx, cancel := context.WithCancel(t.Context())
b := NewQueryReadBuilder(s)
done := make(chan error, 1)
go func() { done <- b.Run(ctx) }()
defer func() { cancel(); <-done; b.Close() }()
wait := func(check func() bool) {
t.Helper()
end := time.Now().Add(20 * time.Second)
for time.Now().Before(end) {
if check() {
return
}
time.Sleep(50 * time.Millisecond)
}
t.Fatal("streamed read model did not converge")
}
wait(func() bool { _, ready := s.querySearchCandidates(ctx, f.org, "FYLX", ""); return ready })
state, err := s.querySearchState(ctx)
if err != nil {
t.Fatal(err)
}
key := queryModelKey("search", f.org, nil)
generation := state.Heads[key].Generation
item, err := s.GetHumanItem(ctx, "item-000001")
if err != nil {
t.Fatal(err)
}
item.Version++
item.Status = domain.HumanItemClaimed
item.AssignedUserID = "synthetic-owner"
item.UpdatedAt = time.Now().UTC().Format(time.RFC3339Nano)
if err = s.UpsertHumanItem(ctx, *item); err != nil {
t.Fatal(err)
}
time.Sleep(1500 * time.Millisecond)
state, err = s.querySearchState(ctx)
if err != nil || state.Heads[key].Generation != generation || state.Revisions[key] != state.Heads[key].Revision {
t.Fatal("ordinary task state invalidated name search", err)
}
if err = s.UpsertProject(ctx, domain.ProjectRecord{ID: "p", OrganizationID: f.org, Name: "新项目名称"}); err != nil {
t.Fatal(err)
}
wait(func() bool {
ids, ready := s.querySearchCandidates(ctx, f.org, "新项目名称", "")
return ready && len(ids) == 12
})
before, err := s.GetHumanItem(ctx, "item-000000")
if err != nil {
t.Fatal(err)
}
if _, err = s.humanItems.DeleteOne(ctx, bson.M{"id": before.ID}); err != nil {
t.Fatal(err)
}
wait(func() bool {
ids, ready := s.querySearchCandidates(ctx, f.org, "新项目名称", "")
return ready && len(ids) == 11
})
// Old generations survive publication only as unreachable, bounded derived
// records; no original annotation or audit records are rewritten.
remaining, err := s.GetHumanItem(ctx, "item-000002")
if err != nil {
t.Fatal(err)
}
raw, _ := json.Marshal(remaining.Subject.Payload)
if !strings.Contains(string(raw), "synthetic annotation must remain unchanged") {
t.Fatal("background rebuild lost annotation")
}
}

View File

@ -0,0 +1,231 @@
package store
import (
"context"
"droboticflow/internal/domain"
"droboticflow/internal/queryserving"
"go.mongodb.org/mongo-driver/bson"
"reflect"
"testing"
"time"
)
func TestQueryReadModelsRestartExpiryAndPrivateGarbageCollectionLocal(t *testing.T) {
f := newQueryModelFixture(t, 24)
s, ctx := f.s, t.Context()
b := NewQueryReadBuilder(s)
defer b.Close()
organizationsBefore, _ := s.organizations.CountDocuments(ctx, bson.M{})
if err := b.RunOnce(ctx); err != nil {
t.Fatal(err)
}
organizationsAfter, _ := s.organizations.CountDocuments(ctx, bson.M{})
if organizationsAfter != organizationsBefore {
t.Fatal("builder bootstrapped business organizations")
}
st, err := s.querySearchState(ctx)
if err != nil {
t.Fatal(err)
}
key := queryModelKey("search", f.org, nil)
head := st.Heads[key]
// Stale observation and known source changes cannot authorize an empty hit.
_, err = s.queryControl().UpdateOne(ctx, bson.M{"_id": "global"}, bson.M{"$set": bson.M{"observedAt": time.Now().Add(-6 * time.Second)}})
if err != nil {
t.Fatal(err)
}
if _, ready := s.querySearchCandidates(ctx, f.org, "TEST", ""); ready {
t.Fatal("expired observer used as complete catalog")
}
_, err = s.queryControl().UpdateOne(ctx, bson.M{"_id": "global"}, bson.M{"$set": bson.M{"observedAt": time.Now()}, "$inc": bson.M{"searchRevisions." + key: 1}})
if err != nil {
t.Fatal(err)
}
if _, ready := s.querySearchCandidates(ctx, f.org, "TEST", ""); ready {
t.Fatal("known obsolete catalog used")
}
before, err := s.GetHumanItem(ctx, "item-000000")
if err != nil {
t.Fatal(err)
}
if _, err = s.humanItems.DeleteOne(ctx, bson.M{"id": "item-000001"}); err != nil {
t.Fatal(err)
}
// Reconciliation after a stopped worker includes missed deletes.
if err = b.RunOnce(ctx); err != nil {
t.Fatal(err)
}
ids, ready := s.querySearchCandidates(ctx, f.org, "item-000001", "")
if !ready {
t.Fatal("restart catalog not ready")
}
for _, id := range ids {
if id == "item-000001" {
t.Fatal("restart retained deleted identity")
}
}
matches, e := s.ListHumanItems(ctx, f.org, domain.HumanItemQuery{Search: "item-000001", Limit: 100})
if e != nil || len(matches) != 0 {
t.Fatal("deleted exact search result retained", e)
}
current, err := s.querySearchState(ctx)
if err != nil {
t.Fatal(err)
}
if current.Heads[key].Generation == head.Generation {
t.Fatal("restart reused unverified generation")
}
// Availability failure: Typesense may be gone while Mongo materialized rows
// remain complete. The old source path is still used if observation expires.
unavailable, err := queryserving.NewTypesense("http://127.0.0.1:1", "synthetic")
if err != nil {
t.Fatal(err)
}
s.queryServing.search = unavailable
s.queryServing.cache = nil
got, ready := s.querySearchCandidates(ctx, f.org, "FYLX_B07", "")
if !ready || !reflect.DeepEqual(got, []string{"item-000000"}) {
t.Fatal("search outage lost existing task", got, ready)
}
s.queryServing.search = f.search
// GC must retain current generations regardless of their creation date,
// while removing only this database's old unreachable derivatives.
_, err = s.queryData().UpdateMany(ctx, bson.M{}, bson.M{"$set": bson.M{"createdAt": time.Now().Add(-time.Hour)}})
if err != nil {
t.Fatal(err)
}
_, err = s.queryData().InsertOne(ctx, bson.M{"_id": "unreachable-test", "generation": "orphan-test", "createdAt": time.Now().Add(-time.Hour)})
if err != nil {
t.Fatal(err)
}
if err = s.cleanQueryModelOrphans(ctx); err != nil {
t.Fatal(err)
}
n, err := s.queryData().CountDocuments(ctx, bson.M{"generation": current.Heads[key].Generation})
if err != nil || n != 24 {
t.Fatal("current generation lost", n, err)
} // 23 rows plus collection registration
n, err = s.queryData().CountDocuments(ctx, bson.M{"_id": "unreachable-test"})
if err != nil || n != 0 {
t.Fatal("orphan not reclaimed", err)
}
after, err := s.GetHumanItem(ctx, before.ID)
if err != nil || !reflect.DeepEqual(after, before) {
t.Fatal("recovery or GC modified annotation")
}
}
func TestQuerySearchPaginationAndLiveScopeEquivalenceLocal(t *testing.T) {
f := newQueryModelFixture(t, 241)
s, ctx := f.s, t.Context()
b := NewQueryReadBuilder(s)
defer b.Close()
if err := b.RunOnce(ctx); err != nil {
t.Fatal(err)
}
// The name catalog never contains assignment or canClaim. Current filters
// apply equally after a task changes supplier, owner, stage or version.
item, err := s.GetHumanItem(ctx, "item-000000")
if err != nil {
t.Fatal(err)
}
item.SupplierID = "changed-supplier"
item.AssignedUserID = "current-owner"
item.Status = domain.HumanItemClaimed
item.Version++
if err = s.UpsertHumanItem(ctx, *item); err != nil {
t.Fatal(err)
}
options := s.queryServing
queries := []domain.HumanItemQuery{
{Search: "part0002", Limit: 17},
{Search: "part0002", Limit: 17, AvailableProjectIDs: []string{}},
{Search: "part0002", Limit: 17, AvailableProjectIDs: []string{"foreign"}},
{Search: "FYLX_B07", ActorUserID: "current-owner", Limit: 17},
{Search: "part0002", UnassignedOnly: true, Limit: 17},
}
for _, q := range queries {
q.PluginID = "annotation-qa"
for page := 0; page < 20; page++ {
s.queryServing = nil
old, e := s.ListHumanItems(ctx, f.org, q)
if e != nil {
t.Fatal(e)
}
s.queryServing = options
got, e := s.ListHumanItems(ctx, f.org, q)
if e != nil || !reflect.DeepEqual(queryIDs(old), queryIDs(got)) {
t.Fatal("paged live-scope mismatch", page, e)
}
if len(old) == 0 {
break
}
last := old[len(old)-1]
q.Cursor = &domain.HumanItemCursor{CreatedAt: last.CreatedAt, ID: last.ID}
if page == 19 {
t.Fatal("pagination did not exhaust")
}
}
}
if rows, e := s.ListHumanItems(ctx, "foreign-org", domain.HumanItemQuery{Search: "part0002", Limit: 500}); e != nil || len(rows) != 0 {
t.Fatal("foreign organization leaked", e)
}
// Cancellation is propagated instead of publishing/caching empty results.
cancelled, stop := context.WithCancel(ctx)
stop()
if _, ready := s.querySearchCandidates(cancelled, f.org, "part0002", ""); ready {
t.Fatal("cancelled candidate lookup became authoritative")
}
}
func TestQuerySearchCandidateBudgetNeverTruncates(t *testing.T) {
if queryCandidateBudget([]string{""}) || queryCandidateBudget(make([]string, 100001)) {
t.Fatal("invalid or oversized candidates accepted")
}
if !queryCandidateBudget(nil) || !queryCandidateBudget([]string{"valid-id"}) {
t.Fatal("valid complete candidates rejected")
}
}
func TestQueryReadModelsCachedResponseScopeAndStaleFallbackLocal(t *testing.T) {
f := newQueryModelFixture(t, 8)
s, ctx := f.s, t.Context()
b := NewQueryReadBuilder(s)
defer b.Close()
if e := b.RunOnce(ctx); e != nil {
t.Fatal(e)
}
_, asOf, e := s.LoadProductionReport(ctx, f.org, []string{"p"})
if e != nil {
t.Fatal(e)
}
s.RememberProductionResponse(ctx, f.org, []string{"p"}, "dates-and-filter", asOf, map[string]int{"proof": 42})
raw, ok := s.PreparedProductionResponse(ctx, f.org, []string{"p"}, "dates-and-filter")
if !ok || string(raw) != "{\"proof\":42}" {
t.Fatal("real shared response not reusable", string(raw), ok)
}
for _, scope := range []struct {
org string
projects []string
query string
}{{"foreign", []string{"p"}, "dates-and-filter"}, {f.org, []string{"foreign"}, "dates-and-filter"}, {f.org, []string{"p"}, "other-dates"}} {
if _, hit := s.PreparedProductionResponse(ctx, scope.org, scope.projects, scope.query); hit {
t.Fatal("response scope leaked")
}
}
key := queryModelKey("production", f.org, []string{"p"})
_, e = s.queryControl().UpdateOne(ctx, bson.M{"_id": "global"}, bson.M{"$set": bson.M{"heads." + key + ".asOf": time.Now().Add(-time.Minute), "productionRevision": int64(10), "observedAt": time.Now()}})
if e != nil {
t.Fatal(e)
}
if _, hit := s.PreparedProductionResponse(ctx, f.org, []string{"p"}, "dates-and-filter"); hit {
t.Fatal("changed source reused report beyond original freshness budget")
}
before := s.QueryServingStatistics()["report_hits"]
if _, _, e = s.LoadProductionReport(ctx, f.org, []string{"p"}); e != nil {
t.Fatal("stale model did not retain old loader", e)
}
if s.QueryServingStatistics()["report_hits"] != before {
t.Fatal("stale model counted as fresh")
}
}

View File

@ -0,0 +1,231 @@
package store
import (
"context"
"crypto/sha256"
"encoding/hex"
"errors"
"net/url"
"regexp"
"strings"
"time"
"droboticflow/internal/domain"
"droboticflow/internal/queryserving"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo/options"
)
type querySearchState struct {
Heads map[string]QueryModelHead `bson:"heads"`
Revisions map[string]int64 `bson:"searchRevisions"`
Global int64 `bson:"searchGlobalRevision"`
ObservedAt time.Time `bson:"observedAt"`
}
func (s *MongoStore) querySearchState(ctx context.Context) (querySearchState, error) {
var st querySearchState
err := s.queryControl().FindOne(ctx, bson.M{"_id": "global"}, options.FindOne().SetProjection(bson.M{"heads": 1, "searchRevisions": 1, "searchGlobalRevision": 1, "observedAt": 1})).Decode(&st)
return st, err
}
func querySafePath(value string) string {
u, err := url.Parse(value)
if err == nil && u.Scheme != "" && u.Host != "" {
u.RawQuery = ""
u.Fragment = ""
u.User = nil
return u.String()
}
return value
}
func searchRowText(row adminPoolRow) string {
// Both exact historical search corpora remain contiguous; adding fields in
// between could otherwise lose cross-field substring matches.
return adminPoolSearchText(row) + "\n" + humanPoolFallbackSearchText(row)
}
func (s *MongoStore) buildQuerySearchModel(ctx context.Context, l queryModelLease, org string) error {
ctx, stop := context.WithTimeout(ctx, 5*time.Minute)
defer stop()
key := queryModelKey("search", org, nil)
state, err := s.querySearchState(ctx)
if err != nil {
return err
}
h := QueryModelHead{Generation: domain.NewID("query-search"), AsOf: time.Now().UTC(), Revision: state.Revisions[key], GlobalRevision: state.Global}
if s.queryServing != nil && s.queryServing.search != nil {
sum := sha256.Sum256([]byte(s.db.Name() + ":" + key + ":" + h.Generation))
h.Collection = "dfq_" + hex.EncodeToString(sum[:])
// Register intent first. Even a canceled/ambiguous create remains
// discoverable by this database's orphan collector.
if _, err = s.queryData().InsertOne(ctx, bson.M{"_id": "search_collection:" + h.Collection, "kind": "search_collection", "collection": h.Collection, "generation": h.Generation, "createdAt": time.Now().UTC()}); err != nil {
return err
}
if err = s.queryServing.search.Create(ctx, h.Collection); err != nil {
return err
}
}
complete := false
defer func() {
if !complete {
cleanup, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
_, _ = s.queryData().DeleteMany(cleanup, bson.M{"generation": h.Generation, "kind": bson.M{"$ne": "search_collection"}})
if h.Collection != "" {
_ = s.queryServing.search.Delete(cleanup, h.Collection)
}
}
}()
pipeline := humanPoolSearchPipeline(org, domain.HumanItemQuery{Search: "_build_names"}, bson.M{"organizationId": normalizedOrganizationID(org)})
cur, err := s.humanItems.Aggregate(ctx, pipeline, options.Aggregate().SetAllowDiskUse(true).SetBatchSize(200))
if err != nil {
return err
}
defer cur.Close(ctx)
for {
rows := []adminPoolRow{}
for len(rows) < 200 && cur.Next(ctx) {
var row adminPoolRow
if err = cur.Decode(&row); err != nil {
return err
}
rows = append(rows, row)
}
if err = cur.Err(); err != nil {
return err
}
if len(rows) == 0 {
break
}
if err = s.humanPoolSearchLegacyPayloads(ctx, org, domain.HumanItemQuery{Search: "_build_names"}, rows); err != nil {
return err
}
docs := make([]any, 0, len(rows))
searchDocs := make([]queryserving.SearchDocument, 0, len(rows))
for _, row := range rows {
for _, task := range row.Tasks {
if querySafePath(task.SourceVideo) != task.SourceVideo {
return errors.New("credential-bearing source requires legacy search")
}
}
text := searchRowText(row)
if len(text) > 64<<10 {
return errors.New("search names exceed materialization budget")
}
// Store only the conservative searchable text and identity. No
// task body, draft, review opinion, eligibility or credential.
id := queryserving.Key("row", h.Generation, row.ID)
docs = append(docs, bson.M{"_id": id, "generation": h.Generation, "organizationId": normalizedOrganizationID(org), "itemId": row.ID, "text": text, "createdAt": time.Now().UTC()})
searchDocs = append(searchDocs, queryserving.SearchDocument{ID: id, ItemID: row.ID, Grams: queryserving.IndexGrams(text)})
}
if _, err = s.queryData().InsertMany(ctx, docs); err != nil {
return err
}
if h.Collection != "" {
if err = s.queryServing.search.Import(ctx, h.Collection, searchDocs); err != nil {
return err
}
}
h.Rows += len(rows)
}
filter := queryLeaseFilter(l)
filter["searchGlobalRevision"] = state.Global
// Missing revision zero is equivalent to zero, without writing business data.
filter["$expr"] = bson.M{"$and": bson.A{bson.M{"$eq": bson.A{bson.M{"$ifNull": bson.A{"$searchRevisions." + key, int64(0)}}, h.Revision}}, bson.M{"$eq": bson.A{bson.M{"$ifNull": bson.A{"$searchGlobalRevision", int64(0)}}, h.GlobalRevision}}}}
delete(filter, "searchGlobalRevision")
res, err := s.queryControl().UpdateOne(ctx, filter, bson.M{"$set": bson.M{"heads." + key: h, "observedAt": time.Now().UTC()}})
if err != nil {
return err
}
if res.MatchedCount != 1 {
return errors.New("search source changed during build")
}
complete = true
return nil
}
// A complete, current catalog only narrows candidates. It never authorizes a
// row, changes a cursor, truncates history, or supplies cached task status.
func (s *MongoStore) querySearchCandidates(ctx context.Context, org, search, capture string) ([]string, bool) {
if s.queryServing == nil || search == "" && capture == "" {
return nil, false
}
short, cancel := context.WithTimeout(ctx, 300*time.Millisecond)
defer cancel()
st, err := s.querySearchState(short)
if err != nil {
return nil, false
}
key := queryModelKey("search", org, nil)
h, ok := st.Heads[key]
if !ok || h.Generation == "" || h.Revision != st.Revisions[key] || h.GlobalRevision != st.Global || time.Since(st.ObservedAt) > 5*time.Second || time.Since(st.ObservedAt) < 0 {
return nil, false
}
cacheKey := queryserving.Key("search-candidates/v1", s.db.Name(), org, h.Generation, search, capture)
var ids []string
if queryserving.GetJSON(short, s.SharedQueryCache(), cacheKey, &ids) && queryCandidateBudget(ids) {
s.queryServing.stats.searchHits.Add(1)
return ids, true
}
if h.Collection != "" && s.queryServing.search != nil {
if out, err := s.queryServing.search.Candidates(short, h.Collection, search, capture); err == nil && queryCandidateBudget(out) {
s.queryServing.stats.searchHits.Add(1)
s.queryServing.stats.typesenseReads.Add(1)
queryserving.PutJSON(short, s.SharedQueryCache(), cacheKey, out, 5*time.Minute)
return out, true
}
}
clauses := bson.A{bson.M{"generation": h.Generation, "organizationId": normalizedOrganizationID(org)}}
for _, term := range []string{search, capture} {
if term != "" {
clauses = append(clauses, bson.M{"text": bson.M{"$regex": regexp.QuoteMeta(strings.ToLower(term))}})
}
}
cur, err := s.queryData().Find(short, bson.M{"$and": clauses}, options.Find().SetProjection(bson.M{"itemId": 1, "_id": 0}))
if err != nil {
return nil, false
}
defer cur.Close(short)
ids = []string{}
for cur.Next(short) {
var row struct {
ID string `bson:"itemId"`
}
if cur.Decode(&row) != nil || row.ID == "" {
return nil, false
}
ids = append(ids, row.ID)
if len(ids) > 100000 {
return nil, false
}
}
if cur.Err() != nil {
return nil, false
}
if !queryCandidateBudget(ids) {
return nil, false
}
s.queryServing.stats.mongoSearchReads.Add(1)
queryserving.PutJSON(short, s.SharedQueryCache(), cacheKey, ids, 5*time.Minute)
s.queryServing.stats.searchHits.Add(1)
return ids, true
}
// Keep the authoritative Mongo filter well below its BSON limit. An oversized
// candidate set is a miss, never a truncated inventory.
func queryCandidateBudget(ids []string) bool {
if len(ids) > 100000 {
return false
}
bytes := 0
for _, id := range ids {
if id == "" {
return false
}
bytes += len(id) + 16
if bytes > 4<<20 {
return false
}
}
return true
}

View File

@ -0,0 +1,41 @@
package store
import (
"droboticflow/internal/queryserving"
"errors"
"os"
)
// ConfigureQueryServingFromEnvironment is called only by process entry points.
// No new feature is enabled by default. Namespace separates databases in
// different deployments even when their database names happen to be identical.
func (s *MongoStore) ConfigureQueryServingFromEnvironment() (func(), error) {
close := func() {}
if os.Getenv("QUERY_SERVING_READS") != "1" {
return close, nil
}
ns := os.Getenv("QUERY_SERVING_NAMESPACE")
if ns == "" {
return close, errors.New("QUERY_SERVING_NAMESPACE is required")
}
var cache queryserving.Cache
if uri := os.Getenv("QUERY_VALKEY_URL"); uri != "" {
v, err := queryserving.NewManagedValkey(uri, ns+":"+s.db.Name())
if err != nil {
return close, err
}
cache = v
close = v.Close
}
var search *queryserving.Typesense
if uri := os.Getenv("QUERY_TYPESENSE_URL"); uri != "" {
var err error
search, err = queryserving.NewTypesense(uri, os.Getenv("QUERY_TYPESENSE_API_KEY"))
if err != nil {
close()
return func() {}, err
}
}
s.ConfigureQueryServing(cache, search)
return close, nil
}

View File

@ -0,0 +1,172 @@
package store
import (
"context"
"crypto/sha256"
"droboticflow/internal/domain"
"encoding/json"
"fmt"
"go.mongodb.org/mongo-driver/mongo/options"
"time"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo"
)
type querySourceDigest struct {
Digest [32]byte
Organization string
}
var querySearchSources = map[string]bool{"humanItems": true, "annotationTasks": true, "captureRecords": true, "runs": true, "projects": true}
// Only names, source identities and references affect search. Saving a draft,
// renewing a lease or changing task status must not invalidate the name index.
func querySearchSourceProjection(collection string) bson.M {
p := bson.M{"_id": 1, "organizationId": 1, "projectId": 1, "id": 1, "poolSearch.text": 1}
fields := map[string][]string{
"humanItems": {"legacyTaskId", "runId", "subject.itemKey", "subject.artifactId"},
"annotationTasks": {"artifactId", "rightArtifactId", "sourceVideo"},
"captureRecords": {"captureId", "prefix"},
"projects": {"name"},
"runs": {"taskId", "projectName", "captureDirectories", "inputs", "artifacts.id", "artifacts.name", "artifacts.path", "nodeExecutions.artifacts.id", "nodeExecutions.artifacts.name", "nodeExecutions.artifacts.path"},
}
for _, f := range fields[collection] {
p[f] = 1
}
if collection == "runs" {
p["inputs"] = bson.M{"$cond": bson.A{poolSearchValidExpression("runs"), bson.M{}, "$inputs"}}
}
if collection == "humanItems" {
p["legacyPayload"] = bson.M{"$cond": bson.A{poolSearchValidExpression("humanItems"), bson.M{}, "$subject.payload"}}
}
return p
}
func querySourceIdentity(collection string, id any) string {
b, _ := json.Marshal(id)
return collection + ":" + string(b)
}
func querySourceHash(doc bson.M) querySourceDigest {
if payload, ok := doc["legacyPayload"]; ok {
names := []string{}
domain.AppendCaptureDirectoriesFromPayload(payload, &names)
doc["legacyPayload"] = names
}
b, _ := json.Marshal(doc)
org, _ := doc["organizationId"].(string)
return querySourceDigest{Digest: sha256.Sum256(b), Organization: org}
}
func (b *QueryReadBuilder) seedSourceDigests(ctx context.Context) error {
for collection := range querySearchSources {
cur, err := b.store.db.Collection(collection).Aggregate(ctx, mongo.Pipeline{{{Key: "$project", Value: querySearchSourceProjection(collection)}}})
if err != nil {
return err
}
for cur.Next(ctx) {
var doc bson.M
if err = cur.Decode(&doc); err != nil {
_ = cur.Close(ctx)
return err
}
b.sourceDigests[querySourceIdentity(collection, doc["_id"])] = querySourceHash(doc)
}
err = cur.Err()
_ = cur.Close(ctx)
if err != nil {
return err
}
}
return nil
}
func (b *QueryReadBuilder) searchSourceChanged(ctx context.Context, collection string, key bson.M) (bool, string, error) {
identity := querySourceIdentity(collection, key["_id"])
old, exists := b.sourceDigests[identity]
cur, err := b.store.db.Collection(collection).Aggregate(ctx, mongo.Pipeline{{{Key: "$match", Value: key}}, {{Key: "$project", Value: querySearchSourceProjection(collection)}}})
if err != nil {
return false, "", err
}
defer cur.Close(ctx)
if !cur.Next(ctx) {
if err = cur.Err(); err != nil {
return false, "", err
}
delete(b.sourceDigests, identity)
return true, old.Organization, nil
}
var doc bson.M
if err = cur.Decode(&doc); err != nil {
return false, "", err
}
next := querySourceHash(doc)
b.sourceDigests[identity] = next
if exists && old.Digest == next.Digest {
return false, next.Organization, nil
}
if exists && old.Organization != next.Organization {
return true, "", nil
}
return true, next.Organization, nil
}
func (b *QueryReadBuilder) observeQuerySourceEvent(ctx context.Context, l queryModelLease, collection, operation string, key bson.M) error {
if operation == "drop" || operation == "rename" || operation == "invalidate" {
b.mu.Lock()
b.all, b.allSearch = true, true
b.mu.Unlock()
return b.store.markQuerySearchDirty(ctx, l, "")
}
org := ""
searchChanged := false
if querySearchSources[collection] {
var err error
searchChanged, org, err = b.searchSourceChanged(ctx, collection, key)
if err != nil {
return err
}
} else if operation != "delete" && operation != "drop" && operation != "rename" {
var doc struct {
Org string `bson:"organizationId"`
}
if err := b.store.db.Collection(collection).FindOne(ctx, key, options.FindOne().SetProjection(bson.M{"organizationId": 1})).Decode(&doc); err != nil && err != mongo.ErrNoDocuments {
return err
}
org = doc.Org
}
if operation == "drop" || operation == "rename" || operation == "invalidate" {
org = ""
searchChanged = true
}
b.mu.Lock()
if org == "" {
b.all = true
} else {
b.dirty[org] = true
}
if searchChanged {
if org == "" {
b.allSearch = true
} else {
b.searchDirty[org] = true
}
}
b.mu.Unlock()
if searchChanged {
return b.store.markQuerySearchDirty(ctx, l, org)
}
return nil
}
func (b *QueryReadBuilder) recordObserved(ctx context.Context, l queryModelLease, at time.Time, changed bool) error {
update := bson.M{"$set": bson.M{"observedAt": at}}
if changed {
update["$inc"] = bson.M{"productionRevision": int64(1)}
}
r, err := b.store.queryControl().UpdateOne(ctx, queryLeaseFilter(l), update)
if err != nil {
return err
}
if r.MatchedCount != 1 {
return fmt.Errorf("query observer fenced")
}
return nil
}

View File

@ -0,0 +1,36 @@
"""Strict comparison of isolated query-serving measurements; no production IO."""
def compare(base, candidate):
problems=[]
if base.get('fixture_digest')!=candidate.get('fixture_digest') or not base.get('fixture_digest'):
problems.append('fixture identity mismatch')
if not base.get('metrics') or not candidate.get('metrics'):
problems.append('missing request measurements')
if candidate.get('actors')!=81 or candidate.get('errors')!=0 or candidate.get('mongo_write_commands')!=0:
problems.append('concurrency, request errors or business writes failed')
counters=candidate.get('query_serving_statistics',{})
if not all(counters.get(k,0)>0 for k in ('report_hits','search_hits','typesense_reads','cache_hits')):
problems.append('accelerated paths not proven active')
for key,m in candidate.get('metrics',{}).items():
limit=2000 if '.search' in key else 500 if key.endswith('.panel') or key=='admin.production' else None
if limit and m.get('p95_ms',float('inf'))>limit:problems.append('latency exceeded: '+key)
for row in candidate.get('cold_samples',[]):
if (row['Key'].endswith('.panel') or row['Key']=='admin.production') and (row['Code']!=200 or row['MS']>2000):
problems.append('prewarmed first visit exceeded: '+row['Key'])
def hashes(report):
out={}
for r in report.get('cold_samples',[])+report.get('samples',[]):
if r.get('Code')==200:out.setdefault((r['Actor'],r['Key']),set()).add(r['Hash'])
return out
old,new=hashes(base),hashes(candidate)
overlap=old.keys() & new.keys()
if not overlap:problems.append('no comparable successful results')
for key in overlap:
if old[key]!=new[key]:problems.append('business result mismatch: '+':'.join(key))
missing=sorted(':'.join(k) for k in new.keys()-old.keys())
# Baseline timeouts cannot demonstrate equivalence; functional fixtures must
# cover those scopes separately. Keep them visible rather than marking equal.
return {'passed':not problems and not missing,'problems':sorted(set(problems)),
'comparableActorEndpoints':len(overlap),'baselineUnavailableComparisons':missing,
'cpuRatio':candidate.get('process_cpu_seconds',0)/max(base.get('process_cpu_seconds',0),.00001),
'retainedHeapRatio':candidate.get('go_heap_after_gc',0)/max(base.get('go_heap_after_gc',0),1)}

View File

@ -0,0 +1,31 @@
name: fanyu-query-local
services:
mongo:
image: mongo@sha256:5d7043a4ffe02b9ed1b6e0bab057546981af5ca0a79107e9c461e49bc44c0a7b
command: ["mongod", "--replSet", "query-rs", "--bind_ip_all", "--wiredTigerCacheSizeGB", "0.5"]
ports: ["127.0.0.1:27019:27017"]
mem_limit: 4g
cpus: 2
ulimits:
nofile:
soft: 65536
hard: 65536
volumes: ["mongo-query:/data/db"]
valkey:
image: valkey/valkey@sha256:e51a82741b780e4bf315db10753edf07eea6496d405fa7df2a8677a18f5e7464
command: ["valkey-server", "--maxmemory", "128mb", "--maxmemory-policy", "allkeys-lru", "--save", "", "--appendonly", "no"]
ports: ["127.0.0.1:16379:6379"]
mem_limit: 256m
cpus: 1
typesense:
image: typesense/typesense@sha256:610f2d34b1f93d00762869da2c67736775e5798d19a2c8b91b014b8a0cc1e110
environment:
TYPESENSE_API_KEY: "${QUERY_TEST_TYPESENSE_API_KEY:?synthetic local key required}"
TYPESENSE_DATA_DIR: /data
ports: ["127.0.0.1:18108:8108"]
mem_limit: 1g
cpus: 2
volumes: ["typesense-query:/data"]
volumes:
mongo-query:
typesense-query:

View File

@ -0,0 +1,97 @@
#!/usr/bin/env python3
"""Fixed loopback synthetic dependencies. Never accepts a customer database URI."""
import argparse
import fcntl
import json
import os
import re
from pathlib import Path
import secrets
import signal
import subprocess
import sys
import time
import urllib.request
from lib.release_checks import check_environment, write_log, source_fingerprint
ROOT = Path(__file__).resolve().parents[1]
PRIVATE = ROOT / 'local-data/query-serving-local'
def environment():
PRIVATE.mkdir(parents=True, exist_ok=True, mode=0o700)
keyfile = PRIVATE / 'synthetic-key'
if not keyfile.exists():
with os.fdopen(os.open(keyfile, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600), 'w') as stream:
stream.write(secrets.token_hex(24))
env = {k:v for k,v in check_environment().items() if not k.startswith('QUERY_')}
env.update(QUERY_TEST_TYPESENSE_API_KEY=keyfile.read_text(), QUERY_SERVING_LOCAL_TEST='1')
return env
def compose(env, *args):
result = subprocess.run(['docker','compose','-f',str(ROOT/'scripts/local/query-serving.compose.yml'),*args],cwd=ROOT,env=env,capture_output=True,text=True,timeout=300)
if result.returncode:
proof=write_log(PRIVATE/'logs', ['docker','compose',*args], result.stdout+'\n'+result.stderr,'failed')
raise RuntimeError('isolated dependencies failed; private log: '+proof['log'])
return result.stdout
def start(env):
compose(env,'up','-d')
for _ in range(60):
try:
compose(env,'exec','-T','mongo','mongosh','--quiet','--eval',"try {rs.status()} catch(e) {rs.initiate({_id:'query-rs',members:[{_id:0,host:'localhost:27017'}]})}")
with urllib.request.urlopen('http://127.0.0.1:18108/health',timeout=1) as r:
if json.load(r).get('ok'):return
except (RuntimeError,OSError,ValueError):
pass
time.sleep(.5)
raise RuntimeError('isolated dependencies not ready')
def run_command(argv, env, timeout):
child=subprocess.Popen(argv,cwd=ROOT,env=env,stdout=subprocess.PIPE,stderr=subprocess.PIPE,text=True,start_new_session=True)
try:
stdout,stderr=child.communicate(timeout=timeout)
except subprocess.TimeoutExpired:
try:os.killpg(child.pid,signal.SIGTERM)
except ProcessLookupError:pass
try:stdout,stderr=child.communicate(timeout=5)
except subprocess.TimeoutExpired:
try:os.killpg(child.pid,signal.SIGKILL)
except ProcessLookupError:pass
stdout,stderr=child.communicate()
return subprocess.CompletedProcess(argv,124,stdout,stderr+'\nphase timed out')
return subprocess.CompletedProcess(argv,child.returncode,stdout,stderr)
def main():
parser=argparse.ArgumentParser()
parser.add_argument('--start',action='store_true')
parser.add_argument('--down',action='store_true',help='stop this test project; retain local volumes')
parser.add_argument('--run',default='^TestQueryServing|^TestQueryReadModel|^TestQuerySearch')
parser.add_argument('--race',action='store_true')
args=parser.parse_args();env=environment()
before=source_fingerprint(ROOT)
with (PRIVATE/'runner.lock').open('w') as lock:
fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
if args.down:compose(env,'down');return 0
if args.start:start(env)
argv=['go','test','./internal/queryserving','./internal/store','./internal/api','./internal/outputsnapshot','-run',args.run,'-count=1','-timeout=10m','-v']
if args.race:argv.append('-race')
result=run_command(argv,env,660)
proof=write_log(PRIVATE/'logs',argv,result.stdout+'\n'+result.stderr,'passed' if result.returncode==0 else 'failed')
print(json.dumps(proof))
# Do not interpret an opt-in skip/empty test run as acceptance.
if result.returncode:
return result.returncode
if not before or source_fingerprint(ROOT)!=before:
print('Source identity changed during checks; acceptance remains pending.');return 2
required=set()
for name in ('internal/queryserving/serving_test.go','internal/store/query_read_models_test.go','internal/store/query_recovery_test.go','internal/api/query_serving_test.go','internal/outputsnapshot/query_shared_cache_test.go'):
required.update(re.findall(r'^func (Test\w+)\(', (ROOT/name).read_text(), re.M))
passed=set(re.findall(r'^--- PASS: (Test\w+)',result.stdout,re.M))
if required-passed:
print('Missing required cases: '+', '.join(sorted(required-passed)));return 2
if '--- SKIP:' in result.stdout or '[no tests to run]' in result.stdout:
print('Unverified skipped/empty coverage; inspect report.');return 2
return result.returncode
if __name__=='__main__':
raise SystemExit(main())

View File

@ -0,0 +1,91 @@
#!/usr/bin/env python3
"""Compare the same synthetic API fixture before/after query serving."""
import argparse
import fcntl
import datetime as dt
import json
from pathlib import Path
import subprocess
import threading
import time
import uuid
from test_query_serving import ROOT, PRIVATE, environment, start, run_command
from lib.query_serving_evidence import compare
from lib.release_checks import write_log, source_fingerprint
def main():
parser=argparse.ArgumentParser()
parser.add_argument('--start',action='store_true')
parser.add_argument('--production-shape',action='store_true')
parser.add_argument('--rounds',type=int,default=1)
parser.add_argument('--warmup',type=int,default=5)
parser.add_argument('--duration',type=int,default=30)
parser.add_argument('--database',help='reuse this script\'s existing synthetic fixture')
args=parser.parse_args()
if not 1<=args.rounds<=3 or not 1<=args.duration<=1800 or not 1<=args.warmup<=300:parser.error('bounded rounds/times required')
if args.database and (not args.database.startswith('poll_pool_read_mix_query_') or not args.database.replace('_','').isalnum()):parser.error('synthetic database only')
env={k:v for k,v in environment().items() if not k.startswith('POLL_POOL_')}
dependency_lock=(PRIVATE/'runner.lock').open('w')
try:fcntl.flock(dependency_lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
except BlockingIOError:
print('Isolated query dependencies already in use; comparison not started.');return 2
if args.start:start(env)
folder=ROOT/'local-data/diagnostics'/('query-serving-'+dt.datetime.now(dt.timezone.utc).strftime('%Y%m%dT%H%M%SZ'))
folder.mkdir(parents=True)
database=args.database or 'poll_pool_read_mix_query_'+uuid.uuid4().hex[:16]
env.update(POLL_POOL_LOAD='1',POLL_POOL_LOCAL_MONGO_URI='mongodb://127.0.0.1:27019/?replicaSet=query-rs&directConnection=true',POLL_POOL_FIXTURE_DATABASE=database,
POLL_POOL_KEEP_FIXTURE='1',POLL_POOL_QUERY_SERVING_PREPARE='1',POLL_POOL_QUERY_PANELS='1',POLL_POOL_READMODELS='1',POLL_POOL_SEARCH='1',POLL_POOL_SEARCH_INDEXES='1',POLL_POOL_PAGE_SHAPE='1',POLL_POOL_ADMIN_TASKS='1' if args.production_shape else '0',POLL_POOL_AUTHORIZATION_STATE='initialized',
POLL_POOL_WARMUP_SECONDS=str(args.warmup),POLL_POOL_DURATION_SECONDS=str(args.duration),ANNOTATION_SOURCE_VALIDATION_MODE='advisory')
if args.production_shape:
env.update(POLL_POOL_PRODUCTION_SHAPE='1', POLL_POOL_RUN_PAYLOAD_KIB='80',
POLL_POOL_TASK_PAYLOAD_KIB='16', POLL_POOL_AVAILABLE_PER_STAGE='215',
POLL_POOL_AUTHOR_HISTORY='100', GOMAXPROCS='2')
before=source_fingerprint(ROOT)
report={'database':database,'inputDigest':before,'shape':'production' if args.production_shape else 'small','rounds':[],'resourceSamples':[],'passed':False,'limits':['synthetic loopback API harness, no media or production writes','process resource sampling includes builder and handler; not Kubernetes Pod limits']}
argv=['go','test','./internal/api','-run','^TestPollPoolReadMixLocal$','-count=1','-timeout=45m','-v']
def run(name,updates):
settings={**env,**updates,'POLL_POOL_OUTPUT':str(folder/(name+'.json'))}
active['phase']=name
started=dt.datetime.now(dt.timezone.utc).isoformat()
result=run_command(argv,settings,args.warmup+args.duration+900)
proof=write_log(folder,argv,result.stdout+'\n'+result.stderr,'passed' if result.returncode==0 else 'failed')
row={'phase':name,'startedAt':started,'endedAt':dt.datetime.now(dt.timezone.utc).isoformat(),'exitCode':result.returncode,**proof,'report':settings['POLL_POOL_OUTPUT']}
report['rounds'].append(row)
(folder/'comparison.json').write_text(json.dumps(report,indent=2)+'\n')
print(json.dumps(row),flush=True)
return result.returncode==0
active={'phase':'prepare'}
if not args.database:
if not run('prepare',{'POLL_POOL_FIXTURE_ONLY':'1'}):return 1
stopped=threading.Event()
def resources():
while not stopped.is_set():
try:
p=subprocess.run(['docker','stats','--no-stream','--format','{{json .}}','fanyu-query-local-mongo-1','fanyu-query-local-valkey-1','fanyu-query-local-typesense-1'],capture_output=True,text=True,timeout=8)
if p.returncode==0:report['resourceSamples'].append({'phase':active['phase'],'at':dt.datetime.now(dt.timezone.utc).isoformat(),'containers':[json.loads(line) for line in p.stdout.splitlines()]})
except (OSError,ValueError,subprocess.TimeoutExpired):pass
(folder/'progress.json').write_text(json.dumps({'phase':active['phase'],'at':dt.datetime.now(dt.timezone.utc).isoformat(),'samples':len(report['resourceSamples'])})+'\n')
stopped.wait(2)
thread=threading.Thread(target=resources,daemon=True);thread.start()
try:
for i in range(args.rounds):
baseline='baseline' if i==0 else 'baseline-'+str(i+1)
run(baseline,{'POLL_POOL_REUSE_FIXTURE':'1','POLL_POOL_ALLOW_BASELINE_FAILURE':'1'})
if not run('candidate-'+str(i+1),{'POLL_POOL_REUSE_FIXTURE':'1','POLL_POOL_QUERY_SERVING':'1','POLL_POOL_MAX_SEARCH_P95_MS':'2000'}):break
finally:
stopped.set();thread.join(timeout=10)
report['inputsChangedDuringRun']=source_fingerprint(ROOT)!=before
candidates=[r for r in report['rounds'] if r['phase'].startswith('candidate-')]
report['comparisons']=[]
try:
for i,row in enumerate(candidates):
basefile='baseline.json' if i==0 else 'baseline-'+str(i+1)+'.json'
report['comparisons'].append(compare(json.loads((folder/basefile).read_text()),json.loads(Path(row['report']).read_text())))
except (OSError,ValueError,KeyError):report['comparisons'].append({'passed':False,'problems':['missing/invalid comparison report']})
report['resourceReviewRequired']=len(report['comparisons'])==3 and any(all(r.get(key,0)>1.1 for r in report['comparisons']) for key in ('cpuRatio','retainedHeapRatio'))
report['passed']=not report['resourceReviewRequired'] and len(candidates)==args.rounds and all(r['exitCode']==0 for r in candidates) and not report['inputsChangedDuringRun'] and all(r['passed'] for r in report['comparisons'])
(folder/'comparison.json').write_text(json.dumps(report,indent=2)+'\n')
print('Comparison: '+str(folder/'comparison.json'),flush=True)
return 0 if report['passed'] else 1
if __name__=='__main__':raise SystemExit(main())

View File

@ -883,6 +883,65 @@
"warnings": [
"工作流门禁仅使用合成媒体和隔离 Mongo;严格模式清单不是校验证书,生产容量、备份恢复和部署需分别验证。"
]
},
{
"name": "query-serving",
"paths": [
"^internal/queryserving/",
"^internal/api/(database_browser|poll_pool_load_test)\\.go$",
"^internal/store/query_.*\\.go$",
"^internal/store/private_derived_collections\\.go$",
"^internal/(api|outputsnapshot)/query_.*\\.go$",
"^cmd/query-read-builder/",
"^scripts/(test_query_serving.*\\.py|lib/query_serving_evidence\\.py|local/query-serving\\.compose\\.yml)$",
"^Dockerfile\\.query-read-builder$",
"^tests/release/test_query_serving_evidence\\.py$",
"^docs/operations/query-serving\\.md$"
],
"frontend": [],
"go": {
"./internal/queryserving": [
"^TestQueryServing"
],
"./internal/store": [
"^TestQueryReadModel",
"^TestQuerySearch"
],
"./internal/outputsnapshot": [
"^TestQueryServing"
],
"./internal/api": [
"^TestQueryServing"
]
},
"task": [
{
"argv": [
"python3",
"scripts/test_query_serving.py",
"--start",
"--race"
],
"tests": true,
"env": {}
},
{
"argv": [
"python3",
"-m",
"unittest",
"discover",
"-s",
"tests/release",
"-p",
"test_query_serving_evidence.py"
],
"tests": true
}
],
"warnings": [
"真实三组件仅使用固定本机合成夹具;完整规模三轮、故障恢复容量及生产部署独立验收。"
]
}
]
}

View File

@ -0,0 +1,29 @@
import sys
import unittest
from pathlib import Path
sys.path.insert(0,str(Path(__file__).resolve().parents[2]/'scripts'))
from lib.query_serving_evidence import compare
class QueryEvidenceTests(unittest.TestCase):
def fixture(self):
sample={'Actor':'reviewer','Key':'acceptance.all.search','Code':200,'MS':20,'Hash':'business-id-claim-counts'}
return {'fixture_digest':'same','actors':81,'errors':0,'mongo_write_commands':0,'metrics':{'acceptance.all.search':{'p95_ms':20}},'samples':[sample], 'cold_samples':[], 'query_serving_statistics':dict(report_hits=1,search_hits=1,typesense_reads=1,cache_hits=1)}
def test_requires_real_measurements_and_active_components(self):
a=self.fixture();b=self.fixture();self.assertTrue(compare(a,b)['passed'])
b['query_serving_statistics']={};self.assertFalse(compare(a,b)['passed'])
self.assertFalse(compare({}, {})['passed'])
def test_results_and_latency_are_both_mandatory(self):
a=self.fixture();b=self.fixture();b['samples'][0]['Hash']='lost-task'
self.assertFalse(compare(a,b)['passed'])
b=self.fixture();b['metrics']['acceptance.all.search']['p95_ms']=2001
self.assertFalse(compare(a,b)['passed'])
def test_business_write_and_cold_panel_failure_are_not_success(self):
a=self.fixture();b=self.fixture();b['mongo_write_commands']=1
self.assertFalse(compare(a,b)['passed'])
b=self.fixture();b['cold_samples']=[{'Actor':'admin','Key':'admin.production','Code':202,'MS':1,'Hash':'empty'}]
self.assertFalse(compare(a,b)['passed'])
def test_baseline_timeout_keeps_equivalence_pending(self):
a=self.fixture();b=self.fixture();a['samples'][0]['Code']=500
r=compare(a,b);self.assertFalse(r['passed']);self.assertTrue(r['baselineUnavailableComparisons'])
if __name__=='__main__':unittest.main()