From 1050f48286344ba13f12a68303c589dd03d73496 Mon Sep 17 00:00:00 2001 From: Codex Date: Thu, 12 Nov 2026 01:18:17 -0800 Subject: [PATCH] =?UTF-8?q?=E2=9C=A8=20Add=20optional=20background=20query?= =?UTF-8?q?=20models,=20shared=20cache=20and=20search?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitea/workflows/web-ci.yml | 22 + .scratch/query-serving-options/spec.md | 22 + Dockerfile.query-read-builder | 17 + cmd/api/main.go | 8 + cmd/query-read-builder/main.go | 65 +++ docs/operations/query-serving.md | 38 ++ go.mod | 1 + go.sum | 4 + internal/api/database_browser.go | 3 +- internal/api/poll_pool_load_test.go | 87 +++- internal/api/production_dashboard.go | 42 +- internal/api/query_serving_test.go | 76 ++++ .../outputsnapshot/query_shared_cache_test.go | 86 ++++ internal/outputsnapshot/reader.go | 27 +- internal/outputsnapshot/shards.go | 2 +- internal/outputsnapshot/types.go | 3 + internal/queryserving/cache.go | 199 +++++++++ internal/queryserving/serving_test.go | 138 +++++++ internal/queryserving/typesense.go | 190 +++++++++ internal/store/admin_annotation_pool.go | 65 +-- internal/store/annotation_output_snapshots.go | 28 ++ internal/store/human_pool_search.go | 11 +- internal/store/memory.go | 4 +- internal/store/mongo.go | 6 +- internal/store/private_derived_collections.go | 9 + internal/store/query_model_maintenance.go | 84 ++++ internal/store/query_read_builder.go | 384 ++++++++++++++++++ internal/store/query_read_models.go | 278 +++++++++++++ internal/store/query_read_models_test.go | 311 ++++++++++++++ internal/store/query_recovery_test.go | 231 +++++++++++ internal/store/query_search_model.go | 231 +++++++++++ internal/store/query_serving_config.go | 41 ++ internal/store/query_source_changes.go | 172 ++++++++ scripts/lib/query_serving_evidence.py | 36 ++ scripts/local/query-serving.compose.yml | 31 ++ scripts/test_query_serving.py | 97 +++++ scripts/test_query_serving_performance.py | 91 +++++ scripts/verification-scopes.json | 59 +++ tests/release/test_query_serving_evidence.py | 29 ++ 39 files changed, 3180 insertions(+), 48 deletions(-) create mode 100644 .scratch/query-serving-options/spec.md create mode 100644 Dockerfile.query-read-builder create mode 100644 cmd/query-read-builder/main.go create mode 100644 docs/operations/query-serving.md create mode 100644 internal/api/query_serving_test.go create mode 100644 internal/outputsnapshot/query_shared_cache_test.go create mode 100644 internal/queryserving/cache.go create mode 100644 internal/queryserving/serving_test.go create mode 100644 internal/queryserving/typesense.go create mode 100644 internal/store/private_derived_collections.go create mode 100644 internal/store/query_model_maintenance.go create mode 100644 internal/store/query_read_builder.go create mode 100644 internal/store/query_read_models.go create mode 100644 internal/store/query_read_models_test.go create mode 100644 internal/store/query_recovery_test.go create mode 100644 internal/store/query_search_model.go create mode 100644 internal/store/query_serving_config.go create mode 100644 internal/store/query_source_changes.go create mode 100644 scripts/lib/query_serving_evidence.py create mode 100644 scripts/local/query-serving.compose.yml create mode 100644 scripts/test_query_serving.py create mode 100644 scripts/test_query_serving_performance.py create mode 100644 tests/release/test_query_serving_evidence.py diff --git a/.gitea/workflows/web-ci.yml b/.gitea/workflows/web-ci.yml index ec2a4beb..2138fc6e 100644 --- a/.gitea/workflows/web-ci.yml +++ b/.gitea/workflows/web-ci.yml @@ -105,3 +105,25 @@ jobs: path: local-data/ if-no-files-found: error retention-days: 14 + + query-serving: + runs-on: ubuntu-24.04 + needs: pool-search-mongo + if: ${{ !cancelled() }} + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version-file: go.mod + cache: true + - run: docker compose version + - run: python3 scripts/test_query_serving.py --start --race + - uses: actions/upload-artifact@v4 + if: always() + with: + name: query-serving-logs + path: local-data/query-serving-local/logs/ + if-no-files-found: error + retention-days: 14 + - run: python3 scripts/test_query_serving.py --down + if: always() diff --git a/.scratch/query-serving-options/spec.md b/.scratch/query-serving-options/spec.md new file mode 100644 index 00000000..e04a7168 --- /dev/null +++ b/.scratch/query-serving-options/spec.md @@ -0,0 +1,22 @@ +# 后台物化读表、Valkey 与 Typesense 实施 + +Status: claimed +Type: task +Base: 9232bace + +用户批准依次实施物化读取与共享缓存,再接搜索;正确性与性能达标后才准备发布。本次仅本地实现、隔离验证和提交,不推送、不部署、不写生产。 + +- 后台独立进程构建完整代次,复用已有统计口径与旧记录解析;读取不触发新组件的全量构建。后台并发有界、分布式租约、原子发布,失败保留上一完整代次,支持重启恢复和删除对账。 +- Valkey 共享不可变统计/查询结果,包含数据库、组织、项目范围、代次与查询参数;缓存故障、丢失与未准备就绪退回现有读取,不能阻断作业。复用既有快照发布协议,不重写标注/审核写路径。 +- Typesense 仅提供搜索候选;保留精确子串、Capture/part、特殊字符、旧记录、排序、完整分页与统计语义。实时权限、可领取、自审禁止、归属、状态、版本与写锁仍由现有服务端代码判定。 +- 新能力默认关闭,运行地址/凭证从环境或现有私有配置读取。新增集合只存可重建派生数据,后台和查询组件的生产安装、索引及回填另列计划。 +- 使用隔离 Mongo、真实 Valkey/Typesense 和合成数据,对照旧路径、撤权/归属变化、删除/重启、组件故障、并发与查询资源;接近线上规模,搜索 p95≤2秒,面板暖态 p95≤500ms、预热首访≤2秒。未实测或未达标保持待验收。 +- 保留原工作树两个 admin pool 测试及旧发布笔记,阶段性本地提交;不修改既有标注和正式审查记录。 + +## 本地进展 + +后台读表、Valkey、Typesense 已实现,默认关闭。真实三组件12项(含竞态、范围、失联、重启、旧数据和缓存损坏)通过;既有五阶段作业/恢复、搜索与统计浏览器回归通过。官方 Go 基线为68项已有失败、无新增;前端基线与构建/生产启动检查通过。范围总报告保留了原生产诊断跳过、原测试债务和最初三组件瞬时回退断言失败,不将其写成全绿;修正的专项已重跑通过。 + +81人、约2.6万任务/34万事件的首轮短测:候选零错误,347组账号/接口业务结果一致;管理员全部项目搜索p95 2592→799ms、生产总览3516→162ms;Mongo内部返回字节199215338→152507258,Go CPU +3.3%。这是短测,不是容量/生产验收。1GiB Mongo 对照OOM已作废,两版统一固定4GiB重新测得上述结果。 + +独立后台镜像构建和启动帮助检查已通过。待完成:冻结源码后的三轮持续对照和资源审查。未达标不准备发布;生产拓扑、新组件及派生索引/构建范围仍需具体计划与单独授权。现有两个受保护文件的哈希保持不变。 diff --git a/Dockerfile.query-read-builder b/Dockerfile.query-read-builder new file mode 100644 index 00000000..3888692f --- /dev/null +++ b/Dockerfile.query-read-builder @@ -0,0 +1,17 @@ +FROM golang:1.24-bookworm@sha256:1a6d4452c65dea36aac2e2d606b01b4a029ec90cc1ae53890540ce6173ea77ac AS build +WORKDIR /src +ARG GOPROXY=https://proxy.golang.org,direct +COPY go.mod go.sum ./ +RUN GOPROXY="${GOPROXY}" go mod download +COPY cmd/query-read-builder ./cmd/query-read-builder +COPY internal ./internal +RUN CGO_ENABLED=0 go build -trimpath -o /out/query-read-builder ./cmd/query-read-builder + +# A static binary only needs trusted CA roots; no media/worker tools or app code. +FROM scratch +COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt +COPY --from=build /out/query-read-builder /query-read-builder +ARG DROBOTICFLOW_BUILD_COMMIT=unknown +LABEL org.opencontainers.image.revision=${DROBOTICFLOW_BUILD_COMMIT} +USER 10001:10001 +ENTRYPOINT ["/query-read-builder"] diff --git a/cmd/api/main.go b/cmd/api/main.go index 457ae6ad..b80010b1 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -173,6 +173,14 @@ func main() { }() dataStore := store.NewMongoStore(client.Database(cfg.MongoDB)) + if os.Getenv("ANNOTATION_PROCESSOR_MODE") != "processor" { + closeServing, servingErr := dataStore.ConfigureQueryServingFromEnvironment() + if servingErr != nil { + fmt.Println("[api] query accelerator configuration unavailable; using existing reads") + } else { + defer closeServing() + } + } if os.Getenv("ANNOTATION_PROCESSOR_MODE") == "processor" { checkIndexes := dataStore.EnsureIndexes mode := os.Getenv("DROBOTICFLOW_DATABASE_BOOTSTRAP") diff --git a/cmd/query-read-builder/main.go b/cmd/query-read-builder/main.go new file mode 100644 index 00000000..673e626f --- /dev/null +++ b/cmd/query-read-builder/main.go @@ -0,0 +1,65 @@ +// query-read-builder only writes derived read models. It never processes work. +package main + +import ( + "context" + "droboticflow/internal/config" + "droboticflow/internal/store" + "flag" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + "log" + "os" + "os/signal" + "syscall" + "time" +) + +func main() { + once := flag.Bool("once", false, "build once; streaming freshness expires after exit") + ensure := flag.Bool("ensure-indexes", false, "explicitly create derived-data indexes before building") + flag.Parse() + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + cfg, err := config.LoadValidated() + if err != nil { + log.Fatal("invalid builder configuration") + } + if os.Getenv("QUERY_SERVING_READS") != "1" { + log.Fatal("explicit QUERY_SERVING_READS=1 is required") + } + client, err := mongo.Connect(ctx, options.Client().ApplyURI(cfg.MongoURI)) + if err != nil { + log.Fatal("builder database unavailable") + } + defer client.Disconnect(context.Background()) + s := store.NewMongoStore(client.Database(cfg.MongoDB)) + closeServing, err := s.ConfigureQueryServingFromEnvironment() + if err != nil { + log.Fatal("builder accelerator configuration invalid") + } + defer closeServing() + if *ensure { + if s.EnsureQueryReadModelIndexes(ctx) != nil { + log.Fatal("derived index creation failed") + } + } + b := store.NewQueryReadBuilder(s) + defer b.Close() + if *once { + if b.RunOnce(ctx) != nil { + log.Fatal("initial materialization incomplete") + } + return + } + for ctx.Err() == nil { + if b.Run(ctx) != nil && ctx.Err() == nil { + log.Print("query builder paused; retrying with full reconciliation") + } + select { + case <-ctx.Done(): + return + case <-time.After(5 * time.Second): + } + } +} diff --git a/docs/operations/query-serving.md b/docs/operations/query-serving.md new file mode 100644 index 00000000..cbc58213 --- /dev/null +++ b/docs/operations/query-serving.md @@ -0,0 +1,38 @@ +# 后台查询读表与可选加速组件 + +状态:本地实现;生产保持关闭。速度与资源结论见同任务证据,未通过的门槛不能据本说明视为通过。 + +## 责任与兼容 + +Mongo 仍是任务、权限、归属、版本与正式审查记录的权威存储。独立 `query-read-builder` 只写 `queryReadModelControl`、`queryReadModelData` 及既有 `annotationOutputSnapshot*` 派生集合,不领取、退回、重置、补验或改写客户任务。读取新集合不触发构建。 + +后台使用租约代次隔离写者,先完整构建再发布清单。单个失败不会发布半份结果。进程重启先打开变更流,再核对源身份和重建;不把遗留索引视为最新。副本集/分片集群变更流是持续搜索加速的前提,普通独立 Mongo 不支持时后台暂停,API 继续旧路径。 + +- 生产总览复用原事实读取与 `productionstats.Build`,保留原日期、供应商、人员和首次送检批次口径。当前配置项目集合生成一份报告;其他未预建项目组合走原路径。来源无变化且观察器新鲜时复用同一代次;有变化按原事实缓存时限停止使用旧结果。 +- 人员与供应商看板复用既有分片快照和聚合器,后台无人访问也预热。Valkey 共享不可变分片及按完整范围、用户、供应商、日期、时区、代次隔离的结果;本进程已算出的结果优先,避免每次重复访问缓存服务器。 +- Typesense 只存任务 ID 与名称搜索窗口,返回保守候选 ID。用 Unicode 窗口交集保持任意子串、Capture/part 和标点语义;最终匹配、排序、分页、实时可领取及权限仍走原服务端规则。候选过大、超时、索引过期或不完整时退回原查询,不能截断候选凑性能指标。 +- 派生集合对通用数据库浏览入口隐藏,防止绕过组织和个人权限。Valkey、Typesense 只允许服务端私网访问,不开放浏览器直连。 + +## 配置与本地验证 + +API、后台分别显式设置 `QUERY_SERVING_READS=1` 与唯一 `QUERY_SERVING_NAMESPACE`,默认关闭。可选 `QUERY_VALKEY_URL`、`QUERY_TYPESENSE_URL`、`QUERY_TYPESENSE_API_KEY` 均通过现有私有环境配置提供;凭证不提交。不同集群即便数据库同名也必须使用不同 namespace。缓存连接异步建立,启动不可用不会阻止 API;缓存请求有短时限和退避。 + +本机合成依赖:`python3 scripts/test_query_serving.py --start`。仅暴露 loopback 27019/16379/18108;随机测试密钥在忽略目录,Mongo 使用独立 query-rs 和独立数据卷。`--down` 只停这一项目,保留卷;不停止共享快照库和其他项目。Mongo 文件句柄上限显式设为65536,避免多轮隔离测试建索引触发默认低限制。 + +功能与故障回归使用真实本机三组件,`--race` 检查并发。规模对照入口为 `python3 scripts/test_query_serving_performance.py --production-shape`,使用同一合成数据、索引、资源及固定压缩设置;可通过 `--database` 复用脚本生成的专用数据库。完整稳态验收另设 `--rounds 3 --warmup 300 --duration 1800`;短测通过不代表线上容量验收。 + +本机规模夹具统一使用 Mongo 2 CPU/4GiB(WiredTiger 0.5GiB)、Valkey 1 CPU/256MiB、Typesense 2 CPU/1GiB;这是隔离对照的固定配置,不是生产扩容建议。1GiB Mongo 的首轮对照因旧路径 OOM 作废,不能计为优化收益。 + +## 运行与清理 + +后台单进程生成、跨副本租约;Valkey 不持久化,清空后仅损失速度。Typesense 索引可重建。清理只删除当前数据库登记、已不可达且超过保留期的派生代次,当前清单和其他部署不在清理范围。固定构建预算和载荷预算超限时保留旧路径,不改任务准入。 + +显式 `query-read-builder --ensure-indexes` 只为派生行创建 `query_model_rows_v1`;API 启动不建索引。生产新增组件、该索引及派生数据构建会产生数据库读取和索引维护开销,需另列具体部署、资源和授权计划。不上线时不向公司集群安装或写入这些数据。 + +修改生产统计或名称匹配的语义时必须递增 `queryModelSchema`,防止新代码复用旧口径的派生代次。API 与后台应发布同一已核验源码版本。 + +## 发布与回退条件 + +先核对 Mongo 拓扑与权限、私网连接及组件资源,再在隔离环境验收完整任务链和查询结果。比较搜索 p95、冷/暖看板延迟、错误、结果/权限等价、后台业务写入为零、Mongo 读取字节与命令、Go/三组件 CPU 和内存。未达到搜索p95≤2秒、暖面板p95≤500ms、预热首访≤2秒,或三轮持续资源恶化未解释时保持待验收。 + +上线采用独立开关,先预热后逐步启用读取。回退关闭 API 的 `QUERY_SERVING_READS` 并停后台,恢复原读取;不改写客户记录,不要求删除新派生集合。组件失联仅是性能降级,不改变标注、切片、送检、质检和验收准入,也不恢复严格源校验。 diff --git a/go.mod b/go.mod index e5fcd607..704120cc 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible github.com/baidubce/bce-sdk-go v0.9.264 github.com/go-chi/chi/v5 v5.2.3 + github.com/valkey-io/valkey-go v1.0.60 go.etcd.io/bbolt v1.4.3 go.mongodb.org/mongo-driver v1.17.6 golang.org/x/crypto v0.36.0 diff --git a/go.sum b/go.sum index 73f7eead..d76792bf 100644 --- a/go.sum +++ b/go.sum @@ -76,6 +76,7 @@ github.com/onsi/ginkgo/v2 v2.21.0 h1:7rg/4f3rB88pb5obDgNZrNHrQ4e6WpjonchcpuBRnZM github.com/onsi/ginkgo/v2 v2.21.0/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo= github.com/onsi/gomega v1.35.1 h1:Cwbd75ZBPxFSuZ6T+rN/WCb/gOc6YgFBXLlZLhC7Ds4= github.com/onsi/gomega v1.35.1/go.mod h1:PvZbdDc8J6XJEpDK4HCuRBm8a6Fzp9/DmhC9C7yFlog= +github.com/onsi/gomega v1.36.2 h1:koNYke6TVk6ZmnyHrCXba/T/MoLBXFjeC1PtvYgw0A8= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -95,6 +96,8 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/valkey-io/valkey-go v1.0.60 h1:idh959D20H5n7D/kwEdTKNaMn5+4HpZTn7bLXnAhQIw= +github.com/valkey-io/valkey-go v1.0.60/go.mod h1:bHmwjIEOrGq/ubOJfh5uMRs7Xj6mV3mQ/ZXUbmqpjqY= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= @@ -169,6 +172,7 @@ golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.26.0 h1:v/60pFQmzmT9ExmjDv2gGIfi3OqfKoEP6I5+umXlbnQ= golang.org/x/tools v0.26.0/go.mod h1:TPVVj70c7JJ3WCazhD8OdXcZg/og+b9+tH/KxylGwH0= +golang.org/x/tools v0.31.0 h1:0EedkvKDbh+qistFTd0Bcwe/YLh4vHwWEkiI0toFIBU= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/internal/api/database_browser.go b/internal/api/database_browser.go index 45fd42f7..a9ea1bc2 100644 --- a/internal/api/database_browser.go +++ b/internal/api/database_browser.go @@ -10,7 +10,6 @@ import ( "time" "droboticflow/internal/domain" - "droboticflow/internal/outputsnapshot" "droboticflow/internal/store" "github.com/go-chi/chi/v5" @@ -148,7 +147,7 @@ func normalizeDatabaseBrowserTarget(raw string) databaseBrowserTarget { func validateDatabaseCollectionName(field string, value string) *validationIssue { name := strings.TrimSpace(value) - if outputsnapshot.IsPrivateCollection(name) { + if store.IsPrivateDerivedCollection(name) { return &validationIssue{Field: field, Message: "internal statistics collection is not accessible"} } label := "collection name" diff --git a/internal/api/poll_pool_load_test.go b/internal/api/poll_pool_load_test.go index 72b5291e..f0a8688e 100644 --- a/internal/api/poll_pool_load_test.go +++ b/internal/api/poll_pool_load_test.go @@ -26,6 +26,8 @@ import ( "time" "droboticflow/internal/domain" + "droboticflow/internal/productionstats" + "droboticflow/internal/queryserving" "droboticflow/internal/store" "go.mongodb.org/mongo-driver/event" "go.mongodb.org/mongo-driver/mongo" @@ -96,6 +98,8 @@ func TestPollPoolReadMixLocal(t *testing.T) { pageShape := os.Getenv("POLL_POOL_PAGE_SHAPE") == "1" allowBaselineFailure := os.Getenv("POLL_POOL_ALLOW_BASELINE_FAILURE") == "1" readModels := os.Getenv("POLL_POOL_READMODELS") == "1" + queryServing := os.Getenv("POLL_POOL_QUERY_SERVING") == "1" + queryPanels := os.Getenv("POLL_POOL_QUERY_PANELS") == "1" mode := os.Getenv("POLL_POOL_MODE") if mode == "" { @@ -108,6 +112,7 @@ func TestPollPoolReadMixLocal(t *testing.T) { defer cancel() var measured atomic.Bool var commands, mongoBytes, mongoNS, writes atomic.Int64 + var derivedWrites atomic.Int64 var probeMu sync.Mutex pending := map[int64]struct { key string @@ -154,10 +159,12 @@ func TestPollPoolReadMixLocal(t *testing.T) { } }, Succeeded: func(_ context.Context, e *event.CommandSucceededEvent) { + derived := false probeMu.Lock() if entry, ok := pending[e.RequestID]; ok { key := entry.key row := observations[key] + derived = strings.HasPrefix(row.Collection, "queryReadModel") || strings.HasPrefix(row.Collection, "annotationOutputSnapshot") row.Count++ row.Bytes += int64(len(e.Reply)) row.Nanoseconds += e.Duration.Nanoseconds() @@ -170,7 +177,11 @@ func TestPollPoolReadMixLocal(t *testing.T) { mongoNS.Add(e.Duration.Nanoseconds()) switch e.CommandName { case "insert", "update", "delete", "findAndModify": - writes.Add(1) + if queryServing && derived { + derivedWrites.Add(1) + } else { + writes.Add(1) + } } } }} @@ -235,6 +246,12 @@ func TestPollPoolReadMixLocal(t *testing.T) { } const org = "load-org" projects := []string{"load-project-a", "load-project-b"} + if os.Getenv("POLL_POOL_QUERY_SERVING_PREPARE") == "1" && !reuseFixture { + must(ds.UpsertOrganization(ctx, domain.OrganizationRecord{ID: org, Status: domain.OrganizationStatusActive})) + _, err := ds.SaveProductionStatsScope(ctx, org, productionstats.Scope{ProjectIDs: projects, Revision: 1}, 0) + must(err) + must(ds.EnsureQueryReadModelIndexes(ctx)) + } if !reuseFixture { for _, p := range projects { must(ds.UpsertProject(ctx, domain.ProjectRecord{ID: p, OrganizationID: org, Name: p})) @@ -487,11 +504,13 @@ func TestPollPoolReadMixLocal(t *testing.T) { elapsed := time.Since(start) completedRequests.Add(1) var body struct { - Count *int `json:"count"` - Counts map[string]int `json:"counts"` - Code string `json:"code"` - Error string `json:"error"` - Items []struct { + Summary *productionstats.Summary `json:"summary"` + Totals json.RawMessage `json:"totals"` + Count *int `json:"count"` + Counts map[string]int `json:"counts"` + Code string `json:"code"` + Error string `json:"error"` + Items []struct { ID string `json:"id"` CanClaim *bool `json:"canClaim"` TraceSummary *domain.HumanTraceSummary `json:"traceSummary"` @@ -534,6 +553,15 @@ func TestPollPoolReadMixLocal(t *testing.T) { completedErrors.Add(1) } rowCount := len(body.Items) + len(body.Summaries) + if body.Summary != nil { + rowCount = len(body.Summary.Rows) + raw, _ := json.Marshal(body.Summary) + ids = append(ids, string(raw)) + } + if strings.HasSuffix(key, ".panel") && len(body.Totals) > 0 { + rowCount = 1 + ids = append(ids, string(body.Totals)) + } if body.Count != nil { rowCount = *body.Count ids = append(ids, fmt.Sprintf("count:%d", *body.Count)) @@ -704,8 +732,23 @@ func TestPollPoolReadMixLocal(t *testing.T) { } return result } + if queryPanels { + original := paths + paths = func(a pollLoadActor, all, countCycle bool, scenario int) map[string]string { + out := original(a, all, countCycle, scenario) + out[a.Role+".panel"] = "/api/dashboard/production-quality?from=2026-10-01&to=2026-11-13" + if a.Role == "admin" { + out["admin.production"] = "/api/dashboard/production?from=2026-10-01&to=2026-11-13" + } + return out + } + } + fixtureSpec := fmt.Sprintf("isolated-preparation-v5|%d|%d|%d|%d|%d|%d|%d|%t|%t|%t|%s|%s", annotators, quality, acceptance, availablePerStage, runPayloadKiB, taskPayloadKiB, authorHistory, productionShape, readModels, claimCounts, authorizationState, os.Getenv("POLL_POOL_SEARCH_INDEXES")) fixtureSpec += "|admin-tasks=" + os.Getenv("POLL_POOL_ADMIN_TASKS") + if os.Getenv("POLL_POOL_QUERY_SERVING_PREPARE") == "1" { + fixtureSpec += "|query-serving-preparation-v1" + } fixtureDigest := fmt.Sprintf("%x", sha256.Sum256([]byte(fixtureSpec))) if reuseFixture { var manifest struct { @@ -742,6 +785,30 @@ func TestPollPoolReadMixLocal(t *testing.T) { if os.Getenv("POLL_POOL_FIXTURE_ONLY") == "1" { return } + var preparationCPU float64 + if queryServing { + startCPU, _ := pollLoadProcessCPU() + if os.Getenv("QUERY_SERVING_LOCAL_TEST") != "1" { + t.Fatal("query acceleration requires isolated local dependencies") + } + cache, err := queryserving.NewManagedValkey("valkey://127.0.0.1:16379", db.Name()) + must(err) + defer cache.Close() + search, err := queryserving.NewTypesense("http://127.0.0.1:18108", os.Getenv("QUERY_TEST_TYPESENSE_API_KEY")) + must(err) + ds.ConfigureQueryServing(cache, search) + builder := store.NewQueryReadBuilder(ds) + buildCtx, stop := context.WithCancel(ctx) + buildDone := make(chan error, 1) + go func() { buildDone <- builder.Run(buildCtx) }() + defer func() { stop(); <-buildDone; builder.Close() }() + prepare, finish := context.WithTimeout(ctx, 4*time.Minute) + err = builder.WaitReady(prepare, org, projects) + finish() + must(err) + endCPU, _ := pollLoadProcessCPU() + preparationCPU = endCPU - startCPU + } // Fail before timing if permissions or fixtures accidentally produce empty work. for _, a := range actors { for key, path := range paths(a, false, true, 0) { @@ -752,7 +819,11 @@ func TestPollPoolReadMixLocal(t *testing.T) { } t.Fatalf("preflight %s status=%d", key, result.Code) } - if (a.Role == "annotation" || a.Role == "admin" && os.Getenv("POLL_POOL_ADMIN_TASKS") != "1" || strings.HasSuffix(key, ".available") || strings.HasSuffix(key, ".mine") && !strings.Contains(key, ".acceptance.")) && result.Rows == 0 { + // This workload seeds assigned annotation work, but no unassigned + // annotation pool unless claim-count coverage was requested. An empty + // admin available pool is then the correct fixture result, not failure. + requireRows := a.Role == "annotation" || a.Role == "admin" && os.Getenv("POLL_POOL_ADMIN_TASKS") != "1" || a.Role != "admin" && (strings.HasSuffix(key, ".available") || strings.HasSuffix(key, ".mine") && !strings.Contains(key, ".acceptance.")) || a.Role == "admin" && claimCounts && strings.HasSuffix(key, ".available") + if requireRows && result.Rows == 0 { t.Fatalf("preflight %s unexpectedly empty for %s", key, a.User.ID) } } @@ -922,7 +993,7 @@ func TestPollPoolReadMixLocal(t *testing.T) { } } } - report := map[string]any{"reused_prepared_fixture": reuseFixture, "fixture_digest": fixtureDigest, "cold_samples": coldSamples, "slow_search_endpoints": slowSearch, "baseline_failure_collection": allowBaselineFailure, "production_shape": productionShape, "page_shape": pageShape, "fixture_counts": fixtureCounts, "search_indexes": os.Getenv("POLL_POOL_SEARCH_INDEXES") == "1", "read_models": readModels, "search_queries": os.Getenv("POLL_POOL_SEARCH") == "1", "schema": 2, "resource_samples": resourceRows, "process_cpu_seconds": cpuAfter - cpuBefore, "authorization_state": authorizationState, "fixture": "read-mix-v5-isolated-preparation", "available_per_stage": availablePerStage, "task_payload_kib": taskPayloadKiB, "run_payload_kib": runPayloadKiB, "author_history_per_reviewer": authorHistory, "mode": mode, "revision": os.Getenv("POLL_POOL_REVISION"), "actors": totalActors, "actor_counts": map[string]int{"annotation": annotators, "quality": quality, "acceptance": acceptance, "admin": 1}, "claim_counts": claimCounts, "warmup_seconds": warmup, "scheduled_seconds": duration, "actual_seconds": actual.Seconds(), "errors": failures, "failure_samples": failureSamples, "metrics": metrics, "mongo_breakdown": observations, "mongo_commands": commands.Load(), "mongo_reply_bytes": mongoBytes.Load(), "mongo_command_ms": float64(mongoNS.Load()) / 1e6, "mongo_write_commands": writes.Load(), "go_heap_before": memBefore.HeapAlloc, "go_heap_after": memAfter.HeapAlloc, "go_heap_after_gc": memAfterGC.HeapAlloc, "go_gc_cycles": memAfter.NumGC - memBefore.NumGC, "go_allocated_bytes": memAfter.TotalAlloc - memBefore.TotalAlloc, "limitations": []string{"handler timing excludes ingress/network/authentication", "gzip computed separately with fixed Go default compression", "no continuation renew, media or workflow writes", "synthetic fixture; not capacity acceptance", "CPU covers handler and in-process load harness, not Mongo/Pod throttling", "resource heap includes retained benchmark request samples; not standalone service RSS"}, "samples": samples} + report := map[string]any{"query_serving": queryServing, "query_preparation_cpu_seconds": preparationCPU, "query_serving_statistics": ds.QueryServingStatistics(), "query_panels": queryPanels, "derived_write_commands": derivedWrites.Load(), "reused_prepared_fixture": reuseFixture, "fixture_digest": fixtureDigest, "cold_samples": coldSamples, "slow_search_endpoints": slowSearch, "baseline_failure_collection": allowBaselineFailure, "production_shape": productionShape, "page_shape": pageShape, "fixture_counts": fixtureCounts, "search_indexes": os.Getenv("POLL_POOL_SEARCH_INDEXES") == "1", "read_models": readModels, "search_queries": os.Getenv("POLL_POOL_SEARCH") == "1", "schema": 2, "resource_samples": resourceRows, "process_cpu_seconds": cpuAfter - cpuBefore, "authorization_state": authorizationState, "fixture": "read-mix-v5-isolated-preparation", "available_per_stage": availablePerStage, "task_payload_kib": taskPayloadKiB, "run_payload_kib": runPayloadKiB, "author_history_per_reviewer": authorHistory, "mode": mode, "revision": os.Getenv("POLL_POOL_REVISION"), "actors": totalActors, "actor_counts": map[string]int{"annotation": annotators, "quality": quality, "acceptance": acceptance, "admin": 1}, "claim_counts": claimCounts, "warmup_seconds": warmup, "scheduled_seconds": duration, "actual_seconds": actual.Seconds(), "errors": failures, "failure_samples": failureSamples, "metrics": metrics, "mongo_breakdown": observations, "mongo_commands": commands.Load(), "mongo_reply_bytes": mongoBytes.Load(), "mongo_command_ms": float64(mongoNS.Load()) / 1e6, "mongo_write_commands": writes.Load(), "go_heap_before": memBefore.HeapAlloc, "go_heap_after": memAfter.HeapAlloc, "go_heap_after_gc": memAfterGC.HeapAlloc, "go_gc_cycles": memAfter.NumGC - memBefore.NumGC, "go_allocated_bytes": memAfter.TotalAlloc - memBefore.TotalAlloc, "limitations": []string{"handler timing excludes ingress/network/authentication", "gzip computed separately with fixed Go default compression", "no continuation renew, media or workflow writes", "synthetic fixture; not capacity acceptance", "CPU covers handler and in-process load harness, not Mongo/Pod throttling", "resource heap includes retained benchmark request samples; not standalone service RSS"}, "samples": samples} encoded, e := json.MarshalIndent(report, "", " ") must(e) must(os.WriteFile(out, encoded, 0600)) diff --git a/internal/api/production_dashboard.go b/internal/api/production_dashboard.go index 95a02710..385cf8ea 100644 --- a/internal/api/production_dashboard.go +++ b/internal/api/production_dashboard.go @@ -5,6 +5,7 @@ import ( "droboticflow/internal/domain" "droboticflow/internal/productionstats" "encoding/json" + "fmt" "net/http" "sort" "strconv" @@ -138,19 +139,42 @@ func (s *server) handleProductionDashboard(w http.ResponseWriter, r *http.Reques return } } - var snapshot productionstats.Snapshot - if cached, ok := s.store.(interface { - LoadProductionStatsCached(context.Context, string, []string) (productionstats.Snapshot, error) + responseIdentity := fmt.Sprint(scope.Revision) + ":" + q.From.Format(time.RFC3339) + ":" + q.To.Format(time.RFC3339) + ":" + params.Encode() + if prepared, ok := s.store.(interface { + PreparedProductionResponse(context.Context, string, []string, string) ([]byte, bool) }); ok { - snapshot, err = cached.LoadProductionStatsCached(ctx, org, ids) + if data, hit := prepared.PreparedProductionResponse(ctx, org, ids, responseIdentity); hit { + w.Header().Set("Content-Type", "application/json") + w.Header().Set("Cache-Control", "no-store") + w.WriteHeader(200) + _, _ = w.Write(data) + return + } + } + var report productionstats.Report + var asOf string + if prepared, ok := s.store.(interface { + LoadProductionReport(context.Context, string, []string) (productionstats.Report, string, error) + }); ok { + report, asOf, err = prepared.LoadProductionReport(ctx, org, ids) } else { - snapshot, err = s.store.LoadProductionStats(ctx, org, ids) + var snapshot productionstats.Snapshot + if cached, ok := s.store.(interface { + LoadProductionStatsCached(context.Context, string, []string) (productionstats.Snapshot, error) + }); ok { + snapshot, err = cached.LoadProductionStatsCached(ctx, org, ids) + } else { + snapshot, err = s.store.LoadProductionStats(ctx, org, ids) + } + if err == nil { + report = productionstats.Build(snapshot) + asOf = snapshot.AsOf + } } if err != nil { renderError(w, 500, err) return } - report := productionstats.Build(snapshot) summary := productionstats.Summarize(report, q) suppliers, err := s.store.ListSuppliers(ctx, org) if err != nil { @@ -196,7 +220,6 @@ func (s *server) handleProductionDashboard(w http.ResponseWriter, r *http.Reques } } } - asOf := snapshot.AsOf if asOf == "" { asOf = started.Format(time.RFC3339Nano) } @@ -223,5 +246,10 @@ func (s *server) handleProductionDashboard(w http.ResponseWriter, r *http.Reques response["total"] = len(items) response["page"] = page } + if prepared, ok := s.store.(interface { + RememberProductionResponse(context.Context, string, []string, string, string, any) + }); ok { + prepared.RememberProductionResponse(ctx, org, ids, responseIdentity, asOf, response) + } renderJSON(w, 200, response) } diff --git a/internal/api/query_serving_test.go b/internal/api/query_serving_test.go new file mode 100644 index 00000000..a6585950 --- /dev/null +++ b/internal/api/query_serving_test.go @@ -0,0 +1,76 @@ +package api + +import ( + "context" + "droboticflow/internal/domain" + "droboticflow/internal/productionstats" + "droboticflow/internal/store" + "fmt" + "strings" + "sync/atomic" + "testing" +) + +type preparedProductionStore struct { + store.Store + calls atomic.Int64 + allowedIdentity string +} + +func (s *preparedProductionStore) PreparedProductionResponse(_ context.Context, org string, projects []string, query string) ([]byte, bool) { + s.calls.Add(1) + key := fmt.Sprint(org, projects, query) + if s.allowedIdentity == "" { + s.allowedIdentity = key + } + if key != s.allowedIdentity { + return nil, false + } + return []byte(`{"cachedFixture":true}`), true +} + +func TestQueryServingProductionResponseChecksCurrentPermissionBeforeCache(t *testing.T) { + f := newCombinedDraftFixture(t) + _, err := f.store.SaveProductionStatsScope(f.ctx, domain.DefaultOrganizationID, productionstats.Scope{ProjectIDs: []string{f.task.ProjectID}, Revision: 1}, 0) + if err != nil { + t.Fatal(err) + } + s := &preparedProductionStore{Store: f.store} + f.handler = NewHandler(s, nil) + path := "/api/dashboard/production?from=2026-11-01&to=2026-11-12" + r := f.request("GET", path, f.admin, nil) + if r.Code != 200 || !strings.Contains(r.Body.String(), "cachedFixture") { + t.Fatalf("cache not used %d %s", r.Code, r.Body.String()) + } + before := s.calls.Load() + for _, user := range []domain.UserRecord{f.annotator, f.reviewer, {ID: "supplier-manager", OrganizationID: domain.DefaultOrganizationID, SupplierID: f.annotator.SupplierID, Role: domain.UserRoleSupplierManager, Status: domain.UserStatusActive}} { + if r = f.request("GET", path, user, nil); r.Code != 403 { + t.Fatalf("unauthorized cache read role=%s status=%d", user.Role, r.Code) + } + } + if s.calls.Load() != before { + t.Fatal("unauthorized request touched prepared response") + } + // Revoking a configured project changes identity before any cached body is + // served, even when its old generation and cache entry still exist. + _, err = f.store.SaveProductionStatsScope(f.ctx, domain.DefaultOrganizationID, productionstats.Scope{ProjectIDs: []string{}, Revision: 2}, 1) + if err != nil { + t.Fatal(err) + } + f.handler = NewHandler(s, nil) + r = f.request("GET", path, f.admin, nil) + if r.Code != 200 || strings.Contains(r.Body.String(), "cachedFixture") { + t.Fatal("revoked project reused old cached body", r.Code) + } +} + +func TestQueryServingPrivateMaterializationsRejectDatabaseBrowser(t *testing.T) { + f := newCombinedDraftFixture(t) + for _, name := range []string{"queryReadModelControl", "queryReadModelData"} { + for _, target := range []string{"platform", "business"} { + if r := f.request("GET", "/api/database/collections/"+name+"/documents?target="+target, f.admin, nil); r.Code < 400 { + t.Fatalf("private read model leaked: %s %s", name, target) + } + } + } +} diff --git a/internal/outputsnapshot/query_shared_cache_test.go b/internal/outputsnapshot/query_shared_cache_test.go new file mode 100644 index 00000000..66868798 --- /dev/null +++ b/internal/outputsnapshot/query_shared_cache_test.go @@ -0,0 +1,86 @@ +package outputsnapshot + +import ( + "context" + "droboticflow/internal/queryserving" + "droboticflow/internal/workforcestats" + "sync" + "sync/atomic" + "testing" + "time" +) + +type queryTestCache struct { + mu sync.Mutex + data map[string][]byte +} + +func (c *queryTestCache) Get(_ context.Context, k string) ([]byte, bool) { + c.mu.Lock() + defer c.mu.Unlock() + v, ok := c.data[k] + return append([]byte(nil), v...), ok +} +func (c *queryTestCache) Put(_ context.Context, k string, v []byte, _ time.Duration) { + c.mu.Lock() + defer c.mu.Unlock() + if c.data == nil { + c.data = map[string][]byte{} + } + c.data[k] = append([]byte(nil), v...) +} + +type queryCachedBackend struct { + MemoryBackend + cache queryTestCache + blobs atomic.Int64 +} + +func (b *queryCachedBackend) SharedQueryCache() queryserving.Cache { return &b.cache } +func (b *queryCachedBackend) OutputSnapshotBlob(ctx context.Context, g Generation) ([]byte, error) { + b.blobs.Add(1) + return b.MemoryBackend.OutputSnapshotBlob(ctx, g) +} + +func TestQueryServingSharedOutputCacheKeepsScopeLocationAndGeneration(t *testing.T) { + now := time.Now().UTC() + backend := &queryCachedBackend{} + source := func(context.Context, string, []string) (workforcestats.OutputSnapshot, error) { + return testData(now), nil + } + one := NewReader(backend, source) + defer one.Close() + if _, err := one.Prewarm(t.Context(), "org", []string{"p"}); err != nil { + t.Fatal(err) + } + _, info := ready(t, one, "") + q := workforcestats.OutputQuery{From: now.Add(-24 * time.Hour), To: now.Add(time.Hour), Location: time.UTC, MatchUser: true, ScopeUserID: "u"} + r, _, err := one.Query(t.Context(), "org", []string{"p"}, info.ID, q) + if err != nil || r.Totals.Submitted.Count != 12 { + t.Fatal("initial report", err) + } + calls := backend.blobs.Load() + two := NewReader(backend, source) + defer two.Close() + q.Location = time.FixedZone("UTC", 0) + r, _, err = two.Query(t.Context(), "org", []string{"p"}, info.ID, q) + if err != nil || r.Totals.Submitted.Count != 12 || backend.blobs.Load() != calls { + t.Fatal("same immutable scope failed cross-reader reuse", err) + } + r.Tasks = nil + r, _, err = two.Query(t.Context(), "org", []string{"p"}, info.ID, q) + if err != nil || len(r.Tasks) == 0 { + t.Fatal("shared response mutated by another caller") + } + q.ScopeUserID = "not-authorized-user" + r, _, err = two.Query(t.Context(), "org", []string{"p"}, info.ID, q) + if err != nil || r.Totals.Submitted.Count != 0 { + t.Fatal("personal cache leaked another user's data", err) + } + if _, _, err = two.Query(t.Context(), "other-org", []string{"p"}, info.ID, q); err != ErrExpired { + t.Fatal("foreign organization borrowed generation", err) + } + if _, _, err = two.Query(t.Context(), "org", []string{"not-visible"}, info.ID, q); err != ErrExpired { + t.Fatal("changed project scope borrowed generation", err) + } +} diff --git a/internal/outputsnapshot/reader.go b/internal/outputsnapshot/reader.go index abbc9359..8697d827 100644 --- a/internal/outputsnapshot/reader.go +++ b/internal/outputsnapshot/reader.go @@ -5,6 +5,7 @@ import ( "compress/gzip" "context" "crypto/sha256" + "droboticflow/internal/queryserving" "droboticflow/internal/workforcestats" "encoding/hex" "encoding/json" @@ -43,6 +44,13 @@ func NewReader(b Backend, source Source) *Reader { return &Reader{queryMemo: map[string][]byte{}, querySlots: make(chan struct{}, 2), backend: b, source: source, now: time.Now, ctx: ctx, cancel: cancel, attempts: map[string]time.Time{}, cache: map[string][]byte{}, reads: make(chan struct{}, 2)} } func (r *Reader) Close() { r.mu.Lock(); r.closed = true; r.cancel(); r.mu.Unlock(); r.wg.Wait() } + +// Prewarm uses the existing fenced publication protocol, without a customer +// request or mutation. Repeated workers cannot duplicate a successful refresh. +func (r *Reader) Prewarm(ctx context.Context, org string, projects []string) (Info, error) { + _, info, err := r.resolve(ctx, org, projects, "") + return info, err +} func scopeKey(org string, projects []string) string { ids := append([]string(nil), projects...) sort.Strings(ids) @@ -256,7 +264,7 @@ func (r *Reader) build(key, org string, projects []string) { return } hash := sha256.Sum256(payload) - gen := Generation{Metrics: &BuildMetrics{DurationMS: time.Since(started).Milliseconds(), DecodedBytes: decodedBytes, Parts: len(data.Parts), Source: raw.SourceStats}, ID: fmt.Sprintf("of1-%s-%d", lease.Owner, lease.Epoch), Slot: lease.Slot, Epoch: lease.Epoch, AsOf: asOf, SHA256: hex.EncodeToString(hash[:]), Bytes: len(payload)} + gen := Generation{Metrics: &BuildMetrics{DurationMS: time.Since(started).Milliseconds(), DecodedBytes: decodedBytes, Parts: len(data.Parts), Source: raw.SourceStats}, ID: GenerationID(lease), Slot: lease.Slot, Epoch: lease.Epoch, AsOf: asOf, SHA256: hex.EncodeToString(hash[:]), Bytes: len(payload)} stage = "publish" published, buildErr = r.backend.PublishOutputSnapshot(ctx, lease, gen, payload, r.now()) if published { @@ -349,6 +357,17 @@ func (r *Reader) Query(ctx context.Context, org string, projects []string, pin s // generation and location identity participate in the key, including personal // and supplier isolation. No permissions or mutable datasets are cached. q.Now = gen.AsOf + var shared queryserving.Cache + if provider, ok := r.backend.(interface{ SharedQueryCache() queryserving.Cache }); ok { + shared = provider.SharedQueryCache() + } + location := "" + if q.Location != nil { + _, fromOffset := q.From.In(q.Location).Zone() + _, toOffset := q.To.In(q.Location).Zone() + location = fmt.Sprintf("%s:%d:%d", q.Location.String(), fromOffset, toOffset) + } + sharedKey := queryserving.Key("output-query/v1", scopeKey(org, projects), gen.ID, q, location) keyData, _ := json.Marshal([]any{gen.ID, q, fmt.Sprintf("%p", q.Location)}) key := string(keyData) select { @@ -385,6 +404,11 @@ func (r *Reader) Query(ctx context.Context, org string, projects []string, pin s if report := r.memoReport(key); report != nil { return answer{report, info}, nil } + var sharedReport workforcestats.OutputReport + if queryserving.GetJSON(buildCtx, shared, sharedKey, &sharedReport) { + r.rememberReport(key, &sharedReport) + return answer{&sharedReport, info}, nil + } report, actual, err := r.queryResolved(buildCtx, gen, info, pin, q) if err != nil || report == nil { return nil, err @@ -392,6 +416,7 @@ func (r *Reader) Query(ctx context.Context, org string, projects []string, pin s // Corrupt-current fallback must not poison the current generation's key. if actual.ID == gen.ID { r.rememberReport(key, report) + queryserving.PutJSON(buildCtx, shared, sharedKey, report, MaxAge) } return answer{report, actual}, nil }) diff --git a/internal/outputsnapshot/shards.go b/internal/outputsnapshot/shards.go index 266ab65b..047c7d6e 100644 --- a/internal/outputsnapshot/shards.go +++ b/internal/outputsnapshot/shards.go @@ -91,7 +91,7 @@ func (r *Reader) buildShards(ctx context.Context, lease Lease, org string, proje } metrics.Source = source metrics.DurationMS = time.Since(started).Milliseconds() - g := Generation{ID: fmt.Sprintf("of1-%s-%d", lease.Owner, lease.Epoch), Slot: lease.Slot, Epoch: lease.Epoch, AsOf: asOf, SHA256: digest(payload), Bytes: total, Shards: len(manifest.Shards), ManifestBytes: len(payload), Metrics: metrics} + g := Generation{ID: GenerationID(lease), Slot: lease.Slot, Epoch: lease.Epoch, AsOf: asOf, SHA256: digest(payload), Bytes: total, Shards: len(manifest.Shards), ManifestBytes: len(payload), Metrics: metrics} return g, payload, nil } func decodeShard(payload []byte, g Generation) (*workforcestats.OutputDataset, error) { diff --git a/internal/outputsnapshot/types.go b/internal/outputsnapshot/types.go index 97fd4f6c..a1ca3a08 100644 --- a/internal/outputsnapshot/types.go +++ b/internal/outputsnapshot/types.go @@ -6,9 +6,12 @@ import ( "context" "droboticflow/internal/workforcestats" "errors" + "fmt" "time" ) +func GenerationID(l Lease) string { return fmt.Sprintf("of1-%s-%d", l.Owner, l.Epoch) } + const ( Schema = "annotation-output-facts/v5" RefreshInterval = 120 * time.Second diff --git a/internal/queryserving/cache.go b/internal/queryserving/cache.go new file mode 100644 index 00000000..82eecd05 --- /dev/null +++ b/internal/queryserving/cache.go @@ -0,0 +1,199 @@ +// Package queryserving contains optional derived-data accelerators. None owns +// business state, permissions or task mutations; every failure is a cache miss. +package queryserving + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "sync" + "sync/atomic" + "time" + + valkey "github.com/valkey-io/valkey-go" +) + +const MaxValueBytes = 4 << 20 + +type Cache interface { + Get(context.Context, string) ([]byte, bool) + Put(context.Context, string, []byte, time.Duration) +} + +// Key includes the schema/database namespace supplied by the caller and only +// hashes its remaining identity; no customer names or query text become keys. +func Key(namespace string, identity ...any) string { + b, _ := json.Marshal(identity) + sum := sha256.Sum256(b) + return namespace + ":" + hex.EncodeToString(sum[:]) +} + +type Valkey struct { + client valkey.Client + prefix string + until atomic.Int64 + Hits, Misses, Errors atomic.Uint64 +} + +func NewValkey(address, namespace string) (*Valkey, error) { + opt, err := valkey.ParseURL(address) + if err != nil { + return nil, errors.New("invalid shared cache configuration") + } + opt.DisableCache = true // no implicit client-side cache or permission reuse + opt.Dialer.Timeout = 200 * time.Millisecond + opt.ConnWriteTimeout = 200 * time.Millisecond + opt.DisableRetry = true + opt.PipelineMultiplex = -1 + opt.BlockingPoolSize = 4 + opt.ReadBufferEachConn = 64 << 10 + opt.WriteBufferEachConn = 64 << 10 + client, err := valkey.NewClient(opt) + if err != nil { + return nil, errors.New("shared cache unavailable") + } + return &Valkey{client: client, prefix: Key("droboticflow-query/v1", namespace)}, nil +} + +// ManagedValkey connects asynchronously so an absent cache at startup cannot +// delay or fail API availability. Connections are retried without request load. +type ManagedValkey struct { + mu sync.RWMutex + value *Valkey + cancel context.CancelFunc + done chan struct{} +} + +func NewManagedValkey(address, namespace string) (*ManagedValkey, error) { + if _, err := valkey.ParseURL(address); err != nil { + return nil, errors.New("invalid shared cache configuration") + } + ctx, cancel := context.WithCancel(context.Background()) + m := &ManagedValkey{cancel: cancel, done: make(chan struct{})} + go func() { + defer close(m.done) + for { + v, err := NewValkey(address, namespace) + if err == nil { + m.mu.Lock() + if ctx.Err() == nil { + m.value = v + m.mu.Unlock() + <-ctx.Done() + v.Close() + return + } + m.mu.Unlock() + v.Close() + return + } + select { + case <-ctx.Done(): + return + case <-time.After(time.Second): + } + } + }() + return m, nil +} +func (m *ManagedValkey) Get(ctx context.Context, key string) ([]byte, bool) { + m.mu.RLock() + v := m.value + m.mu.RUnlock() + if v == nil { + return nil, false + } + return v.Get(ctx, key) +} +func (m *ManagedValkey) Put(ctx context.Context, key string, b []byte, ttl time.Duration) { + m.mu.RLock() + v := m.value + m.mu.RUnlock() + if v != nil { + v.Put(ctx, key, b, ttl) + } +} +func (m *ManagedValkey) Close() { m.cancel(); <-m.done } +func (m *ManagedValkey) Ready() bool { m.mu.RLock(); defer m.mu.RUnlock(); return m.value != nil } + +func (v *Valkey) Close() { v.client.Close() } +func (v *Valkey) failed() { v.Errors.Add(1); v.until.Store(time.Now().Add(time.Second).UnixNano()) } +func (v *Valkey) Get(ctx context.Context, key string) ([]byte, bool) { + parent := ctx + if parent.Err() != nil { + return nil, false + } + if time.Now().UnixNano() < v.until.Load() { + v.Misses.Add(1) + return nil, false + } + ctx, cancel := context.WithTimeout(ctx, 40*time.Millisecond) + defer cancel() + b, err := v.client.Do(ctx, v.client.B().Get().Key(v.prefix+":"+key).Build()).AsBytes() + if err != nil { + if !valkey.IsValkeyNil(err) && parent.Err() == nil { + v.failed() + } + v.Misses.Add(1) + return nil, false + } + if len(b) < 36 || len(b) > MaxValueBytes+36 || string(b[:4]) != "DFQ1" { + v.Misses.Add(1) + return nil, false + } + sum := sha256.Sum256(append([]byte(key+"\x00"), b[36:]...)) + if !bytes.Equal(sum[:], b[4:36]) { + v.Misses.Add(1) + return nil, false + } + v.Hits.Add(1) + return append([]byte(nil), b[36:]...), true +} +func (v *Valkey) Put(ctx context.Context, key string, b []byte, ttl time.Duration) { + parent := ctx + if parent.Err() != nil { + return + } + if len(b) > MaxValueBytes || ttl <= 0 || time.Now().UnixNano() < v.until.Load() { + return + } + ctx, cancel := context.WithTimeout(ctx, 40*time.Millisecond) + defer cancel() + sum := sha256.Sum256(append([]byte(key+"\x00"), b...)) + encoded := append(append([]byte("DFQ1"), sum[:]...), b...) + if err := v.client.Do(ctx, v.client.B().Set().Key(v.prefix+":"+key).Value(string(encoded)).Px(ttl).Build()).Error(); err != nil { + if parent.Err() == nil { + v.failed() + } + } +} + +func GetJSON(ctx context.Context, c Cache, key string, value any) bool { + if c == nil { + return false + } + b, ok := c.Get(ctx, key) + return ok && json.Unmarshal(b, value) == nil +} +func PutJSON(ctx context.Context, c Cache, key string, value any, ttl time.Duration) { + if c == nil { + return + } + b, err := json.Marshal(value) + if err == nil { + c.Put(ctx, key, b, ttl) + } +} + +func (m *ManagedValkey) Statistics() map[string]uint64 { + m.mu.RLock() + v := m.value + m.mu.RUnlock() + if v == nil { + return map[string]uint64{} + } + return map[string]uint64{"hits": v.Hits.Load(), "misses": v.Misses.Load(), "errors": v.Errors.Load()} +} diff --git a/internal/queryserving/serving_test.go b/internal/queryserving/serving_test.go new file mode 100644 index 00000000..3f1c6e8f --- /dev/null +++ b/internal/queryserving/serving_test.go @@ -0,0 +1,138 @@ +package queryserving + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "reflect" + "strings" + "testing" + "time" +) + +func TestQueryServingGramsPreserveSubstringCandidates(t *testing.T) { + for _, text := range []string{"FYLX_B07_AZER764008K_20260927_074440/part0002", "İ中文😀_左右/part", "a.b+?x[1]", "abc def ghi"} { + r := []rune(text) + indexed := map[string]bool{} + for _, g := range IndexGrams(text) { + indexed[g] = true + } + for a := range r { + for z := a + 1; z <= len(r); z++ { + for _, g := range Grams(string(r[a:z])) { + if !indexed[g] { + t.Fatalf("substring lost %q in %q", r[a:z], text) + } + } + } + } + } + if reflect.DeepEqual(Key("namespace-a", "org", []string{"p"}), Key("namespace-b", "org", []string{"p"})) { + t.Fatal("namespace collision") + } +} + +func TestQueryServingTypesenseIncompleteImportAndExportFailClosed(t *testing.T) { + mode := "partial" + secret := "synthetic-secret-only" + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("X-TYPESENSE-API-KEY") != secret { + t.Error("missing backend authentication") + } + switch mode { + case "partial": + _, _ = w.Write([]byte("{\"success\":true}\n{\"success\":false,\"error\":\"" + secret + "\"}\n")) + case "short": + _, _ = w.Write([]byte("{\"success\":true}\n")) + case "malformed": + _, _ = w.Write([]byte("{\"item_id\":\"one\"}\n{bad")) + case "failure": + w.WriteHeader(503) + _, _ = w.Write([]byte(secret)) + case "ok": + _, _ = w.Write([]byte("{\"item_id\":\"one\"}\n{\"item_id\":\"two\"}\n")) + } + })) + defer server.Close() + client, _ := NewTypesense(server.URL, secret) + docs := []SearchDocument{{ID: "one"}, {ID: "two"}} + for _, m := range []string{"partial", "short"} { + mode = m + if err := client.Import(t.Context(), "dfq_test", docs); err == nil || strings.Contains(err.Error(), secret) { + t.Fatal("partial import accepted or secret leaked", err) + } + } + for _, m := range []string{"malformed", "failure"} { + mode = m + if ids, err := client.Candidates(t.Context(), "dfq_test", "_x/"); err == nil || ids != nil || strings.Contains(err.Error(), secret) { + t.Fatal("incomplete export accepted or secret leaked", ids, err) + } + } + mode = "ok" + if ids, err := client.Candidates(t.Context(), "dfq_test", "part"); err != nil || !reflect.DeepEqual(ids, []string{"one", "two"}) { + t.Fatal(ids, err) + } +} + +func TestQueryServingValkeyRealIsolationCorruptionAndColdRecoveryLocal(t *testing.T) { + if os.Getenv("QUERY_SERVING_LOCAL_TEST") != "1" { + t.Skip("isolated loopback query dependencies") + } + namespace := t.Name() + time.Now().Format(time.RFC3339Nano) + a, err := NewValkey("valkey://127.0.0.1:16379", namespace) + if err != nil { + t.Fatal(err) + } + defer a.Close() + b, err := NewValkey("valkey://127.0.0.1:16379", namespace) + if err != nil { + t.Fatal(err) + } + defer b.Close() + foreign, err := NewValkey("valkey://127.0.0.1:16379", namespace+"-foreign") + if err != nil { + t.Fatal(err) + } + defer foreign.Close() + a.Put(t.Context(), "generation-one", []byte(`{"value":42}`), time.Second) + if data, ok := b.Get(t.Context(), "generation-one"); !ok || string(data) != `{"value":42}` { + t.Fatal("cross-instance cache missed") + } + if _, ok := foreign.Get(t.Context(), "generation-one"); ok { + t.Fatal("foreign namespace read cache") + } + if err = a.client.Do(t.Context(), a.client.B().Set().Key(a.prefix+":generation-one").Value(`{}`).Build()).Error(); err != nil { + t.Fatal(err) + } + if _, ok := b.Get(t.Context(), "generation-one"); ok { + t.Fatal("corrupt valid JSON treated as a valid cache value") + } + managed, err := NewManagedValkey("valkey://127.0.0.1:16379", namespace) + if err != nil { + t.Fatal(err) + } + defer managed.Close() + deadline := time.Now().Add(2 * time.Second) + for !managed.Ready() && time.Now().Before(deadline) { + time.Sleep(10 * time.Millisecond) + } + if !managed.Ready() { + t.Fatal("managed cache did not connect") + } + ctx, cancel := context.WithCancel(t.Context()) + cancel() + if _, ok := managed.Get(ctx, "generation-one"); ok { + t.Fatal("cancelled cache read succeeded") + } + managed.Put(t.Context(), "different-query", []byte("[]"), time.Second) + data, ok := managed.Get(t.Context(), "different-query") + if !ok { + t.Fatal("managed cache did not warm") + } + var value []string + if json.Unmarshal(data, &value) != nil { + t.Fatal("cache payload changed") + } +} diff --git a/internal/queryserving/typesense.go b/internal/queryserving/typesense.go new file mode 100644 index 00000000..aa4bc3a4 --- /dev/null +++ b/internal/queryserving/typesense.go @@ -0,0 +1,190 @@ +package queryserving + +import ( + "bufio" + "bytes" + "context" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "sort" + "strings" + "time" +) + +type SearchDocument struct { + ID string `json:"id"` + ItemID string `json:"item_id"` + Grams []string `json:"grams"` +} + +// Grams are hex-encoded Unicode windows, independent of Typesense's tokenizer, +// punctuation/typo/prefix rules. Intersection is only a conservative prefilter; +// the existing Go matcher remains the final authority. No query is truncated. +func Grams(text string) []string { + r := []rune(strings.ToLower(text)) + width := min(3, len(r)) + if width == 0 { + return nil + } + set := map[string]bool{} + for i := 0; i+width <= len(r); i++ { + set[hex.EncodeToString([]byte(string(r[i:i+width])))] = true + } + out := make([]string, 0, len(set)) + for s := range set { + out = append(out, s) + } + sort.Strings(out) + return out +} +func IndexGrams(text string) []string { + r := []rune(strings.ToLower(text)) + set := map[string]bool{} + for width := 1; width <= 3; width++ { + for i := 0; i+width <= len(r); i++ { + set[hex.EncodeToString([]byte(string(r[i:i+width])))] = true + } + } + out := make([]string, 0, len(set)) + for s := range set { + out = append(out, s) + } + sort.Strings(out) + return out +} + +type Typesense struct { + base, key string + client *http.Client +} +type HTTPError struct{ Status int } + +func (e *HTTPError) Error() string { return fmt.Sprintf("search backend status %d", e.Status) } + +func NewTypesense(address, key string) (*Typesense, error) { + u, err := url.Parse(address) + if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" || u.User != nil || u.RawQuery != "" || u.Fragment != "" || (u.Path != "" && u.Path != "/") || key == "" { + return nil, errors.New("invalid search configuration") + } + return &Typesense{base: strings.TrimRight(address, "/"), key: key, client: &http.Client{Timeout: 30 * time.Second, CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }}}, nil +} +func (t *Typesense) request(ctx context.Context, method, path string, body []byte) ([]byte, error) { + r, err := http.NewRequestWithContext(ctx, method, t.base+path, bytes.NewReader(body)) + if err != nil { + return nil, errors.New("search request invalid") + } + r.Header.Set("X-TYPESENSE-API-KEY", t.key) + r.Header.Set("Content-Type", "application/json") + res, err := t.client.Do(r) + if err != nil { + return nil, errors.New("search backend unavailable") + } + defer res.Body.Close() + if res.StatusCode < 200 || res.StatusCode >= 300 { + return nil, &HTTPError{Status: res.StatusCode} + } + b, err := io.ReadAll(io.LimitReader(res.Body, 32<<20+1)) + if err != nil || len(b) > 32<<20 { + return nil, errors.New("search response incomplete") + } + return b, nil +} +func validCollection(c string) bool { + if !strings.HasPrefix(c, "dfq_") || len(c) > 120 { + return false + } + for _, r := range c { + if r != '_' && (r < 'a' || r > 'z') && (r < '0' || r > '9') { + return false + } + } + return true +} +func (t *Typesense) Create(ctx context.Context, c string) error { + if !validCollection(c) { + return errors.New("invalid search collection") + } + b, _ := json.Marshal(map[string]any{"name": c, "fields": []map[string]any{{"name": "item_id", "type": "string", "index": false}, {"name": "grams", "type": "string[]", "facet": true}}}) + _, err := t.request(ctx, "POST", "/collections", b) + return err +} +func (t *Typesense) Import(ctx context.Context, c string, docs []SearchDocument) error { + if !validCollection(c) { + return errors.New("invalid search collection") + } + var data bytes.Buffer + for _, d := range docs { + if err := json.NewEncoder(&data).Encode(d); err != nil { + return err + } + } + b, err := t.request(ctx, "POST", "/collections/"+c+"/documents/import?action=create", data.Bytes()) + if err != nil { + return err + } + s := bufio.NewScanner(bytes.NewReader(b)) + s.Buffer(make([]byte, 4096), 1<<20) + count := 0 + for s.Scan() { + var result struct { + Success bool `json:"success"` + } + if json.Unmarshal(s.Bytes(), &result) != nil || !result.Success { + return errors.New("search import incomplete") + } + count++ + } + if s.Err() != nil || count != len(docs) { + return errors.New("search import response incomplete") + } + return nil +} +func (t *Typesense) Candidates(ctx context.Context, c string, terms ...string) ([]string, error) { + if !validCollection(c) { + return nil, errors.New("invalid search collection") + } + filters := []string{} + for _, term := range terms { + for _, g := range Grams(term) { + filters = append(filters, "grams:=["+g+"]") + } + } + params := url.Values{"include_fields": {"item_id"}} + if len(filters) > 0 { + params.Set("filter_by", strings.Join(filters, " && ")) + } + b, err := t.request(ctx, "GET", "/collections/"+c+"/documents/export?"+params.Encode(), nil) + if err != nil { + return nil, err + } + s := bufio.NewScanner(bytes.NewReader(b)) + s.Buffer(make([]byte, 4096), 1<<20) + ids := []string{} + for s.Scan() { + var d SearchDocument + if json.Unmarshal(s.Bytes(), &d) != nil || d.ItemID == "" { + return nil, errors.New("search candidates incomplete") + } + ids = append(ids, d.ItemID) + } + if s.Err() != nil { + return nil, errors.New("search candidates incomplete") + } + return ids, nil +} +func (t *Typesense) Delete(ctx context.Context, c string) error { + if !validCollection(c) { + return errors.New("invalid search collection") + } + _, err := t.request(ctx, "DELETE", "/collections/"+c, nil) + var status *HTTPError + if errors.As(err, &status) && status.Status == 404 { + return nil + } + return err +} diff --git a/internal/store/admin_annotation_pool.go b/internal/store/admin_annotation_pool.go index 69fcf7f2..830b1546 100644 --- a/internal/store/admin_annotation_pool.go +++ b/internal/store/admin_annotation_pool.go @@ -157,6 +157,29 @@ func adminPoolRowMatches(row adminPoolRow, q AdminAnnotationPoolQuery) bool { if q.Search == "" && q.Capture == "" { return true } + values := adminPoolCaptureValues(row) + if q.Capture != "" { + found := false + for _, v := range values { + if strings.EqualFold(strings.TrimSpace(v), q.Capture) { + found = true + } + } + if !found { + return false + } + } + if q.Search != "" && !strings.Contains(adminPoolSearchText(row), strings.ToLower(q.Search)) { + return false + } + return true +} +func adminPoolCaptureValues(row adminPoolRow) []string { + var task domain.AnnotationTaskRecord + if len(row.Tasks) > 0 { + task = row.Tasks[0] + } + values := []string{} if len(row.Captures) > 0 { c := row.Captures[0] @@ -180,29 +203,18 @@ func adminPoolRowMatches(row adminPoolRow, q AdminAnnotationPoolQuery) bool { if len(row.Runs) > 0 { values = append(values, domain.CaptureDirectoriesFromRun(row.Runs[0])...) } - if q.Capture != "" { - found := false - for _, v := range values { - if strings.EqualFold(strings.TrimSpace(v), q.Capture) { - found = true - } - } - if !found { - return false - } - } - if q.Search != "" { - if len(row.Projects) > 0 { - values = append(values, row.Projects[0].Name) - } else if len(row.Runs) > 0 { - values = append(values, row.Runs[0].ProjectName) - } - if !strings.Contains(strings.ToLower(strings.Join(values, " ")), strings.ToLower(q.Search)) { - return false - } - } - return true + return values } +func adminPoolSearchText(row adminPoolRow) string { + values := adminPoolCaptureValues(row) + if len(row.Projects) > 0 { + values = append(values, row.Projects[0].Name) + } else if len(row.Runs) > 0 { + values = append(values, row.Runs[0].ProjectName) + } + return strings.ToLower(strings.Join(values, " ")) +} + func (s *MemoryStore) adminPoolRowsLocked(org string, q AdminAnnotationPoolQuery) []adminPoolRow { rows := []adminPoolRow{} for _, i := range s.humanItems { @@ -288,7 +300,14 @@ func (s *MongoStore) scanAdminAnnotationPool(ctx context.Context, org string, q } ctx, cancel := context.WithTimeout(ctx, 8*time.Second) defer cancel() - cur, err := s.humanItems.Aggregate(ctx, adminPoolPagePipeline(org, q), options.Aggregate().SetAllowDiskUse(false).SetMaxTime(8*time.Second).SetBatchSize(200)) + pipeline := adminPoolPagePipeline(org, q) + if ids, ready := s.querySearchCandidates(ctx, org, q.Search, q.Capture); ready { + if len(ids) == 0 { + return nil + } + pipeline[0] = bson.D{{Key: "$match", Value: bson.M{"$and": bson.A{pipeline[0][0].Value, bson.M{"id": bson.M{"$in": ids}}}}}} + } + cur, err := s.humanItems.Aggregate(ctx, pipeline, options.Aggregate().SetAllowDiskUse(false).SetMaxTime(8*time.Second).SetBatchSize(200)) if err != nil { return err } diff --git a/internal/store/annotation_output_snapshots.go b/internal/store/annotation_output_snapshots.go index 0216dfa3..b6db1d70 100644 --- a/internal/store/annotation_output_snapshots.go +++ b/internal/store/annotation_output_snapshots.go @@ -4,6 +4,7 @@ import ( "context" "droboticflow/internal/domain" "droboticflow/internal/outputsnapshot" + "droboticflow/internal/queryserving" "fmt" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/mongo" @@ -113,6 +114,12 @@ func (s *MongoStore) FailOutputSnapshot(ctx context.Context, l outputsnapshot.Le return err } func (s *MongoStore) OutputSnapshotBlob(ctx context.Context, g outputsnapshot.Generation) ([]byte, error) { + key := queryserving.Key("output-blob/v1", s.db.Name(), g.ID, g.Slot, g.Epoch, g.SHA256) + if cache := s.SharedQueryCache(); cache != nil { + if b, hit := cache.Get(ctx, key); hit { + return b, nil + } + } var doc struct { Data []byte `bson:"data"` } @@ -120,6 +127,11 @@ func (s *MongoStore) OutputSnapshotBlob(ctx context.Context, g outputsnapshot.Ge if err == mongo.ErrNoDocuments { err = outputsnapshot.ErrExpired } + if err == nil { + if cache := s.SharedQueryCache(); cache != nil { + cache.Put(ctx, key, doc.Data, outputsnapshot.MaxAge) + } + } return doc.Data, err } @@ -143,9 +155,20 @@ func (s *MongoStore) StageOutputSnapshotShard(ctx context.Context, l outputsnaps // Immutable: never overwrite a staged shard. id := outputShardID(l.Slot, l.Epoch, index) _, err = s.outputBlobs().InsertOne(ctx, bson.M{"_id": id, "data": data}) + if err == nil { + if cache := s.SharedQueryCache(); cache != nil { + cache.Put(ctx, queryserving.Key("output-shard/v1", s.db.Name(), outputsnapshot.GenerationID(l), l.Slot, l.Epoch, index), data, outputsnapshot.MaxAge) + } + } return err == nil, err } func (s *MongoStore) OutputSnapshotShard(ctx context.Context, g outputsnapshot.Generation, index int) ([]byte, error) { + key := queryserving.Key("output-shard/v1", s.db.Name(), g.ID, g.Slot, g.Epoch, index) + if cache := s.SharedQueryCache(); cache != nil { + if b, hit := cache.Get(ctx, key); hit { + return b, nil + } + } var doc struct { Data []byte `bson:"data"` } @@ -153,6 +176,11 @@ func (s *MongoStore) OutputSnapshotShard(ctx context.Context, g outputsnapshot.G if err == mongo.ErrNoDocuments { err = outputsnapshot.ErrExpired } + if err == nil { + if cache := s.SharedQueryCache(); cache != nil { + cache.Put(ctx, key, doc.Data, outputsnapshot.MaxAge) + } + } return doc.Data, err } func (s *MemoryStore) StageOutputSnapshotShard(ctx context.Context, l outputsnapshot.Lease, i int, b []byte, t time.Time) (bool, error) { diff --git a/internal/store/human_pool_search.go b/internal/store/human_pool_search.go index 9caa958e..455545f3 100644 --- a/internal/store/human_pool_search.go +++ b/internal/store/human_pool_search.go @@ -46,6 +46,9 @@ func humanPoolSearchRowMatches(row adminPoolRow, q domain.HumanItemQuery) bool { if adminPoolRowMatches(row, AdminAnnotationPoolQuery{Search: q.Search}) { return true } + return strings.Contains(humanPoolFallbackSearchText(row), strings.ToLower(q.Search)) +} +func humanPoolFallbackSearchText(row adminPoolRow) string { values := []string{row.ID, row.LegacyTaskID, row.Subject.ItemKey, row.Subject.ArtifactID} for _, t := range row.Tasks { values = append(values, t.ArtifactID, t.SourceVideo) @@ -76,7 +79,7 @@ func humanPoolSearchRowMatches(row adminPoolRow, q domain.HumanItemQuery) bool { values = append(values, capture+"/"+match[1]) } } - return strings.Contains(strings.ToLower(strings.Join(values, " ")), strings.ToLower(q.Search)) + return strings.ToLower(strings.Join(values, " ")) } // Stream metadata through the existing search rules before applying the page @@ -121,6 +124,12 @@ func (s *MongoStore) prepareHumanPoolSearchFilter(ctx context.Context, org strin } filter = bson.M{"$and": bson.A{filter, visibility}} } + if ids, ready := s.querySearchCandidates(ctx, org, q.Search, q.Capture); ready { + if len(ids) == 0 { + return nil, true, nil + } + return bson.M{"$and": bson.A{filter, bson.M{"id": bson.M{"$in": ids}}}}, false, nil + } candidates, err := s.humanPoolSearchCandidates(ctx, org, q, filter) if err != nil { return nil, false, err diff --git a/internal/store/memory.go b/internal/store/memory.go index 73f632ba..28e95a94 100644 --- a/internal/store/memory.go +++ b/internal/store/memory.go @@ -3266,7 +3266,7 @@ func (store *MemoryStore) GetDatabaseOverview(context.Context) (domain.DatabaseO Collections: make([]domain.DatabaseCollectionSummary, 0, len(names)), } for _, name := range names { - if outputsnapshot.IsPrivateCollection(name) { + if IsPrivateDerivedCollection(name) { continue } documents := collectionsMap[name] @@ -3283,7 +3283,7 @@ func (store *MemoryStore) GetDatabaseOverview(context.Context) (domain.DatabaseO } func (store *MemoryStore) QueryDatabaseCollection(_ context.Context, input domain.DatabaseQueryInput) (domain.DatabaseQueryResult, error) { - if outputsnapshot.IsPrivateCollection(input.Collection) { + if IsPrivateDerivedCollection(input.Collection) { return domain.DatabaseQueryResult{}, fmt.Errorf("collection %s not found", input.Collection) } store.mu.Lock() diff --git a/internal/store/mongo.go b/internal/store/mongo.go index 4b40040b..10fa0575 100644 --- a/internal/store/mongo.go +++ b/internal/store/mongo.go @@ -12,7 +12,6 @@ import ( "time" "droboticflow/internal/domain" - "droboticflow/internal/outputsnapshot" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson/primitive" @@ -21,6 +20,7 @@ import ( ) type MongoStore struct { + queryServing *queryServingOptions productionReadFlights singleflight.Group claimReadFlights singleflight.Group poolSearchReadFlights singleflight.Group @@ -2808,7 +2808,7 @@ func (store *MongoStore) GetDatabaseOverview(ctx context.Context) (domain.Databa Collections: make([]domain.DatabaseCollectionSummary, 0, len(names)), } for _, name := range names { - if outputsnapshot.IsPrivateCollection(name) { + if IsPrivateDerivedCollection(name) { continue } summary, err := mongoBrowserCollectionSummary(ctx, store.db.Collection(name), name) @@ -2825,7 +2825,7 @@ func (store *MongoStore) QueryDatabaseCollection(ctx context.Context, input doma if collectionName == "" { return domain.DatabaseQueryResult{}, fmt.Errorf("collection is required") } - if outputsnapshot.IsPrivateCollection(collectionName) { + if IsPrivateDerivedCollection(collectionName) { return domain.DatabaseQueryResult{}, fmt.Errorf("collection %s not found", collectionName) } limit := input.Limit diff --git a/internal/store/private_derived_collections.go b/internal/store/private_derived_collections.go new file mode 100644 index 00000000..a9375640 --- /dev/null +++ b/internal/store/private_derived_collections.go @@ -0,0 +1,9 @@ +package store + +import "droboticflow/internal/outputsnapshot" + +// Derived generations contain facts from multiple authorization scopes and +// must only be served through their permission-aware reporting/search readers. +func IsPrivateDerivedCollection(name string) bool { + return outputsnapshot.IsPrivateCollection(name) || name == "queryReadModelControl" || name == "queryReadModelData" +} diff --git a/internal/store/query_model_maintenance.go b/internal/store/query_model_maintenance.go new file mode 100644 index 00000000..c95ff414 --- /dev/null +++ b/internal/store/query_model_maintenance.go @@ -0,0 +1,84 @@ +package store + +import ( + "context" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + "time" +) + +// This is an explicit derived-data index operation, never API startup behavior. +// Production execution requires its own reviewed migration/index authorization. +func (s *MongoStore) EnsureQueryReadModelIndexes(ctx context.Context) error { + _, err := s.queryData().Indexes().CreateOne(ctx, mongo.IndexModel{Keys: bson.D{{Key: "generation", Value: 1}, {Key: "organizationId", Value: 1}, {Key: "itemId", Value: 1}}, Options: options.Index().SetName("query_model_rows_v1")}) + return err +} +func (s *MongoStore) cleanQueryModelOrphans(ctx context.Context) error { + var state struct { + Heads map[string]QueryModelHead `bson:"heads"` + } + if err := s.queryControl().FindOne(ctx, bson.M{"_id": "global"}).Decode(&state); err != nil { + return err + } + keep := []string{} + for _, h := range state.Heads { + keep = append(keep, h.Generation) + } + filter := bson.M{"generation": bson.M{"$nin": keep}, "createdAt": bson.M{"$lt": time.Now().UTC().Add(-15 * time.Minute)}} + // Only this database's registered, unreachable search generations are + // eligible; other deployments and current generations remain untouched. + cur, err := s.queryData().Find(ctx, bson.M{"$and": bson.A{filter, bson.M{"kind": "search_collection"}}}, options.Find().SetProjection(bson.M{"_id": 1, "collection": 1})) + if err != nil { + return err + } + defer cur.Close(ctx) + for cur.Next(ctx) { + var row struct { + ID string `bson:"_id"` + Collection string `bson:"collection"` + } + if err = cur.Decode(&row); err != nil { + return err + } + if s.queryServing != nil && s.queryServing.search != nil { + if err = s.queryServing.search.Delete(ctx, row.Collection); err != nil { + return err + } + } else { + return nil + } + } + if err = cur.Err(); err != nil { + return err + } + _, err = s.queryData().DeleteMany(ctx, filter) + return err +} + +// Obsolete configured scopes cannot pin an unbounded history of generations. +// Only the current builder may change this derived manifest directory. +func (s *MongoStore) pruneQueryModelScopes(ctx context.Context, l queryModelLease, scopes map[string][]string) error { + var st struct { + Heads map[string]QueryModelHead `bson:"heads"` + } + if err := s.queryControl().FindOne(ctx, queryLeaseFilter(l)).Decode(&st); err != nil { + return err + } + keep := map[string]bool{} + for org, projects := range scopes { + keep[queryModelKey("production", org, projects)] = true + keep[queryModelKey("search", org, nil)] = true + } + unset := bson.M{} + for key := range st.Heads { + if !keep[key] { + unset["heads."+key] = "" + } + } + if len(unset) == 0 { + return nil + } + _, err := s.queryControl().UpdateOne(ctx, queryLeaseFilter(l), bson.M{"$unset": unset}) + return err +} diff --git a/internal/store/query_read_builder.go b/internal/store/query_read_builder.go new file mode 100644 index 00000000..f404d085 --- /dev/null +++ b/internal/store/query_read_builder.go @@ -0,0 +1,384 @@ +package store + +import ( + "context" + "errors" + "fmt" + "log" + "strings" + "sync" + "time" + + "droboticflow/internal/outputsnapshot" + "droboticflow/internal/productionstats" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" +) + +var querySourceCollections = []string{"organizations", "projects", "runs", "captureRecords", "humanItems", "humanBatches", "annotationTasks", "annotationWorklogEvents", "humanEvents", "deliveryPackages", "annotationRevisions", "sliceRevisions", "annotationReviews", "users", "suppliers", "productionStatsScopes"} + +// QueryReadBuilder runs only in a dedicated, explicitly configured process. +// It uses the existing source loaders and publication protocol, never claims or +// mutates a business task. Restart starts a fresh stream BEFORE the full rebuild. +type QueryReadBuilder struct { + store *MongoStore + mu sync.Mutex + dirty map[string]bool + all bool + reader *outputsnapshot.Reader + searchDirty map[string]bool + allSearch bool + sourceDigests map[string]querySourceDigest +} + +func NewQueryReadBuilder(s *MongoStore) *QueryReadBuilder { + r := outputsnapshot.NewStreamReader(s, s.StreamAnnotationOutput) + return &QueryReadBuilder{store: s, dirty: map[string]bool{}, all: true, reader: r, searchDirty: map[string]bool{}, allSearch: true, sourceDigests: map[string]querySourceDigest{}} +} +func (b *QueryReadBuilder) Close() { b.reader.Close() } + +// WaitReady is for explicit prewarming and isolated acceptance, not HTTP reads. +func (b *QueryReadBuilder) WaitReady(ctx context.Context, org string, projects []string) error { + tick := time.NewTicker(100 * time.Millisecond) + defer tick.Stop() + for { + _, production := b.store.readQueryModelHead(ctx, queryModelKey("production", org, projects), 5*time.Minute) + st, err := b.store.querySearchState(ctx) + h, exists := st.Heads[queryModelKey("search", org, nil)] + search := err == nil && exists && h.Generation != "" && h.Revision == st.Revisions[queryModelKey("search", org, nil)] && h.GlobalRevision == st.Global && time.Since(st.ObservedAt) >= 0 && time.Since(st.ObservedAt) < 5*time.Second + output := len(projects) == 0 + if !output { + health, e := b.reader.Inspect(ctx, org, projects) + output = e == nil && health.Available + } + if production && search && output { + return nil + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-tick.C: + } + } +} + +func (b *QueryReadBuilder) scopes(ctx context.Context) (map[string][]string, error) { + // Public ListOrganizations also bootstraps a default business organization; + // the read builder must enumerate without that write side effect. + cur, err := b.store.organizations.Find(ctx, bson.M{}, options.Find().SetProjection(bson.M{"id": 1, "_id": 0})) + if err != nil { + return nil, err + } + defer cur.Close(ctx) + var orgs []struct { + ID string `bson:"id"` + } + if err = cur.All(ctx, &orgs); err != nil { + return nil, err + } + out := map[string][]string{} + for _, org := range orgs { + scope, err := b.store.GetProductionStatsScope(ctx, org.ID) + if err != nil { + return nil, err + } + projects, err := b.store.ListProjectsByOrganization(ctx, org.ID) + if err != nil { + return nil, err + } + selected := map[string]bool{} + for _, id := range scope.ProjectIDs { + selected[id] = true + } + ids := []string{} + for _, p := range projects { + if selected[p.ID] { + ids = append(ids, p.ID) + } + } + out[org.ID] = ids + } + return out, nil +} +func (b *QueryReadBuilder) rebuild(ctx context.Context, l queryModelLease, org string, projects []string) error { + started := time.Now() + var source struct { + Revision int64 `bson:"productionRevision"` + } + if err := b.store.queryControl().FindOne(ctx, bson.M{"_id": "global"}, options.FindOne().SetProjection(bson.M{"productionRevision": 1})).Decode(&source); err != nil { + return err + } + buildCtx, finish := context.WithTimeout(ctx, 5*time.Minute) + defer finish() + snapshot, err := b.store.loadProductionStats(buildCtx, org, projects) + if err != nil { + return err + } + asOf, err := time.Parse(time.RFC3339Nano, snapshot.AsOf) + if err != nil { + return err + } + key := queryModelKey("production", org, projects) + head, err := b.store.stageQueryModel(ctx, key, productionstats.Build(snapshot), asOf) + if err != nil { + return err + } + head.SourceRevision = source.Revision + if err = b.store.publishQueryModelHead(ctx, l, key, head); err != nil { + return err + } + if len(projects) > 0 { + if _, err = b.reader.Prewarm(ctx, org, projects); err != nil { + return err + } + } + + log.Printf("query_model_build result=complete duration_ms=%d", time.Since(started).Milliseconds()) + return nil +} + +// RunOnce is for bounded initial prewarming and isolated tests, not a freshness +// substitute for Run. After the heartbeat expires search falls back to old reads. +func (b *QueryReadBuilder) RunOnce(ctx context.Context) error { + l, ok, err := b.store.acquireQueryModelLease(ctx) + if err != nil { + return err + } + if !ok { + return errors.New("query builder already active") + } + return b.withLease(ctx, l, func(ctx context.Context) error { + scopes, err := b.scopes(ctx) + if err != nil { + return err + } + for org, ids := range scopes { + if err = b.rebuild(ctx, l, org, ids); err != nil { + return err + } + if err = b.store.buildQuerySearchModel(ctx, l, org); err != nil { + return err + } + } + return nil + }) +} +func (b *QueryReadBuilder) withLease(ctx context.Context, l queryModelLease, run func(context.Context) error) error { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + done := make(chan struct{}) + go func() { + defer close(done) + tick := time.NewTicker(10 * time.Second) + defer tick.Stop() + for { + select { + case <-ctx.Done(): + return + case <-tick.C: + if b.store.renewQueryModelLease(ctx, l) != nil { + cancel() + return + } + } + } + }() + err := run(ctx) + cancel() + <-done + cleanup, stop := context.WithTimeout(context.Background(), time.Second) + defer stop() + _, _ = b.store.queryControl().UpdateOne(cleanup, queryLeaseFilter(l), bson.M{"$set": bson.M{"owner": "", "until": time.Now().UTC()}}) + return err +} + +func (b *QueryReadBuilder) Run(ctx context.Context) error { + b.mu.Lock() + b.all, b.allSearch = true, true + b.sourceDigests = map[string]querySourceDigest{} + b.mu.Unlock() + l, ok, err := b.store.acquireQueryModelLease(ctx) + if err != nil { + return err + } + if !ok { + return errors.New("query builder already active") + } + return b.withLease(ctx, l, func(ctx context.Context) error { + // No fullDocument lookup: projecting only event identity avoids copying + // annotation bodies, credentials or full video payloads on each save. + pipeline := mongo.Pipeline{{{Key: "$match", Value: bson.M{"ns.coll": bson.M{"$in": querySourceCollections}}}}, {{Key: "$project", Value: bson.M{"_id": 1, "ns": 1, "documentKey": 1, "operationType": 1, "clusterTime": 1}}}} + stream, err := b.store.db.Watch(ctx, pipeline, options.ChangeStream().SetMaxAwaitTime(time.Second)) + if err != nil { + return fmt.Errorf("query builder needs available Mongo change streams: %w", err) + } + defer stream.Close(context.Background()) + // Seed before consuming: events already captured by the open stream + // close the initial snapshot/change gap without retaining customer data. + seedCtx, seedCancel := context.WithTimeout(ctx, 2*time.Minute) + err = b.seedSourceDigests(seedCtx) + seedCancel() + if err != nil { + return err + } + watchCtx, stop := context.WithCancel(ctx) + defer stop() + watchDone := make(chan error, 1) + watchExited := make(chan struct{}) + go func() { + defer close(watchExited) + lastHeartbeat := time.Time{} + pendingChange := false + for watchCtx.Err() == nil { + observed := time.Now().UTC() + if stream.TryNext(watchCtx) { + var event struct { + NS struct { + Collection string `bson:"coll"` + } `bson:"ns"` + Operation string `bson:"operationType"` + Key bson.M `bson:"documentKey"` + ClusterTime primitive.Timestamp `bson:"clusterTime"` + } + if e := stream.Decode(&event); e != nil { + watchDone <- e + return + } + pendingChange = true + if event.ClusterTime.T > 0 { + observed = time.Unix(int64(event.ClusterTime.T), 0).UTC() + } + if e := b.observeQuerySourceEvent(watchCtx, l, event.NS.Collection, event.Operation, event.Key); e != nil { + watchDone <- e + return + } + } else if e := stream.Err(); e != nil { + watchDone <- e + return + } + if time.Since(lastHeartbeat) < time.Second { + continue + } + if e := b.recordObserved(watchCtx, l, observed, pendingChange); e != nil { + watchDone <- e + return + } + pendingChange = false + lastHeartbeat = time.Now() + } + watchDone <- watchCtx.Err() + }() + defer func() { stop(); <-watchExited }() + tick := time.NewTicker(time.Second) + defer tick.Stop() + pendingStats, pendingSearch := map[string]bool{}, map[string]bool{} + lastStats := map[string]time.Time{} + var scopes map[string][]string + refreshScopes := true + lastScopeRead := time.Time{} + lastCleanup := time.Now() + for { + b.mu.Lock() + all, allSearch := b.all, b.allSearch + for org := range b.dirty { + pendingStats[org] = true + } + for org := range b.searchDirty { + pendingSearch[org] = true + } + b.dirty, b.searchDirty = map[string]bool{}, map[string]bool{} + b.all, b.allSearch = false, false + b.mu.Unlock() + if refreshScopes || all || time.Since(lastScopeRead) > 30*time.Second { + var e error + scopes, e = b.scopes(ctx) + if e != nil { + return e + } + if e = b.store.pruneQueryModelScopes(ctx, l, scopes); e != nil { + return e + } + refreshScopes = false + lastScopeRead = time.Now() + } + for org := range scopes { + if all { + pendingStats[org] = true + } + if allSearch { + pendingSearch[org] = true + } + } + for org := range pendingSearch { + if _, exists := scopes[org]; !exists { + delete(pendingSearch, org) + continue + } + if e := b.store.buildQuerySearchModel(ctx, l, org); e == nil { + delete(pendingSearch, org) + } else { + log.Print("query_search_build result=retry category=source_or_publication") + } + } + for org := range pendingStats { + ids, exists := scopes[org] + if !exists { + delete(pendingStats, org) + continue + } + if time.Since(lastStats[org]) < 30*time.Second { + continue + } + if e := b.rebuild(ctx, l, org, ids); e == nil { + delete(pendingStats, org) + lastStats[org] = time.Now() + } else { + log.Print("query_model_build result=retry category=source_or_publication") + } + } + if time.Since(lastCleanup) > time.Minute { + cleanup, finish := context.WithTimeout(ctx, 5*time.Second) + _ = b.store.cleanQueryModelOrphans(cleanup) + finish() + lastCleanup = time.Now() + } + // Prewarm due generations even when no browser is connected. + for org, ids := range scopes { + if len(ids) > 0 { + _, _ = b.reader.Prewarm(ctx, org, ids) + } + } + select { + case <-ctx.Done(): + stop() + <-watchDone + return ctx.Err() + case e := <-watchDone: + return e + case <-tick.C: + } + } + }) +} + +func (s *MongoStore) markQuerySearchDirty(ctx context.Context, l queryModelLease, org string) error { + if org != "" { + _, err := s.queryControl().UpdateOne(ctx, queryLeaseFilter(l), bson.M{"$inc": bson.M{"searchRevisions." + queryModelKey("search", org, nil): int64(1)}}) + return err + } + _, err := s.queryControl().UpdateOne(ctx, queryLeaseFilter(l), bson.M{"$inc": bson.M{"searchGlobalRevision": int64(1)}}) + return err +} + +// The builder's source collection list excludes all derived collections so its +// own publications cannot recursively schedule rebuilds. +func querySourceCollection(name string) bool { + for _, c := range querySourceCollections { + if strings.EqualFold(c, name) { + return true + } + } + return false +} diff --git a/internal/store/query_read_models.go b/internal/store/query_read_models.go new file mode 100644 index 00000000..39c66521 --- /dev/null +++ b/internal/store/query_read_models.go @@ -0,0 +1,278 @@ +package store + +import ( + "bytes" + "compress/gzip" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "sort" + "sync/atomic" + "time" + + "droboticflow/internal/domain" + "droboticflow/internal/productionstats" + "droboticflow/internal/queryserving" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" +) + +const queryModelSchema = "query-model/v1" +const queryModelChunkBytes = 2 << 20 +const queryModelMaxBytes = 64 << 20 + +type queryServingOptions struct { + cache queryserving.Cache + search *queryserving.Typesense + stats queryServingCounters +} + +type queryServingCounters struct{ reportHits, responseHits, searchHits, typesenseReads, mongoSearchReads atomic.Uint64 } + +// QueryServingStatistics exposes counters only, never names or query contents. +func (s *MongoStore) QueryServingStatistics() map[string]uint64 { + if s.queryServing == nil { + return map[string]uint64{} + } + c := &s.queryServing.stats + out := map[string]uint64{"report_hits": c.reportHits.Load(), "response_hits": c.responseHits.Load(), "search_hits": c.searchHits.Load(), "typesense_reads": c.typesenseReads.Load(), "materialized_search_reads": c.mongoSearchReads.Load()} + if cache, ok := s.SharedQueryCache().(interface{ Statistics() map[string]uint64 }); ok { + for k, v := range cache.Statistics() { + out["cache_"+k] = v + } + } + return out +} + +// ConfigureQueryServing is startup-only. Optional accelerators are never +// created by a business read or mutation. Nil dependencies preserve old reads. +func (s *MongoStore) ConfigureQueryServing(cache queryserving.Cache, search *queryserving.Typesense) { + s.queryServing = &queryServingOptions{cache: cache, search: search} +} +func (s *MongoStore) SharedQueryCache() queryserving.Cache { + if s.queryServing == nil { + return nil + } + return s.queryServing.cache +} + +type QueryModelHead struct { + SourceRevision int64 `bson:"sourceRevision" json:"sourceRevision"` + Revision int64 `bson:"revision" json:"revision"` + GlobalRevision int64 `bson:"globalRevision" json:"globalRevision"` + Generation string `bson:"generation" json:"generation"` + AsOf time.Time `bson:"asOf" json:"asOf"` + Chunks int `bson:"chunks" json:"chunks"` + Bytes int `bson:"bytes" json:"bytes"` + Digest string `bson:"digest" json:"digest"` + Collection string `bson:"searchCollection,omitempty" json:"searchCollection,omitempty"` + Rows int `bson:"rows" json:"rows"` +} +type queryModelLease struct { + Owner string + Epoch int64 +} + +func queryModelKey(kind, org string, projects []string) string { + return queryserving.Key(queryModelSchema, kind, normalizedOrganizationID(org), normalizedIDs(projects)) +} +func normalizedIDs(ids []string) []string { + out := append([]string{}, ids...) + sort.Strings(out) + return out +} +func (s *MongoStore) queryControl() *mongo.Collection { + return s.db.Collection("queryReadModelControl") +} +func (s *MongoStore) queryData() *mongo.Collection { return s.db.Collection("queryReadModelData") } +func (s *MongoStore) acquireQueryModelLease(ctx context.Context) (queryModelLease, bool, error) { + now := time.Now().UTC() + owner := domain.NewID("query-builder") + var st struct { + Owner string `bson:"owner"` + Epoch int64 `bson:"epoch"` + } + err := s.queryControl().FindOneAndUpdate(ctx, bson.M{"_id": "global", "$or": bson.A{bson.M{"owner": ""}, bson.M{"until": bson.M{"$lte": now}}, bson.M{"owner": bson.M{"$exists": false}}}}, bson.M{"$set": bson.M{"owner": owner, "until": now.Add(60 * time.Second)}, "$inc": bson.M{"epoch": int64(1)}}, options.FindOneAndUpdate().SetUpsert(true).SetReturnDocument(options.After)).Decode(&st) + if mongo.IsDuplicateKeyError(err) { + return queryModelLease{}, false, nil + } + return queryModelLease{Owner: st.Owner, Epoch: st.Epoch}, err == nil, err +} +func queryLeaseFilter(l queryModelLease) bson.M { + return bson.M{"_id": "global", "owner": l.Owner, "epoch": l.Epoch, "until": bson.M{"$gt": time.Now().UTC()}} +} +func (s *MongoStore) renewQueryModelLease(ctx context.Context, l queryModelLease) error { + r, err := s.queryControl().UpdateOne(ctx, queryLeaseFilter(l), bson.M{"$set": bson.M{"until": time.Now().UTC().Add(60 * time.Second)}}) + if err != nil { + return err + } + if r.MatchedCount != 1 { + return errors.New("query builder lease lost") + } + return nil +} +func (s *MongoStore) readQueryModelHead(ctx context.Context, key string, maxAge time.Duration) (QueryModelHead, bool) { + var st struct { + Heads map[string]QueryModelHead `bson:"heads"` + ObservedAt time.Time `bson:"observedAt"` + SourceRevision int64 `bson:"productionRevision"` + } + short, cancel := context.WithTimeout(ctx, 200*time.Millisecond) + defer cancel() + if s.queryControl().FindOne(short, bson.M{"_id": "global"}, options.FindOne().SetProjection(bson.M{"heads." + key: 1, "observedAt": 1, "productionRevision": 1})).Decode(&st) != nil { + return QueryModelHead{}, false + } + h, ok := st.Heads[key] + age := time.Since(h.AsOf) + verifiedUnchanged := st.SourceRevision == h.SourceRevision && time.Since(st.ObservedAt) >= 0 && time.Since(st.ObservedAt) <= 5*time.Second + return h, ok && h.Generation != "" && age >= 0 && (age <= maxAge || verifiedUnchanged) +} +func (s *MongoStore) publishQueryModelHead(ctx context.Context, l queryModelLease, key string, h QueryModelHead) error { + r, err := s.queryControl().UpdateOne(ctx, queryLeaseFilter(l), bson.M{"$set": bson.M{"heads." + key: h}}) + if err != nil { + return err + } + if r.MatchedCount != 1 { + return errors.New("query publication fenced") + } + return nil +} +func queryModelBlobKey(key, generation string, index int) string { + return fmt.Sprintf("%s:%s:%09d", key, generation, index) +} +func (s *MongoStore) stageQueryModel(ctx context.Context, key string, value any, asOf time.Time) (QueryModelHead, error) { + var data bytes.Buffer + zw := gzip.NewWriter(&data) + budget := &queryModelBudgetWriter{writer: zw} + if err := json.NewEncoder(budget).Encode(value); err != nil { + return QueryModelHead{}, err + } + if err := zw.Close(); err != nil { + return QueryModelHead{}, err + } + if data.Len() > queryModelMaxBytes { + return QueryModelHead{}, errors.New("query materialization exceeds budget") + } + h := QueryModelHead{Generation: domain.NewID("query-generation"), AsOf: asOf, Bytes: data.Len(), Chunks: (data.Len() + queryModelChunkBytes - 1) / queryModelChunkBytes} + sum := sha256.Sum256(data.Bytes()) + h.Digest = hex.EncodeToString(sum[:]) + for i := 0; i < h.Chunks; i++ { + b := data.Bytes()[i*queryModelChunkBytes : min((i+1)*queryModelChunkBytes, data.Len())] + _, err := s.queryData().InsertOne(ctx, bson.M{"_id": queryModelBlobKey(key, h.Generation, i), "generation": h.Generation, "data": b, "createdAt": time.Now().UTC()}) + if err != nil { + return QueryModelHead{}, err + } + if cache := s.SharedQueryCache(); cache != nil { + cache.Put(ctx, queryModelBlobKey(key, h.Generation, i), b, 15*time.Minute) + } + } + return h, nil +} +func (s *MongoStore) loadQueryModel(ctx context.Context, key string, h QueryModelHead, out any) error { + if h.Chunks < 1 || h.Bytes < 1 || h.Bytes > queryModelMaxBytes || h.Chunks != (h.Bytes+queryModelChunkBytes-1)/queryModelChunkBytes { + return errors.New("invalid query manifest") + } + var data bytes.Buffer + for i := 0; i < h.Chunks; i++ { + blobKey := queryModelBlobKey(key, h.Generation, i) + var b []byte + cache := s.SharedQueryCache() + if cache != nil { + b, _ = cache.Get(ctx, blobKey) + } + if b == nil { + var doc struct { + Data []byte `bson:"data"` + } + if err := s.queryData().FindOne(ctx, bson.M{"_id": blobKey}).Decode(&doc); err != nil { + return err + } + b = doc.Data + if cache != nil { + cache.Put(ctx, blobKey, b, 15*time.Minute) + } + } + if len(b) > queryModelChunkBytes { + return errors.New("invalid query chunk") + } + data.Write(b) + } + sum := sha256.Sum256(data.Bytes()) + if data.Len() != h.Bytes || hex.EncodeToString(sum[:]) != h.Digest { + return errors.New("query materialization corrupt") + } + zr, err := gzip.NewReader(bytes.NewReader(data.Bytes())) + if err != nil { + return err + } + defer zr.Close() + b, err := io.ReadAll(io.LimitReader(zr, 128<<20+1)) + if err != nil || len(b) > 128<<20 { + return errors.New("query decode exceeds budget") + } + return json.Unmarshal(b, out) +} + +// LoadProductionReport serves a derived, scope-qualified report if available. +// Legacy reads remain the compatibility/failure path. No refresh is triggered. +func (s *MongoStore) LoadProductionReport(ctx context.Context, org string, projects []string) (productionstats.Report, string, error) { + if s.queryServing != nil { + key := queryModelKey("production", org, projects) + if head, ok := s.readQueryModelHead(ctx, key, dashboardStatsCacheTTL); ok { + var report productionstats.Report + if err := s.loadQueryModel(ctx, key, head, &report); err == nil { + s.queryServing.stats.reportHits.Add(1) + return report, head.AsOf.Format(time.RFC3339Nano), nil + } + } + } + snapshot, err := s.LoadProductionStatsCached(ctx, org, projects) + if err != nil { + return productionstats.Report{}, "", err + } + return productionstats.Build(snapshot), snapshot.AsOf, nil +} + +func (s *MongoStore) PreparedProductionResponse(ctx context.Context, org string, projects []string, query string) ([]byte, bool) { + if s.queryServing == nil { + return nil, false + } + head, ok := s.readQueryModelHead(ctx, queryModelKey("production", org, projects), dashboardStatsCacheTTL) + if !ok || s.SharedQueryCache() == nil { + return nil, false + } + b, hit := s.SharedQueryCache().Get(ctx, queryserving.Key("production-response/v1", s.db.Name(), org, normalizedIDs(projects), head.Generation, query)) + if hit { + s.queryServing.stats.responseHits.Add(1) + } + return b, hit +} +func (s *MongoStore) RememberProductionResponse(ctx context.Context, org string, projects []string, query, asOf string, value any) { + if s.queryServing == nil || s.SharedQueryCache() == nil { + return + } + head, ok := s.readQueryModelHead(ctx, queryModelKey("production", org, projects), dashboardStatsCacheTTL) + if !ok || head.AsOf.Format(time.RFC3339Nano) != asOf { + return + } + queryserving.PutJSON(ctx, s.SharedQueryCache(), queryserving.Key("production-response/v1", s.db.Name(), org, normalizedIDs(projects), head.Generation, query), value, 5*time.Minute) +} + +type queryModelBudgetWriter struct { + writer io.Writer + bytes int +} + +func (w *queryModelBudgetWriter) Write(p []byte) (int, error) { + if len(p) > (128<<20)-w.bytes { + return 0, errors.New("query decode exceeds budget") + } + n, e := w.writer.Write(p) + w.bytes += n + return n, e +} diff --git a/internal/store/query_read_models_test.go b/internal/store/query_read_models_test.go new file mode 100644 index 00000000..5e2b7c5d --- /dev/null +++ b/internal/store/query_read_models_test.go @@ -0,0 +1,311 @@ +package store + +import ( + "context" + "encoding/json" + "fmt" + "os" + "reflect" + "strings" + "testing" + "time" + + "droboticflow/internal/domain" + "droboticflow/internal/productionstats" + "droboticflow/internal/queryserving" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" +) + +type queryModelFixture struct { + s *MongoStore + cache *queryserving.ManagedValkey + search *queryserving.Typesense + org string +} + +func newQueryModelFixture(t *testing.T, n int) queryModelFixture { + t.Helper() + if os.Getenv("QUERY_SERVING_LOCAL_TEST") != "1" { + t.Skip("isolated query-serving dependencies") + } + ctx := t.Context() + client, err := mongo.Connect(ctx, options.Client().ApplyURI("mongodb://127.0.0.1:27019/?replicaSet=query-rs&directConnection=true").SetServerSelectionTimeout(3*time.Second)) + if err != nil { + t.Fatal(err) + } + db := client.Database(fmt.Sprintf("query_serving_test_%d", time.Now().UnixNano())) + s := NewMongoStore(db) + search, err := queryserving.NewTypesense("http://127.0.0.1:18108", os.Getenv("QUERY_TEST_TYPESENSE_API_KEY")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + cleanup, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + var control struct { + Heads map[string]QueryModelHead `bson:"heads"` + } + if s.queryControl().FindOne(cleanup, bson.M{"_id": "global"}).Decode(&control) == nil { + for _, h := range control.Heads { + if h.Collection != "" { + _ = search.Delete(cleanup, h.Collection) + } + } + } + _ = db.Drop(cleanup) + _ = client.Disconnect(cleanup) + }) + cache, err := queryserving.NewManagedValkey("valkey://127.0.0.1:16379", db.Name()) + if err != nil { + t.Fatal(err) + } + t.Cleanup(cache.Close) + deadline := time.Now().Add(3 * time.Second) + for !cache.Ready() && time.Now().Before(deadline) { + time.Sleep(10 * time.Millisecond) + } + if !cache.Ready() { + t.Fatal("isolated cache unavailable") + } + s.ConfigureQueryServing(cache, search) + must := func(err error) { + t.Helper() + if err != nil { + t.Fatal(err) + } + } + for _, c := range []string{"humanItems", "runs", "captureRecords", "annotationTasks", "projects", "humanBatches"} { + _, err := db.Collection(c).Indexes().CreateOne(ctx, mongo.IndexModel{Keys: bson.D{{Key: "id", Value: 1}}, Options: options.Index().SetUnique(true)}) + must(err) + } + must(s.EnsureQueryReadModelIndexes(ctx)) + org := "query-org" + must(s.UpsertOrganization(ctx, domain.OrganizationRecord{ID: org, Status: domain.OrganizationStatusActive})) + must(s.UpsertProject(ctx, domain.ProjectRecord{ID: "p", OrganizationID: org, Name: "历史项目"})) + must(s.UpsertHumanBatch(ctx, domain.HumanBatchRecord{ID: "batch", OrganizationID: org, ProjectID: "p", PluginID: "annotation-qa", Stages: domain.DefaultFanyuV58HumanStages(), Status: domain.HumanBatchActive})) + _, err = s.SaveProductionStatsScope(ctx, org, productionstats.Scope{ProjectIDs: []string{"p"}, Revision: 1}, 0) + must(err) + docs := map[string][]any{} + push := func(c string, value any, legacy bool) { + raw, e := bson.Marshal(value) + must(e) + var d bson.M + must(bson.Unmarshal(raw, &d)) + if !legacy { + stampPoolSearchMetadata(c, d) + } + docs[c] = append(docs[c], d) + } + flush := func() { + for c, rows := range docs { + if len(rows) > 0 { + _, e := db.Collection(c).InsertMany(ctx, rows) + must(e) + } + } + docs = map[string][]any{} + } + for i := 0; i < n; i++ { + id := fmt.Sprintf("item-%06d", i) + capture := fmt.Sprintf("FYLX_TEST_%06d", i) + if i == 0 { + capture = "FYLX_B07_AZER764008K_20260927_074440" + } + if i == 1 { + capture = "İ中文😀" + } + run := domain.RunRecord{ID: "run-" + id, OrganizationID: org, ProjectID: "p", TaskID: "capture-" + id, CaptureDirectories: []string{capture}, CreatedAt: "2026-11-10T00:00:00Z", Status: domain.RunStatusWaitingForHuman} + task := domain.AnnotationTaskRecord{ID: "task-" + id, OrganizationID: org, ProjectID: "p", RunID: run.ID, ArtifactID: capture + "/part0002/camera_left.mp4", SourceVideo: capture + "/part0002/camera_left.mp4", ClipEnd: 10} + item := domain.HumanItemRecord{ID: id, OrganizationID: org, ProjectID: "p", BatchID: "batch", PluginID: "annotation-qa", RunID: run.ID, LegacyTaskID: task.ID, CurrentStageID: "quality_review", Status: domain.HumanItemReady, SupplierID: fmt.Sprintf("supplier-%d", i%2), Version: 1, CreatedAt: time.Date(2026, 11, 10, 0, 0, i, 0, time.UTC).Format(time.RFC3339), Subject: domain.HumanSubjectRef{ItemKey: task.ArtifactID, Payload: map[string]any{"captureDirectory": capture, "annotationDraft": map[string]any{"description": "synthetic annotation must remain unchanged"}}}} + if i < 3 { + item.Subject.Payload["unusedBody"] = strings.Repeat("x", 1<<20) + } + push("runs", run, i%3 == 0) + push("annotationTasks", task, i%3 == 0) + push("captureRecords", domain.CaptureRecord{ID: run.TaskID, OrganizationID: org, ProjectID: "p", CaptureID: capture, RunID: run.ID}, i%3 == 0) + push("humanItems", item, i%3 == 0) + if (i+1)%1000 == 0 { + flush() + } + } + flush() + return queryModelFixture{s: s, cache: cache, search: search, org: org} +} + +func queryIDs(items []domain.HumanItemRecord) []string { + out := []string{} + for _, i := range items { + out = append(out, i.ID) + } + return out +} + +func TestQueryReadModelsMongoEquivalenceFencingAndFailure(t *testing.T) { + f := newQueryModelFixture(t, 241) + ctx := t.Context() + s := f.s + before, err := s.GetHumanItem(ctx, "item-000000") + if err != nil { + t.Fatal(err) + } + old := s.queryServing + s.queryServing = nil + queries := []string{"FYLX_B07_AZER764008K_20260927_074440/part0002", "TEST_0000", "中文", "İ", "item-000001", "历史项目", "[missing]", "part0002", "camera_left", "000000/part"} + expected := map[string][]string{} + for _, q := range queries { + items, e := s.ListHumanItems(ctx, f.org, domain.HumanItemQuery{PluginID: "annotation-qa", Search: q, Limit: 500}) + if e != nil { + t.Fatal(e) + } + expected[q] = queryIDs(items) + } + base, e := s.LoadProductionStats(ctx, f.org, []string{"p"}) + if e != nil { + t.Fatal(e) + } + wantReport := productionstats.Build(base) + s.queryServing = old + b := NewQueryReadBuilder(s) + defer b.Close() + if err = b.RunOnce(ctx); err != nil { + t.Fatal(err) + } + for _, q := range queries { + candidateIDs, ready := s.querySearchCandidates(ctx, f.org, q, "") + if !ready { + t.Logf("accelerator unavailable for %q; verifying compatibility result", q) + } + set := map[string]bool{} + for _, id := range candidateIDs { + set[id] = true + } + for _, id := range expected[q] { + if ready && !set[id] { + t.Fatalf("candidate false negative query=%q id=%s", q, id) + } + } + items, e := s.ListHumanItems(ctx, f.org, domain.HumanItemQuery{PluginID: "annotation-qa", Search: q, Limit: 500}) + if e != nil || !reflect.DeepEqual(queryIDs(items), expected[q]) { + t.Fatalf("result mismatch query=%q got=%v expected=%v err=%v", q, queryIDs(items), expected[q], e) + } + } + if stats := s.QueryServingStatistics(); stats["search_hits"] == 0 || stats["typesense_reads"] == 0 { + t.Fatal("new search path never exercised", stats) + } + report, asOf, err := s.LoadProductionReport(ctx, f.org, []string{"p"}) + if err != nil || asOf == "" || !reflect.DeepEqual(report, wantReport) { + t.Fatal("materialized production report differs", err) + } + after, err := s.GetHumanItem(ctx, before.ID) + if err != nil || !reflect.DeepEqual(before, after) { + t.Fatal("read model changed annotation, ownership or version") + } + // A late writer can stage data but cannot replace a newer owner's head. + l1, ok, err := s.acquireQueryModelLease(ctx) + if err != nil || !ok { + t.Fatal(err) + } + _, err = s.queryControl().UpdateOne(ctx, bson.M{"_id": "global"}, bson.M{"$set": bson.M{"until": time.Now().Add(-time.Second)}}) + if err != nil { + t.Fatal(err) + } + l2, ok, err := s.acquireQueryModelLease(ctx) + if err != nil || !ok { + t.Fatal(err) + } + key := queryModelKey("production", f.org, []string{"p"}) + head, ok := s.readQueryModelHead(ctx, key, time.Hour) + if !ok { + t.Fatal("missing production head") + } + if s.publishQueryModelHead(ctx, l1, key, head) == nil { + t.Fatal("expired writer published") + } + if err = s.publishQueryModelHead(ctx, l2, key, head); err != nil { + t.Fatal(err) + } + // Incomplete/corrupt derived storage never becomes zero business data. + _, err = s.queryData().UpdateOne(ctx, bson.M{"_id": queryModelBlobKey(key, head.Generation, 0)}, bson.M{"$set": bson.M{"data": []byte("corrupt")}}) + if err != nil { + t.Fatal(err) + } + s.queryServing.cache = nil + got, _, err := s.LoadProductionReport(ctx, f.org, []string{"p"}) + if err != nil || !reflect.DeepEqual(got, wantReport) { + t.Fatal("corrupt materialization failed to retain compatibility path", err) + } +} + +func TestQuerySearchChangeObservationIgnoresDraftsAndRecoversNamesLocal(t *testing.T) { + f := newQueryModelFixture(t, 12) + s := f.s + ctx, cancel := context.WithCancel(t.Context()) + b := NewQueryReadBuilder(s) + done := make(chan error, 1) + go func() { done <- b.Run(ctx) }() + defer func() { cancel(); <-done; b.Close() }() + wait := func(check func() bool) { + t.Helper() + end := time.Now().Add(20 * time.Second) + for time.Now().Before(end) { + if check() { + return + } + time.Sleep(50 * time.Millisecond) + } + t.Fatal("streamed read model did not converge") + } + wait(func() bool { _, ready := s.querySearchCandidates(ctx, f.org, "FYLX", ""); return ready }) + state, err := s.querySearchState(ctx) + if err != nil { + t.Fatal(err) + } + key := queryModelKey("search", f.org, nil) + generation := state.Heads[key].Generation + item, err := s.GetHumanItem(ctx, "item-000001") + if err != nil { + t.Fatal(err) + } + item.Version++ + item.Status = domain.HumanItemClaimed + item.AssignedUserID = "synthetic-owner" + item.UpdatedAt = time.Now().UTC().Format(time.RFC3339Nano) + if err = s.UpsertHumanItem(ctx, *item); err != nil { + t.Fatal(err) + } + time.Sleep(1500 * time.Millisecond) + state, err = s.querySearchState(ctx) + if err != nil || state.Heads[key].Generation != generation || state.Revisions[key] != state.Heads[key].Revision { + t.Fatal("ordinary task state invalidated name search", err) + } + if err = s.UpsertProject(ctx, domain.ProjectRecord{ID: "p", OrganizationID: f.org, Name: "新项目名称"}); err != nil { + t.Fatal(err) + } + wait(func() bool { + ids, ready := s.querySearchCandidates(ctx, f.org, "新项目名称", "") + return ready && len(ids) == 12 + }) + before, err := s.GetHumanItem(ctx, "item-000000") + if err != nil { + t.Fatal(err) + } + if _, err = s.humanItems.DeleteOne(ctx, bson.M{"id": before.ID}); err != nil { + t.Fatal(err) + } + wait(func() bool { + ids, ready := s.querySearchCandidates(ctx, f.org, "新项目名称", "") + return ready && len(ids) == 11 + }) + // Old generations survive publication only as unreachable, bounded derived + // records; no original annotation or audit records are rewritten. + remaining, err := s.GetHumanItem(ctx, "item-000002") + if err != nil { + t.Fatal(err) + } + raw, _ := json.Marshal(remaining.Subject.Payload) + if !strings.Contains(string(raw), "synthetic annotation must remain unchanged") { + t.Fatal("background rebuild lost annotation") + } +} diff --git a/internal/store/query_recovery_test.go b/internal/store/query_recovery_test.go new file mode 100644 index 00000000..79bd4d74 --- /dev/null +++ b/internal/store/query_recovery_test.go @@ -0,0 +1,231 @@ +package store + +import ( + "context" + "droboticflow/internal/domain" + "droboticflow/internal/queryserving" + "go.mongodb.org/mongo-driver/bson" + "reflect" + "testing" + "time" +) + +func TestQueryReadModelsRestartExpiryAndPrivateGarbageCollectionLocal(t *testing.T) { + f := newQueryModelFixture(t, 24) + s, ctx := f.s, t.Context() + b := NewQueryReadBuilder(s) + defer b.Close() + organizationsBefore, _ := s.organizations.CountDocuments(ctx, bson.M{}) + if err := b.RunOnce(ctx); err != nil { + t.Fatal(err) + } + organizationsAfter, _ := s.organizations.CountDocuments(ctx, bson.M{}) + if organizationsAfter != organizationsBefore { + t.Fatal("builder bootstrapped business organizations") + } + st, err := s.querySearchState(ctx) + if err != nil { + t.Fatal(err) + } + key := queryModelKey("search", f.org, nil) + head := st.Heads[key] + // Stale observation and known source changes cannot authorize an empty hit. + _, err = s.queryControl().UpdateOne(ctx, bson.M{"_id": "global"}, bson.M{"$set": bson.M{"observedAt": time.Now().Add(-6 * time.Second)}}) + if err != nil { + t.Fatal(err) + } + if _, ready := s.querySearchCandidates(ctx, f.org, "TEST", ""); ready { + t.Fatal("expired observer used as complete catalog") + } + _, err = s.queryControl().UpdateOne(ctx, bson.M{"_id": "global"}, bson.M{"$set": bson.M{"observedAt": time.Now()}, "$inc": bson.M{"searchRevisions." + key: 1}}) + if err != nil { + t.Fatal(err) + } + if _, ready := s.querySearchCandidates(ctx, f.org, "TEST", ""); ready { + t.Fatal("known obsolete catalog used") + } + before, err := s.GetHumanItem(ctx, "item-000000") + if err != nil { + t.Fatal(err) + } + if _, err = s.humanItems.DeleteOne(ctx, bson.M{"id": "item-000001"}); err != nil { + t.Fatal(err) + } + // Reconciliation after a stopped worker includes missed deletes. + if err = b.RunOnce(ctx); err != nil { + t.Fatal(err) + } + ids, ready := s.querySearchCandidates(ctx, f.org, "item-000001", "") + if !ready { + t.Fatal("restart catalog not ready") + } + for _, id := range ids { + if id == "item-000001" { + t.Fatal("restart retained deleted identity") + } + } + matches, e := s.ListHumanItems(ctx, f.org, domain.HumanItemQuery{Search: "item-000001", Limit: 100}) + if e != nil || len(matches) != 0 { + t.Fatal("deleted exact search result retained", e) + } + current, err := s.querySearchState(ctx) + if err != nil { + t.Fatal(err) + } + if current.Heads[key].Generation == head.Generation { + t.Fatal("restart reused unverified generation") + } + // Availability failure: Typesense may be gone while Mongo materialized rows + // remain complete. The old source path is still used if observation expires. + unavailable, err := queryserving.NewTypesense("http://127.0.0.1:1", "synthetic") + if err != nil { + t.Fatal(err) + } + s.queryServing.search = unavailable + s.queryServing.cache = nil + got, ready := s.querySearchCandidates(ctx, f.org, "FYLX_B07", "") + if !ready || !reflect.DeepEqual(got, []string{"item-000000"}) { + t.Fatal("search outage lost existing task", got, ready) + } + s.queryServing.search = f.search + // GC must retain current generations regardless of their creation date, + // while removing only this database's old unreachable derivatives. + _, err = s.queryData().UpdateMany(ctx, bson.M{}, bson.M{"$set": bson.M{"createdAt": time.Now().Add(-time.Hour)}}) + if err != nil { + t.Fatal(err) + } + _, err = s.queryData().InsertOne(ctx, bson.M{"_id": "unreachable-test", "generation": "orphan-test", "createdAt": time.Now().Add(-time.Hour)}) + if err != nil { + t.Fatal(err) + } + if err = s.cleanQueryModelOrphans(ctx); err != nil { + t.Fatal(err) + } + n, err := s.queryData().CountDocuments(ctx, bson.M{"generation": current.Heads[key].Generation}) + if err != nil || n != 24 { + t.Fatal("current generation lost", n, err) + } // 23 rows plus collection registration + n, err = s.queryData().CountDocuments(ctx, bson.M{"_id": "unreachable-test"}) + if err != nil || n != 0 { + t.Fatal("orphan not reclaimed", err) + } + after, err := s.GetHumanItem(ctx, before.ID) + if err != nil || !reflect.DeepEqual(after, before) { + t.Fatal("recovery or GC modified annotation") + } +} + +func TestQuerySearchPaginationAndLiveScopeEquivalenceLocal(t *testing.T) { + f := newQueryModelFixture(t, 241) + s, ctx := f.s, t.Context() + b := NewQueryReadBuilder(s) + defer b.Close() + if err := b.RunOnce(ctx); err != nil { + t.Fatal(err) + } + // The name catalog never contains assignment or canClaim. Current filters + // apply equally after a task changes supplier, owner, stage or version. + item, err := s.GetHumanItem(ctx, "item-000000") + if err != nil { + t.Fatal(err) + } + item.SupplierID = "changed-supplier" + item.AssignedUserID = "current-owner" + item.Status = domain.HumanItemClaimed + item.Version++ + if err = s.UpsertHumanItem(ctx, *item); err != nil { + t.Fatal(err) + } + options := s.queryServing + queries := []domain.HumanItemQuery{ + {Search: "part0002", Limit: 17}, + {Search: "part0002", Limit: 17, AvailableProjectIDs: []string{}}, + {Search: "part0002", Limit: 17, AvailableProjectIDs: []string{"foreign"}}, + {Search: "FYLX_B07", ActorUserID: "current-owner", Limit: 17}, + {Search: "part0002", UnassignedOnly: true, Limit: 17}, + } + for _, q := range queries { + q.PluginID = "annotation-qa" + for page := 0; page < 20; page++ { + s.queryServing = nil + old, e := s.ListHumanItems(ctx, f.org, q) + if e != nil { + t.Fatal(e) + } + s.queryServing = options + got, e := s.ListHumanItems(ctx, f.org, q) + if e != nil || !reflect.DeepEqual(queryIDs(old), queryIDs(got)) { + t.Fatal("paged live-scope mismatch", page, e) + } + if len(old) == 0 { + break + } + last := old[len(old)-1] + q.Cursor = &domain.HumanItemCursor{CreatedAt: last.CreatedAt, ID: last.ID} + if page == 19 { + t.Fatal("pagination did not exhaust") + } + } + } + if rows, e := s.ListHumanItems(ctx, "foreign-org", domain.HumanItemQuery{Search: "part0002", Limit: 500}); e != nil || len(rows) != 0 { + t.Fatal("foreign organization leaked", e) + } + // Cancellation is propagated instead of publishing/caching empty results. + cancelled, stop := context.WithCancel(ctx) + stop() + if _, ready := s.querySearchCandidates(cancelled, f.org, "part0002", ""); ready { + t.Fatal("cancelled candidate lookup became authoritative") + } +} + +func TestQuerySearchCandidateBudgetNeverTruncates(t *testing.T) { + if queryCandidateBudget([]string{""}) || queryCandidateBudget(make([]string, 100001)) { + t.Fatal("invalid or oversized candidates accepted") + } + if !queryCandidateBudget(nil) || !queryCandidateBudget([]string{"valid-id"}) { + t.Fatal("valid complete candidates rejected") + } +} + +func TestQueryReadModelsCachedResponseScopeAndStaleFallbackLocal(t *testing.T) { + f := newQueryModelFixture(t, 8) + s, ctx := f.s, t.Context() + b := NewQueryReadBuilder(s) + defer b.Close() + if e := b.RunOnce(ctx); e != nil { + t.Fatal(e) + } + _, asOf, e := s.LoadProductionReport(ctx, f.org, []string{"p"}) + if e != nil { + t.Fatal(e) + } + s.RememberProductionResponse(ctx, f.org, []string{"p"}, "dates-and-filter", asOf, map[string]int{"proof": 42}) + raw, ok := s.PreparedProductionResponse(ctx, f.org, []string{"p"}, "dates-and-filter") + if !ok || string(raw) != "{\"proof\":42}" { + t.Fatal("real shared response not reusable", string(raw), ok) + } + for _, scope := range []struct { + org string + projects []string + query string + }{{"foreign", []string{"p"}, "dates-and-filter"}, {f.org, []string{"foreign"}, "dates-and-filter"}, {f.org, []string{"p"}, "other-dates"}} { + if _, hit := s.PreparedProductionResponse(ctx, scope.org, scope.projects, scope.query); hit { + t.Fatal("response scope leaked") + } + } + key := queryModelKey("production", f.org, []string{"p"}) + _, e = s.queryControl().UpdateOne(ctx, bson.M{"_id": "global"}, bson.M{"$set": bson.M{"heads." + key + ".asOf": time.Now().Add(-time.Minute), "productionRevision": int64(10), "observedAt": time.Now()}}) + if e != nil { + t.Fatal(e) + } + if _, hit := s.PreparedProductionResponse(ctx, f.org, []string{"p"}, "dates-and-filter"); hit { + t.Fatal("changed source reused report beyond original freshness budget") + } + before := s.QueryServingStatistics()["report_hits"] + if _, _, e = s.LoadProductionReport(ctx, f.org, []string{"p"}); e != nil { + t.Fatal("stale model did not retain old loader", e) + } + if s.QueryServingStatistics()["report_hits"] != before { + t.Fatal("stale model counted as fresh") + } +} diff --git a/internal/store/query_search_model.go b/internal/store/query_search_model.go new file mode 100644 index 00000000..afb98c54 --- /dev/null +++ b/internal/store/query_search_model.go @@ -0,0 +1,231 @@ +package store + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "net/url" + "regexp" + "strings" + "time" + + "droboticflow/internal/domain" + "droboticflow/internal/queryserving" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo/options" +) + +type querySearchState struct { + Heads map[string]QueryModelHead `bson:"heads"` + Revisions map[string]int64 `bson:"searchRevisions"` + Global int64 `bson:"searchGlobalRevision"` + ObservedAt time.Time `bson:"observedAt"` +} + +func (s *MongoStore) querySearchState(ctx context.Context) (querySearchState, error) { + var st querySearchState + err := s.queryControl().FindOne(ctx, bson.M{"_id": "global"}, options.FindOne().SetProjection(bson.M{"heads": 1, "searchRevisions": 1, "searchGlobalRevision": 1, "observedAt": 1})).Decode(&st) + return st, err +} +func querySafePath(value string) string { + u, err := url.Parse(value) + if err == nil && u.Scheme != "" && u.Host != "" { + u.RawQuery = "" + u.Fragment = "" + u.User = nil + return u.String() + } + return value +} +func searchRowText(row adminPoolRow) string { + // Both exact historical search corpora remain contiguous; adding fields in + // between could otherwise lose cross-field substring matches. + return adminPoolSearchText(row) + "\n" + humanPoolFallbackSearchText(row) +} + +func (s *MongoStore) buildQuerySearchModel(ctx context.Context, l queryModelLease, org string) error { + ctx, stop := context.WithTimeout(ctx, 5*time.Minute) + defer stop() + key := queryModelKey("search", org, nil) + state, err := s.querySearchState(ctx) + if err != nil { + return err + } + h := QueryModelHead{Generation: domain.NewID("query-search"), AsOf: time.Now().UTC(), Revision: state.Revisions[key], GlobalRevision: state.Global} + if s.queryServing != nil && s.queryServing.search != nil { + sum := sha256.Sum256([]byte(s.db.Name() + ":" + key + ":" + h.Generation)) + h.Collection = "dfq_" + hex.EncodeToString(sum[:]) + // Register intent first. Even a canceled/ambiguous create remains + // discoverable by this database's orphan collector. + if _, err = s.queryData().InsertOne(ctx, bson.M{"_id": "search_collection:" + h.Collection, "kind": "search_collection", "collection": h.Collection, "generation": h.Generation, "createdAt": time.Now().UTC()}); err != nil { + return err + } + if err = s.queryServing.search.Create(ctx, h.Collection); err != nil { + return err + } + } + complete := false + defer func() { + if !complete { + cleanup, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + _, _ = s.queryData().DeleteMany(cleanup, bson.M{"generation": h.Generation, "kind": bson.M{"$ne": "search_collection"}}) + if h.Collection != "" { + _ = s.queryServing.search.Delete(cleanup, h.Collection) + } + } + }() + pipeline := humanPoolSearchPipeline(org, domain.HumanItemQuery{Search: "_build_names"}, bson.M{"organizationId": normalizedOrganizationID(org)}) + cur, err := s.humanItems.Aggregate(ctx, pipeline, options.Aggregate().SetAllowDiskUse(true).SetBatchSize(200)) + if err != nil { + return err + } + defer cur.Close(ctx) + for { + rows := []adminPoolRow{} + for len(rows) < 200 && cur.Next(ctx) { + var row adminPoolRow + if err = cur.Decode(&row); err != nil { + return err + } + rows = append(rows, row) + } + if err = cur.Err(); err != nil { + return err + } + if len(rows) == 0 { + break + } + if err = s.humanPoolSearchLegacyPayloads(ctx, org, domain.HumanItemQuery{Search: "_build_names"}, rows); err != nil { + return err + } + docs := make([]any, 0, len(rows)) + searchDocs := make([]queryserving.SearchDocument, 0, len(rows)) + for _, row := range rows { + for _, task := range row.Tasks { + if querySafePath(task.SourceVideo) != task.SourceVideo { + return errors.New("credential-bearing source requires legacy search") + } + } + text := searchRowText(row) + if len(text) > 64<<10 { + return errors.New("search names exceed materialization budget") + } + // Store only the conservative searchable text and identity. No + // task body, draft, review opinion, eligibility or credential. + id := queryserving.Key("row", h.Generation, row.ID) + docs = append(docs, bson.M{"_id": id, "generation": h.Generation, "organizationId": normalizedOrganizationID(org), "itemId": row.ID, "text": text, "createdAt": time.Now().UTC()}) + searchDocs = append(searchDocs, queryserving.SearchDocument{ID: id, ItemID: row.ID, Grams: queryserving.IndexGrams(text)}) + } + if _, err = s.queryData().InsertMany(ctx, docs); err != nil { + return err + } + if h.Collection != "" { + if err = s.queryServing.search.Import(ctx, h.Collection, searchDocs); err != nil { + return err + } + } + h.Rows += len(rows) + } + filter := queryLeaseFilter(l) + filter["searchGlobalRevision"] = state.Global + // Missing revision zero is equivalent to zero, without writing business data. + filter["$expr"] = bson.M{"$and": bson.A{bson.M{"$eq": bson.A{bson.M{"$ifNull": bson.A{"$searchRevisions." + key, int64(0)}}, h.Revision}}, bson.M{"$eq": bson.A{bson.M{"$ifNull": bson.A{"$searchGlobalRevision", int64(0)}}, h.GlobalRevision}}}} + delete(filter, "searchGlobalRevision") + res, err := s.queryControl().UpdateOne(ctx, filter, bson.M{"$set": bson.M{"heads." + key: h, "observedAt": time.Now().UTC()}}) + if err != nil { + return err + } + if res.MatchedCount != 1 { + return errors.New("search source changed during build") + } + complete = true + return nil +} + +// A complete, current catalog only narrows candidates. It never authorizes a +// row, changes a cursor, truncates history, or supplies cached task status. +func (s *MongoStore) querySearchCandidates(ctx context.Context, org, search, capture string) ([]string, bool) { + if s.queryServing == nil || search == "" && capture == "" { + return nil, false + } + short, cancel := context.WithTimeout(ctx, 300*time.Millisecond) + defer cancel() + st, err := s.querySearchState(short) + if err != nil { + return nil, false + } + key := queryModelKey("search", org, nil) + h, ok := st.Heads[key] + if !ok || h.Generation == "" || h.Revision != st.Revisions[key] || h.GlobalRevision != st.Global || time.Since(st.ObservedAt) > 5*time.Second || time.Since(st.ObservedAt) < 0 { + return nil, false + } + cacheKey := queryserving.Key("search-candidates/v1", s.db.Name(), org, h.Generation, search, capture) + var ids []string + if queryserving.GetJSON(short, s.SharedQueryCache(), cacheKey, &ids) && queryCandidateBudget(ids) { + s.queryServing.stats.searchHits.Add(1) + return ids, true + } + if h.Collection != "" && s.queryServing.search != nil { + if out, err := s.queryServing.search.Candidates(short, h.Collection, search, capture); err == nil && queryCandidateBudget(out) { + s.queryServing.stats.searchHits.Add(1) + s.queryServing.stats.typesenseReads.Add(1) + queryserving.PutJSON(short, s.SharedQueryCache(), cacheKey, out, 5*time.Minute) + return out, true + } + } + clauses := bson.A{bson.M{"generation": h.Generation, "organizationId": normalizedOrganizationID(org)}} + for _, term := range []string{search, capture} { + if term != "" { + clauses = append(clauses, bson.M{"text": bson.M{"$regex": regexp.QuoteMeta(strings.ToLower(term))}}) + } + } + cur, err := s.queryData().Find(short, bson.M{"$and": clauses}, options.Find().SetProjection(bson.M{"itemId": 1, "_id": 0})) + if err != nil { + return nil, false + } + defer cur.Close(short) + ids = []string{} + for cur.Next(short) { + var row struct { + ID string `bson:"itemId"` + } + if cur.Decode(&row) != nil || row.ID == "" { + return nil, false + } + ids = append(ids, row.ID) + if len(ids) > 100000 { + return nil, false + } + } + if cur.Err() != nil { + return nil, false + } + if !queryCandidateBudget(ids) { + return nil, false + } + s.queryServing.stats.mongoSearchReads.Add(1) + queryserving.PutJSON(short, s.SharedQueryCache(), cacheKey, ids, 5*time.Minute) + s.queryServing.stats.searchHits.Add(1) + return ids, true +} + +// Keep the authoritative Mongo filter well below its BSON limit. An oversized +// candidate set is a miss, never a truncated inventory. +func queryCandidateBudget(ids []string) bool { + if len(ids) > 100000 { + return false + } + bytes := 0 + for _, id := range ids { + if id == "" { + return false + } + bytes += len(id) + 16 + if bytes > 4<<20 { + return false + } + } + return true +} diff --git a/internal/store/query_serving_config.go b/internal/store/query_serving_config.go new file mode 100644 index 00000000..156f3b83 --- /dev/null +++ b/internal/store/query_serving_config.go @@ -0,0 +1,41 @@ +package store + +import ( + "droboticflow/internal/queryserving" + "errors" + "os" +) + +// ConfigureQueryServingFromEnvironment is called only by process entry points. +// No new feature is enabled by default. Namespace separates databases in +// different deployments even when their database names happen to be identical. +func (s *MongoStore) ConfigureQueryServingFromEnvironment() (func(), error) { + close := func() {} + if os.Getenv("QUERY_SERVING_READS") != "1" { + return close, nil + } + ns := os.Getenv("QUERY_SERVING_NAMESPACE") + if ns == "" { + return close, errors.New("QUERY_SERVING_NAMESPACE is required") + } + var cache queryserving.Cache + if uri := os.Getenv("QUERY_VALKEY_URL"); uri != "" { + v, err := queryserving.NewManagedValkey(uri, ns+":"+s.db.Name()) + if err != nil { + return close, err + } + cache = v + close = v.Close + } + var search *queryserving.Typesense + if uri := os.Getenv("QUERY_TYPESENSE_URL"); uri != "" { + var err error + search, err = queryserving.NewTypesense(uri, os.Getenv("QUERY_TYPESENSE_API_KEY")) + if err != nil { + close() + return func() {}, err + } + } + s.ConfigureQueryServing(cache, search) + return close, nil +} diff --git a/internal/store/query_source_changes.go b/internal/store/query_source_changes.go new file mode 100644 index 00000000..160aba16 --- /dev/null +++ b/internal/store/query_source_changes.go @@ -0,0 +1,172 @@ +package store + +import ( + "context" + "crypto/sha256" + "droboticflow/internal/domain" + "encoding/json" + "fmt" + "go.mongodb.org/mongo-driver/mongo/options" + "time" + + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" +) + +type querySourceDigest struct { + Digest [32]byte + Organization string +} + +var querySearchSources = map[string]bool{"humanItems": true, "annotationTasks": true, "captureRecords": true, "runs": true, "projects": true} + +// Only names, source identities and references affect search. Saving a draft, +// renewing a lease or changing task status must not invalidate the name index. +func querySearchSourceProjection(collection string) bson.M { + p := bson.M{"_id": 1, "organizationId": 1, "projectId": 1, "id": 1, "poolSearch.text": 1} + fields := map[string][]string{ + "humanItems": {"legacyTaskId", "runId", "subject.itemKey", "subject.artifactId"}, + "annotationTasks": {"artifactId", "rightArtifactId", "sourceVideo"}, + "captureRecords": {"captureId", "prefix"}, + "projects": {"name"}, + "runs": {"taskId", "projectName", "captureDirectories", "inputs", "artifacts.id", "artifacts.name", "artifacts.path", "nodeExecutions.artifacts.id", "nodeExecutions.artifacts.name", "nodeExecutions.artifacts.path"}, + } + for _, f := range fields[collection] { + p[f] = 1 + } + if collection == "runs" { + p["inputs"] = bson.M{"$cond": bson.A{poolSearchValidExpression("runs"), bson.M{}, "$inputs"}} + } + if collection == "humanItems" { + p["legacyPayload"] = bson.M{"$cond": bson.A{poolSearchValidExpression("humanItems"), bson.M{}, "$subject.payload"}} + } + return p +} +func querySourceIdentity(collection string, id any) string { + b, _ := json.Marshal(id) + return collection + ":" + string(b) +} +func querySourceHash(doc bson.M) querySourceDigest { + if payload, ok := doc["legacyPayload"]; ok { + names := []string{} + domain.AppendCaptureDirectoriesFromPayload(payload, &names) + doc["legacyPayload"] = names + } + b, _ := json.Marshal(doc) + org, _ := doc["organizationId"].(string) + return querySourceDigest{Digest: sha256.Sum256(b), Organization: org} +} +func (b *QueryReadBuilder) seedSourceDigests(ctx context.Context) error { + for collection := range querySearchSources { + cur, err := b.store.db.Collection(collection).Aggregate(ctx, mongo.Pipeline{{{Key: "$project", Value: querySearchSourceProjection(collection)}}}) + if err != nil { + return err + } + for cur.Next(ctx) { + var doc bson.M + if err = cur.Decode(&doc); err != nil { + _ = cur.Close(ctx) + return err + } + b.sourceDigests[querySourceIdentity(collection, doc["_id"])] = querySourceHash(doc) + } + err = cur.Err() + _ = cur.Close(ctx) + if err != nil { + return err + } + } + return nil +} +func (b *QueryReadBuilder) searchSourceChanged(ctx context.Context, collection string, key bson.M) (bool, string, error) { + identity := querySourceIdentity(collection, key["_id"]) + old, exists := b.sourceDigests[identity] + cur, err := b.store.db.Collection(collection).Aggregate(ctx, mongo.Pipeline{{{Key: "$match", Value: key}}, {{Key: "$project", Value: querySearchSourceProjection(collection)}}}) + if err != nil { + return false, "", err + } + defer cur.Close(ctx) + if !cur.Next(ctx) { + if err = cur.Err(); err != nil { + return false, "", err + } + delete(b.sourceDigests, identity) + return true, old.Organization, nil + } + var doc bson.M + if err = cur.Decode(&doc); err != nil { + return false, "", err + } + next := querySourceHash(doc) + b.sourceDigests[identity] = next + if exists && old.Digest == next.Digest { + return false, next.Organization, nil + } + if exists && old.Organization != next.Organization { + return true, "", nil + } + return true, next.Organization, nil +} + +func (b *QueryReadBuilder) observeQuerySourceEvent(ctx context.Context, l queryModelLease, collection, operation string, key bson.M) error { + if operation == "drop" || operation == "rename" || operation == "invalidate" { + b.mu.Lock() + b.all, b.allSearch = true, true + b.mu.Unlock() + return b.store.markQuerySearchDirty(ctx, l, "") + } + org := "" + searchChanged := false + if querySearchSources[collection] { + var err error + searchChanged, org, err = b.searchSourceChanged(ctx, collection, key) + if err != nil { + return err + } + } else if operation != "delete" && operation != "drop" && operation != "rename" { + var doc struct { + Org string `bson:"organizationId"` + } + if err := b.store.db.Collection(collection).FindOne(ctx, key, options.FindOne().SetProjection(bson.M{"organizationId": 1})).Decode(&doc); err != nil && err != mongo.ErrNoDocuments { + return err + } + org = doc.Org + } + if operation == "drop" || operation == "rename" || operation == "invalidate" { + org = "" + searchChanged = true + } + b.mu.Lock() + if org == "" { + b.all = true + } else { + b.dirty[org] = true + } + if searchChanged { + if org == "" { + b.allSearch = true + } else { + b.searchDirty[org] = true + } + } + b.mu.Unlock() + if searchChanged { + return b.store.markQuerySearchDirty(ctx, l, org) + } + return nil +} + +func (b *QueryReadBuilder) recordObserved(ctx context.Context, l queryModelLease, at time.Time, changed bool) error { + update := bson.M{"$set": bson.M{"observedAt": at}} + if changed { + update["$inc"] = bson.M{"productionRevision": int64(1)} + } + r, err := b.store.queryControl().UpdateOne(ctx, queryLeaseFilter(l), update) + if err != nil { + return err + } + if r.MatchedCount != 1 { + return fmt.Errorf("query observer fenced") + } + return nil +} diff --git a/scripts/lib/query_serving_evidence.py b/scripts/lib/query_serving_evidence.py new file mode 100644 index 00000000..c02578b3 --- /dev/null +++ b/scripts/lib/query_serving_evidence.py @@ -0,0 +1,36 @@ +"""Strict comparison of isolated query-serving measurements; no production IO.""" + +def compare(base, candidate): + problems=[] + if base.get('fixture_digest')!=candidate.get('fixture_digest') or not base.get('fixture_digest'): + problems.append('fixture identity mismatch') + if not base.get('metrics') or not candidate.get('metrics'): + problems.append('missing request measurements') + if candidate.get('actors')!=81 or candidate.get('errors')!=0 or candidate.get('mongo_write_commands')!=0: + problems.append('concurrency, request errors or business writes failed') + counters=candidate.get('query_serving_statistics',{}) + if not all(counters.get(k,0)>0 for k in ('report_hits','search_hits','typesense_reads','cache_hits')): + problems.append('accelerated paths not proven active') + for key,m in candidate.get('metrics',{}).items(): + limit=2000 if '.search' in key else 500 if key.endswith('.panel') or key=='admin.production' else None + if limit and m.get('p95_ms',float('inf'))>limit:problems.append('latency exceeded: '+key) + for row in candidate.get('cold_samples',[]): + if (row['Key'].endswith('.panel') or row['Key']=='admin.production') and (row['Code']!=200 or row['MS']>2000): + problems.append('prewarmed first visit exceeded: '+row['Key']) + def hashes(report): + out={} + for r in report.get('cold_samples',[])+report.get('samples',[]): + if r.get('Code')==200:out.setdefault((r['Actor'],r['Key']),set()).add(r['Hash']) + return out + old,new=hashes(base),hashes(candidate) + overlap=old.keys() & new.keys() + if not overlap:problems.append('no comparable successful results') + for key in overlap: + if old[key]!=new[key]:problems.append('business result mismatch: '+':'.join(key)) + missing=sorted(':'.join(k) for k in new.keys()-old.keys()) + # Baseline timeouts cannot demonstrate equivalence; functional fixtures must + # cover those scopes separately. Keep them visible rather than marking equal. + return {'passed':not problems and not missing,'problems':sorted(set(problems)), + 'comparableActorEndpoints':len(overlap),'baselineUnavailableComparisons':missing, + 'cpuRatio':candidate.get('process_cpu_seconds',0)/max(base.get('process_cpu_seconds',0),.00001), + 'retainedHeapRatio':candidate.get('go_heap_after_gc',0)/max(base.get('go_heap_after_gc',0),1)} diff --git a/scripts/local/query-serving.compose.yml b/scripts/local/query-serving.compose.yml new file mode 100644 index 00000000..3f7b59e8 --- /dev/null +++ b/scripts/local/query-serving.compose.yml @@ -0,0 +1,31 @@ +name: fanyu-query-local +services: + mongo: + image: mongo@sha256:5d7043a4ffe02b9ed1b6e0bab057546981af5ca0a79107e9c461e49bc44c0a7b + command: ["mongod", "--replSet", "query-rs", "--bind_ip_all", "--wiredTigerCacheSizeGB", "0.5"] + ports: ["127.0.0.1:27019:27017"] + mem_limit: 4g + cpus: 2 + ulimits: + nofile: + soft: 65536 + hard: 65536 + volumes: ["mongo-query:/data/db"] + valkey: + image: valkey/valkey@sha256:e51a82741b780e4bf315db10753edf07eea6496d405fa7df2a8677a18f5e7464 + command: ["valkey-server", "--maxmemory", "128mb", "--maxmemory-policy", "allkeys-lru", "--save", "", "--appendonly", "no"] + ports: ["127.0.0.1:16379:6379"] + mem_limit: 256m + cpus: 1 + typesense: + image: typesense/typesense@sha256:610f2d34b1f93d00762869da2c67736775e5798d19a2c8b91b014b8a0cc1e110 + environment: + TYPESENSE_API_KEY: "${QUERY_TEST_TYPESENSE_API_KEY:?synthetic local key required}" + TYPESENSE_DATA_DIR: /data + ports: ["127.0.0.1:18108:8108"] + mem_limit: 1g + cpus: 2 + volumes: ["typesense-query:/data"] +volumes: + mongo-query: + typesense-query: diff --git a/scripts/test_query_serving.py b/scripts/test_query_serving.py new file mode 100644 index 00000000..2410c2a7 --- /dev/null +++ b/scripts/test_query_serving.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python3 +"""Fixed loopback synthetic dependencies. Never accepts a customer database URI.""" +import argparse +import fcntl +import json +import os +import re +from pathlib import Path +import secrets +import signal +import subprocess +import sys +import time +import urllib.request +from lib.release_checks import check_environment, write_log, source_fingerprint + +ROOT = Path(__file__).resolve().parents[1] +PRIVATE = ROOT / 'local-data/query-serving-local' + +def environment(): + PRIVATE.mkdir(parents=True, exist_ok=True, mode=0o700) + keyfile = PRIVATE / 'synthetic-key' + if not keyfile.exists(): + with os.fdopen(os.open(keyfile, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600), 'w') as stream: + stream.write(secrets.token_hex(24)) + env = {k:v for k,v in check_environment().items() if not k.startswith('QUERY_')} + env.update(QUERY_TEST_TYPESENSE_API_KEY=keyfile.read_text(), QUERY_SERVING_LOCAL_TEST='1') + return env + +def compose(env, *args): + result = subprocess.run(['docker','compose','-f',str(ROOT/'scripts/local/query-serving.compose.yml'),*args],cwd=ROOT,env=env,capture_output=True,text=True,timeout=300) + if result.returncode: + proof=write_log(PRIVATE/'logs', ['docker','compose',*args], result.stdout+'\n'+result.stderr,'failed') + raise RuntimeError('isolated dependencies failed; private log: '+proof['log']) + return result.stdout + +def start(env): + compose(env,'up','-d') + for _ in range(60): + try: + compose(env,'exec','-T','mongo','mongosh','--quiet','--eval',"try {rs.status()} catch(e) {rs.initiate({_id:'query-rs',members:[{_id:0,host:'localhost:27017'}]})}") + with urllib.request.urlopen('http://127.0.0.1:18108/health',timeout=1) as r: + if json.load(r).get('ok'):return + except (RuntimeError,OSError,ValueError): + pass + time.sleep(.5) + raise RuntimeError('isolated dependencies not ready') + +def run_command(argv, env, timeout): + child=subprocess.Popen(argv,cwd=ROOT,env=env,stdout=subprocess.PIPE,stderr=subprocess.PIPE,text=True,start_new_session=True) + try: + stdout,stderr=child.communicate(timeout=timeout) + except subprocess.TimeoutExpired: + try:os.killpg(child.pid,signal.SIGTERM) + except ProcessLookupError:pass + try:stdout,stderr=child.communicate(timeout=5) + except subprocess.TimeoutExpired: + try:os.killpg(child.pid,signal.SIGKILL) + except ProcessLookupError:pass + stdout,stderr=child.communicate() + return subprocess.CompletedProcess(argv,124,stdout,stderr+'\nphase timed out') + return subprocess.CompletedProcess(argv,child.returncode,stdout,stderr) + +def main(): + parser=argparse.ArgumentParser() + parser.add_argument('--start',action='store_true') + parser.add_argument('--down',action='store_true',help='stop this test project; retain local volumes') + parser.add_argument('--run',default='^TestQueryServing|^TestQueryReadModel|^TestQuerySearch') + parser.add_argument('--race',action='store_true') + args=parser.parse_args();env=environment() + before=source_fingerprint(ROOT) + with (PRIVATE/'runner.lock').open('w') as lock: + fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB) + if args.down:compose(env,'down');return 0 + if args.start:start(env) + argv=['go','test','./internal/queryserving','./internal/store','./internal/api','./internal/outputsnapshot','-run',args.run,'-count=1','-timeout=10m','-v'] + if args.race:argv.append('-race') + result=run_command(argv,env,660) + proof=write_log(PRIVATE/'logs',argv,result.stdout+'\n'+result.stderr,'passed' if result.returncode==0 else 'failed') + print(json.dumps(proof)) + # Do not interpret an opt-in skip/empty test run as acceptance. + if result.returncode: + return result.returncode + if not before or source_fingerprint(ROOT)!=before: + print('Source identity changed during checks; acceptance remains pending.');return 2 + required=set() + for name in ('internal/queryserving/serving_test.go','internal/store/query_read_models_test.go','internal/store/query_recovery_test.go','internal/api/query_serving_test.go','internal/outputsnapshot/query_shared_cache_test.go'): + required.update(re.findall(r'^func (Test\w+)\(', (ROOT/name).read_text(), re.M)) + passed=set(re.findall(r'^--- PASS: (Test\w+)',result.stdout,re.M)) + if required-passed: + print('Missing required cases: '+', '.join(sorted(required-passed)));return 2 + if '--- SKIP:' in result.stdout or '[no tests to run]' in result.stdout: + print('Unverified skipped/empty coverage; inspect report.');return 2 + return result.returncode + +if __name__=='__main__': + raise SystemExit(main()) diff --git a/scripts/test_query_serving_performance.py b/scripts/test_query_serving_performance.py new file mode 100644 index 00000000..6341ecb3 --- /dev/null +++ b/scripts/test_query_serving_performance.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python3 +"""Compare the same synthetic API fixture before/after query serving.""" +import argparse +import fcntl +import datetime as dt +import json +from pathlib import Path +import subprocess +import threading +import time +import uuid +from test_query_serving import ROOT, PRIVATE, environment, start, run_command +from lib.query_serving_evidence import compare +from lib.release_checks import write_log, source_fingerprint + +def main(): + parser=argparse.ArgumentParser() + parser.add_argument('--start',action='store_true') + parser.add_argument('--production-shape',action='store_true') + parser.add_argument('--rounds',type=int,default=1) + parser.add_argument('--warmup',type=int,default=5) + parser.add_argument('--duration',type=int,default=30) + parser.add_argument('--database',help='reuse this script\'s existing synthetic fixture') + args=parser.parse_args() + if not 1<=args.rounds<=3 or not 1<=args.duration<=1800 or not 1<=args.warmup<=300:parser.error('bounded rounds/times required') + if args.database and (not args.database.startswith('poll_pool_read_mix_query_') or not args.database.replace('_','').isalnum()):parser.error('synthetic database only') + env={k:v for k,v in environment().items() if not k.startswith('POLL_POOL_')} + dependency_lock=(PRIVATE/'runner.lock').open('w') + try:fcntl.flock(dependency_lock,fcntl.LOCK_EX|fcntl.LOCK_NB) + except BlockingIOError: + print('Isolated query dependencies already in use; comparison not started.');return 2 + if args.start:start(env) + folder=ROOT/'local-data/diagnostics'/('query-serving-'+dt.datetime.now(dt.timezone.utc).strftime('%Y%m%dT%H%M%SZ')) + folder.mkdir(parents=True) + database=args.database or 'poll_pool_read_mix_query_'+uuid.uuid4().hex[:16] + env.update(POLL_POOL_LOAD='1',POLL_POOL_LOCAL_MONGO_URI='mongodb://127.0.0.1:27019/?replicaSet=query-rs&directConnection=true',POLL_POOL_FIXTURE_DATABASE=database, + POLL_POOL_KEEP_FIXTURE='1',POLL_POOL_QUERY_SERVING_PREPARE='1',POLL_POOL_QUERY_PANELS='1',POLL_POOL_READMODELS='1',POLL_POOL_SEARCH='1',POLL_POOL_SEARCH_INDEXES='1',POLL_POOL_PAGE_SHAPE='1',POLL_POOL_ADMIN_TASKS='1' if args.production_shape else '0',POLL_POOL_AUTHORIZATION_STATE='initialized', + POLL_POOL_WARMUP_SECONDS=str(args.warmup),POLL_POOL_DURATION_SECONDS=str(args.duration),ANNOTATION_SOURCE_VALIDATION_MODE='advisory') + if args.production_shape: + env.update(POLL_POOL_PRODUCTION_SHAPE='1', POLL_POOL_RUN_PAYLOAD_KIB='80', + POLL_POOL_TASK_PAYLOAD_KIB='16', POLL_POOL_AVAILABLE_PER_STAGE='215', + POLL_POOL_AUTHOR_HISTORY='100', GOMAXPROCS='2') + before=source_fingerprint(ROOT) + report={'database':database,'inputDigest':before,'shape':'production' if args.production_shape else 'small','rounds':[],'resourceSamples':[],'passed':False,'limits':['synthetic loopback API harness, no media or production writes','process resource sampling includes builder and handler; not Kubernetes Pod limits']} + argv=['go','test','./internal/api','-run','^TestPollPoolReadMixLocal$','-count=1','-timeout=45m','-v'] + def run(name,updates): + settings={**env,**updates,'POLL_POOL_OUTPUT':str(folder/(name+'.json'))} + active['phase']=name + started=dt.datetime.now(dt.timezone.utc).isoformat() + result=run_command(argv,settings,args.warmup+args.duration+900) + proof=write_log(folder,argv,result.stdout+'\n'+result.stderr,'passed' if result.returncode==0 else 'failed') + row={'phase':name,'startedAt':started,'endedAt':dt.datetime.now(dt.timezone.utc).isoformat(),'exitCode':result.returncode,**proof,'report':settings['POLL_POOL_OUTPUT']} + report['rounds'].append(row) + (folder/'comparison.json').write_text(json.dumps(report,indent=2)+'\n') + print(json.dumps(row),flush=True) + return result.returncode==0 + active={'phase':'prepare'} + if not args.database: + if not run('prepare',{'POLL_POOL_FIXTURE_ONLY':'1'}):return 1 + stopped=threading.Event() + def resources(): + while not stopped.is_set(): + try: + p=subprocess.run(['docker','stats','--no-stream','--format','{{json .}}','fanyu-query-local-mongo-1','fanyu-query-local-valkey-1','fanyu-query-local-typesense-1'],capture_output=True,text=True,timeout=8) + if p.returncode==0:report['resourceSamples'].append({'phase':active['phase'],'at':dt.datetime.now(dt.timezone.utc).isoformat(),'containers':[json.loads(line) for line in p.stdout.splitlines()]}) + except (OSError,ValueError,subprocess.TimeoutExpired):pass + (folder/'progress.json').write_text(json.dumps({'phase':active['phase'],'at':dt.datetime.now(dt.timezone.utc).isoformat(),'samples':len(report['resourceSamples'])})+'\n') + stopped.wait(2) + thread=threading.Thread(target=resources,daemon=True);thread.start() + try: + for i in range(args.rounds): + baseline='baseline' if i==0 else 'baseline-'+str(i+1) + run(baseline,{'POLL_POOL_REUSE_FIXTURE':'1','POLL_POOL_ALLOW_BASELINE_FAILURE':'1'}) + if not run('candidate-'+str(i+1),{'POLL_POOL_REUSE_FIXTURE':'1','POLL_POOL_QUERY_SERVING':'1','POLL_POOL_MAX_SEARCH_P95_MS':'2000'}):break + finally: + stopped.set();thread.join(timeout=10) + report['inputsChangedDuringRun']=source_fingerprint(ROOT)!=before + candidates=[r for r in report['rounds'] if r['phase'].startswith('candidate-')] + report['comparisons']=[] + try: + for i,row in enumerate(candidates): + basefile='baseline.json' if i==0 else 'baseline-'+str(i+1)+'.json' + report['comparisons'].append(compare(json.loads((folder/basefile).read_text()),json.loads(Path(row['report']).read_text()))) + except (OSError,ValueError,KeyError):report['comparisons'].append({'passed':False,'problems':['missing/invalid comparison report']}) + report['resourceReviewRequired']=len(report['comparisons'])==3 and any(all(r.get(key,0)>1.1 for r in report['comparisons']) for key in ('cpuRatio','retainedHeapRatio')) + report['passed']=not report['resourceReviewRequired'] and len(candidates)==args.rounds and all(r['exitCode']==0 for r in candidates) and not report['inputsChangedDuringRun'] and all(r['passed'] for r in report['comparisons']) + (folder/'comparison.json').write_text(json.dumps(report,indent=2)+'\n') + print('Comparison: '+str(folder/'comparison.json'),flush=True) + return 0 if report['passed'] else 1 + +if __name__=='__main__':raise SystemExit(main()) diff --git a/scripts/verification-scopes.json b/scripts/verification-scopes.json index 57f6574e..1f3c19dc 100644 --- a/scripts/verification-scopes.json +++ b/scripts/verification-scopes.json @@ -883,6 +883,65 @@ "warnings": [ "工作流门禁仅使用合成媒体和隔离 Mongo;严格模式清单不是校验证书,生产容量、备份恢复和部署需分别验证。" ] + }, + { + "name": "query-serving", + "paths": [ + "^internal/queryserving/", + "^internal/api/(database_browser|poll_pool_load_test)\\.go$", + "^internal/store/query_.*\\.go$", + "^internal/store/private_derived_collections\\.go$", + "^internal/(api|outputsnapshot)/query_.*\\.go$", + "^cmd/query-read-builder/", + "^scripts/(test_query_serving.*\\.py|lib/query_serving_evidence\\.py|local/query-serving\\.compose\\.yml)$", + "^Dockerfile\\.query-read-builder$", + "^tests/release/test_query_serving_evidence\\.py$", + "^docs/operations/query-serving\\.md$" + ], + "frontend": [], + "go": { + "./internal/queryserving": [ + "^TestQueryServing" + ], + "./internal/store": [ + "^TestQueryReadModel", + "^TestQuerySearch" + ], + "./internal/outputsnapshot": [ + "^TestQueryServing" + ], + "./internal/api": [ + "^TestQueryServing" + ] + }, + "task": [ + { + "argv": [ + "python3", + "scripts/test_query_serving.py", + "--start", + "--race" + ], + "tests": true, + "env": {} + }, + { + "argv": [ + "python3", + "-m", + "unittest", + "discover", + "-s", + "tests/release", + "-p", + "test_query_serving_evidence.py" + ], + "tests": true + } + ], + "warnings": [ + "真实三组件仅使用固定本机合成夹具;完整规模三轮、故障恢复容量及生产部署独立验收。" + ] } ] } diff --git a/tests/release/test_query_serving_evidence.py b/tests/release/test_query_serving_evidence.py new file mode 100644 index 00000000..1b0af24d --- /dev/null +++ b/tests/release/test_query_serving_evidence.py @@ -0,0 +1,29 @@ +import sys +import unittest +from pathlib import Path +sys.path.insert(0,str(Path(__file__).resolve().parents[2]/'scripts')) +from lib.query_serving_evidence import compare + +class QueryEvidenceTests(unittest.TestCase): + def fixture(self): + sample={'Actor':'reviewer','Key':'acceptance.all.search','Code':200,'MS':20,'Hash':'business-id-claim-counts'} + return {'fixture_digest':'same','actors':81,'errors':0,'mongo_write_commands':0,'metrics':{'acceptance.all.search':{'p95_ms':20}},'samples':[sample], 'cold_samples':[], 'query_serving_statistics':dict(report_hits=1,search_hits=1,typesense_reads=1,cache_hits=1)} + def test_requires_real_measurements_and_active_components(self): + a=self.fixture();b=self.fixture();self.assertTrue(compare(a,b)['passed']) + b['query_serving_statistics']={};self.assertFalse(compare(a,b)['passed']) + self.assertFalse(compare({}, {})['passed']) + def test_results_and_latency_are_both_mandatory(self): + a=self.fixture();b=self.fixture();b['samples'][0]['Hash']='lost-task' + self.assertFalse(compare(a,b)['passed']) + b=self.fixture();b['metrics']['acceptance.all.search']['p95_ms']=2001 + self.assertFalse(compare(a,b)['passed']) + def test_business_write_and_cold_panel_failure_are_not_success(self): + a=self.fixture();b=self.fixture();b['mongo_write_commands']=1 + self.assertFalse(compare(a,b)['passed']) + b=self.fixture();b['cold_samples']=[{'Actor':'admin','Key':'admin.production','Code':202,'MS':1,'Hash':'empty'}] + self.assertFalse(compare(a,b)['passed']) + def test_baseline_timeout_keeps_equivalence_pending(self): + a=self.fixture();b=self.fixture();a['samples'][0]['Code']=500 + r=compare(a,b);self.assertFalse(r['passed']);self.assertTrue(r['baselineUnavailableComparisons']) + +if __name__=='__main__':unittest.main()