forked from longtao.wu/droboticflow
✨ Add optional background query models, shared cache and search
This commit is contained in:
parent
9232bace60
commit
1050f48286
@ -105,3 +105,25 @@ jobs:
|
||||
path: local-data/
|
||||
if-no-files-found: error
|
||||
retention-days: 14
|
||||
|
||||
query-serving:
|
||||
runs-on: ubuntu-24.04
|
||||
needs: pool-search-mongo
|
||||
if: ${{ !cancelled() }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version-file: go.mod
|
||||
cache: true
|
||||
- run: docker compose version
|
||||
- run: python3 scripts/test_query_serving.py --start --race
|
||||
- uses: actions/upload-artifact@v4
|
||||
if: always()
|
||||
with:
|
||||
name: query-serving-logs
|
||||
path: local-data/query-serving-local/logs/
|
||||
if-no-files-found: error
|
||||
retention-days: 14
|
||||
- run: python3 scripts/test_query_serving.py --down
|
||||
if: always()
|
||||
|
||||
22
.scratch/query-serving-options/spec.md
Normal file
22
.scratch/query-serving-options/spec.md
Normal file
@ -0,0 +1,22 @@
|
||||
# 后台物化读表、Valkey 与 Typesense 实施
|
||||
|
||||
Status: claimed
|
||||
Type: task
|
||||
Base: 9232bace
|
||||
|
||||
用户批准依次实施物化读取与共享缓存,再接搜索;正确性与性能达标后才准备发布。本次仅本地实现、隔离验证和提交,不推送、不部署、不写生产。
|
||||
|
||||
- 后台独立进程构建完整代次,复用已有统计口径与旧记录解析;读取不触发新组件的全量构建。后台并发有界、分布式租约、原子发布,失败保留上一完整代次,支持重启恢复和删除对账。
|
||||
- Valkey 共享不可变统计/查询结果,包含数据库、组织、项目范围、代次与查询参数;缓存故障、丢失与未准备就绪退回现有读取,不能阻断作业。复用既有快照发布协议,不重写标注/审核写路径。
|
||||
- Typesense 仅提供搜索候选;保留精确子串、Capture/part、特殊字符、旧记录、排序、完整分页与统计语义。实时权限、可领取、自审禁止、归属、状态、版本与写锁仍由现有服务端代码判定。
|
||||
- 新能力默认关闭,运行地址/凭证从环境或现有私有配置读取。新增集合只存可重建派生数据,后台和查询组件的生产安装、索引及回填另列计划。
|
||||
- 使用隔离 Mongo、真实 Valkey/Typesense 和合成数据,对照旧路径、撤权/归属变化、删除/重启、组件故障、并发与查询资源;接近线上规模,搜索 p95≤2秒,面板暖态 p95≤500ms、预热首访≤2秒。未实测或未达标保持待验收。
|
||||
- 保留原工作树两个 admin pool 测试及旧发布笔记,阶段性本地提交;不修改既有标注和正式审查记录。
|
||||
|
||||
## 本地进展
|
||||
|
||||
后台读表、Valkey、Typesense 已实现,默认关闭。真实三组件12项(含竞态、范围、失联、重启、旧数据和缓存损坏)通过;既有五阶段作业/恢复、搜索与统计浏览器回归通过。官方 Go 基线为68项已有失败、无新增;前端基线与构建/生产启动检查通过。范围总报告保留了原生产诊断跳过、原测试债务和最初三组件瞬时回退断言失败,不将其写成全绿;修正的专项已重跑通过。
|
||||
|
||||
81人、约2.6万任务/34万事件的首轮短测:候选零错误,347组账号/接口业务结果一致;管理员全部项目搜索p95 2592→799ms、生产总览3516→162ms;Mongo内部返回字节199215338→152507258,Go CPU +3.3%。这是短测,不是容量/生产验收。1GiB Mongo 对照OOM已作废,两版统一固定4GiB重新测得上述结果。
|
||||
|
||||
独立后台镜像构建和启动帮助检查已通过。待完成:冻结源码后的三轮持续对照和资源审查。未达标不准备发布;生产拓扑、新组件及派生索引/构建范围仍需具体计划与单独授权。现有两个受保护文件的哈希保持不变。
|
||||
17
Dockerfile.query-read-builder
Normal file
17
Dockerfile.query-read-builder
Normal file
@ -0,0 +1,17 @@
|
||||
FROM golang:1.24-bookworm@sha256:1a6d4452c65dea36aac2e2d606b01b4a029ec90cc1ae53890540ce6173ea77ac AS build
|
||||
WORKDIR /src
|
||||
ARG GOPROXY=https://proxy.golang.org,direct
|
||||
COPY go.mod go.sum ./
|
||||
RUN GOPROXY="${GOPROXY}" go mod download
|
||||
COPY cmd/query-read-builder ./cmd/query-read-builder
|
||||
COPY internal ./internal
|
||||
RUN CGO_ENABLED=0 go build -trimpath -o /out/query-read-builder ./cmd/query-read-builder
|
||||
|
||||
# A static binary only needs trusted CA roots; no media/worker tools or app code.
|
||||
FROM scratch
|
||||
COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
|
||||
COPY --from=build /out/query-read-builder /query-read-builder
|
||||
ARG DROBOTICFLOW_BUILD_COMMIT=unknown
|
||||
LABEL org.opencontainers.image.revision=${DROBOTICFLOW_BUILD_COMMIT}
|
||||
USER 10001:10001
|
||||
ENTRYPOINT ["/query-read-builder"]
|
||||
@ -173,6 +173,14 @@ func main() {
|
||||
}()
|
||||
|
||||
dataStore := store.NewMongoStore(client.Database(cfg.MongoDB))
|
||||
if os.Getenv("ANNOTATION_PROCESSOR_MODE") != "processor" {
|
||||
closeServing, servingErr := dataStore.ConfigureQueryServingFromEnvironment()
|
||||
if servingErr != nil {
|
||||
fmt.Println("[api] query accelerator configuration unavailable; using existing reads")
|
||||
} else {
|
||||
defer closeServing()
|
||||
}
|
||||
}
|
||||
if os.Getenv("ANNOTATION_PROCESSOR_MODE") == "processor" {
|
||||
checkIndexes := dataStore.EnsureIndexes
|
||||
mode := os.Getenv("DROBOTICFLOW_DATABASE_BOOTSTRAP")
|
||||
|
||||
65
cmd/query-read-builder/main.go
Normal file
65
cmd/query-read-builder/main.go
Normal file
@ -0,0 +1,65 @@
|
||||
// query-read-builder only writes derived read models. It never processes work.
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"droboticflow/internal/config"
|
||||
"droboticflow/internal/store"
|
||||
"flag"
|
||||
"go.mongodb.org/mongo-driver/mongo"
|
||||
"go.mongodb.org/mongo-driver/mongo/options"
|
||||
"log"
|
||||
"os"
|
||||
"os/signal"
|
||||
"syscall"
|
||||
"time"
|
||||
)
|
||||
|
||||
func main() {
|
||||
once := flag.Bool("once", false, "build once; streaming freshness expires after exit")
|
||||
ensure := flag.Bool("ensure-indexes", false, "explicitly create derived-data indexes before building")
|
||||
flag.Parse()
|
||||
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
||||
defer stop()
|
||||
cfg, err := config.LoadValidated()
|
||||
if err != nil {
|
||||
log.Fatal("invalid builder configuration")
|
||||
}
|
||||
if os.Getenv("QUERY_SERVING_READS") != "1" {
|
||||
log.Fatal("explicit QUERY_SERVING_READS=1 is required")
|
||||
}
|
||||
client, err := mongo.Connect(ctx, options.Client().ApplyURI(cfg.MongoURI))
|
||||
if err != nil {
|
||||
log.Fatal("builder database unavailable")
|
||||
}
|
||||
defer client.Disconnect(context.Background())
|
||||
s := store.NewMongoStore(client.Database(cfg.MongoDB))
|
||||
closeServing, err := s.ConfigureQueryServingFromEnvironment()
|
||||
if err != nil {
|
||||
log.Fatal("builder accelerator configuration invalid")
|
||||
}
|
||||
defer closeServing()
|
||||
if *ensure {
|
||||
if s.EnsureQueryReadModelIndexes(ctx) != nil {
|
||||
log.Fatal("derived index creation failed")
|
||||
}
|
||||
}
|
||||
b := store.NewQueryReadBuilder(s)
|
||||
defer b.Close()
|
||||
if *once {
|
||||
if b.RunOnce(ctx) != nil {
|
||||
log.Fatal("initial materialization incomplete")
|
||||
}
|
||||
return
|
||||
}
|
||||
for ctx.Err() == nil {
|
||||
if b.Run(ctx) != nil && ctx.Err() == nil {
|
||||
log.Print("query builder paused; retrying with full reconciliation")
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-time.After(5 * time.Second):
|
||||
}
|
||||
}
|
||||
}
|
||||
38
docs/operations/query-serving.md
Normal file
38
docs/operations/query-serving.md
Normal file
@ -0,0 +1,38 @@
|
||||
# 后台查询读表与可选加速组件
|
||||
|
||||
状态:本地实现;生产保持关闭。速度与资源结论见同任务证据,未通过的门槛不能据本说明视为通过。
|
||||
|
||||
## 责任与兼容
|
||||
|
||||
Mongo 仍是任务、权限、归属、版本与正式审查记录的权威存储。独立 `query-read-builder` 只写 `queryReadModelControl`、`queryReadModelData` 及既有 `annotationOutputSnapshot*` 派生集合,不领取、退回、重置、补验或改写客户任务。读取新集合不触发构建。
|
||||
|
||||
后台使用租约代次隔离写者,先完整构建再发布清单。单个失败不会发布半份结果。进程重启先打开变更流,再核对源身份和重建;不把遗留索引视为最新。副本集/分片集群变更流是持续搜索加速的前提,普通独立 Mongo 不支持时后台暂停,API 继续旧路径。
|
||||
|
||||
- 生产总览复用原事实读取与 `productionstats.Build`,保留原日期、供应商、人员和首次送检批次口径。当前配置项目集合生成一份报告;其他未预建项目组合走原路径。来源无变化且观察器新鲜时复用同一代次;有变化按原事实缓存时限停止使用旧结果。
|
||||
- 人员与供应商看板复用既有分片快照和聚合器,后台无人访问也预热。Valkey 共享不可变分片及按完整范围、用户、供应商、日期、时区、代次隔离的结果;本进程已算出的结果优先,避免每次重复访问缓存服务器。
|
||||
- Typesense 只存任务 ID 与名称搜索窗口,返回保守候选 ID。用 Unicode 窗口交集保持任意子串、Capture/part 和标点语义;最终匹配、排序、分页、实时可领取及权限仍走原服务端规则。候选过大、超时、索引过期或不完整时退回原查询,不能截断候选凑性能指标。
|
||||
- 派生集合对通用数据库浏览入口隐藏,防止绕过组织和个人权限。Valkey、Typesense 只允许服务端私网访问,不开放浏览器直连。
|
||||
|
||||
## 配置与本地验证
|
||||
|
||||
API、后台分别显式设置 `QUERY_SERVING_READS=1` 与唯一 `QUERY_SERVING_NAMESPACE`,默认关闭。可选 `QUERY_VALKEY_URL`、`QUERY_TYPESENSE_URL`、`QUERY_TYPESENSE_API_KEY` 均通过现有私有环境配置提供;凭证不提交。不同集群即便数据库同名也必须使用不同 namespace。缓存连接异步建立,启动不可用不会阻止 API;缓存请求有短时限和退避。
|
||||
|
||||
本机合成依赖:`python3 scripts/test_query_serving.py --start`。仅暴露 loopback 27019/16379/18108;随机测试密钥在忽略目录,Mongo 使用独立 query-rs 和独立数据卷。`--down` 只停这一项目,保留卷;不停止共享快照库和其他项目。Mongo 文件句柄上限显式设为65536,避免多轮隔离测试建索引触发默认低限制。
|
||||
|
||||
功能与故障回归使用真实本机三组件,`--race` 检查并发。规模对照入口为 `python3 scripts/test_query_serving_performance.py --production-shape`,使用同一合成数据、索引、资源及固定压缩设置;可通过 `--database` 复用脚本生成的专用数据库。完整稳态验收另设 `--rounds 3 --warmup 300 --duration 1800`;短测通过不代表线上容量验收。
|
||||
|
||||
本机规模夹具统一使用 Mongo 2 CPU/4GiB(WiredTiger 0.5GiB)、Valkey 1 CPU/256MiB、Typesense 2 CPU/1GiB;这是隔离对照的固定配置,不是生产扩容建议。1GiB Mongo 的首轮对照因旧路径 OOM 作废,不能计为优化收益。
|
||||
|
||||
## 运行与清理
|
||||
|
||||
后台单进程生成、跨副本租约;Valkey 不持久化,清空后仅损失速度。Typesense 索引可重建。清理只删除当前数据库登记、已不可达且超过保留期的派生代次,当前清单和其他部署不在清理范围。固定构建预算和载荷预算超限时保留旧路径,不改任务准入。
|
||||
|
||||
显式 `query-read-builder --ensure-indexes` 只为派生行创建 `query_model_rows_v1`;API 启动不建索引。生产新增组件、该索引及派生数据构建会产生数据库读取和索引维护开销,需另列具体部署、资源和授权计划。不上线时不向公司集群安装或写入这些数据。
|
||||
|
||||
修改生产统计或名称匹配的语义时必须递增 `queryModelSchema`,防止新代码复用旧口径的派生代次。API 与后台应发布同一已核验源码版本。
|
||||
|
||||
## 发布与回退条件
|
||||
|
||||
先核对 Mongo 拓扑与权限、私网连接及组件资源,再在隔离环境验收完整任务链和查询结果。比较搜索 p95、冷/暖看板延迟、错误、结果/权限等价、后台业务写入为零、Mongo 读取字节与命令、Go/三组件 CPU 和内存。未达到搜索p95≤2秒、暖面板p95≤500ms、预热首访≤2秒,或三轮持续资源恶化未解释时保持待验收。
|
||||
|
||||
上线采用独立开关,先预热后逐步启用读取。回退关闭 API 的 `QUERY_SERVING_READS` 并停后台,恢复原读取;不改写客户记录,不要求删除新派生集合。组件失联仅是性能降级,不改变标注、切片、送检、质检和验收准入,也不恢复严格源校验。
|
||||
1
go.mod
1
go.mod
@ -6,6 +6,7 @@ require (
|
||||
github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible
|
||||
github.com/baidubce/bce-sdk-go v0.9.264
|
||||
github.com/go-chi/chi/v5 v5.2.3
|
||||
github.com/valkey-io/valkey-go v1.0.60
|
||||
go.etcd.io/bbolt v1.4.3
|
||||
go.mongodb.org/mongo-driver v1.17.6
|
||||
golang.org/x/crypto v0.36.0
|
||||
|
||||
4
go.sum
4
go.sum
@ -76,6 +76,7 @@ github.com/onsi/ginkgo/v2 v2.21.0 h1:7rg/4f3rB88pb5obDgNZrNHrQ4e6WpjonchcpuBRnZM
|
||||
github.com/onsi/ginkgo/v2 v2.21.0/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo=
|
||||
github.com/onsi/gomega v1.35.1 h1:Cwbd75ZBPxFSuZ6T+rN/WCb/gOc6YgFBXLlZLhC7Ds4=
|
||||
github.com/onsi/gomega v1.35.1/go.mod h1:PvZbdDc8J6XJEpDK4HCuRBm8a6Fzp9/DmhC9C7yFlog=
|
||||
github.com/onsi/gomega v1.36.2 h1:koNYke6TVk6ZmnyHrCXba/T/MoLBXFjeC1PtvYgw0A8=
|
||||
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
|
||||
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
@ -95,6 +96,8 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO
|
||||
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
|
||||
github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
|
||||
github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
|
||||
github.com/valkey-io/valkey-go v1.0.60 h1:idh959D20H5n7D/kwEdTKNaMn5+4HpZTn7bLXnAhQIw=
|
||||
github.com/valkey-io/valkey-go v1.0.60/go.mod h1:bHmwjIEOrGq/ubOJfh5uMRs7Xj6mV3mQ/ZXUbmqpjqY=
|
||||
github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM=
|
||||
github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg=
|
||||
github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
|
||||
@ -169,6 +172,7 @@ golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4f
|
||||
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
|
||||
golang.org/x/tools v0.26.0 h1:v/60pFQmzmT9ExmjDv2gGIfi3OqfKoEP6I5+umXlbnQ=
|
||||
golang.org/x/tools v0.26.0/go.mod h1:TPVVj70c7JJ3WCazhD8OdXcZg/og+b9+tH/KxylGwH0=
|
||||
golang.org/x/tools v0.31.0 h1:0EedkvKDbh+qistFTd0Bcwe/YLh4vHwWEkiI0toFIBU=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
|
||||
@ -10,7 +10,6 @@ import (
|
||||
"time"
|
||||
|
||||
"droboticflow/internal/domain"
|
||||
"droboticflow/internal/outputsnapshot"
|
||||
"droboticflow/internal/store"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
@ -148,7 +147,7 @@ func normalizeDatabaseBrowserTarget(raw string) databaseBrowserTarget {
|
||||
|
||||
func validateDatabaseCollectionName(field string, value string) *validationIssue {
|
||||
name := strings.TrimSpace(value)
|
||||
if outputsnapshot.IsPrivateCollection(name) {
|
||||
if store.IsPrivateDerivedCollection(name) {
|
||||
return &validationIssue{Field: field, Message: "internal statistics collection is not accessible"}
|
||||
}
|
||||
label := "collection name"
|
||||
|
||||
@ -26,6 +26,8 @@ import (
|
||||
"time"
|
||||
|
||||
"droboticflow/internal/domain"
|
||||
"droboticflow/internal/productionstats"
|
||||
"droboticflow/internal/queryserving"
|
||||
"droboticflow/internal/store"
|
||||
"go.mongodb.org/mongo-driver/event"
|
||||
"go.mongodb.org/mongo-driver/mongo"
|
||||
@ -96,6 +98,8 @@ func TestPollPoolReadMixLocal(t *testing.T) {
|
||||
pageShape := os.Getenv("POLL_POOL_PAGE_SHAPE") == "1"
|
||||
allowBaselineFailure := os.Getenv("POLL_POOL_ALLOW_BASELINE_FAILURE") == "1"
|
||||
readModels := os.Getenv("POLL_POOL_READMODELS") == "1"
|
||||
queryServing := os.Getenv("POLL_POOL_QUERY_SERVING") == "1"
|
||||
queryPanels := os.Getenv("POLL_POOL_QUERY_PANELS") == "1"
|
||||
|
||||
mode := os.Getenv("POLL_POOL_MODE")
|
||||
if mode == "" {
|
||||
@ -108,6 +112,7 @@ func TestPollPoolReadMixLocal(t *testing.T) {
|
||||
defer cancel()
|
||||
var measured atomic.Bool
|
||||
var commands, mongoBytes, mongoNS, writes atomic.Int64
|
||||
var derivedWrites atomic.Int64
|
||||
var probeMu sync.Mutex
|
||||
pending := map[int64]struct {
|
||||
key string
|
||||
@ -154,10 +159,12 @@ func TestPollPoolReadMixLocal(t *testing.T) {
|
||||
}
|
||||
},
|
||||
Succeeded: func(_ context.Context, e *event.CommandSucceededEvent) {
|
||||
derived := false
|
||||
probeMu.Lock()
|
||||
if entry, ok := pending[e.RequestID]; ok {
|
||||
key := entry.key
|
||||
row := observations[key]
|
||||
derived = strings.HasPrefix(row.Collection, "queryReadModel") || strings.HasPrefix(row.Collection, "annotationOutputSnapshot")
|
||||
row.Count++
|
||||
row.Bytes += int64(len(e.Reply))
|
||||
row.Nanoseconds += e.Duration.Nanoseconds()
|
||||
@ -170,7 +177,11 @@ func TestPollPoolReadMixLocal(t *testing.T) {
|
||||
mongoNS.Add(e.Duration.Nanoseconds())
|
||||
switch e.CommandName {
|
||||
case "insert", "update", "delete", "findAndModify":
|
||||
writes.Add(1)
|
||||
if queryServing && derived {
|
||||
derivedWrites.Add(1)
|
||||
} else {
|
||||
writes.Add(1)
|
||||
}
|
||||
}
|
||||
}
|
||||
}}
|
||||
@ -235,6 +246,12 @@ func TestPollPoolReadMixLocal(t *testing.T) {
|
||||
}
|
||||
const org = "load-org"
|
||||
projects := []string{"load-project-a", "load-project-b"}
|
||||
if os.Getenv("POLL_POOL_QUERY_SERVING_PREPARE") == "1" && !reuseFixture {
|
||||
must(ds.UpsertOrganization(ctx, domain.OrganizationRecord{ID: org, Status: domain.OrganizationStatusActive}))
|
||||
_, err := ds.SaveProductionStatsScope(ctx, org, productionstats.Scope{ProjectIDs: projects, Revision: 1}, 0)
|
||||
must(err)
|
||||
must(ds.EnsureQueryReadModelIndexes(ctx))
|
||||
}
|
||||
if !reuseFixture {
|
||||
for _, p := range projects {
|
||||
must(ds.UpsertProject(ctx, domain.ProjectRecord{ID: p, OrganizationID: org, Name: p}))
|
||||
@ -487,11 +504,13 @@ func TestPollPoolReadMixLocal(t *testing.T) {
|
||||
elapsed := time.Since(start)
|
||||
completedRequests.Add(1)
|
||||
var body struct {
|
||||
Count *int `json:"count"`
|
||||
Counts map[string]int `json:"counts"`
|
||||
Code string `json:"code"`
|
||||
Error string `json:"error"`
|
||||
Items []struct {
|
||||
Summary *productionstats.Summary `json:"summary"`
|
||||
Totals json.RawMessage `json:"totals"`
|
||||
Count *int `json:"count"`
|
||||
Counts map[string]int `json:"counts"`
|
||||
Code string `json:"code"`
|
||||
Error string `json:"error"`
|
||||
Items []struct {
|
||||
ID string `json:"id"`
|
||||
CanClaim *bool `json:"canClaim"`
|
||||
TraceSummary *domain.HumanTraceSummary `json:"traceSummary"`
|
||||
@ -534,6 +553,15 @@ func TestPollPoolReadMixLocal(t *testing.T) {
|
||||
completedErrors.Add(1)
|
||||
}
|
||||
rowCount := len(body.Items) + len(body.Summaries)
|
||||
if body.Summary != nil {
|
||||
rowCount = len(body.Summary.Rows)
|
||||
raw, _ := json.Marshal(body.Summary)
|
||||
ids = append(ids, string(raw))
|
||||
}
|
||||
if strings.HasSuffix(key, ".panel") && len(body.Totals) > 0 {
|
||||
rowCount = 1
|
||||
ids = append(ids, string(body.Totals))
|
||||
}
|
||||
if body.Count != nil {
|
||||
rowCount = *body.Count
|
||||
ids = append(ids, fmt.Sprintf("count:%d", *body.Count))
|
||||
@ -704,8 +732,23 @@ func TestPollPoolReadMixLocal(t *testing.T) {
|
||||
}
|
||||
return result
|
||||
}
|
||||
if queryPanels {
|
||||
original := paths
|
||||
paths = func(a pollLoadActor, all, countCycle bool, scenario int) map[string]string {
|
||||
out := original(a, all, countCycle, scenario)
|
||||
out[a.Role+".panel"] = "/api/dashboard/production-quality?from=2026-10-01&to=2026-11-13"
|
||||
if a.Role == "admin" {
|
||||
out["admin.production"] = "/api/dashboard/production?from=2026-10-01&to=2026-11-13"
|
||||
}
|
||||
return out
|
||||
}
|
||||
}
|
||||
|
||||
fixtureSpec := fmt.Sprintf("isolated-preparation-v5|%d|%d|%d|%d|%d|%d|%d|%t|%t|%t|%s|%s", annotators, quality, acceptance, availablePerStage, runPayloadKiB, taskPayloadKiB, authorHistory, productionShape, readModels, claimCounts, authorizationState, os.Getenv("POLL_POOL_SEARCH_INDEXES"))
|
||||
fixtureSpec += "|admin-tasks=" + os.Getenv("POLL_POOL_ADMIN_TASKS")
|
||||
if os.Getenv("POLL_POOL_QUERY_SERVING_PREPARE") == "1" {
|
||||
fixtureSpec += "|query-serving-preparation-v1"
|
||||
}
|
||||
fixtureDigest := fmt.Sprintf("%x", sha256.Sum256([]byte(fixtureSpec)))
|
||||
if reuseFixture {
|
||||
var manifest struct {
|
||||
@ -742,6 +785,30 @@ func TestPollPoolReadMixLocal(t *testing.T) {
|
||||
if os.Getenv("POLL_POOL_FIXTURE_ONLY") == "1" {
|
||||
return
|
||||
}
|
||||
var preparationCPU float64
|
||||
if queryServing {
|
||||
startCPU, _ := pollLoadProcessCPU()
|
||||
if os.Getenv("QUERY_SERVING_LOCAL_TEST") != "1" {
|
||||
t.Fatal("query acceleration requires isolated local dependencies")
|
||||
}
|
||||
cache, err := queryserving.NewManagedValkey("valkey://127.0.0.1:16379", db.Name())
|
||||
must(err)
|
||||
defer cache.Close()
|
||||
search, err := queryserving.NewTypesense("http://127.0.0.1:18108", os.Getenv("QUERY_TEST_TYPESENSE_API_KEY"))
|
||||
must(err)
|
||||
ds.ConfigureQueryServing(cache, search)
|
||||
builder := store.NewQueryReadBuilder(ds)
|
||||
buildCtx, stop := context.WithCancel(ctx)
|
||||
buildDone := make(chan error, 1)
|
||||
go func() { buildDone <- builder.Run(buildCtx) }()
|
||||
defer func() { stop(); <-buildDone; builder.Close() }()
|
||||
prepare, finish := context.WithTimeout(ctx, 4*time.Minute)
|
||||
err = builder.WaitReady(prepare, org, projects)
|
||||
finish()
|
||||
must(err)
|
||||
endCPU, _ := pollLoadProcessCPU()
|
||||
preparationCPU = endCPU - startCPU
|
||||
}
|
||||
// Fail before timing if permissions or fixtures accidentally produce empty work.
|
||||
for _, a := range actors {
|
||||
for key, path := range paths(a, false, true, 0) {
|
||||
@ -752,7 +819,11 @@ func TestPollPoolReadMixLocal(t *testing.T) {
|
||||
}
|
||||
t.Fatalf("preflight %s status=%d", key, result.Code)
|
||||
}
|
||||
if (a.Role == "annotation" || a.Role == "admin" && os.Getenv("POLL_POOL_ADMIN_TASKS") != "1" || strings.HasSuffix(key, ".available") || strings.HasSuffix(key, ".mine") && !strings.Contains(key, ".acceptance.")) && result.Rows == 0 {
|
||||
// This workload seeds assigned annotation work, but no unassigned
|
||||
// annotation pool unless claim-count coverage was requested. An empty
|
||||
// admin available pool is then the correct fixture result, not failure.
|
||||
requireRows := a.Role == "annotation" || a.Role == "admin" && os.Getenv("POLL_POOL_ADMIN_TASKS") != "1" || a.Role != "admin" && (strings.HasSuffix(key, ".available") || strings.HasSuffix(key, ".mine") && !strings.Contains(key, ".acceptance.")) || a.Role == "admin" && claimCounts && strings.HasSuffix(key, ".available")
|
||||
if requireRows && result.Rows == 0 {
|
||||
t.Fatalf("preflight %s unexpectedly empty for %s", key, a.User.ID)
|
||||
}
|
||||
}
|
||||
@ -922,7 +993,7 @@ func TestPollPoolReadMixLocal(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
report := map[string]any{"reused_prepared_fixture": reuseFixture, "fixture_digest": fixtureDigest, "cold_samples": coldSamples, "slow_search_endpoints": slowSearch, "baseline_failure_collection": allowBaselineFailure, "production_shape": productionShape, "page_shape": pageShape, "fixture_counts": fixtureCounts, "search_indexes": os.Getenv("POLL_POOL_SEARCH_INDEXES") == "1", "read_models": readModels, "search_queries": os.Getenv("POLL_POOL_SEARCH") == "1", "schema": 2, "resource_samples": resourceRows, "process_cpu_seconds": cpuAfter - cpuBefore, "authorization_state": authorizationState, "fixture": "read-mix-v5-isolated-preparation", "available_per_stage": availablePerStage, "task_payload_kib": taskPayloadKiB, "run_payload_kib": runPayloadKiB, "author_history_per_reviewer": authorHistory, "mode": mode, "revision": os.Getenv("POLL_POOL_REVISION"), "actors": totalActors, "actor_counts": map[string]int{"annotation": annotators, "quality": quality, "acceptance": acceptance, "admin": 1}, "claim_counts": claimCounts, "warmup_seconds": warmup, "scheduled_seconds": duration, "actual_seconds": actual.Seconds(), "errors": failures, "failure_samples": failureSamples, "metrics": metrics, "mongo_breakdown": observations, "mongo_commands": commands.Load(), "mongo_reply_bytes": mongoBytes.Load(), "mongo_command_ms": float64(mongoNS.Load()) / 1e6, "mongo_write_commands": writes.Load(), "go_heap_before": memBefore.HeapAlloc, "go_heap_after": memAfter.HeapAlloc, "go_heap_after_gc": memAfterGC.HeapAlloc, "go_gc_cycles": memAfter.NumGC - memBefore.NumGC, "go_allocated_bytes": memAfter.TotalAlloc - memBefore.TotalAlloc, "limitations": []string{"handler timing excludes ingress/network/authentication", "gzip computed separately with fixed Go default compression", "no continuation renew, media or workflow writes", "synthetic fixture; not capacity acceptance", "CPU covers handler and in-process load harness, not Mongo/Pod throttling", "resource heap includes retained benchmark request samples; not standalone service RSS"}, "samples": samples}
|
||||
report := map[string]any{"query_serving": queryServing, "query_preparation_cpu_seconds": preparationCPU, "query_serving_statistics": ds.QueryServingStatistics(), "query_panels": queryPanels, "derived_write_commands": derivedWrites.Load(), "reused_prepared_fixture": reuseFixture, "fixture_digest": fixtureDigest, "cold_samples": coldSamples, "slow_search_endpoints": slowSearch, "baseline_failure_collection": allowBaselineFailure, "production_shape": productionShape, "page_shape": pageShape, "fixture_counts": fixtureCounts, "search_indexes": os.Getenv("POLL_POOL_SEARCH_INDEXES") == "1", "read_models": readModels, "search_queries": os.Getenv("POLL_POOL_SEARCH") == "1", "schema": 2, "resource_samples": resourceRows, "process_cpu_seconds": cpuAfter - cpuBefore, "authorization_state": authorizationState, "fixture": "read-mix-v5-isolated-preparation", "available_per_stage": availablePerStage, "task_payload_kib": taskPayloadKiB, "run_payload_kib": runPayloadKiB, "author_history_per_reviewer": authorHistory, "mode": mode, "revision": os.Getenv("POLL_POOL_REVISION"), "actors": totalActors, "actor_counts": map[string]int{"annotation": annotators, "quality": quality, "acceptance": acceptance, "admin": 1}, "claim_counts": claimCounts, "warmup_seconds": warmup, "scheduled_seconds": duration, "actual_seconds": actual.Seconds(), "errors": failures, "failure_samples": failureSamples, "metrics": metrics, "mongo_breakdown": observations, "mongo_commands": commands.Load(), "mongo_reply_bytes": mongoBytes.Load(), "mongo_command_ms": float64(mongoNS.Load()) / 1e6, "mongo_write_commands": writes.Load(), "go_heap_before": memBefore.HeapAlloc, "go_heap_after": memAfter.HeapAlloc, "go_heap_after_gc": memAfterGC.HeapAlloc, "go_gc_cycles": memAfter.NumGC - memBefore.NumGC, "go_allocated_bytes": memAfter.TotalAlloc - memBefore.TotalAlloc, "limitations": []string{"handler timing excludes ingress/network/authentication", "gzip computed separately with fixed Go default compression", "no continuation renew, media or workflow writes", "synthetic fixture; not capacity acceptance", "CPU covers handler and in-process load harness, not Mongo/Pod throttling", "resource heap includes retained benchmark request samples; not standalone service RSS"}, "samples": samples}
|
||||
encoded, e := json.MarshalIndent(report, "", " ")
|
||||
must(e)
|
||||
must(os.WriteFile(out, encoded, 0600))
|
||||
|
||||
@ -5,6 +5,7 @@ import (
|
||||
"droboticflow/internal/domain"
|
||||
"droboticflow/internal/productionstats"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"sort"
|
||||
"strconv"
|
||||
@ -138,19 +139,42 @@ func (s *server) handleProductionDashboard(w http.ResponseWriter, r *http.Reques
|
||||
return
|
||||
}
|
||||
}
|
||||
var snapshot productionstats.Snapshot
|
||||
if cached, ok := s.store.(interface {
|
||||
LoadProductionStatsCached(context.Context, string, []string) (productionstats.Snapshot, error)
|
||||
responseIdentity := fmt.Sprint(scope.Revision) + ":" + q.From.Format(time.RFC3339) + ":" + q.To.Format(time.RFC3339) + ":" + params.Encode()
|
||||
if prepared, ok := s.store.(interface {
|
||||
PreparedProductionResponse(context.Context, string, []string, string) ([]byte, bool)
|
||||
}); ok {
|
||||
snapshot, err = cached.LoadProductionStatsCached(ctx, org, ids)
|
||||
if data, hit := prepared.PreparedProductionResponse(ctx, org, ids, responseIdentity); hit {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.Header().Set("Cache-Control", "no-store")
|
||||
w.WriteHeader(200)
|
||||
_, _ = w.Write(data)
|
||||
return
|
||||
}
|
||||
}
|
||||
var report productionstats.Report
|
||||
var asOf string
|
||||
if prepared, ok := s.store.(interface {
|
||||
LoadProductionReport(context.Context, string, []string) (productionstats.Report, string, error)
|
||||
}); ok {
|
||||
report, asOf, err = prepared.LoadProductionReport(ctx, org, ids)
|
||||
} else {
|
||||
snapshot, err = s.store.LoadProductionStats(ctx, org, ids)
|
||||
var snapshot productionstats.Snapshot
|
||||
if cached, ok := s.store.(interface {
|
||||
LoadProductionStatsCached(context.Context, string, []string) (productionstats.Snapshot, error)
|
||||
}); ok {
|
||||
snapshot, err = cached.LoadProductionStatsCached(ctx, org, ids)
|
||||
} else {
|
||||
snapshot, err = s.store.LoadProductionStats(ctx, org, ids)
|
||||
}
|
||||
if err == nil {
|
||||
report = productionstats.Build(snapshot)
|
||||
asOf = snapshot.AsOf
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
renderError(w, 500, err)
|
||||
return
|
||||
}
|
||||
report := productionstats.Build(snapshot)
|
||||
summary := productionstats.Summarize(report, q)
|
||||
suppliers, err := s.store.ListSuppliers(ctx, org)
|
||||
if err != nil {
|
||||
@ -196,7 +220,6 @@ func (s *server) handleProductionDashboard(w http.ResponseWriter, r *http.Reques
|
||||
}
|
||||
}
|
||||
}
|
||||
asOf := snapshot.AsOf
|
||||
if asOf == "" {
|
||||
asOf = started.Format(time.RFC3339Nano)
|
||||
}
|
||||
@ -223,5 +246,10 @@ func (s *server) handleProductionDashboard(w http.ResponseWriter, r *http.Reques
|
||||
response["total"] = len(items)
|
||||
response["page"] = page
|
||||
}
|
||||
if prepared, ok := s.store.(interface {
|
||||
RememberProductionResponse(context.Context, string, []string, string, string, any)
|
||||
}); ok {
|
||||
prepared.RememberProductionResponse(ctx, org, ids, responseIdentity, asOf, response)
|
||||
}
|
||||
renderJSON(w, 200, response)
|
||||
}
|
||||
|
||||
76
internal/api/query_serving_test.go
Normal file
76
internal/api/query_serving_test.go
Normal file
@ -0,0 +1,76 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"context"
|
||||
"droboticflow/internal/domain"
|
||||
"droboticflow/internal/productionstats"
|
||||
"droboticflow/internal/store"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
)
|
||||
|
||||
type preparedProductionStore struct {
|
||||
store.Store
|
||||
calls atomic.Int64
|
||||
allowedIdentity string
|
||||
}
|
||||
|
||||
func (s *preparedProductionStore) PreparedProductionResponse(_ context.Context, org string, projects []string, query string) ([]byte, bool) {
|
||||
s.calls.Add(1)
|
||||
key := fmt.Sprint(org, projects, query)
|
||||
if s.allowedIdentity == "" {
|
||||
s.allowedIdentity = key
|
||||
}
|
||||
if key != s.allowedIdentity {
|
||||
return nil, false
|
||||
}
|
||||
return []byte(`{"cachedFixture":true}`), true
|
||||
}
|
||||
|
||||
func TestQueryServingProductionResponseChecksCurrentPermissionBeforeCache(t *testing.T) {
|
||||
f := newCombinedDraftFixture(t)
|
||||
_, err := f.store.SaveProductionStatsScope(f.ctx, domain.DefaultOrganizationID, productionstats.Scope{ProjectIDs: []string{f.task.ProjectID}, Revision: 1}, 0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
s := &preparedProductionStore{Store: f.store}
|
||||
f.handler = NewHandler(s, nil)
|
||||
path := "/api/dashboard/production?from=2026-11-01&to=2026-11-12"
|
||||
r := f.request("GET", path, f.admin, nil)
|
||||
if r.Code != 200 || !strings.Contains(r.Body.String(), "cachedFixture") {
|
||||
t.Fatalf("cache not used %d %s", r.Code, r.Body.String())
|
||||
}
|
||||
before := s.calls.Load()
|
||||
for _, user := range []domain.UserRecord{f.annotator, f.reviewer, {ID: "supplier-manager", OrganizationID: domain.DefaultOrganizationID, SupplierID: f.annotator.SupplierID, Role: domain.UserRoleSupplierManager, Status: domain.UserStatusActive}} {
|
||||
if r = f.request("GET", path, user, nil); r.Code != 403 {
|
||||
t.Fatalf("unauthorized cache read role=%s status=%d", user.Role, r.Code)
|
||||
}
|
||||
}
|
||||
if s.calls.Load() != before {
|
||||
t.Fatal("unauthorized request touched prepared response")
|
||||
}
|
||||
// Revoking a configured project changes identity before any cached body is
|
||||
// served, even when its old generation and cache entry still exist.
|
||||
_, err = f.store.SaveProductionStatsScope(f.ctx, domain.DefaultOrganizationID, productionstats.Scope{ProjectIDs: []string{}, Revision: 2}, 1)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f.handler = NewHandler(s, nil)
|
||||
r = f.request("GET", path, f.admin, nil)
|
||||
if r.Code != 200 || strings.Contains(r.Body.String(), "cachedFixture") {
|
||||
t.Fatal("revoked project reused old cached body", r.Code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestQueryServingPrivateMaterializationsRejectDatabaseBrowser(t *testing.T) {
|
||||
f := newCombinedDraftFixture(t)
|
||||
for _, name := range []string{"queryReadModelControl", "queryReadModelData"} {
|
||||
for _, target := range []string{"platform", "business"} {
|
||||
if r := f.request("GET", "/api/database/collections/"+name+"/documents?target="+target, f.admin, nil); r.Code < 400 {
|
||||
t.Fatalf("private read model leaked: %s %s", name, target)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
86
internal/outputsnapshot/query_shared_cache_test.go
Normal file
86
internal/outputsnapshot/query_shared_cache_test.go
Normal file
@ -0,0 +1,86 @@
|
||||
package outputsnapshot
|
||||
|
||||
import (
|
||||
"context"
|
||||
"droboticflow/internal/queryserving"
|
||||
"droboticflow/internal/workforcestats"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
type queryTestCache struct {
|
||||
mu sync.Mutex
|
||||
data map[string][]byte
|
||||
}
|
||||
|
||||
func (c *queryTestCache) Get(_ context.Context, k string) ([]byte, bool) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
v, ok := c.data[k]
|
||||
return append([]byte(nil), v...), ok
|
||||
}
|
||||
func (c *queryTestCache) Put(_ context.Context, k string, v []byte, _ time.Duration) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
if c.data == nil {
|
||||
c.data = map[string][]byte{}
|
||||
}
|
||||
c.data[k] = append([]byte(nil), v...)
|
||||
}
|
||||
|
||||
type queryCachedBackend struct {
|
||||
MemoryBackend
|
||||
cache queryTestCache
|
||||
blobs atomic.Int64
|
||||
}
|
||||
|
||||
func (b *queryCachedBackend) SharedQueryCache() queryserving.Cache { return &b.cache }
|
||||
func (b *queryCachedBackend) OutputSnapshotBlob(ctx context.Context, g Generation) ([]byte, error) {
|
||||
b.blobs.Add(1)
|
||||
return b.MemoryBackend.OutputSnapshotBlob(ctx, g)
|
||||
}
|
||||
|
||||
func TestQueryServingSharedOutputCacheKeepsScopeLocationAndGeneration(t *testing.T) {
|
||||
now := time.Now().UTC()
|
||||
backend := &queryCachedBackend{}
|
||||
source := func(context.Context, string, []string) (workforcestats.OutputSnapshot, error) {
|
||||
return testData(now), nil
|
||||
}
|
||||
one := NewReader(backend, source)
|
||||
defer one.Close()
|
||||
if _, err := one.Prewarm(t.Context(), "org", []string{"p"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, info := ready(t, one, "")
|
||||
q := workforcestats.OutputQuery{From: now.Add(-24 * time.Hour), To: now.Add(time.Hour), Location: time.UTC, MatchUser: true, ScopeUserID: "u"}
|
||||
r, _, err := one.Query(t.Context(), "org", []string{"p"}, info.ID, q)
|
||||
if err != nil || r.Totals.Submitted.Count != 12 {
|
||||
t.Fatal("initial report", err)
|
||||
}
|
||||
calls := backend.blobs.Load()
|
||||
two := NewReader(backend, source)
|
||||
defer two.Close()
|
||||
q.Location = time.FixedZone("UTC", 0)
|
||||
r, _, err = two.Query(t.Context(), "org", []string{"p"}, info.ID, q)
|
||||
if err != nil || r.Totals.Submitted.Count != 12 || backend.blobs.Load() != calls {
|
||||
t.Fatal("same immutable scope failed cross-reader reuse", err)
|
||||
}
|
||||
r.Tasks = nil
|
||||
r, _, err = two.Query(t.Context(), "org", []string{"p"}, info.ID, q)
|
||||
if err != nil || len(r.Tasks) == 0 {
|
||||
t.Fatal("shared response mutated by another caller")
|
||||
}
|
||||
q.ScopeUserID = "not-authorized-user"
|
||||
r, _, err = two.Query(t.Context(), "org", []string{"p"}, info.ID, q)
|
||||
if err != nil || r.Totals.Submitted.Count != 0 {
|
||||
t.Fatal("personal cache leaked another user's data", err)
|
||||
}
|
||||
if _, _, err = two.Query(t.Context(), "other-org", []string{"p"}, info.ID, q); err != ErrExpired {
|
||||
t.Fatal("foreign organization borrowed generation", err)
|
||||
}
|
||||
if _, _, err = two.Query(t.Context(), "org", []string{"not-visible"}, info.ID, q); err != ErrExpired {
|
||||
t.Fatal("changed project scope borrowed generation", err)
|
||||
}
|
||||
}
|
||||
@ -5,6 +5,7 @@ import (
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"droboticflow/internal/queryserving"
|
||||
"droboticflow/internal/workforcestats"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
@ -43,6 +44,13 @@ func NewReader(b Backend, source Source) *Reader {
|
||||
return &Reader{queryMemo: map[string][]byte{}, querySlots: make(chan struct{}, 2), backend: b, source: source, now: time.Now, ctx: ctx, cancel: cancel, attempts: map[string]time.Time{}, cache: map[string][]byte{}, reads: make(chan struct{}, 2)}
|
||||
}
|
||||
func (r *Reader) Close() { r.mu.Lock(); r.closed = true; r.cancel(); r.mu.Unlock(); r.wg.Wait() }
|
||||
|
||||
// Prewarm uses the existing fenced publication protocol, without a customer
|
||||
// request or mutation. Repeated workers cannot duplicate a successful refresh.
|
||||
func (r *Reader) Prewarm(ctx context.Context, org string, projects []string) (Info, error) {
|
||||
_, info, err := r.resolve(ctx, org, projects, "")
|
||||
return info, err
|
||||
}
|
||||
func scopeKey(org string, projects []string) string {
|
||||
ids := append([]string(nil), projects...)
|
||||
sort.Strings(ids)
|
||||
@ -256,7 +264,7 @@ func (r *Reader) build(key, org string, projects []string) {
|
||||
return
|
||||
}
|
||||
hash := sha256.Sum256(payload)
|
||||
gen := Generation{Metrics: &BuildMetrics{DurationMS: time.Since(started).Milliseconds(), DecodedBytes: decodedBytes, Parts: len(data.Parts), Source: raw.SourceStats}, ID: fmt.Sprintf("of1-%s-%d", lease.Owner, lease.Epoch), Slot: lease.Slot, Epoch: lease.Epoch, AsOf: asOf, SHA256: hex.EncodeToString(hash[:]), Bytes: len(payload)}
|
||||
gen := Generation{Metrics: &BuildMetrics{DurationMS: time.Since(started).Milliseconds(), DecodedBytes: decodedBytes, Parts: len(data.Parts), Source: raw.SourceStats}, ID: GenerationID(lease), Slot: lease.Slot, Epoch: lease.Epoch, AsOf: asOf, SHA256: hex.EncodeToString(hash[:]), Bytes: len(payload)}
|
||||
stage = "publish"
|
||||
published, buildErr = r.backend.PublishOutputSnapshot(ctx, lease, gen, payload, r.now())
|
||||
if published {
|
||||
@ -349,6 +357,17 @@ func (r *Reader) Query(ctx context.Context, org string, projects []string, pin s
|
||||
// generation and location identity participate in the key, including personal
|
||||
// and supplier isolation. No permissions or mutable datasets are cached.
|
||||
q.Now = gen.AsOf
|
||||
var shared queryserving.Cache
|
||||
if provider, ok := r.backend.(interface{ SharedQueryCache() queryserving.Cache }); ok {
|
||||
shared = provider.SharedQueryCache()
|
||||
}
|
||||
location := ""
|
||||
if q.Location != nil {
|
||||
_, fromOffset := q.From.In(q.Location).Zone()
|
||||
_, toOffset := q.To.In(q.Location).Zone()
|
||||
location = fmt.Sprintf("%s:%d:%d", q.Location.String(), fromOffset, toOffset)
|
||||
}
|
||||
sharedKey := queryserving.Key("output-query/v1", scopeKey(org, projects), gen.ID, q, location)
|
||||
keyData, _ := json.Marshal([]any{gen.ID, q, fmt.Sprintf("%p", q.Location)})
|
||||
key := string(keyData)
|
||||
select {
|
||||
@ -385,6 +404,11 @@ func (r *Reader) Query(ctx context.Context, org string, projects []string, pin s
|
||||
if report := r.memoReport(key); report != nil {
|
||||
return answer{report, info}, nil
|
||||
}
|
||||
var sharedReport workforcestats.OutputReport
|
||||
if queryserving.GetJSON(buildCtx, shared, sharedKey, &sharedReport) {
|
||||
r.rememberReport(key, &sharedReport)
|
||||
return answer{&sharedReport, info}, nil
|
||||
}
|
||||
report, actual, err := r.queryResolved(buildCtx, gen, info, pin, q)
|
||||
if err != nil || report == nil {
|
||||
return nil, err
|
||||
@ -392,6 +416,7 @@ func (r *Reader) Query(ctx context.Context, org string, projects []string, pin s
|
||||
// Corrupt-current fallback must not poison the current generation's key.
|
||||
if actual.ID == gen.ID {
|
||||
r.rememberReport(key, report)
|
||||
queryserving.PutJSON(buildCtx, shared, sharedKey, report, MaxAge)
|
||||
}
|
||||
return answer{report, actual}, nil
|
||||
})
|
||||
|
||||
@ -91,7 +91,7 @@ func (r *Reader) buildShards(ctx context.Context, lease Lease, org string, proje
|
||||
}
|
||||
metrics.Source = source
|
||||
metrics.DurationMS = time.Since(started).Milliseconds()
|
||||
g := Generation{ID: fmt.Sprintf("of1-%s-%d", lease.Owner, lease.Epoch), Slot: lease.Slot, Epoch: lease.Epoch, AsOf: asOf, SHA256: digest(payload), Bytes: total, Shards: len(manifest.Shards), ManifestBytes: len(payload), Metrics: metrics}
|
||||
g := Generation{ID: GenerationID(lease), Slot: lease.Slot, Epoch: lease.Epoch, AsOf: asOf, SHA256: digest(payload), Bytes: total, Shards: len(manifest.Shards), ManifestBytes: len(payload), Metrics: metrics}
|
||||
return g, payload, nil
|
||||
}
|
||||
func decodeShard(payload []byte, g Generation) (*workforcestats.OutputDataset, error) {
|
||||
|
||||
@ -6,9 +6,12 @@ import (
|
||||
"context"
|
||||
"droboticflow/internal/workforcestats"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
)
|
||||
|
||||
func GenerationID(l Lease) string { return fmt.Sprintf("of1-%s-%d", l.Owner, l.Epoch) }
|
||||
|
||||
const (
|
||||
Schema = "annotation-output-facts/v5"
|
||||
RefreshInterval = 120 * time.Second
|
||||
|
||||
199
internal/queryserving/cache.go
Normal file
199
internal/queryserving/cache.go
Normal file
@ -0,0 +1,199 @@
|
||||
// Package queryserving contains optional derived-data accelerators. None owns
|
||||
// business state, permissions or task mutations; every failure is a cache miss.
|
||||
package queryserving
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
valkey "github.com/valkey-io/valkey-go"
|
||||
)
|
||||
|
||||
const MaxValueBytes = 4 << 20
|
||||
|
||||
type Cache interface {
|
||||
Get(context.Context, string) ([]byte, bool)
|
||||
Put(context.Context, string, []byte, time.Duration)
|
||||
}
|
||||
|
||||
// Key includes the schema/database namespace supplied by the caller and only
|
||||
// hashes its remaining identity; no customer names or query text become keys.
|
||||
func Key(namespace string, identity ...any) string {
|
||||
b, _ := json.Marshal(identity)
|
||||
sum := sha256.Sum256(b)
|
||||
return namespace + ":" + hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
type Valkey struct {
|
||||
client valkey.Client
|
||||
prefix string
|
||||
until atomic.Int64
|
||||
Hits, Misses, Errors atomic.Uint64
|
||||
}
|
||||
|
||||
func NewValkey(address, namespace string) (*Valkey, error) {
|
||||
opt, err := valkey.ParseURL(address)
|
||||
if err != nil {
|
||||
return nil, errors.New("invalid shared cache configuration")
|
||||
}
|
||||
opt.DisableCache = true // no implicit client-side cache or permission reuse
|
||||
opt.Dialer.Timeout = 200 * time.Millisecond
|
||||
opt.ConnWriteTimeout = 200 * time.Millisecond
|
||||
opt.DisableRetry = true
|
||||
opt.PipelineMultiplex = -1
|
||||
opt.BlockingPoolSize = 4
|
||||
opt.ReadBufferEachConn = 64 << 10
|
||||
opt.WriteBufferEachConn = 64 << 10
|
||||
client, err := valkey.NewClient(opt)
|
||||
if err != nil {
|
||||
return nil, errors.New("shared cache unavailable")
|
||||
}
|
||||
return &Valkey{client: client, prefix: Key("droboticflow-query/v1", namespace)}, nil
|
||||
}
|
||||
|
||||
// ManagedValkey connects asynchronously so an absent cache at startup cannot
|
||||
// delay or fail API availability. Connections are retried without request load.
|
||||
type ManagedValkey struct {
|
||||
mu sync.RWMutex
|
||||
value *Valkey
|
||||
cancel context.CancelFunc
|
||||
done chan struct{}
|
||||
}
|
||||
|
||||
func NewManagedValkey(address, namespace string) (*ManagedValkey, error) {
|
||||
if _, err := valkey.ParseURL(address); err != nil {
|
||||
return nil, errors.New("invalid shared cache configuration")
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
m := &ManagedValkey{cancel: cancel, done: make(chan struct{})}
|
||||
go func() {
|
||||
defer close(m.done)
|
||||
for {
|
||||
v, err := NewValkey(address, namespace)
|
||||
if err == nil {
|
||||
m.mu.Lock()
|
||||
if ctx.Err() == nil {
|
||||
m.value = v
|
||||
m.mu.Unlock()
|
||||
<-ctx.Done()
|
||||
v.Close()
|
||||
return
|
||||
}
|
||||
m.mu.Unlock()
|
||||
v.Close()
|
||||
return
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-time.After(time.Second):
|
||||
}
|
||||
}
|
||||
}()
|
||||
return m, nil
|
||||
}
|
||||
func (m *ManagedValkey) Get(ctx context.Context, key string) ([]byte, bool) {
|
||||
m.mu.RLock()
|
||||
v := m.value
|
||||
m.mu.RUnlock()
|
||||
if v == nil {
|
||||
return nil, false
|
||||
}
|
||||
return v.Get(ctx, key)
|
||||
}
|
||||
func (m *ManagedValkey) Put(ctx context.Context, key string, b []byte, ttl time.Duration) {
|
||||
m.mu.RLock()
|
||||
v := m.value
|
||||
m.mu.RUnlock()
|
||||
if v != nil {
|
||||
v.Put(ctx, key, b, ttl)
|
||||
}
|
||||
}
|
||||
func (m *ManagedValkey) Close() { m.cancel(); <-m.done }
|
||||
func (m *ManagedValkey) Ready() bool { m.mu.RLock(); defer m.mu.RUnlock(); return m.value != nil }
|
||||
|
||||
func (v *Valkey) Close() { v.client.Close() }
|
||||
func (v *Valkey) failed() { v.Errors.Add(1); v.until.Store(time.Now().Add(time.Second).UnixNano()) }
|
||||
func (v *Valkey) Get(ctx context.Context, key string) ([]byte, bool) {
|
||||
parent := ctx
|
||||
if parent.Err() != nil {
|
||||
return nil, false
|
||||
}
|
||||
if time.Now().UnixNano() < v.until.Load() {
|
||||
v.Misses.Add(1)
|
||||
return nil, false
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(ctx, 40*time.Millisecond)
|
||||
defer cancel()
|
||||
b, err := v.client.Do(ctx, v.client.B().Get().Key(v.prefix+":"+key).Build()).AsBytes()
|
||||
if err != nil {
|
||||
if !valkey.IsValkeyNil(err) && parent.Err() == nil {
|
||||
v.failed()
|
||||
}
|
||||
v.Misses.Add(1)
|
||||
return nil, false
|
||||
}
|
||||
if len(b) < 36 || len(b) > MaxValueBytes+36 || string(b[:4]) != "DFQ1" {
|
||||
v.Misses.Add(1)
|
||||
return nil, false
|
||||
}
|
||||
sum := sha256.Sum256(append([]byte(key+"\x00"), b[36:]...))
|
||||
if !bytes.Equal(sum[:], b[4:36]) {
|
||||
v.Misses.Add(1)
|
||||
return nil, false
|
||||
}
|
||||
v.Hits.Add(1)
|
||||
return append([]byte(nil), b[36:]...), true
|
||||
}
|
||||
func (v *Valkey) Put(ctx context.Context, key string, b []byte, ttl time.Duration) {
|
||||
parent := ctx
|
||||
if parent.Err() != nil {
|
||||
return
|
||||
}
|
||||
if len(b) > MaxValueBytes || ttl <= 0 || time.Now().UnixNano() < v.until.Load() {
|
||||
return
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(ctx, 40*time.Millisecond)
|
||||
defer cancel()
|
||||
sum := sha256.Sum256(append([]byte(key+"\x00"), b...))
|
||||
encoded := append(append([]byte("DFQ1"), sum[:]...), b...)
|
||||
if err := v.client.Do(ctx, v.client.B().Set().Key(v.prefix+":"+key).Value(string(encoded)).Px(ttl).Build()).Error(); err != nil {
|
||||
if parent.Err() == nil {
|
||||
v.failed()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func GetJSON(ctx context.Context, c Cache, key string, value any) bool {
|
||||
if c == nil {
|
||||
return false
|
||||
}
|
||||
b, ok := c.Get(ctx, key)
|
||||
return ok && json.Unmarshal(b, value) == nil
|
||||
}
|
||||
func PutJSON(ctx context.Context, c Cache, key string, value any, ttl time.Duration) {
|
||||
if c == nil {
|
||||
return
|
||||
}
|
||||
b, err := json.Marshal(value)
|
||||
if err == nil {
|
||||
c.Put(ctx, key, b, ttl)
|
||||
}
|
||||
}
|
||||
|
||||
func (m *ManagedValkey) Statistics() map[string]uint64 {
|
||||
m.mu.RLock()
|
||||
v := m.value
|
||||
m.mu.RUnlock()
|
||||
if v == nil {
|
||||
return map[string]uint64{}
|
||||
}
|
||||
return map[string]uint64{"hits": v.Hits.Load(), "misses": v.Misses.Load(), "errors": v.Errors.Load()}
|
||||
}
|
||||
138
internal/queryserving/serving_test.go
Normal file
138
internal/queryserving/serving_test.go
Normal file
@ -0,0 +1,138 @@
|
||||
package queryserving
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestQueryServingGramsPreserveSubstringCandidates(t *testing.T) {
|
||||
for _, text := range []string{"FYLX_B07_AZER764008K_20260927_074440/part0002", "İ中文😀_左右/part", "a.b+?x[1]", "abc def ghi"} {
|
||||
r := []rune(text)
|
||||
indexed := map[string]bool{}
|
||||
for _, g := range IndexGrams(text) {
|
||||
indexed[g] = true
|
||||
}
|
||||
for a := range r {
|
||||
for z := a + 1; z <= len(r); z++ {
|
||||
for _, g := range Grams(string(r[a:z])) {
|
||||
if !indexed[g] {
|
||||
t.Fatalf("substring lost %q in %q", r[a:z], text)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if reflect.DeepEqual(Key("namespace-a", "org", []string{"p"}), Key("namespace-b", "org", []string{"p"})) {
|
||||
t.Fatal("namespace collision")
|
||||
}
|
||||
}
|
||||
|
||||
func TestQueryServingTypesenseIncompleteImportAndExportFailClosed(t *testing.T) {
|
||||
mode := "partial"
|
||||
secret := "synthetic-secret-only"
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Header.Get("X-TYPESENSE-API-KEY") != secret {
|
||||
t.Error("missing backend authentication")
|
||||
}
|
||||
switch mode {
|
||||
case "partial":
|
||||
_, _ = w.Write([]byte("{\"success\":true}\n{\"success\":false,\"error\":\"" + secret + "\"}\n"))
|
||||
case "short":
|
||||
_, _ = w.Write([]byte("{\"success\":true}\n"))
|
||||
case "malformed":
|
||||
_, _ = w.Write([]byte("{\"item_id\":\"one\"}\n{bad"))
|
||||
case "failure":
|
||||
w.WriteHeader(503)
|
||||
_, _ = w.Write([]byte(secret))
|
||||
case "ok":
|
||||
_, _ = w.Write([]byte("{\"item_id\":\"one\"}\n{\"item_id\":\"two\"}\n"))
|
||||
}
|
||||
}))
|
||||
defer server.Close()
|
||||
client, _ := NewTypesense(server.URL, secret)
|
||||
docs := []SearchDocument{{ID: "one"}, {ID: "two"}}
|
||||
for _, m := range []string{"partial", "short"} {
|
||||
mode = m
|
||||
if err := client.Import(t.Context(), "dfq_test", docs); err == nil || strings.Contains(err.Error(), secret) {
|
||||
t.Fatal("partial import accepted or secret leaked", err)
|
||||
}
|
||||
}
|
||||
for _, m := range []string{"malformed", "failure"} {
|
||||
mode = m
|
||||
if ids, err := client.Candidates(t.Context(), "dfq_test", "_x/"); err == nil || ids != nil || strings.Contains(err.Error(), secret) {
|
||||
t.Fatal("incomplete export accepted or secret leaked", ids, err)
|
||||
}
|
||||
}
|
||||
mode = "ok"
|
||||
if ids, err := client.Candidates(t.Context(), "dfq_test", "part"); err != nil || !reflect.DeepEqual(ids, []string{"one", "two"}) {
|
||||
t.Fatal(ids, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestQueryServingValkeyRealIsolationCorruptionAndColdRecoveryLocal(t *testing.T) {
|
||||
if os.Getenv("QUERY_SERVING_LOCAL_TEST") != "1" {
|
||||
t.Skip("isolated loopback query dependencies")
|
||||
}
|
||||
namespace := t.Name() + time.Now().Format(time.RFC3339Nano)
|
||||
a, err := NewValkey("valkey://127.0.0.1:16379", namespace)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer a.Close()
|
||||
b, err := NewValkey("valkey://127.0.0.1:16379", namespace)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer b.Close()
|
||||
foreign, err := NewValkey("valkey://127.0.0.1:16379", namespace+"-foreign")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer foreign.Close()
|
||||
a.Put(t.Context(), "generation-one", []byte(`{"value":42}`), time.Second)
|
||||
if data, ok := b.Get(t.Context(), "generation-one"); !ok || string(data) != `{"value":42}` {
|
||||
t.Fatal("cross-instance cache missed")
|
||||
}
|
||||
if _, ok := foreign.Get(t.Context(), "generation-one"); ok {
|
||||
t.Fatal("foreign namespace read cache")
|
||||
}
|
||||
if err = a.client.Do(t.Context(), a.client.B().Set().Key(a.prefix+":generation-one").Value(`{}`).Build()).Error(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, ok := b.Get(t.Context(), "generation-one"); ok {
|
||||
t.Fatal("corrupt valid JSON treated as a valid cache value")
|
||||
}
|
||||
managed, err := NewManagedValkey("valkey://127.0.0.1:16379", namespace)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer managed.Close()
|
||||
deadline := time.Now().Add(2 * time.Second)
|
||||
for !managed.Ready() && time.Now().Before(deadline) {
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
if !managed.Ready() {
|
||||
t.Fatal("managed cache did not connect")
|
||||
}
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
cancel()
|
||||
if _, ok := managed.Get(ctx, "generation-one"); ok {
|
||||
t.Fatal("cancelled cache read succeeded")
|
||||
}
|
||||
managed.Put(t.Context(), "different-query", []byte("[]"), time.Second)
|
||||
data, ok := managed.Get(t.Context(), "different-query")
|
||||
if !ok {
|
||||
t.Fatal("managed cache did not warm")
|
||||
}
|
||||
var value []string
|
||||
if json.Unmarshal(data, &value) != nil {
|
||||
t.Fatal("cache payload changed")
|
||||
}
|
||||
}
|
||||
190
internal/queryserving/typesense.go
Normal file
190
internal/queryserving/typesense.go
Normal file
@ -0,0 +1,190 @@
|
||||
package queryserving
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
type SearchDocument struct {
|
||||
ID string `json:"id"`
|
||||
ItemID string `json:"item_id"`
|
||||
Grams []string `json:"grams"`
|
||||
}
|
||||
|
||||
// Grams are hex-encoded Unicode windows, independent of Typesense's tokenizer,
|
||||
// punctuation/typo/prefix rules. Intersection is only a conservative prefilter;
|
||||
// the existing Go matcher remains the final authority. No query is truncated.
|
||||
func Grams(text string) []string {
|
||||
r := []rune(strings.ToLower(text))
|
||||
width := min(3, len(r))
|
||||
if width == 0 {
|
||||
return nil
|
||||
}
|
||||
set := map[string]bool{}
|
||||
for i := 0; i+width <= len(r); i++ {
|
||||
set[hex.EncodeToString([]byte(string(r[i:i+width])))] = true
|
||||
}
|
||||
out := make([]string, 0, len(set))
|
||||
for s := range set {
|
||||
out = append(out, s)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
func IndexGrams(text string) []string {
|
||||
r := []rune(strings.ToLower(text))
|
||||
set := map[string]bool{}
|
||||
for width := 1; width <= 3; width++ {
|
||||
for i := 0; i+width <= len(r); i++ {
|
||||
set[hex.EncodeToString([]byte(string(r[i:i+width])))] = true
|
||||
}
|
||||
}
|
||||
out := make([]string, 0, len(set))
|
||||
for s := range set {
|
||||
out = append(out, s)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
type Typesense struct {
|
||||
base, key string
|
||||
client *http.Client
|
||||
}
|
||||
type HTTPError struct{ Status int }
|
||||
|
||||
func (e *HTTPError) Error() string { return fmt.Sprintf("search backend status %d", e.Status) }
|
||||
|
||||
func NewTypesense(address, key string) (*Typesense, error) {
|
||||
u, err := url.Parse(address)
|
||||
if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" || u.User != nil || u.RawQuery != "" || u.Fragment != "" || (u.Path != "" && u.Path != "/") || key == "" {
|
||||
return nil, errors.New("invalid search configuration")
|
||||
}
|
||||
return &Typesense{base: strings.TrimRight(address, "/"), key: key, client: &http.Client{Timeout: 30 * time.Second, CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }}}, nil
|
||||
}
|
||||
func (t *Typesense) request(ctx context.Context, method, path string, body []byte) ([]byte, error) {
|
||||
r, err := http.NewRequestWithContext(ctx, method, t.base+path, bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return nil, errors.New("search request invalid")
|
||||
}
|
||||
r.Header.Set("X-TYPESENSE-API-KEY", t.key)
|
||||
r.Header.Set("Content-Type", "application/json")
|
||||
res, err := t.client.Do(r)
|
||||
if err != nil {
|
||||
return nil, errors.New("search backend unavailable")
|
||||
}
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode < 200 || res.StatusCode >= 300 {
|
||||
return nil, &HTTPError{Status: res.StatusCode}
|
||||
}
|
||||
b, err := io.ReadAll(io.LimitReader(res.Body, 32<<20+1))
|
||||
if err != nil || len(b) > 32<<20 {
|
||||
return nil, errors.New("search response incomplete")
|
||||
}
|
||||
return b, nil
|
||||
}
|
||||
func validCollection(c string) bool {
|
||||
if !strings.HasPrefix(c, "dfq_") || len(c) > 120 {
|
||||
return false
|
||||
}
|
||||
for _, r := range c {
|
||||
if r != '_' && (r < 'a' || r > 'z') && (r < '0' || r > '9') {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
func (t *Typesense) Create(ctx context.Context, c string) error {
|
||||
if !validCollection(c) {
|
||||
return errors.New("invalid search collection")
|
||||
}
|
||||
b, _ := json.Marshal(map[string]any{"name": c, "fields": []map[string]any{{"name": "item_id", "type": "string", "index": false}, {"name": "grams", "type": "string[]", "facet": true}}})
|
||||
_, err := t.request(ctx, "POST", "/collections", b)
|
||||
return err
|
||||
}
|
||||
func (t *Typesense) Import(ctx context.Context, c string, docs []SearchDocument) error {
|
||||
if !validCollection(c) {
|
||||
return errors.New("invalid search collection")
|
||||
}
|
||||
var data bytes.Buffer
|
||||
for _, d := range docs {
|
||||
if err := json.NewEncoder(&data).Encode(d); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
b, err := t.request(ctx, "POST", "/collections/"+c+"/documents/import?action=create", data.Bytes())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
s := bufio.NewScanner(bytes.NewReader(b))
|
||||
s.Buffer(make([]byte, 4096), 1<<20)
|
||||
count := 0
|
||||
for s.Scan() {
|
||||
var result struct {
|
||||
Success bool `json:"success"`
|
||||
}
|
||||
if json.Unmarshal(s.Bytes(), &result) != nil || !result.Success {
|
||||
return errors.New("search import incomplete")
|
||||
}
|
||||
count++
|
||||
}
|
||||
if s.Err() != nil || count != len(docs) {
|
||||
return errors.New("search import response incomplete")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
func (t *Typesense) Candidates(ctx context.Context, c string, terms ...string) ([]string, error) {
|
||||
if !validCollection(c) {
|
||||
return nil, errors.New("invalid search collection")
|
||||
}
|
||||
filters := []string{}
|
||||
for _, term := range terms {
|
||||
for _, g := range Grams(term) {
|
||||
filters = append(filters, "grams:=["+g+"]")
|
||||
}
|
||||
}
|
||||
params := url.Values{"include_fields": {"item_id"}}
|
||||
if len(filters) > 0 {
|
||||
params.Set("filter_by", strings.Join(filters, " && "))
|
||||
}
|
||||
b, err := t.request(ctx, "GET", "/collections/"+c+"/documents/export?"+params.Encode(), nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
s := bufio.NewScanner(bytes.NewReader(b))
|
||||
s.Buffer(make([]byte, 4096), 1<<20)
|
||||
ids := []string{}
|
||||
for s.Scan() {
|
||||
var d SearchDocument
|
||||
if json.Unmarshal(s.Bytes(), &d) != nil || d.ItemID == "" {
|
||||
return nil, errors.New("search candidates incomplete")
|
||||
}
|
||||
ids = append(ids, d.ItemID)
|
||||
}
|
||||
if s.Err() != nil {
|
||||
return nil, errors.New("search candidates incomplete")
|
||||
}
|
||||
return ids, nil
|
||||
}
|
||||
func (t *Typesense) Delete(ctx context.Context, c string) error {
|
||||
if !validCollection(c) {
|
||||
return errors.New("invalid search collection")
|
||||
}
|
||||
_, err := t.request(ctx, "DELETE", "/collections/"+c, nil)
|
||||
var status *HTTPError
|
||||
if errors.As(err, &status) && status.Status == 404 {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
@ -157,6 +157,29 @@ func adminPoolRowMatches(row adminPoolRow, q AdminAnnotationPoolQuery) bool {
|
||||
if q.Search == "" && q.Capture == "" {
|
||||
return true
|
||||
}
|
||||
values := adminPoolCaptureValues(row)
|
||||
if q.Capture != "" {
|
||||
found := false
|
||||
for _, v := range values {
|
||||
if strings.EqualFold(strings.TrimSpace(v), q.Capture) {
|
||||
found = true
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
return false
|
||||
}
|
||||
}
|
||||
if q.Search != "" && !strings.Contains(adminPoolSearchText(row), strings.ToLower(q.Search)) {
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
func adminPoolCaptureValues(row adminPoolRow) []string {
|
||||
var task domain.AnnotationTaskRecord
|
||||
if len(row.Tasks) > 0 {
|
||||
task = row.Tasks[0]
|
||||
}
|
||||
|
||||
values := []string{}
|
||||
if len(row.Captures) > 0 {
|
||||
c := row.Captures[0]
|
||||
@ -180,29 +203,18 @@ func adminPoolRowMatches(row adminPoolRow, q AdminAnnotationPoolQuery) bool {
|
||||
if len(row.Runs) > 0 {
|
||||
values = append(values, domain.CaptureDirectoriesFromRun(row.Runs[0])...)
|
||||
}
|
||||
if q.Capture != "" {
|
||||
found := false
|
||||
for _, v := range values {
|
||||
if strings.EqualFold(strings.TrimSpace(v), q.Capture) {
|
||||
found = true
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
return false
|
||||
}
|
||||
}
|
||||
if q.Search != "" {
|
||||
if len(row.Projects) > 0 {
|
||||
values = append(values, row.Projects[0].Name)
|
||||
} else if len(row.Runs) > 0 {
|
||||
values = append(values, row.Runs[0].ProjectName)
|
||||
}
|
||||
if !strings.Contains(strings.ToLower(strings.Join(values, " ")), strings.ToLower(q.Search)) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
return values
|
||||
}
|
||||
func adminPoolSearchText(row adminPoolRow) string {
|
||||
values := adminPoolCaptureValues(row)
|
||||
if len(row.Projects) > 0 {
|
||||
values = append(values, row.Projects[0].Name)
|
||||
} else if len(row.Runs) > 0 {
|
||||
values = append(values, row.Runs[0].ProjectName)
|
||||
}
|
||||
return strings.ToLower(strings.Join(values, " "))
|
||||
}
|
||||
|
||||
func (s *MemoryStore) adminPoolRowsLocked(org string, q AdminAnnotationPoolQuery) []adminPoolRow {
|
||||
rows := []adminPoolRow{}
|
||||
for _, i := range s.humanItems {
|
||||
@ -288,7 +300,14 @@ func (s *MongoStore) scanAdminAnnotationPool(ctx context.Context, org string, q
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(ctx, 8*time.Second)
|
||||
defer cancel()
|
||||
cur, err := s.humanItems.Aggregate(ctx, adminPoolPagePipeline(org, q), options.Aggregate().SetAllowDiskUse(false).SetMaxTime(8*time.Second).SetBatchSize(200))
|
||||
pipeline := adminPoolPagePipeline(org, q)
|
||||
if ids, ready := s.querySearchCandidates(ctx, org, q.Search, q.Capture); ready {
|
||||
if len(ids) == 0 {
|
||||
return nil
|
||||
}
|
||||
pipeline[0] = bson.D{{Key: "$match", Value: bson.M{"$and": bson.A{pipeline[0][0].Value, bson.M{"id": bson.M{"$in": ids}}}}}}
|
||||
}
|
||||
cur, err := s.humanItems.Aggregate(ctx, pipeline, options.Aggregate().SetAllowDiskUse(false).SetMaxTime(8*time.Second).SetBatchSize(200))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"droboticflow/internal/domain"
|
||||
"droboticflow/internal/outputsnapshot"
|
||||
"droboticflow/internal/queryserving"
|
||||
"fmt"
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
"go.mongodb.org/mongo-driver/mongo"
|
||||
@ -113,6 +114,12 @@ func (s *MongoStore) FailOutputSnapshot(ctx context.Context, l outputsnapshot.Le
|
||||
return err
|
||||
}
|
||||
func (s *MongoStore) OutputSnapshotBlob(ctx context.Context, g outputsnapshot.Generation) ([]byte, error) {
|
||||
key := queryserving.Key("output-blob/v1", s.db.Name(), g.ID, g.Slot, g.Epoch, g.SHA256)
|
||||
if cache := s.SharedQueryCache(); cache != nil {
|
||||
if b, hit := cache.Get(ctx, key); hit {
|
||||
return b, nil
|
||||
}
|
||||
}
|
||||
var doc struct {
|
||||
Data []byte `bson:"data"`
|
||||
}
|
||||
@ -120,6 +127,11 @@ func (s *MongoStore) OutputSnapshotBlob(ctx context.Context, g outputsnapshot.Ge
|
||||
if err == mongo.ErrNoDocuments {
|
||||
err = outputsnapshot.ErrExpired
|
||||
}
|
||||
if err == nil {
|
||||
if cache := s.SharedQueryCache(); cache != nil {
|
||||
cache.Put(ctx, key, doc.Data, outputsnapshot.MaxAge)
|
||||
}
|
||||
}
|
||||
return doc.Data, err
|
||||
}
|
||||
|
||||
@ -143,9 +155,20 @@ func (s *MongoStore) StageOutputSnapshotShard(ctx context.Context, l outputsnaps
|
||||
// Immutable: never overwrite a staged shard.
|
||||
id := outputShardID(l.Slot, l.Epoch, index)
|
||||
_, err = s.outputBlobs().InsertOne(ctx, bson.M{"_id": id, "data": data})
|
||||
if err == nil {
|
||||
if cache := s.SharedQueryCache(); cache != nil {
|
||||
cache.Put(ctx, queryserving.Key("output-shard/v1", s.db.Name(), outputsnapshot.GenerationID(l), l.Slot, l.Epoch, index), data, outputsnapshot.MaxAge)
|
||||
}
|
||||
}
|
||||
return err == nil, err
|
||||
}
|
||||
func (s *MongoStore) OutputSnapshotShard(ctx context.Context, g outputsnapshot.Generation, index int) ([]byte, error) {
|
||||
key := queryserving.Key("output-shard/v1", s.db.Name(), g.ID, g.Slot, g.Epoch, index)
|
||||
if cache := s.SharedQueryCache(); cache != nil {
|
||||
if b, hit := cache.Get(ctx, key); hit {
|
||||
return b, nil
|
||||
}
|
||||
}
|
||||
var doc struct {
|
||||
Data []byte `bson:"data"`
|
||||
}
|
||||
@ -153,6 +176,11 @@ func (s *MongoStore) OutputSnapshotShard(ctx context.Context, g outputsnapshot.G
|
||||
if err == mongo.ErrNoDocuments {
|
||||
err = outputsnapshot.ErrExpired
|
||||
}
|
||||
if err == nil {
|
||||
if cache := s.SharedQueryCache(); cache != nil {
|
||||
cache.Put(ctx, key, doc.Data, outputsnapshot.MaxAge)
|
||||
}
|
||||
}
|
||||
return doc.Data, err
|
||||
}
|
||||
func (s *MemoryStore) StageOutputSnapshotShard(ctx context.Context, l outputsnapshot.Lease, i int, b []byte, t time.Time) (bool, error) {
|
||||
|
||||
@ -46,6 +46,9 @@ func humanPoolSearchRowMatches(row adminPoolRow, q domain.HumanItemQuery) bool {
|
||||
if adminPoolRowMatches(row, AdminAnnotationPoolQuery{Search: q.Search}) {
|
||||
return true
|
||||
}
|
||||
return strings.Contains(humanPoolFallbackSearchText(row), strings.ToLower(q.Search))
|
||||
}
|
||||
func humanPoolFallbackSearchText(row adminPoolRow) string {
|
||||
values := []string{row.ID, row.LegacyTaskID, row.Subject.ItemKey, row.Subject.ArtifactID}
|
||||
for _, t := range row.Tasks {
|
||||
values = append(values, t.ArtifactID, t.SourceVideo)
|
||||
@ -76,7 +79,7 @@ func humanPoolSearchRowMatches(row adminPoolRow, q domain.HumanItemQuery) bool {
|
||||
values = append(values, capture+"/"+match[1])
|
||||
}
|
||||
}
|
||||
return strings.Contains(strings.ToLower(strings.Join(values, " ")), strings.ToLower(q.Search))
|
||||
return strings.ToLower(strings.Join(values, " "))
|
||||
}
|
||||
|
||||
// Stream metadata through the existing search rules before applying the page
|
||||
@ -121,6 +124,12 @@ func (s *MongoStore) prepareHumanPoolSearchFilter(ctx context.Context, org strin
|
||||
}
|
||||
filter = bson.M{"$and": bson.A{filter, visibility}}
|
||||
}
|
||||
if ids, ready := s.querySearchCandidates(ctx, org, q.Search, q.Capture); ready {
|
||||
if len(ids) == 0 {
|
||||
return nil, true, nil
|
||||
}
|
||||
return bson.M{"$and": bson.A{filter, bson.M{"id": bson.M{"$in": ids}}}}, false, nil
|
||||
}
|
||||
candidates, err := s.humanPoolSearchCandidates(ctx, org, q, filter)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
|
||||
@ -3266,7 +3266,7 @@ func (store *MemoryStore) GetDatabaseOverview(context.Context) (domain.DatabaseO
|
||||
Collections: make([]domain.DatabaseCollectionSummary, 0, len(names)),
|
||||
}
|
||||
for _, name := range names {
|
||||
if outputsnapshot.IsPrivateCollection(name) {
|
||||
if IsPrivateDerivedCollection(name) {
|
||||
continue
|
||||
}
|
||||
documents := collectionsMap[name]
|
||||
@ -3283,7 +3283,7 @@ func (store *MemoryStore) GetDatabaseOverview(context.Context) (domain.DatabaseO
|
||||
}
|
||||
|
||||
func (store *MemoryStore) QueryDatabaseCollection(_ context.Context, input domain.DatabaseQueryInput) (domain.DatabaseQueryResult, error) {
|
||||
if outputsnapshot.IsPrivateCollection(input.Collection) {
|
||||
if IsPrivateDerivedCollection(input.Collection) {
|
||||
return domain.DatabaseQueryResult{}, fmt.Errorf("collection %s not found", input.Collection)
|
||||
}
|
||||
store.mu.Lock()
|
||||
|
||||
@ -12,7 +12,6 @@ import (
|
||||
"time"
|
||||
|
||||
"droboticflow/internal/domain"
|
||||
"droboticflow/internal/outputsnapshot"
|
||||
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
"go.mongodb.org/mongo-driver/bson/primitive"
|
||||
@ -21,6 +20,7 @@ import (
|
||||
)
|
||||
|
||||
type MongoStore struct {
|
||||
queryServing *queryServingOptions
|
||||
productionReadFlights singleflight.Group
|
||||
claimReadFlights singleflight.Group
|
||||
poolSearchReadFlights singleflight.Group
|
||||
@ -2808,7 +2808,7 @@ func (store *MongoStore) GetDatabaseOverview(ctx context.Context) (domain.Databa
|
||||
Collections: make([]domain.DatabaseCollectionSummary, 0, len(names)),
|
||||
}
|
||||
for _, name := range names {
|
||||
if outputsnapshot.IsPrivateCollection(name) {
|
||||
if IsPrivateDerivedCollection(name) {
|
||||
continue
|
||||
}
|
||||
summary, err := mongoBrowserCollectionSummary(ctx, store.db.Collection(name), name)
|
||||
@ -2825,7 +2825,7 @@ func (store *MongoStore) QueryDatabaseCollection(ctx context.Context, input doma
|
||||
if collectionName == "" {
|
||||
return domain.DatabaseQueryResult{}, fmt.Errorf("collection is required")
|
||||
}
|
||||
if outputsnapshot.IsPrivateCollection(collectionName) {
|
||||
if IsPrivateDerivedCollection(collectionName) {
|
||||
return domain.DatabaseQueryResult{}, fmt.Errorf("collection %s not found", collectionName)
|
||||
}
|
||||
limit := input.Limit
|
||||
|
||||
9
internal/store/private_derived_collections.go
Normal file
9
internal/store/private_derived_collections.go
Normal file
@ -0,0 +1,9 @@
|
||||
package store
|
||||
|
||||
import "droboticflow/internal/outputsnapshot"
|
||||
|
||||
// Derived generations contain facts from multiple authorization scopes and
|
||||
// must only be served through their permission-aware reporting/search readers.
|
||||
func IsPrivateDerivedCollection(name string) bool {
|
||||
return outputsnapshot.IsPrivateCollection(name) || name == "queryReadModelControl" || name == "queryReadModelData"
|
||||
}
|
||||
84
internal/store/query_model_maintenance.go
Normal file
84
internal/store/query_model_maintenance.go
Normal file
@ -0,0 +1,84 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
"go.mongodb.org/mongo-driver/mongo"
|
||||
"go.mongodb.org/mongo-driver/mongo/options"
|
||||
"time"
|
||||
)
|
||||
|
||||
// This is an explicit derived-data index operation, never API startup behavior.
|
||||
// Production execution requires its own reviewed migration/index authorization.
|
||||
func (s *MongoStore) EnsureQueryReadModelIndexes(ctx context.Context) error {
|
||||
_, err := s.queryData().Indexes().CreateOne(ctx, mongo.IndexModel{Keys: bson.D{{Key: "generation", Value: 1}, {Key: "organizationId", Value: 1}, {Key: "itemId", Value: 1}}, Options: options.Index().SetName("query_model_rows_v1")})
|
||||
return err
|
||||
}
|
||||
func (s *MongoStore) cleanQueryModelOrphans(ctx context.Context) error {
|
||||
var state struct {
|
||||
Heads map[string]QueryModelHead `bson:"heads"`
|
||||
}
|
||||
if err := s.queryControl().FindOne(ctx, bson.M{"_id": "global"}).Decode(&state); err != nil {
|
||||
return err
|
||||
}
|
||||
keep := []string{}
|
||||
for _, h := range state.Heads {
|
||||
keep = append(keep, h.Generation)
|
||||
}
|
||||
filter := bson.M{"generation": bson.M{"$nin": keep}, "createdAt": bson.M{"$lt": time.Now().UTC().Add(-15 * time.Minute)}}
|
||||
// Only this database's registered, unreachable search generations are
|
||||
// eligible; other deployments and current generations remain untouched.
|
||||
cur, err := s.queryData().Find(ctx, bson.M{"$and": bson.A{filter, bson.M{"kind": "search_collection"}}}, options.Find().SetProjection(bson.M{"_id": 1, "collection": 1}))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer cur.Close(ctx)
|
||||
for cur.Next(ctx) {
|
||||
var row struct {
|
||||
ID string `bson:"_id"`
|
||||
Collection string `bson:"collection"`
|
||||
}
|
||||
if err = cur.Decode(&row); err != nil {
|
||||
return err
|
||||
}
|
||||
if s.queryServing != nil && s.queryServing.search != nil {
|
||||
if err = s.queryServing.search.Delete(ctx, row.Collection); err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
if err = cur.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = s.queryData().DeleteMany(ctx, filter)
|
||||
return err
|
||||
}
|
||||
|
||||
// Obsolete configured scopes cannot pin an unbounded history of generations.
|
||||
// Only the current builder may change this derived manifest directory.
|
||||
func (s *MongoStore) pruneQueryModelScopes(ctx context.Context, l queryModelLease, scopes map[string][]string) error {
|
||||
var st struct {
|
||||
Heads map[string]QueryModelHead `bson:"heads"`
|
||||
}
|
||||
if err := s.queryControl().FindOne(ctx, queryLeaseFilter(l)).Decode(&st); err != nil {
|
||||
return err
|
||||
}
|
||||
keep := map[string]bool{}
|
||||
for org, projects := range scopes {
|
||||
keep[queryModelKey("production", org, projects)] = true
|
||||
keep[queryModelKey("search", org, nil)] = true
|
||||
}
|
||||
unset := bson.M{}
|
||||
for key := range st.Heads {
|
||||
if !keep[key] {
|
||||
unset["heads."+key] = ""
|
||||
}
|
||||
}
|
||||
if len(unset) == 0 {
|
||||
return nil
|
||||
}
|
||||
_, err := s.queryControl().UpdateOne(ctx, queryLeaseFilter(l), bson.M{"$unset": unset})
|
||||
return err
|
||||
}
|
||||
384
internal/store/query_read_builder.go
Normal file
384
internal/store/query_read_builder.go
Normal file
@ -0,0 +1,384 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"droboticflow/internal/outputsnapshot"
|
||||
"droboticflow/internal/productionstats"
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
"go.mongodb.org/mongo-driver/bson/primitive"
|
||||
"go.mongodb.org/mongo-driver/mongo"
|
||||
"go.mongodb.org/mongo-driver/mongo/options"
|
||||
)
|
||||
|
||||
var querySourceCollections = []string{"organizations", "projects", "runs", "captureRecords", "humanItems", "humanBatches", "annotationTasks", "annotationWorklogEvents", "humanEvents", "deliveryPackages", "annotationRevisions", "sliceRevisions", "annotationReviews", "users", "suppliers", "productionStatsScopes"}
|
||||
|
||||
// QueryReadBuilder runs only in a dedicated, explicitly configured process.
|
||||
// It uses the existing source loaders and publication protocol, never claims or
|
||||
// mutates a business task. Restart starts a fresh stream BEFORE the full rebuild.
|
||||
type QueryReadBuilder struct {
|
||||
store *MongoStore
|
||||
mu sync.Mutex
|
||||
dirty map[string]bool
|
||||
all bool
|
||||
reader *outputsnapshot.Reader
|
||||
searchDirty map[string]bool
|
||||
allSearch bool
|
||||
sourceDigests map[string]querySourceDigest
|
||||
}
|
||||
|
||||
func NewQueryReadBuilder(s *MongoStore) *QueryReadBuilder {
|
||||
r := outputsnapshot.NewStreamReader(s, s.StreamAnnotationOutput)
|
||||
return &QueryReadBuilder{store: s, dirty: map[string]bool{}, all: true, reader: r, searchDirty: map[string]bool{}, allSearch: true, sourceDigests: map[string]querySourceDigest{}}
|
||||
}
|
||||
func (b *QueryReadBuilder) Close() { b.reader.Close() }
|
||||
|
||||
// WaitReady is for explicit prewarming and isolated acceptance, not HTTP reads.
|
||||
func (b *QueryReadBuilder) WaitReady(ctx context.Context, org string, projects []string) error {
|
||||
tick := time.NewTicker(100 * time.Millisecond)
|
||||
defer tick.Stop()
|
||||
for {
|
||||
_, production := b.store.readQueryModelHead(ctx, queryModelKey("production", org, projects), 5*time.Minute)
|
||||
st, err := b.store.querySearchState(ctx)
|
||||
h, exists := st.Heads[queryModelKey("search", org, nil)]
|
||||
search := err == nil && exists && h.Generation != "" && h.Revision == st.Revisions[queryModelKey("search", org, nil)] && h.GlobalRevision == st.Global && time.Since(st.ObservedAt) >= 0 && time.Since(st.ObservedAt) < 5*time.Second
|
||||
output := len(projects) == 0
|
||||
if !output {
|
||||
health, e := b.reader.Inspect(ctx, org, projects)
|
||||
output = e == nil && health.Available
|
||||
}
|
||||
if production && search && output {
|
||||
return nil
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-tick.C:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (b *QueryReadBuilder) scopes(ctx context.Context) (map[string][]string, error) {
|
||||
// Public ListOrganizations also bootstraps a default business organization;
|
||||
// the read builder must enumerate without that write side effect.
|
||||
cur, err := b.store.organizations.Find(ctx, bson.M{}, options.Find().SetProjection(bson.M{"id": 1, "_id": 0}))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer cur.Close(ctx)
|
||||
var orgs []struct {
|
||||
ID string `bson:"id"`
|
||||
}
|
||||
if err = cur.All(ctx, &orgs); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := map[string][]string{}
|
||||
for _, org := range orgs {
|
||||
scope, err := b.store.GetProductionStatsScope(ctx, org.ID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
projects, err := b.store.ListProjectsByOrganization(ctx, org.ID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
selected := map[string]bool{}
|
||||
for _, id := range scope.ProjectIDs {
|
||||
selected[id] = true
|
||||
}
|
||||
ids := []string{}
|
||||
for _, p := range projects {
|
||||
if selected[p.ID] {
|
||||
ids = append(ids, p.ID)
|
||||
}
|
||||
}
|
||||
out[org.ID] = ids
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
func (b *QueryReadBuilder) rebuild(ctx context.Context, l queryModelLease, org string, projects []string) error {
|
||||
started := time.Now()
|
||||
var source struct {
|
||||
Revision int64 `bson:"productionRevision"`
|
||||
}
|
||||
if err := b.store.queryControl().FindOne(ctx, bson.M{"_id": "global"}, options.FindOne().SetProjection(bson.M{"productionRevision": 1})).Decode(&source); err != nil {
|
||||
return err
|
||||
}
|
||||
buildCtx, finish := context.WithTimeout(ctx, 5*time.Minute)
|
||||
defer finish()
|
||||
snapshot, err := b.store.loadProductionStats(buildCtx, org, projects)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
asOf, err := time.Parse(time.RFC3339Nano, snapshot.AsOf)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
key := queryModelKey("production", org, projects)
|
||||
head, err := b.store.stageQueryModel(ctx, key, productionstats.Build(snapshot), asOf)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
head.SourceRevision = source.Revision
|
||||
if err = b.store.publishQueryModelHead(ctx, l, key, head); err != nil {
|
||||
return err
|
||||
}
|
||||
if len(projects) > 0 {
|
||||
if _, err = b.reader.Prewarm(ctx, org, projects); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
log.Printf("query_model_build result=complete duration_ms=%d", time.Since(started).Milliseconds())
|
||||
return nil
|
||||
}
|
||||
|
||||
// RunOnce is for bounded initial prewarming and isolated tests, not a freshness
|
||||
// substitute for Run. After the heartbeat expires search falls back to old reads.
|
||||
func (b *QueryReadBuilder) RunOnce(ctx context.Context) error {
|
||||
l, ok, err := b.store.acquireQueryModelLease(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !ok {
|
||||
return errors.New("query builder already active")
|
||||
}
|
||||
return b.withLease(ctx, l, func(ctx context.Context) error {
|
||||
scopes, err := b.scopes(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for org, ids := range scopes {
|
||||
if err = b.rebuild(ctx, l, org, ids); err != nil {
|
||||
return err
|
||||
}
|
||||
if err = b.store.buildQuerySearchModel(ctx, l, org); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
func (b *QueryReadBuilder) withLease(ctx context.Context, l queryModelLease, run func(context.Context) error) error {
|
||||
ctx, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
defer close(done)
|
||||
tick := time.NewTicker(10 * time.Second)
|
||||
defer tick.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-tick.C:
|
||||
if b.store.renewQueryModelLease(ctx, l) != nil {
|
||||
cancel()
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
err := run(ctx)
|
||||
cancel()
|
||||
<-done
|
||||
cleanup, stop := context.WithTimeout(context.Background(), time.Second)
|
||||
defer stop()
|
||||
_, _ = b.store.queryControl().UpdateOne(cleanup, queryLeaseFilter(l), bson.M{"$set": bson.M{"owner": "", "until": time.Now().UTC()}})
|
||||
return err
|
||||
}
|
||||
|
||||
func (b *QueryReadBuilder) Run(ctx context.Context) error {
|
||||
b.mu.Lock()
|
||||
b.all, b.allSearch = true, true
|
||||
b.sourceDigests = map[string]querySourceDigest{}
|
||||
b.mu.Unlock()
|
||||
l, ok, err := b.store.acquireQueryModelLease(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !ok {
|
||||
return errors.New("query builder already active")
|
||||
}
|
||||
return b.withLease(ctx, l, func(ctx context.Context) error {
|
||||
// No fullDocument lookup: projecting only event identity avoids copying
|
||||
// annotation bodies, credentials or full video payloads on each save.
|
||||
pipeline := mongo.Pipeline{{{Key: "$match", Value: bson.M{"ns.coll": bson.M{"$in": querySourceCollections}}}}, {{Key: "$project", Value: bson.M{"_id": 1, "ns": 1, "documentKey": 1, "operationType": 1, "clusterTime": 1}}}}
|
||||
stream, err := b.store.db.Watch(ctx, pipeline, options.ChangeStream().SetMaxAwaitTime(time.Second))
|
||||
if err != nil {
|
||||
return fmt.Errorf("query builder needs available Mongo change streams: %w", err)
|
||||
}
|
||||
defer stream.Close(context.Background())
|
||||
// Seed before consuming: events already captured by the open stream
|
||||
// close the initial snapshot/change gap without retaining customer data.
|
||||
seedCtx, seedCancel := context.WithTimeout(ctx, 2*time.Minute)
|
||||
err = b.seedSourceDigests(seedCtx)
|
||||
seedCancel()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
watchCtx, stop := context.WithCancel(ctx)
|
||||
defer stop()
|
||||
watchDone := make(chan error, 1)
|
||||
watchExited := make(chan struct{})
|
||||
go func() {
|
||||
defer close(watchExited)
|
||||
lastHeartbeat := time.Time{}
|
||||
pendingChange := false
|
||||
for watchCtx.Err() == nil {
|
||||
observed := time.Now().UTC()
|
||||
if stream.TryNext(watchCtx) {
|
||||
var event struct {
|
||||
NS struct {
|
||||
Collection string `bson:"coll"`
|
||||
} `bson:"ns"`
|
||||
Operation string `bson:"operationType"`
|
||||
Key bson.M `bson:"documentKey"`
|
||||
ClusterTime primitive.Timestamp `bson:"clusterTime"`
|
||||
}
|
||||
if e := stream.Decode(&event); e != nil {
|
||||
watchDone <- e
|
||||
return
|
||||
}
|
||||
pendingChange = true
|
||||
if event.ClusterTime.T > 0 {
|
||||
observed = time.Unix(int64(event.ClusterTime.T), 0).UTC()
|
||||
}
|
||||
if e := b.observeQuerySourceEvent(watchCtx, l, event.NS.Collection, event.Operation, event.Key); e != nil {
|
||||
watchDone <- e
|
||||
return
|
||||
}
|
||||
} else if e := stream.Err(); e != nil {
|
||||
watchDone <- e
|
||||
return
|
||||
}
|
||||
if time.Since(lastHeartbeat) < time.Second {
|
||||
continue
|
||||
}
|
||||
if e := b.recordObserved(watchCtx, l, observed, pendingChange); e != nil {
|
||||
watchDone <- e
|
||||
return
|
||||
}
|
||||
pendingChange = false
|
||||
lastHeartbeat = time.Now()
|
||||
}
|
||||
watchDone <- watchCtx.Err()
|
||||
}()
|
||||
defer func() { stop(); <-watchExited }()
|
||||
tick := time.NewTicker(time.Second)
|
||||
defer tick.Stop()
|
||||
pendingStats, pendingSearch := map[string]bool{}, map[string]bool{}
|
||||
lastStats := map[string]time.Time{}
|
||||
var scopes map[string][]string
|
||||
refreshScopes := true
|
||||
lastScopeRead := time.Time{}
|
||||
lastCleanup := time.Now()
|
||||
for {
|
||||
b.mu.Lock()
|
||||
all, allSearch := b.all, b.allSearch
|
||||
for org := range b.dirty {
|
||||
pendingStats[org] = true
|
||||
}
|
||||
for org := range b.searchDirty {
|
||||
pendingSearch[org] = true
|
||||
}
|
||||
b.dirty, b.searchDirty = map[string]bool{}, map[string]bool{}
|
||||
b.all, b.allSearch = false, false
|
||||
b.mu.Unlock()
|
||||
if refreshScopes || all || time.Since(lastScopeRead) > 30*time.Second {
|
||||
var e error
|
||||
scopes, e = b.scopes(ctx)
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
if e = b.store.pruneQueryModelScopes(ctx, l, scopes); e != nil {
|
||||
return e
|
||||
}
|
||||
refreshScopes = false
|
||||
lastScopeRead = time.Now()
|
||||
}
|
||||
for org := range scopes {
|
||||
if all {
|
||||
pendingStats[org] = true
|
||||
}
|
||||
if allSearch {
|
||||
pendingSearch[org] = true
|
||||
}
|
||||
}
|
||||
for org := range pendingSearch {
|
||||
if _, exists := scopes[org]; !exists {
|
||||
delete(pendingSearch, org)
|
||||
continue
|
||||
}
|
||||
if e := b.store.buildQuerySearchModel(ctx, l, org); e == nil {
|
||||
delete(pendingSearch, org)
|
||||
} else {
|
||||
log.Print("query_search_build result=retry category=source_or_publication")
|
||||
}
|
||||
}
|
||||
for org := range pendingStats {
|
||||
ids, exists := scopes[org]
|
||||
if !exists {
|
||||
delete(pendingStats, org)
|
||||
continue
|
||||
}
|
||||
if time.Since(lastStats[org]) < 30*time.Second {
|
||||
continue
|
||||
}
|
||||
if e := b.rebuild(ctx, l, org, ids); e == nil {
|
||||
delete(pendingStats, org)
|
||||
lastStats[org] = time.Now()
|
||||
} else {
|
||||
log.Print("query_model_build result=retry category=source_or_publication")
|
||||
}
|
||||
}
|
||||
if time.Since(lastCleanup) > time.Minute {
|
||||
cleanup, finish := context.WithTimeout(ctx, 5*time.Second)
|
||||
_ = b.store.cleanQueryModelOrphans(cleanup)
|
||||
finish()
|
||||
lastCleanup = time.Now()
|
||||
}
|
||||
// Prewarm due generations even when no browser is connected.
|
||||
for org, ids := range scopes {
|
||||
if len(ids) > 0 {
|
||||
_, _ = b.reader.Prewarm(ctx, org, ids)
|
||||
}
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
stop()
|
||||
<-watchDone
|
||||
return ctx.Err()
|
||||
case e := <-watchDone:
|
||||
return e
|
||||
case <-tick.C:
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func (s *MongoStore) markQuerySearchDirty(ctx context.Context, l queryModelLease, org string) error {
|
||||
if org != "" {
|
||||
_, err := s.queryControl().UpdateOne(ctx, queryLeaseFilter(l), bson.M{"$inc": bson.M{"searchRevisions." + queryModelKey("search", org, nil): int64(1)}})
|
||||
return err
|
||||
}
|
||||
_, err := s.queryControl().UpdateOne(ctx, queryLeaseFilter(l), bson.M{"$inc": bson.M{"searchGlobalRevision": int64(1)}})
|
||||
return err
|
||||
}
|
||||
|
||||
// The builder's source collection list excludes all derived collections so its
|
||||
// own publications cannot recursively schedule rebuilds.
|
||||
func querySourceCollection(name string) bool {
|
||||
for _, c := range querySourceCollections {
|
||||
if strings.EqualFold(c, name) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
278
internal/store/query_read_models.go
Normal file
278
internal/store/query_read_models.go
Normal file
@ -0,0 +1,278 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"sort"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"droboticflow/internal/domain"
|
||||
"droboticflow/internal/productionstats"
|
||||
"droboticflow/internal/queryserving"
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
"go.mongodb.org/mongo-driver/mongo"
|
||||
"go.mongodb.org/mongo-driver/mongo/options"
|
||||
)
|
||||
|
||||
const queryModelSchema = "query-model/v1"
|
||||
const queryModelChunkBytes = 2 << 20
|
||||
const queryModelMaxBytes = 64 << 20
|
||||
|
||||
type queryServingOptions struct {
|
||||
cache queryserving.Cache
|
||||
search *queryserving.Typesense
|
||||
stats queryServingCounters
|
||||
}
|
||||
|
||||
type queryServingCounters struct{ reportHits, responseHits, searchHits, typesenseReads, mongoSearchReads atomic.Uint64 }
|
||||
|
||||
// QueryServingStatistics exposes counters only, never names or query contents.
|
||||
func (s *MongoStore) QueryServingStatistics() map[string]uint64 {
|
||||
if s.queryServing == nil {
|
||||
return map[string]uint64{}
|
||||
}
|
||||
c := &s.queryServing.stats
|
||||
out := map[string]uint64{"report_hits": c.reportHits.Load(), "response_hits": c.responseHits.Load(), "search_hits": c.searchHits.Load(), "typesense_reads": c.typesenseReads.Load(), "materialized_search_reads": c.mongoSearchReads.Load()}
|
||||
if cache, ok := s.SharedQueryCache().(interface{ Statistics() map[string]uint64 }); ok {
|
||||
for k, v := range cache.Statistics() {
|
||||
out["cache_"+k] = v
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// ConfigureQueryServing is startup-only. Optional accelerators are never
|
||||
// created by a business read or mutation. Nil dependencies preserve old reads.
|
||||
func (s *MongoStore) ConfigureQueryServing(cache queryserving.Cache, search *queryserving.Typesense) {
|
||||
s.queryServing = &queryServingOptions{cache: cache, search: search}
|
||||
}
|
||||
func (s *MongoStore) SharedQueryCache() queryserving.Cache {
|
||||
if s.queryServing == nil {
|
||||
return nil
|
||||
}
|
||||
return s.queryServing.cache
|
||||
}
|
||||
|
||||
type QueryModelHead struct {
|
||||
SourceRevision int64 `bson:"sourceRevision" json:"sourceRevision"`
|
||||
Revision int64 `bson:"revision" json:"revision"`
|
||||
GlobalRevision int64 `bson:"globalRevision" json:"globalRevision"`
|
||||
Generation string `bson:"generation" json:"generation"`
|
||||
AsOf time.Time `bson:"asOf" json:"asOf"`
|
||||
Chunks int `bson:"chunks" json:"chunks"`
|
||||
Bytes int `bson:"bytes" json:"bytes"`
|
||||
Digest string `bson:"digest" json:"digest"`
|
||||
Collection string `bson:"searchCollection,omitempty" json:"searchCollection,omitempty"`
|
||||
Rows int `bson:"rows" json:"rows"`
|
||||
}
|
||||
type queryModelLease struct {
|
||||
Owner string
|
||||
Epoch int64
|
||||
}
|
||||
|
||||
func queryModelKey(kind, org string, projects []string) string {
|
||||
return queryserving.Key(queryModelSchema, kind, normalizedOrganizationID(org), normalizedIDs(projects))
|
||||
}
|
||||
func normalizedIDs(ids []string) []string {
|
||||
out := append([]string{}, ids...)
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
func (s *MongoStore) queryControl() *mongo.Collection {
|
||||
return s.db.Collection("queryReadModelControl")
|
||||
}
|
||||
func (s *MongoStore) queryData() *mongo.Collection { return s.db.Collection("queryReadModelData") }
|
||||
func (s *MongoStore) acquireQueryModelLease(ctx context.Context) (queryModelLease, bool, error) {
|
||||
now := time.Now().UTC()
|
||||
owner := domain.NewID("query-builder")
|
||||
var st struct {
|
||||
Owner string `bson:"owner"`
|
||||
Epoch int64 `bson:"epoch"`
|
||||
}
|
||||
err := s.queryControl().FindOneAndUpdate(ctx, bson.M{"_id": "global", "$or": bson.A{bson.M{"owner": ""}, bson.M{"until": bson.M{"$lte": now}}, bson.M{"owner": bson.M{"$exists": false}}}}, bson.M{"$set": bson.M{"owner": owner, "until": now.Add(60 * time.Second)}, "$inc": bson.M{"epoch": int64(1)}}, options.FindOneAndUpdate().SetUpsert(true).SetReturnDocument(options.After)).Decode(&st)
|
||||
if mongo.IsDuplicateKeyError(err) {
|
||||
return queryModelLease{}, false, nil
|
||||
}
|
||||
return queryModelLease{Owner: st.Owner, Epoch: st.Epoch}, err == nil, err
|
||||
}
|
||||
func queryLeaseFilter(l queryModelLease) bson.M {
|
||||
return bson.M{"_id": "global", "owner": l.Owner, "epoch": l.Epoch, "until": bson.M{"$gt": time.Now().UTC()}}
|
||||
}
|
||||
func (s *MongoStore) renewQueryModelLease(ctx context.Context, l queryModelLease) error {
|
||||
r, err := s.queryControl().UpdateOne(ctx, queryLeaseFilter(l), bson.M{"$set": bson.M{"until": time.Now().UTC().Add(60 * time.Second)}})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if r.MatchedCount != 1 {
|
||||
return errors.New("query builder lease lost")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
func (s *MongoStore) readQueryModelHead(ctx context.Context, key string, maxAge time.Duration) (QueryModelHead, bool) {
|
||||
var st struct {
|
||||
Heads map[string]QueryModelHead `bson:"heads"`
|
||||
ObservedAt time.Time `bson:"observedAt"`
|
||||
SourceRevision int64 `bson:"productionRevision"`
|
||||
}
|
||||
short, cancel := context.WithTimeout(ctx, 200*time.Millisecond)
|
||||
defer cancel()
|
||||
if s.queryControl().FindOne(short, bson.M{"_id": "global"}, options.FindOne().SetProjection(bson.M{"heads." + key: 1, "observedAt": 1, "productionRevision": 1})).Decode(&st) != nil {
|
||||
return QueryModelHead{}, false
|
||||
}
|
||||
h, ok := st.Heads[key]
|
||||
age := time.Since(h.AsOf)
|
||||
verifiedUnchanged := st.SourceRevision == h.SourceRevision && time.Since(st.ObservedAt) >= 0 && time.Since(st.ObservedAt) <= 5*time.Second
|
||||
return h, ok && h.Generation != "" && age >= 0 && (age <= maxAge || verifiedUnchanged)
|
||||
}
|
||||
func (s *MongoStore) publishQueryModelHead(ctx context.Context, l queryModelLease, key string, h QueryModelHead) error {
|
||||
r, err := s.queryControl().UpdateOne(ctx, queryLeaseFilter(l), bson.M{"$set": bson.M{"heads." + key: h}})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if r.MatchedCount != 1 {
|
||||
return errors.New("query publication fenced")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
func queryModelBlobKey(key, generation string, index int) string {
|
||||
return fmt.Sprintf("%s:%s:%09d", key, generation, index)
|
||||
}
|
||||
func (s *MongoStore) stageQueryModel(ctx context.Context, key string, value any, asOf time.Time) (QueryModelHead, error) {
|
||||
var data bytes.Buffer
|
||||
zw := gzip.NewWriter(&data)
|
||||
budget := &queryModelBudgetWriter{writer: zw}
|
||||
if err := json.NewEncoder(budget).Encode(value); err != nil {
|
||||
return QueryModelHead{}, err
|
||||
}
|
||||
if err := zw.Close(); err != nil {
|
||||
return QueryModelHead{}, err
|
||||
}
|
||||
if data.Len() > queryModelMaxBytes {
|
||||
return QueryModelHead{}, errors.New("query materialization exceeds budget")
|
||||
}
|
||||
h := QueryModelHead{Generation: domain.NewID("query-generation"), AsOf: asOf, Bytes: data.Len(), Chunks: (data.Len() + queryModelChunkBytes - 1) / queryModelChunkBytes}
|
||||
sum := sha256.Sum256(data.Bytes())
|
||||
h.Digest = hex.EncodeToString(sum[:])
|
||||
for i := 0; i < h.Chunks; i++ {
|
||||
b := data.Bytes()[i*queryModelChunkBytes : min((i+1)*queryModelChunkBytes, data.Len())]
|
||||
_, err := s.queryData().InsertOne(ctx, bson.M{"_id": queryModelBlobKey(key, h.Generation, i), "generation": h.Generation, "data": b, "createdAt": time.Now().UTC()})
|
||||
if err != nil {
|
||||
return QueryModelHead{}, err
|
||||
}
|
||||
if cache := s.SharedQueryCache(); cache != nil {
|
||||
cache.Put(ctx, queryModelBlobKey(key, h.Generation, i), b, 15*time.Minute)
|
||||
}
|
||||
}
|
||||
return h, nil
|
||||
}
|
||||
func (s *MongoStore) loadQueryModel(ctx context.Context, key string, h QueryModelHead, out any) error {
|
||||
if h.Chunks < 1 || h.Bytes < 1 || h.Bytes > queryModelMaxBytes || h.Chunks != (h.Bytes+queryModelChunkBytes-1)/queryModelChunkBytes {
|
||||
return errors.New("invalid query manifest")
|
||||
}
|
||||
var data bytes.Buffer
|
||||
for i := 0; i < h.Chunks; i++ {
|
||||
blobKey := queryModelBlobKey(key, h.Generation, i)
|
||||
var b []byte
|
||||
cache := s.SharedQueryCache()
|
||||
if cache != nil {
|
||||
b, _ = cache.Get(ctx, blobKey)
|
||||
}
|
||||
if b == nil {
|
||||
var doc struct {
|
||||
Data []byte `bson:"data"`
|
||||
}
|
||||
if err := s.queryData().FindOne(ctx, bson.M{"_id": blobKey}).Decode(&doc); err != nil {
|
||||
return err
|
||||
}
|
||||
b = doc.Data
|
||||
if cache != nil {
|
||||
cache.Put(ctx, blobKey, b, 15*time.Minute)
|
||||
}
|
||||
}
|
||||
if len(b) > queryModelChunkBytes {
|
||||
return errors.New("invalid query chunk")
|
||||
}
|
||||
data.Write(b)
|
||||
}
|
||||
sum := sha256.Sum256(data.Bytes())
|
||||
if data.Len() != h.Bytes || hex.EncodeToString(sum[:]) != h.Digest {
|
||||
return errors.New("query materialization corrupt")
|
||||
}
|
||||
zr, err := gzip.NewReader(bytes.NewReader(data.Bytes()))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer zr.Close()
|
||||
b, err := io.ReadAll(io.LimitReader(zr, 128<<20+1))
|
||||
if err != nil || len(b) > 128<<20 {
|
||||
return errors.New("query decode exceeds budget")
|
||||
}
|
||||
return json.Unmarshal(b, out)
|
||||
}
|
||||
|
||||
// LoadProductionReport serves a derived, scope-qualified report if available.
|
||||
// Legacy reads remain the compatibility/failure path. No refresh is triggered.
|
||||
func (s *MongoStore) LoadProductionReport(ctx context.Context, org string, projects []string) (productionstats.Report, string, error) {
|
||||
if s.queryServing != nil {
|
||||
key := queryModelKey("production", org, projects)
|
||||
if head, ok := s.readQueryModelHead(ctx, key, dashboardStatsCacheTTL); ok {
|
||||
var report productionstats.Report
|
||||
if err := s.loadQueryModel(ctx, key, head, &report); err == nil {
|
||||
s.queryServing.stats.reportHits.Add(1)
|
||||
return report, head.AsOf.Format(time.RFC3339Nano), nil
|
||||
}
|
||||
}
|
||||
}
|
||||
snapshot, err := s.LoadProductionStatsCached(ctx, org, projects)
|
||||
if err != nil {
|
||||
return productionstats.Report{}, "", err
|
||||
}
|
||||
return productionstats.Build(snapshot), snapshot.AsOf, nil
|
||||
}
|
||||
|
||||
func (s *MongoStore) PreparedProductionResponse(ctx context.Context, org string, projects []string, query string) ([]byte, bool) {
|
||||
if s.queryServing == nil {
|
||||
return nil, false
|
||||
}
|
||||
head, ok := s.readQueryModelHead(ctx, queryModelKey("production", org, projects), dashboardStatsCacheTTL)
|
||||
if !ok || s.SharedQueryCache() == nil {
|
||||
return nil, false
|
||||
}
|
||||
b, hit := s.SharedQueryCache().Get(ctx, queryserving.Key("production-response/v1", s.db.Name(), org, normalizedIDs(projects), head.Generation, query))
|
||||
if hit {
|
||||
s.queryServing.stats.responseHits.Add(1)
|
||||
}
|
||||
return b, hit
|
||||
}
|
||||
func (s *MongoStore) RememberProductionResponse(ctx context.Context, org string, projects []string, query, asOf string, value any) {
|
||||
if s.queryServing == nil || s.SharedQueryCache() == nil {
|
||||
return
|
||||
}
|
||||
head, ok := s.readQueryModelHead(ctx, queryModelKey("production", org, projects), dashboardStatsCacheTTL)
|
||||
if !ok || head.AsOf.Format(time.RFC3339Nano) != asOf {
|
||||
return
|
||||
}
|
||||
queryserving.PutJSON(ctx, s.SharedQueryCache(), queryserving.Key("production-response/v1", s.db.Name(), org, normalizedIDs(projects), head.Generation, query), value, 5*time.Minute)
|
||||
}
|
||||
|
||||
type queryModelBudgetWriter struct {
|
||||
writer io.Writer
|
||||
bytes int
|
||||
}
|
||||
|
||||
func (w *queryModelBudgetWriter) Write(p []byte) (int, error) {
|
||||
if len(p) > (128<<20)-w.bytes {
|
||||
return 0, errors.New("query decode exceeds budget")
|
||||
}
|
||||
n, e := w.writer.Write(p)
|
||||
w.bytes += n
|
||||
return n, e
|
||||
}
|
||||
311
internal/store/query_read_models_test.go
Normal file
311
internal/store/query_read_models_test.go
Normal file
@ -0,0 +1,311 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"droboticflow/internal/domain"
|
||||
"droboticflow/internal/productionstats"
|
||||
"droboticflow/internal/queryserving"
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
"go.mongodb.org/mongo-driver/mongo"
|
||||
"go.mongodb.org/mongo-driver/mongo/options"
|
||||
)
|
||||
|
||||
type queryModelFixture struct {
|
||||
s *MongoStore
|
||||
cache *queryserving.ManagedValkey
|
||||
search *queryserving.Typesense
|
||||
org string
|
||||
}
|
||||
|
||||
func newQueryModelFixture(t *testing.T, n int) queryModelFixture {
|
||||
t.Helper()
|
||||
if os.Getenv("QUERY_SERVING_LOCAL_TEST") != "1" {
|
||||
t.Skip("isolated query-serving dependencies")
|
||||
}
|
||||
ctx := t.Context()
|
||||
client, err := mongo.Connect(ctx, options.Client().ApplyURI("mongodb://127.0.0.1:27019/?replicaSet=query-rs&directConnection=true").SetServerSelectionTimeout(3*time.Second))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
db := client.Database(fmt.Sprintf("query_serving_test_%d", time.Now().UnixNano()))
|
||||
s := NewMongoStore(db)
|
||||
search, err := queryserving.NewTypesense("http://127.0.0.1:18108", os.Getenv("QUERY_TEST_TYPESENSE_API_KEY"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
cleanup, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
var control struct {
|
||||
Heads map[string]QueryModelHead `bson:"heads"`
|
||||
}
|
||||
if s.queryControl().FindOne(cleanup, bson.M{"_id": "global"}).Decode(&control) == nil {
|
||||
for _, h := range control.Heads {
|
||||
if h.Collection != "" {
|
||||
_ = search.Delete(cleanup, h.Collection)
|
||||
}
|
||||
}
|
||||
}
|
||||
_ = db.Drop(cleanup)
|
||||
_ = client.Disconnect(cleanup)
|
||||
})
|
||||
cache, err := queryserving.NewManagedValkey("valkey://127.0.0.1:16379", db.Name())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(cache.Close)
|
||||
deadline := time.Now().Add(3 * time.Second)
|
||||
for !cache.Ready() && time.Now().Before(deadline) {
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
if !cache.Ready() {
|
||||
t.Fatal("isolated cache unavailable")
|
||||
}
|
||||
s.ConfigureQueryServing(cache, search)
|
||||
must := func(err error) {
|
||||
t.Helper()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
for _, c := range []string{"humanItems", "runs", "captureRecords", "annotationTasks", "projects", "humanBatches"} {
|
||||
_, err := db.Collection(c).Indexes().CreateOne(ctx, mongo.IndexModel{Keys: bson.D{{Key: "id", Value: 1}}, Options: options.Index().SetUnique(true)})
|
||||
must(err)
|
||||
}
|
||||
must(s.EnsureQueryReadModelIndexes(ctx))
|
||||
org := "query-org"
|
||||
must(s.UpsertOrganization(ctx, domain.OrganizationRecord{ID: org, Status: domain.OrganizationStatusActive}))
|
||||
must(s.UpsertProject(ctx, domain.ProjectRecord{ID: "p", OrganizationID: org, Name: "历史项目"}))
|
||||
must(s.UpsertHumanBatch(ctx, domain.HumanBatchRecord{ID: "batch", OrganizationID: org, ProjectID: "p", PluginID: "annotation-qa", Stages: domain.DefaultFanyuV58HumanStages(), Status: domain.HumanBatchActive}))
|
||||
_, err = s.SaveProductionStatsScope(ctx, org, productionstats.Scope{ProjectIDs: []string{"p"}, Revision: 1}, 0)
|
||||
must(err)
|
||||
docs := map[string][]any{}
|
||||
push := func(c string, value any, legacy bool) {
|
||||
raw, e := bson.Marshal(value)
|
||||
must(e)
|
||||
var d bson.M
|
||||
must(bson.Unmarshal(raw, &d))
|
||||
if !legacy {
|
||||
stampPoolSearchMetadata(c, d)
|
||||
}
|
||||
docs[c] = append(docs[c], d)
|
||||
}
|
||||
flush := func() {
|
||||
for c, rows := range docs {
|
||||
if len(rows) > 0 {
|
||||
_, e := db.Collection(c).InsertMany(ctx, rows)
|
||||
must(e)
|
||||
}
|
||||
}
|
||||
docs = map[string][]any{}
|
||||
}
|
||||
for i := 0; i < n; i++ {
|
||||
id := fmt.Sprintf("item-%06d", i)
|
||||
capture := fmt.Sprintf("FYLX_TEST_%06d", i)
|
||||
if i == 0 {
|
||||
capture = "FYLX_B07_AZER764008K_20260927_074440"
|
||||
}
|
||||
if i == 1 {
|
||||
capture = "İ中文😀"
|
||||
}
|
||||
run := domain.RunRecord{ID: "run-" + id, OrganizationID: org, ProjectID: "p", TaskID: "capture-" + id, CaptureDirectories: []string{capture}, CreatedAt: "2026-11-10T00:00:00Z", Status: domain.RunStatusWaitingForHuman}
|
||||
task := domain.AnnotationTaskRecord{ID: "task-" + id, OrganizationID: org, ProjectID: "p", RunID: run.ID, ArtifactID: capture + "/part0002/camera_left.mp4", SourceVideo: capture + "/part0002/camera_left.mp4", ClipEnd: 10}
|
||||
item := domain.HumanItemRecord{ID: id, OrganizationID: org, ProjectID: "p", BatchID: "batch", PluginID: "annotation-qa", RunID: run.ID, LegacyTaskID: task.ID, CurrentStageID: "quality_review", Status: domain.HumanItemReady, SupplierID: fmt.Sprintf("supplier-%d", i%2), Version: 1, CreatedAt: time.Date(2026, 11, 10, 0, 0, i, 0, time.UTC).Format(time.RFC3339), Subject: domain.HumanSubjectRef{ItemKey: task.ArtifactID, Payload: map[string]any{"captureDirectory": capture, "annotationDraft": map[string]any{"description": "synthetic annotation must remain unchanged"}}}}
|
||||
if i < 3 {
|
||||
item.Subject.Payload["unusedBody"] = strings.Repeat("x", 1<<20)
|
||||
}
|
||||
push("runs", run, i%3 == 0)
|
||||
push("annotationTasks", task, i%3 == 0)
|
||||
push("captureRecords", domain.CaptureRecord{ID: run.TaskID, OrganizationID: org, ProjectID: "p", CaptureID: capture, RunID: run.ID}, i%3 == 0)
|
||||
push("humanItems", item, i%3 == 0)
|
||||
if (i+1)%1000 == 0 {
|
||||
flush()
|
||||
}
|
||||
}
|
||||
flush()
|
||||
return queryModelFixture{s: s, cache: cache, search: search, org: org}
|
||||
}
|
||||
|
||||
func queryIDs(items []domain.HumanItemRecord) []string {
|
||||
out := []string{}
|
||||
for _, i := range items {
|
||||
out = append(out, i.ID)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func TestQueryReadModelsMongoEquivalenceFencingAndFailure(t *testing.T) {
|
||||
f := newQueryModelFixture(t, 241)
|
||||
ctx := t.Context()
|
||||
s := f.s
|
||||
before, err := s.GetHumanItem(ctx, "item-000000")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
old := s.queryServing
|
||||
s.queryServing = nil
|
||||
queries := []string{"FYLX_B07_AZER764008K_20260927_074440/part0002", "TEST_0000", "中文", "İ", "item-000001", "历史项目", "[missing]", "part0002", "camera_left", "000000/part"}
|
||||
expected := map[string][]string{}
|
||||
for _, q := range queries {
|
||||
items, e := s.ListHumanItems(ctx, f.org, domain.HumanItemQuery{PluginID: "annotation-qa", Search: q, Limit: 500})
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
expected[q] = queryIDs(items)
|
||||
}
|
||||
base, e := s.LoadProductionStats(ctx, f.org, []string{"p"})
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
wantReport := productionstats.Build(base)
|
||||
s.queryServing = old
|
||||
b := NewQueryReadBuilder(s)
|
||||
defer b.Close()
|
||||
if err = b.RunOnce(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, q := range queries {
|
||||
candidateIDs, ready := s.querySearchCandidates(ctx, f.org, q, "")
|
||||
if !ready {
|
||||
t.Logf("accelerator unavailable for %q; verifying compatibility result", q)
|
||||
}
|
||||
set := map[string]bool{}
|
||||
for _, id := range candidateIDs {
|
||||
set[id] = true
|
||||
}
|
||||
for _, id := range expected[q] {
|
||||
if ready && !set[id] {
|
||||
t.Fatalf("candidate false negative query=%q id=%s", q, id)
|
||||
}
|
||||
}
|
||||
items, e := s.ListHumanItems(ctx, f.org, domain.HumanItemQuery{PluginID: "annotation-qa", Search: q, Limit: 500})
|
||||
if e != nil || !reflect.DeepEqual(queryIDs(items), expected[q]) {
|
||||
t.Fatalf("result mismatch query=%q got=%v expected=%v err=%v", q, queryIDs(items), expected[q], e)
|
||||
}
|
||||
}
|
||||
if stats := s.QueryServingStatistics(); stats["search_hits"] == 0 || stats["typesense_reads"] == 0 {
|
||||
t.Fatal("new search path never exercised", stats)
|
||||
}
|
||||
report, asOf, err := s.LoadProductionReport(ctx, f.org, []string{"p"})
|
||||
if err != nil || asOf == "" || !reflect.DeepEqual(report, wantReport) {
|
||||
t.Fatal("materialized production report differs", err)
|
||||
}
|
||||
after, err := s.GetHumanItem(ctx, before.ID)
|
||||
if err != nil || !reflect.DeepEqual(before, after) {
|
||||
t.Fatal("read model changed annotation, ownership or version")
|
||||
}
|
||||
// A late writer can stage data but cannot replace a newer owner's head.
|
||||
l1, ok, err := s.acquireQueryModelLease(ctx)
|
||||
if err != nil || !ok {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err = s.queryControl().UpdateOne(ctx, bson.M{"_id": "global"}, bson.M{"$set": bson.M{"until": time.Now().Add(-time.Second)}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
l2, ok, err := s.acquireQueryModelLease(ctx)
|
||||
if err != nil || !ok {
|
||||
t.Fatal(err)
|
||||
}
|
||||
key := queryModelKey("production", f.org, []string{"p"})
|
||||
head, ok := s.readQueryModelHead(ctx, key, time.Hour)
|
||||
if !ok {
|
||||
t.Fatal("missing production head")
|
||||
}
|
||||
if s.publishQueryModelHead(ctx, l1, key, head) == nil {
|
||||
t.Fatal("expired writer published")
|
||||
}
|
||||
if err = s.publishQueryModelHead(ctx, l2, key, head); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Incomplete/corrupt derived storage never becomes zero business data.
|
||||
_, err = s.queryData().UpdateOne(ctx, bson.M{"_id": queryModelBlobKey(key, head.Generation, 0)}, bson.M{"$set": bson.M{"data": []byte("corrupt")}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
s.queryServing.cache = nil
|
||||
got, _, err := s.LoadProductionReport(ctx, f.org, []string{"p"})
|
||||
if err != nil || !reflect.DeepEqual(got, wantReport) {
|
||||
t.Fatal("corrupt materialization failed to retain compatibility path", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestQuerySearchChangeObservationIgnoresDraftsAndRecoversNamesLocal(t *testing.T) {
|
||||
f := newQueryModelFixture(t, 12)
|
||||
s := f.s
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
b := NewQueryReadBuilder(s)
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- b.Run(ctx) }()
|
||||
defer func() { cancel(); <-done; b.Close() }()
|
||||
wait := func(check func() bool) {
|
||||
t.Helper()
|
||||
end := time.Now().Add(20 * time.Second)
|
||||
for time.Now().Before(end) {
|
||||
if check() {
|
||||
return
|
||||
}
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
}
|
||||
t.Fatal("streamed read model did not converge")
|
||||
}
|
||||
wait(func() bool { _, ready := s.querySearchCandidates(ctx, f.org, "FYLX", ""); return ready })
|
||||
state, err := s.querySearchState(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
key := queryModelKey("search", f.org, nil)
|
||||
generation := state.Heads[key].Generation
|
||||
item, err := s.GetHumanItem(ctx, "item-000001")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
item.Version++
|
||||
item.Status = domain.HumanItemClaimed
|
||||
item.AssignedUserID = "synthetic-owner"
|
||||
item.UpdatedAt = time.Now().UTC().Format(time.RFC3339Nano)
|
||||
if err = s.UpsertHumanItem(ctx, *item); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
time.Sleep(1500 * time.Millisecond)
|
||||
state, err = s.querySearchState(ctx)
|
||||
if err != nil || state.Heads[key].Generation != generation || state.Revisions[key] != state.Heads[key].Revision {
|
||||
t.Fatal("ordinary task state invalidated name search", err)
|
||||
}
|
||||
if err = s.UpsertProject(ctx, domain.ProjectRecord{ID: "p", OrganizationID: f.org, Name: "新项目名称"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
wait(func() bool {
|
||||
ids, ready := s.querySearchCandidates(ctx, f.org, "新项目名称", "")
|
||||
return ready && len(ids) == 12
|
||||
})
|
||||
before, err := s.GetHumanItem(ctx, "item-000000")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err = s.humanItems.DeleteOne(ctx, bson.M{"id": before.ID}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
wait(func() bool {
|
||||
ids, ready := s.querySearchCandidates(ctx, f.org, "新项目名称", "")
|
||||
return ready && len(ids) == 11
|
||||
})
|
||||
// Old generations survive publication only as unreachable, bounded derived
|
||||
// records; no original annotation or audit records are rewritten.
|
||||
remaining, err := s.GetHumanItem(ctx, "item-000002")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
raw, _ := json.Marshal(remaining.Subject.Payload)
|
||||
if !strings.Contains(string(raw), "synthetic annotation must remain unchanged") {
|
||||
t.Fatal("background rebuild lost annotation")
|
||||
}
|
||||
}
|
||||
231
internal/store/query_recovery_test.go
Normal file
231
internal/store/query_recovery_test.go
Normal file
@ -0,0 +1,231 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"droboticflow/internal/domain"
|
||||
"droboticflow/internal/queryserving"
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
"reflect"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestQueryReadModelsRestartExpiryAndPrivateGarbageCollectionLocal(t *testing.T) {
|
||||
f := newQueryModelFixture(t, 24)
|
||||
s, ctx := f.s, t.Context()
|
||||
b := NewQueryReadBuilder(s)
|
||||
defer b.Close()
|
||||
organizationsBefore, _ := s.organizations.CountDocuments(ctx, bson.M{})
|
||||
if err := b.RunOnce(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
organizationsAfter, _ := s.organizations.CountDocuments(ctx, bson.M{})
|
||||
if organizationsAfter != organizationsBefore {
|
||||
t.Fatal("builder bootstrapped business organizations")
|
||||
}
|
||||
st, err := s.querySearchState(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
key := queryModelKey("search", f.org, nil)
|
||||
head := st.Heads[key]
|
||||
// Stale observation and known source changes cannot authorize an empty hit.
|
||||
_, err = s.queryControl().UpdateOne(ctx, bson.M{"_id": "global"}, bson.M{"$set": bson.M{"observedAt": time.Now().Add(-6 * time.Second)}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, ready := s.querySearchCandidates(ctx, f.org, "TEST", ""); ready {
|
||||
t.Fatal("expired observer used as complete catalog")
|
||||
}
|
||||
_, err = s.queryControl().UpdateOne(ctx, bson.M{"_id": "global"}, bson.M{"$set": bson.M{"observedAt": time.Now()}, "$inc": bson.M{"searchRevisions." + key: 1}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, ready := s.querySearchCandidates(ctx, f.org, "TEST", ""); ready {
|
||||
t.Fatal("known obsolete catalog used")
|
||||
}
|
||||
before, err := s.GetHumanItem(ctx, "item-000000")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err = s.humanItems.DeleteOne(ctx, bson.M{"id": "item-000001"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Reconciliation after a stopped worker includes missed deletes.
|
||||
if err = b.RunOnce(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ids, ready := s.querySearchCandidates(ctx, f.org, "item-000001", "")
|
||||
if !ready {
|
||||
t.Fatal("restart catalog not ready")
|
||||
}
|
||||
for _, id := range ids {
|
||||
if id == "item-000001" {
|
||||
t.Fatal("restart retained deleted identity")
|
||||
}
|
||||
}
|
||||
matches, e := s.ListHumanItems(ctx, f.org, domain.HumanItemQuery{Search: "item-000001", Limit: 100})
|
||||
if e != nil || len(matches) != 0 {
|
||||
t.Fatal("deleted exact search result retained", e)
|
||||
}
|
||||
current, err := s.querySearchState(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if current.Heads[key].Generation == head.Generation {
|
||||
t.Fatal("restart reused unverified generation")
|
||||
}
|
||||
// Availability failure: Typesense may be gone while Mongo materialized rows
|
||||
// remain complete. The old source path is still used if observation expires.
|
||||
unavailable, err := queryserving.NewTypesense("http://127.0.0.1:1", "synthetic")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
s.queryServing.search = unavailable
|
||||
s.queryServing.cache = nil
|
||||
got, ready := s.querySearchCandidates(ctx, f.org, "FYLX_B07", "")
|
||||
if !ready || !reflect.DeepEqual(got, []string{"item-000000"}) {
|
||||
t.Fatal("search outage lost existing task", got, ready)
|
||||
}
|
||||
s.queryServing.search = f.search
|
||||
// GC must retain current generations regardless of their creation date,
|
||||
// while removing only this database's old unreachable derivatives.
|
||||
_, err = s.queryData().UpdateMany(ctx, bson.M{}, bson.M{"$set": bson.M{"createdAt": time.Now().Add(-time.Hour)}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err = s.queryData().InsertOne(ctx, bson.M{"_id": "unreachable-test", "generation": "orphan-test", "createdAt": time.Now().Add(-time.Hour)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = s.cleanQueryModelOrphans(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
n, err := s.queryData().CountDocuments(ctx, bson.M{"generation": current.Heads[key].Generation})
|
||||
if err != nil || n != 24 {
|
||||
t.Fatal("current generation lost", n, err)
|
||||
} // 23 rows plus collection registration
|
||||
n, err = s.queryData().CountDocuments(ctx, bson.M{"_id": "unreachable-test"})
|
||||
if err != nil || n != 0 {
|
||||
t.Fatal("orphan not reclaimed", err)
|
||||
}
|
||||
after, err := s.GetHumanItem(ctx, before.ID)
|
||||
if err != nil || !reflect.DeepEqual(after, before) {
|
||||
t.Fatal("recovery or GC modified annotation")
|
||||
}
|
||||
}
|
||||
|
||||
func TestQuerySearchPaginationAndLiveScopeEquivalenceLocal(t *testing.T) {
|
||||
f := newQueryModelFixture(t, 241)
|
||||
s, ctx := f.s, t.Context()
|
||||
b := NewQueryReadBuilder(s)
|
||||
defer b.Close()
|
||||
if err := b.RunOnce(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The name catalog never contains assignment or canClaim. Current filters
|
||||
// apply equally after a task changes supplier, owner, stage or version.
|
||||
item, err := s.GetHumanItem(ctx, "item-000000")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
item.SupplierID = "changed-supplier"
|
||||
item.AssignedUserID = "current-owner"
|
||||
item.Status = domain.HumanItemClaimed
|
||||
item.Version++
|
||||
if err = s.UpsertHumanItem(ctx, *item); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
options := s.queryServing
|
||||
queries := []domain.HumanItemQuery{
|
||||
{Search: "part0002", Limit: 17},
|
||||
{Search: "part0002", Limit: 17, AvailableProjectIDs: []string{}},
|
||||
{Search: "part0002", Limit: 17, AvailableProjectIDs: []string{"foreign"}},
|
||||
{Search: "FYLX_B07", ActorUserID: "current-owner", Limit: 17},
|
||||
{Search: "part0002", UnassignedOnly: true, Limit: 17},
|
||||
}
|
||||
for _, q := range queries {
|
||||
q.PluginID = "annotation-qa"
|
||||
for page := 0; page < 20; page++ {
|
||||
s.queryServing = nil
|
||||
old, e := s.ListHumanItems(ctx, f.org, q)
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
s.queryServing = options
|
||||
got, e := s.ListHumanItems(ctx, f.org, q)
|
||||
if e != nil || !reflect.DeepEqual(queryIDs(old), queryIDs(got)) {
|
||||
t.Fatal("paged live-scope mismatch", page, e)
|
||||
}
|
||||
if len(old) == 0 {
|
||||
break
|
||||
}
|
||||
last := old[len(old)-1]
|
||||
q.Cursor = &domain.HumanItemCursor{CreatedAt: last.CreatedAt, ID: last.ID}
|
||||
if page == 19 {
|
||||
t.Fatal("pagination did not exhaust")
|
||||
}
|
||||
}
|
||||
}
|
||||
if rows, e := s.ListHumanItems(ctx, "foreign-org", domain.HumanItemQuery{Search: "part0002", Limit: 500}); e != nil || len(rows) != 0 {
|
||||
t.Fatal("foreign organization leaked", e)
|
||||
}
|
||||
// Cancellation is propagated instead of publishing/caching empty results.
|
||||
cancelled, stop := context.WithCancel(ctx)
|
||||
stop()
|
||||
if _, ready := s.querySearchCandidates(cancelled, f.org, "part0002", ""); ready {
|
||||
t.Fatal("cancelled candidate lookup became authoritative")
|
||||
}
|
||||
}
|
||||
|
||||
func TestQuerySearchCandidateBudgetNeverTruncates(t *testing.T) {
|
||||
if queryCandidateBudget([]string{""}) || queryCandidateBudget(make([]string, 100001)) {
|
||||
t.Fatal("invalid or oversized candidates accepted")
|
||||
}
|
||||
if !queryCandidateBudget(nil) || !queryCandidateBudget([]string{"valid-id"}) {
|
||||
t.Fatal("valid complete candidates rejected")
|
||||
}
|
||||
}
|
||||
|
||||
func TestQueryReadModelsCachedResponseScopeAndStaleFallbackLocal(t *testing.T) {
|
||||
f := newQueryModelFixture(t, 8)
|
||||
s, ctx := f.s, t.Context()
|
||||
b := NewQueryReadBuilder(s)
|
||||
defer b.Close()
|
||||
if e := b.RunOnce(ctx); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
_, asOf, e := s.LoadProductionReport(ctx, f.org, []string{"p"})
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
s.RememberProductionResponse(ctx, f.org, []string{"p"}, "dates-and-filter", asOf, map[string]int{"proof": 42})
|
||||
raw, ok := s.PreparedProductionResponse(ctx, f.org, []string{"p"}, "dates-and-filter")
|
||||
if !ok || string(raw) != "{\"proof\":42}" {
|
||||
t.Fatal("real shared response not reusable", string(raw), ok)
|
||||
}
|
||||
for _, scope := range []struct {
|
||||
org string
|
||||
projects []string
|
||||
query string
|
||||
}{{"foreign", []string{"p"}, "dates-and-filter"}, {f.org, []string{"foreign"}, "dates-and-filter"}, {f.org, []string{"p"}, "other-dates"}} {
|
||||
if _, hit := s.PreparedProductionResponse(ctx, scope.org, scope.projects, scope.query); hit {
|
||||
t.Fatal("response scope leaked")
|
||||
}
|
||||
}
|
||||
key := queryModelKey("production", f.org, []string{"p"})
|
||||
_, e = s.queryControl().UpdateOne(ctx, bson.M{"_id": "global"}, bson.M{"$set": bson.M{"heads." + key + ".asOf": time.Now().Add(-time.Minute), "productionRevision": int64(10), "observedAt": time.Now()}})
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if _, hit := s.PreparedProductionResponse(ctx, f.org, []string{"p"}, "dates-and-filter"); hit {
|
||||
t.Fatal("changed source reused report beyond original freshness budget")
|
||||
}
|
||||
before := s.QueryServingStatistics()["report_hits"]
|
||||
if _, _, e = s.LoadProductionReport(ctx, f.org, []string{"p"}); e != nil {
|
||||
t.Fatal("stale model did not retain old loader", e)
|
||||
}
|
||||
if s.QueryServingStatistics()["report_hits"] != before {
|
||||
t.Fatal("stale model counted as fresh")
|
||||
}
|
||||
}
|
||||
231
internal/store/query_search_model.go
Normal file
231
internal/store/query_search_model.go
Normal file
@ -0,0 +1,231 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"droboticflow/internal/domain"
|
||||
"droboticflow/internal/queryserving"
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
"go.mongodb.org/mongo-driver/mongo/options"
|
||||
)
|
||||
|
||||
type querySearchState struct {
|
||||
Heads map[string]QueryModelHead `bson:"heads"`
|
||||
Revisions map[string]int64 `bson:"searchRevisions"`
|
||||
Global int64 `bson:"searchGlobalRevision"`
|
||||
ObservedAt time.Time `bson:"observedAt"`
|
||||
}
|
||||
|
||||
func (s *MongoStore) querySearchState(ctx context.Context) (querySearchState, error) {
|
||||
var st querySearchState
|
||||
err := s.queryControl().FindOne(ctx, bson.M{"_id": "global"}, options.FindOne().SetProjection(bson.M{"heads": 1, "searchRevisions": 1, "searchGlobalRevision": 1, "observedAt": 1})).Decode(&st)
|
||||
return st, err
|
||||
}
|
||||
func querySafePath(value string) string {
|
||||
u, err := url.Parse(value)
|
||||
if err == nil && u.Scheme != "" && u.Host != "" {
|
||||
u.RawQuery = ""
|
||||
u.Fragment = ""
|
||||
u.User = nil
|
||||
return u.String()
|
||||
}
|
||||
return value
|
||||
}
|
||||
func searchRowText(row adminPoolRow) string {
|
||||
// Both exact historical search corpora remain contiguous; adding fields in
|
||||
// between could otherwise lose cross-field substring matches.
|
||||
return adminPoolSearchText(row) + "\n" + humanPoolFallbackSearchText(row)
|
||||
}
|
||||
|
||||
func (s *MongoStore) buildQuerySearchModel(ctx context.Context, l queryModelLease, org string) error {
|
||||
ctx, stop := context.WithTimeout(ctx, 5*time.Minute)
|
||||
defer stop()
|
||||
key := queryModelKey("search", org, nil)
|
||||
state, err := s.querySearchState(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
h := QueryModelHead{Generation: domain.NewID("query-search"), AsOf: time.Now().UTC(), Revision: state.Revisions[key], GlobalRevision: state.Global}
|
||||
if s.queryServing != nil && s.queryServing.search != nil {
|
||||
sum := sha256.Sum256([]byte(s.db.Name() + ":" + key + ":" + h.Generation))
|
||||
h.Collection = "dfq_" + hex.EncodeToString(sum[:])
|
||||
// Register intent first. Even a canceled/ambiguous create remains
|
||||
// discoverable by this database's orphan collector.
|
||||
if _, err = s.queryData().InsertOne(ctx, bson.M{"_id": "search_collection:" + h.Collection, "kind": "search_collection", "collection": h.Collection, "generation": h.Generation, "createdAt": time.Now().UTC()}); err != nil {
|
||||
return err
|
||||
}
|
||||
if err = s.queryServing.search.Create(ctx, h.Collection); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
complete := false
|
||||
defer func() {
|
||||
if !complete {
|
||||
cleanup, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
_, _ = s.queryData().DeleteMany(cleanup, bson.M{"generation": h.Generation, "kind": bson.M{"$ne": "search_collection"}})
|
||||
if h.Collection != "" {
|
||||
_ = s.queryServing.search.Delete(cleanup, h.Collection)
|
||||
}
|
||||
}
|
||||
}()
|
||||
pipeline := humanPoolSearchPipeline(org, domain.HumanItemQuery{Search: "_build_names"}, bson.M{"organizationId": normalizedOrganizationID(org)})
|
||||
cur, err := s.humanItems.Aggregate(ctx, pipeline, options.Aggregate().SetAllowDiskUse(true).SetBatchSize(200))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer cur.Close(ctx)
|
||||
for {
|
||||
rows := []adminPoolRow{}
|
||||
for len(rows) < 200 && cur.Next(ctx) {
|
||||
var row adminPoolRow
|
||||
if err = cur.Decode(&row); err != nil {
|
||||
return err
|
||||
}
|
||||
rows = append(rows, row)
|
||||
}
|
||||
if err = cur.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if len(rows) == 0 {
|
||||
break
|
||||
}
|
||||
if err = s.humanPoolSearchLegacyPayloads(ctx, org, domain.HumanItemQuery{Search: "_build_names"}, rows); err != nil {
|
||||
return err
|
||||
}
|
||||
docs := make([]any, 0, len(rows))
|
||||
searchDocs := make([]queryserving.SearchDocument, 0, len(rows))
|
||||
for _, row := range rows {
|
||||
for _, task := range row.Tasks {
|
||||
if querySafePath(task.SourceVideo) != task.SourceVideo {
|
||||
return errors.New("credential-bearing source requires legacy search")
|
||||
}
|
||||
}
|
||||
text := searchRowText(row)
|
||||
if len(text) > 64<<10 {
|
||||
return errors.New("search names exceed materialization budget")
|
||||
}
|
||||
// Store only the conservative searchable text and identity. No
|
||||
// task body, draft, review opinion, eligibility or credential.
|
||||
id := queryserving.Key("row", h.Generation, row.ID)
|
||||
docs = append(docs, bson.M{"_id": id, "generation": h.Generation, "organizationId": normalizedOrganizationID(org), "itemId": row.ID, "text": text, "createdAt": time.Now().UTC()})
|
||||
searchDocs = append(searchDocs, queryserving.SearchDocument{ID: id, ItemID: row.ID, Grams: queryserving.IndexGrams(text)})
|
||||
}
|
||||
if _, err = s.queryData().InsertMany(ctx, docs); err != nil {
|
||||
return err
|
||||
}
|
||||
if h.Collection != "" {
|
||||
if err = s.queryServing.search.Import(ctx, h.Collection, searchDocs); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
h.Rows += len(rows)
|
||||
}
|
||||
filter := queryLeaseFilter(l)
|
||||
filter["searchGlobalRevision"] = state.Global
|
||||
// Missing revision zero is equivalent to zero, without writing business data.
|
||||
filter["$expr"] = bson.M{"$and": bson.A{bson.M{"$eq": bson.A{bson.M{"$ifNull": bson.A{"$searchRevisions." + key, int64(0)}}, h.Revision}}, bson.M{"$eq": bson.A{bson.M{"$ifNull": bson.A{"$searchGlobalRevision", int64(0)}}, h.GlobalRevision}}}}
|
||||
delete(filter, "searchGlobalRevision")
|
||||
res, err := s.queryControl().UpdateOne(ctx, filter, bson.M{"$set": bson.M{"heads." + key: h, "observedAt": time.Now().UTC()}})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if res.MatchedCount != 1 {
|
||||
return errors.New("search source changed during build")
|
||||
}
|
||||
complete = true
|
||||
return nil
|
||||
}
|
||||
|
||||
// A complete, current catalog only narrows candidates. It never authorizes a
|
||||
// row, changes a cursor, truncates history, or supplies cached task status.
|
||||
func (s *MongoStore) querySearchCandidates(ctx context.Context, org, search, capture string) ([]string, bool) {
|
||||
if s.queryServing == nil || search == "" && capture == "" {
|
||||
return nil, false
|
||||
}
|
||||
short, cancel := context.WithTimeout(ctx, 300*time.Millisecond)
|
||||
defer cancel()
|
||||
st, err := s.querySearchState(short)
|
||||
if err != nil {
|
||||
return nil, false
|
||||
}
|
||||
key := queryModelKey("search", org, nil)
|
||||
h, ok := st.Heads[key]
|
||||
if !ok || h.Generation == "" || h.Revision != st.Revisions[key] || h.GlobalRevision != st.Global || time.Since(st.ObservedAt) > 5*time.Second || time.Since(st.ObservedAt) < 0 {
|
||||
return nil, false
|
||||
}
|
||||
cacheKey := queryserving.Key("search-candidates/v1", s.db.Name(), org, h.Generation, search, capture)
|
||||
var ids []string
|
||||
if queryserving.GetJSON(short, s.SharedQueryCache(), cacheKey, &ids) && queryCandidateBudget(ids) {
|
||||
s.queryServing.stats.searchHits.Add(1)
|
||||
return ids, true
|
||||
}
|
||||
if h.Collection != "" && s.queryServing.search != nil {
|
||||
if out, err := s.queryServing.search.Candidates(short, h.Collection, search, capture); err == nil && queryCandidateBudget(out) {
|
||||
s.queryServing.stats.searchHits.Add(1)
|
||||
s.queryServing.stats.typesenseReads.Add(1)
|
||||
queryserving.PutJSON(short, s.SharedQueryCache(), cacheKey, out, 5*time.Minute)
|
||||
return out, true
|
||||
}
|
||||
}
|
||||
clauses := bson.A{bson.M{"generation": h.Generation, "organizationId": normalizedOrganizationID(org)}}
|
||||
for _, term := range []string{search, capture} {
|
||||
if term != "" {
|
||||
clauses = append(clauses, bson.M{"text": bson.M{"$regex": regexp.QuoteMeta(strings.ToLower(term))}})
|
||||
}
|
||||
}
|
||||
cur, err := s.queryData().Find(short, bson.M{"$and": clauses}, options.Find().SetProjection(bson.M{"itemId": 1, "_id": 0}))
|
||||
if err != nil {
|
||||
return nil, false
|
||||
}
|
||||
defer cur.Close(short)
|
||||
ids = []string{}
|
||||
for cur.Next(short) {
|
||||
var row struct {
|
||||
ID string `bson:"itemId"`
|
||||
}
|
||||
if cur.Decode(&row) != nil || row.ID == "" {
|
||||
return nil, false
|
||||
}
|
||||
ids = append(ids, row.ID)
|
||||
if len(ids) > 100000 {
|
||||
return nil, false
|
||||
}
|
||||
}
|
||||
if cur.Err() != nil {
|
||||
return nil, false
|
||||
}
|
||||
if !queryCandidateBudget(ids) {
|
||||
return nil, false
|
||||
}
|
||||
s.queryServing.stats.mongoSearchReads.Add(1)
|
||||
queryserving.PutJSON(short, s.SharedQueryCache(), cacheKey, ids, 5*time.Minute)
|
||||
s.queryServing.stats.searchHits.Add(1)
|
||||
return ids, true
|
||||
}
|
||||
|
||||
// Keep the authoritative Mongo filter well below its BSON limit. An oversized
|
||||
// candidate set is a miss, never a truncated inventory.
|
||||
func queryCandidateBudget(ids []string) bool {
|
||||
if len(ids) > 100000 {
|
||||
return false
|
||||
}
|
||||
bytes := 0
|
||||
for _, id := range ids {
|
||||
if id == "" {
|
||||
return false
|
||||
}
|
||||
bytes += len(id) + 16
|
||||
if bytes > 4<<20 {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
41
internal/store/query_serving_config.go
Normal file
41
internal/store/query_serving_config.go
Normal file
@ -0,0 +1,41 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"droboticflow/internal/queryserving"
|
||||
"errors"
|
||||
"os"
|
||||
)
|
||||
|
||||
// ConfigureQueryServingFromEnvironment is called only by process entry points.
|
||||
// No new feature is enabled by default. Namespace separates databases in
|
||||
// different deployments even when their database names happen to be identical.
|
||||
func (s *MongoStore) ConfigureQueryServingFromEnvironment() (func(), error) {
|
||||
close := func() {}
|
||||
if os.Getenv("QUERY_SERVING_READS") != "1" {
|
||||
return close, nil
|
||||
}
|
||||
ns := os.Getenv("QUERY_SERVING_NAMESPACE")
|
||||
if ns == "" {
|
||||
return close, errors.New("QUERY_SERVING_NAMESPACE is required")
|
||||
}
|
||||
var cache queryserving.Cache
|
||||
if uri := os.Getenv("QUERY_VALKEY_URL"); uri != "" {
|
||||
v, err := queryserving.NewManagedValkey(uri, ns+":"+s.db.Name())
|
||||
if err != nil {
|
||||
return close, err
|
||||
}
|
||||
cache = v
|
||||
close = v.Close
|
||||
}
|
||||
var search *queryserving.Typesense
|
||||
if uri := os.Getenv("QUERY_TYPESENSE_URL"); uri != "" {
|
||||
var err error
|
||||
search, err = queryserving.NewTypesense(uri, os.Getenv("QUERY_TYPESENSE_API_KEY"))
|
||||
if err != nil {
|
||||
close()
|
||||
return func() {}, err
|
||||
}
|
||||
}
|
||||
s.ConfigureQueryServing(cache, search)
|
||||
return close, nil
|
||||
}
|
||||
172
internal/store/query_source_changes.go
Normal file
172
internal/store/query_source_changes.go
Normal file
@ -0,0 +1,172 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"droboticflow/internal/domain"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"go.mongodb.org/mongo-driver/mongo/options"
|
||||
"time"
|
||||
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
"go.mongodb.org/mongo-driver/mongo"
|
||||
)
|
||||
|
||||
type querySourceDigest struct {
|
||||
Digest [32]byte
|
||||
Organization string
|
||||
}
|
||||
|
||||
var querySearchSources = map[string]bool{"humanItems": true, "annotationTasks": true, "captureRecords": true, "runs": true, "projects": true}
|
||||
|
||||
// Only names, source identities and references affect search. Saving a draft,
|
||||
// renewing a lease or changing task status must not invalidate the name index.
|
||||
func querySearchSourceProjection(collection string) bson.M {
|
||||
p := bson.M{"_id": 1, "organizationId": 1, "projectId": 1, "id": 1, "poolSearch.text": 1}
|
||||
fields := map[string][]string{
|
||||
"humanItems": {"legacyTaskId", "runId", "subject.itemKey", "subject.artifactId"},
|
||||
"annotationTasks": {"artifactId", "rightArtifactId", "sourceVideo"},
|
||||
"captureRecords": {"captureId", "prefix"},
|
||||
"projects": {"name"},
|
||||
"runs": {"taskId", "projectName", "captureDirectories", "inputs", "artifacts.id", "artifacts.name", "artifacts.path", "nodeExecutions.artifacts.id", "nodeExecutions.artifacts.name", "nodeExecutions.artifacts.path"},
|
||||
}
|
||||
for _, f := range fields[collection] {
|
||||
p[f] = 1
|
||||
}
|
||||
if collection == "runs" {
|
||||
p["inputs"] = bson.M{"$cond": bson.A{poolSearchValidExpression("runs"), bson.M{}, "$inputs"}}
|
||||
}
|
||||
if collection == "humanItems" {
|
||||
p["legacyPayload"] = bson.M{"$cond": bson.A{poolSearchValidExpression("humanItems"), bson.M{}, "$subject.payload"}}
|
||||
}
|
||||
return p
|
||||
}
|
||||
func querySourceIdentity(collection string, id any) string {
|
||||
b, _ := json.Marshal(id)
|
||||
return collection + ":" + string(b)
|
||||
}
|
||||
func querySourceHash(doc bson.M) querySourceDigest {
|
||||
if payload, ok := doc["legacyPayload"]; ok {
|
||||
names := []string{}
|
||||
domain.AppendCaptureDirectoriesFromPayload(payload, &names)
|
||||
doc["legacyPayload"] = names
|
||||
}
|
||||
b, _ := json.Marshal(doc)
|
||||
org, _ := doc["organizationId"].(string)
|
||||
return querySourceDigest{Digest: sha256.Sum256(b), Organization: org}
|
||||
}
|
||||
func (b *QueryReadBuilder) seedSourceDigests(ctx context.Context) error {
|
||||
for collection := range querySearchSources {
|
||||
cur, err := b.store.db.Collection(collection).Aggregate(ctx, mongo.Pipeline{{{Key: "$project", Value: querySearchSourceProjection(collection)}}})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for cur.Next(ctx) {
|
||||
var doc bson.M
|
||||
if err = cur.Decode(&doc); err != nil {
|
||||
_ = cur.Close(ctx)
|
||||
return err
|
||||
}
|
||||
b.sourceDigests[querySourceIdentity(collection, doc["_id"])] = querySourceHash(doc)
|
||||
}
|
||||
err = cur.Err()
|
||||
_ = cur.Close(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
func (b *QueryReadBuilder) searchSourceChanged(ctx context.Context, collection string, key bson.M) (bool, string, error) {
|
||||
identity := querySourceIdentity(collection, key["_id"])
|
||||
old, exists := b.sourceDigests[identity]
|
||||
cur, err := b.store.db.Collection(collection).Aggregate(ctx, mongo.Pipeline{{{Key: "$match", Value: key}}, {{Key: "$project", Value: querySearchSourceProjection(collection)}}})
|
||||
if err != nil {
|
||||
return false, "", err
|
||||
}
|
||||
defer cur.Close(ctx)
|
||||
if !cur.Next(ctx) {
|
||||
if err = cur.Err(); err != nil {
|
||||
return false, "", err
|
||||
}
|
||||
delete(b.sourceDigests, identity)
|
||||
return true, old.Organization, nil
|
||||
}
|
||||
var doc bson.M
|
||||
if err = cur.Decode(&doc); err != nil {
|
||||
return false, "", err
|
||||
}
|
||||
next := querySourceHash(doc)
|
||||
b.sourceDigests[identity] = next
|
||||
if exists && old.Digest == next.Digest {
|
||||
return false, next.Organization, nil
|
||||
}
|
||||
if exists && old.Organization != next.Organization {
|
||||
return true, "", nil
|
||||
}
|
||||
return true, next.Organization, nil
|
||||
}
|
||||
|
||||
func (b *QueryReadBuilder) observeQuerySourceEvent(ctx context.Context, l queryModelLease, collection, operation string, key bson.M) error {
|
||||
if operation == "drop" || operation == "rename" || operation == "invalidate" {
|
||||
b.mu.Lock()
|
||||
b.all, b.allSearch = true, true
|
||||
b.mu.Unlock()
|
||||
return b.store.markQuerySearchDirty(ctx, l, "")
|
||||
}
|
||||
org := ""
|
||||
searchChanged := false
|
||||
if querySearchSources[collection] {
|
||||
var err error
|
||||
searchChanged, org, err = b.searchSourceChanged(ctx, collection, key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
} else if operation != "delete" && operation != "drop" && operation != "rename" {
|
||||
var doc struct {
|
||||
Org string `bson:"organizationId"`
|
||||
}
|
||||
if err := b.store.db.Collection(collection).FindOne(ctx, key, options.FindOne().SetProjection(bson.M{"organizationId": 1})).Decode(&doc); err != nil && err != mongo.ErrNoDocuments {
|
||||
return err
|
||||
}
|
||||
org = doc.Org
|
||||
}
|
||||
if operation == "drop" || operation == "rename" || operation == "invalidate" {
|
||||
org = ""
|
||||
searchChanged = true
|
||||
}
|
||||
b.mu.Lock()
|
||||
if org == "" {
|
||||
b.all = true
|
||||
} else {
|
||||
b.dirty[org] = true
|
||||
}
|
||||
if searchChanged {
|
||||
if org == "" {
|
||||
b.allSearch = true
|
||||
} else {
|
||||
b.searchDirty[org] = true
|
||||
}
|
||||
}
|
||||
b.mu.Unlock()
|
||||
if searchChanged {
|
||||
return b.store.markQuerySearchDirty(ctx, l, org)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *QueryReadBuilder) recordObserved(ctx context.Context, l queryModelLease, at time.Time, changed bool) error {
|
||||
update := bson.M{"$set": bson.M{"observedAt": at}}
|
||||
if changed {
|
||||
update["$inc"] = bson.M{"productionRevision": int64(1)}
|
||||
}
|
||||
r, err := b.store.queryControl().UpdateOne(ctx, queryLeaseFilter(l), update)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if r.MatchedCount != 1 {
|
||||
return fmt.Errorf("query observer fenced")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
36
scripts/lib/query_serving_evidence.py
Normal file
36
scripts/lib/query_serving_evidence.py
Normal file
@ -0,0 +1,36 @@
|
||||
"""Strict comparison of isolated query-serving measurements; no production IO."""
|
||||
|
||||
def compare(base, candidate):
|
||||
problems=[]
|
||||
if base.get('fixture_digest')!=candidate.get('fixture_digest') or not base.get('fixture_digest'):
|
||||
problems.append('fixture identity mismatch')
|
||||
if not base.get('metrics') or not candidate.get('metrics'):
|
||||
problems.append('missing request measurements')
|
||||
if candidate.get('actors')!=81 or candidate.get('errors')!=0 or candidate.get('mongo_write_commands')!=0:
|
||||
problems.append('concurrency, request errors or business writes failed')
|
||||
counters=candidate.get('query_serving_statistics',{})
|
||||
if not all(counters.get(k,0)>0 for k in ('report_hits','search_hits','typesense_reads','cache_hits')):
|
||||
problems.append('accelerated paths not proven active')
|
||||
for key,m in candidate.get('metrics',{}).items():
|
||||
limit=2000 if '.search' in key else 500 if key.endswith('.panel') or key=='admin.production' else None
|
||||
if limit and m.get('p95_ms',float('inf'))>limit:problems.append('latency exceeded: '+key)
|
||||
for row in candidate.get('cold_samples',[]):
|
||||
if (row['Key'].endswith('.panel') or row['Key']=='admin.production') and (row['Code']!=200 or row['MS']>2000):
|
||||
problems.append('prewarmed first visit exceeded: '+row['Key'])
|
||||
def hashes(report):
|
||||
out={}
|
||||
for r in report.get('cold_samples',[])+report.get('samples',[]):
|
||||
if r.get('Code')==200:out.setdefault((r['Actor'],r['Key']),set()).add(r['Hash'])
|
||||
return out
|
||||
old,new=hashes(base),hashes(candidate)
|
||||
overlap=old.keys() & new.keys()
|
||||
if not overlap:problems.append('no comparable successful results')
|
||||
for key in overlap:
|
||||
if old[key]!=new[key]:problems.append('business result mismatch: '+':'.join(key))
|
||||
missing=sorted(':'.join(k) for k in new.keys()-old.keys())
|
||||
# Baseline timeouts cannot demonstrate equivalence; functional fixtures must
|
||||
# cover those scopes separately. Keep them visible rather than marking equal.
|
||||
return {'passed':not problems and not missing,'problems':sorted(set(problems)),
|
||||
'comparableActorEndpoints':len(overlap),'baselineUnavailableComparisons':missing,
|
||||
'cpuRatio':candidate.get('process_cpu_seconds',0)/max(base.get('process_cpu_seconds',0),.00001),
|
||||
'retainedHeapRatio':candidate.get('go_heap_after_gc',0)/max(base.get('go_heap_after_gc',0),1)}
|
||||
31
scripts/local/query-serving.compose.yml
Normal file
31
scripts/local/query-serving.compose.yml
Normal file
@ -0,0 +1,31 @@
|
||||
name: fanyu-query-local
|
||||
services:
|
||||
mongo:
|
||||
image: mongo@sha256:5d7043a4ffe02b9ed1b6e0bab057546981af5ca0a79107e9c461e49bc44c0a7b
|
||||
command: ["mongod", "--replSet", "query-rs", "--bind_ip_all", "--wiredTigerCacheSizeGB", "0.5"]
|
||||
ports: ["127.0.0.1:27019:27017"]
|
||||
mem_limit: 4g
|
||||
cpus: 2
|
||||
ulimits:
|
||||
nofile:
|
||||
soft: 65536
|
||||
hard: 65536
|
||||
volumes: ["mongo-query:/data/db"]
|
||||
valkey:
|
||||
image: valkey/valkey@sha256:e51a82741b780e4bf315db10753edf07eea6496d405fa7df2a8677a18f5e7464
|
||||
command: ["valkey-server", "--maxmemory", "128mb", "--maxmemory-policy", "allkeys-lru", "--save", "", "--appendonly", "no"]
|
||||
ports: ["127.0.0.1:16379:6379"]
|
||||
mem_limit: 256m
|
||||
cpus: 1
|
||||
typesense:
|
||||
image: typesense/typesense@sha256:610f2d34b1f93d00762869da2c67736775e5798d19a2c8b91b014b8a0cc1e110
|
||||
environment:
|
||||
TYPESENSE_API_KEY: "${QUERY_TEST_TYPESENSE_API_KEY:?synthetic local key required}"
|
||||
TYPESENSE_DATA_DIR: /data
|
||||
ports: ["127.0.0.1:18108:8108"]
|
||||
mem_limit: 1g
|
||||
cpus: 2
|
||||
volumes: ["typesense-query:/data"]
|
||||
volumes:
|
||||
mongo-query:
|
||||
typesense-query:
|
||||
97
scripts/test_query_serving.py
Normal file
97
scripts/test_query_serving.py
Normal file
@ -0,0 +1,97 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Fixed loopback synthetic dependencies. Never accepts a customer database URI."""
|
||||
import argparse
|
||||
import fcntl
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
from pathlib import Path
|
||||
import secrets
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import urllib.request
|
||||
from lib.release_checks import check_environment, write_log, source_fingerprint
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
PRIVATE = ROOT / 'local-data/query-serving-local'
|
||||
|
||||
def environment():
|
||||
PRIVATE.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||
keyfile = PRIVATE / 'synthetic-key'
|
||||
if not keyfile.exists():
|
||||
with os.fdopen(os.open(keyfile, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600), 'w') as stream:
|
||||
stream.write(secrets.token_hex(24))
|
||||
env = {k:v for k,v in check_environment().items() if not k.startswith('QUERY_')}
|
||||
env.update(QUERY_TEST_TYPESENSE_API_KEY=keyfile.read_text(), QUERY_SERVING_LOCAL_TEST='1')
|
||||
return env
|
||||
|
||||
def compose(env, *args):
|
||||
result = subprocess.run(['docker','compose','-f',str(ROOT/'scripts/local/query-serving.compose.yml'),*args],cwd=ROOT,env=env,capture_output=True,text=True,timeout=300)
|
||||
if result.returncode:
|
||||
proof=write_log(PRIVATE/'logs', ['docker','compose',*args], result.stdout+'\n'+result.stderr,'failed')
|
||||
raise RuntimeError('isolated dependencies failed; private log: '+proof['log'])
|
||||
return result.stdout
|
||||
|
||||
def start(env):
|
||||
compose(env,'up','-d')
|
||||
for _ in range(60):
|
||||
try:
|
||||
compose(env,'exec','-T','mongo','mongosh','--quiet','--eval',"try {rs.status()} catch(e) {rs.initiate({_id:'query-rs',members:[{_id:0,host:'localhost:27017'}]})}")
|
||||
with urllib.request.urlopen('http://127.0.0.1:18108/health',timeout=1) as r:
|
||||
if json.load(r).get('ok'):return
|
||||
except (RuntimeError,OSError,ValueError):
|
||||
pass
|
||||
time.sleep(.5)
|
||||
raise RuntimeError('isolated dependencies not ready')
|
||||
|
||||
def run_command(argv, env, timeout):
|
||||
child=subprocess.Popen(argv,cwd=ROOT,env=env,stdout=subprocess.PIPE,stderr=subprocess.PIPE,text=True,start_new_session=True)
|
||||
try:
|
||||
stdout,stderr=child.communicate(timeout=timeout)
|
||||
except subprocess.TimeoutExpired:
|
||||
try:os.killpg(child.pid,signal.SIGTERM)
|
||||
except ProcessLookupError:pass
|
||||
try:stdout,stderr=child.communicate(timeout=5)
|
||||
except subprocess.TimeoutExpired:
|
||||
try:os.killpg(child.pid,signal.SIGKILL)
|
||||
except ProcessLookupError:pass
|
||||
stdout,stderr=child.communicate()
|
||||
return subprocess.CompletedProcess(argv,124,stdout,stderr+'\nphase timed out')
|
||||
return subprocess.CompletedProcess(argv,child.returncode,stdout,stderr)
|
||||
|
||||
def main():
|
||||
parser=argparse.ArgumentParser()
|
||||
parser.add_argument('--start',action='store_true')
|
||||
parser.add_argument('--down',action='store_true',help='stop this test project; retain local volumes')
|
||||
parser.add_argument('--run',default='^TestQueryServing|^TestQueryReadModel|^TestQuerySearch')
|
||||
parser.add_argument('--race',action='store_true')
|
||||
args=parser.parse_args();env=environment()
|
||||
before=source_fingerprint(ROOT)
|
||||
with (PRIVATE/'runner.lock').open('w') as lock:
|
||||
fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
|
||||
if args.down:compose(env,'down');return 0
|
||||
if args.start:start(env)
|
||||
argv=['go','test','./internal/queryserving','./internal/store','./internal/api','./internal/outputsnapshot','-run',args.run,'-count=1','-timeout=10m','-v']
|
||||
if args.race:argv.append('-race')
|
||||
result=run_command(argv,env,660)
|
||||
proof=write_log(PRIVATE/'logs',argv,result.stdout+'\n'+result.stderr,'passed' if result.returncode==0 else 'failed')
|
||||
print(json.dumps(proof))
|
||||
# Do not interpret an opt-in skip/empty test run as acceptance.
|
||||
if result.returncode:
|
||||
return result.returncode
|
||||
if not before or source_fingerprint(ROOT)!=before:
|
||||
print('Source identity changed during checks; acceptance remains pending.');return 2
|
||||
required=set()
|
||||
for name in ('internal/queryserving/serving_test.go','internal/store/query_read_models_test.go','internal/store/query_recovery_test.go','internal/api/query_serving_test.go','internal/outputsnapshot/query_shared_cache_test.go'):
|
||||
required.update(re.findall(r'^func (Test\w+)\(', (ROOT/name).read_text(), re.M))
|
||||
passed=set(re.findall(r'^--- PASS: (Test\w+)',result.stdout,re.M))
|
||||
if required-passed:
|
||||
print('Missing required cases: '+', '.join(sorted(required-passed)));return 2
|
||||
if '--- SKIP:' in result.stdout or '[no tests to run]' in result.stdout:
|
||||
print('Unverified skipped/empty coverage; inspect report.');return 2
|
||||
return result.returncode
|
||||
|
||||
if __name__=='__main__':
|
||||
raise SystemExit(main())
|
||||
91
scripts/test_query_serving_performance.py
Normal file
91
scripts/test_query_serving_performance.py
Normal file
@ -0,0 +1,91 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Compare the same synthetic API fixture before/after query serving."""
|
||||
import argparse
|
||||
import fcntl
|
||||
import datetime as dt
|
||||
import json
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from test_query_serving import ROOT, PRIVATE, environment, start, run_command
|
||||
from lib.query_serving_evidence import compare
|
||||
from lib.release_checks import write_log, source_fingerprint
|
||||
|
||||
def main():
|
||||
parser=argparse.ArgumentParser()
|
||||
parser.add_argument('--start',action='store_true')
|
||||
parser.add_argument('--production-shape',action='store_true')
|
||||
parser.add_argument('--rounds',type=int,default=1)
|
||||
parser.add_argument('--warmup',type=int,default=5)
|
||||
parser.add_argument('--duration',type=int,default=30)
|
||||
parser.add_argument('--database',help='reuse this script\'s existing synthetic fixture')
|
||||
args=parser.parse_args()
|
||||
if not 1<=args.rounds<=3 or not 1<=args.duration<=1800 or not 1<=args.warmup<=300:parser.error('bounded rounds/times required')
|
||||
if args.database and (not args.database.startswith('poll_pool_read_mix_query_') or not args.database.replace('_','').isalnum()):parser.error('synthetic database only')
|
||||
env={k:v for k,v in environment().items() if not k.startswith('POLL_POOL_')}
|
||||
dependency_lock=(PRIVATE/'runner.lock').open('w')
|
||||
try:fcntl.flock(dependency_lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
|
||||
except BlockingIOError:
|
||||
print('Isolated query dependencies already in use; comparison not started.');return 2
|
||||
if args.start:start(env)
|
||||
folder=ROOT/'local-data/diagnostics'/('query-serving-'+dt.datetime.now(dt.timezone.utc).strftime('%Y%m%dT%H%M%SZ'))
|
||||
folder.mkdir(parents=True)
|
||||
database=args.database or 'poll_pool_read_mix_query_'+uuid.uuid4().hex[:16]
|
||||
env.update(POLL_POOL_LOAD='1',POLL_POOL_LOCAL_MONGO_URI='mongodb://127.0.0.1:27019/?replicaSet=query-rs&directConnection=true',POLL_POOL_FIXTURE_DATABASE=database,
|
||||
POLL_POOL_KEEP_FIXTURE='1',POLL_POOL_QUERY_SERVING_PREPARE='1',POLL_POOL_QUERY_PANELS='1',POLL_POOL_READMODELS='1',POLL_POOL_SEARCH='1',POLL_POOL_SEARCH_INDEXES='1',POLL_POOL_PAGE_SHAPE='1',POLL_POOL_ADMIN_TASKS='1' if args.production_shape else '0',POLL_POOL_AUTHORIZATION_STATE='initialized',
|
||||
POLL_POOL_WARMUP_SECONDS=str(args.warmup),POLL_POOL_DURATION_SECONDS=str(args.duration),ANNOTATION_SOURCE_VALIDATION_MODE='advisory')
|
||||
if args.production_shape:
|
||||
env.update(POLL_POOL_PRODUCTION_SHAPE='1', POLL_POOL_RUN_PAYLOAD_KIB='80',
|
||||
POLL_POOL_TASK_PAYLOAD_KIB='16', POLL_POOL_AVAILABLE_PER_STAGE='215',
|
||||
POLL_POOL_AUTHOR_HISTORY='100', GOMAXPROCS='2')
|
||||
before=source_fingerprint(ROOT)
|
||||
report={'database':database,'inputDigest':before,'shape':'production' if args.production_shape else 'small','rounds':[],'resourceSamples':[],'passed':False,'limits':['synthetic loopback API harness, no media or production writes','process resource sampling includes builder and handler; not Kubernetes Pod limits']}
|
||||
argv=['go','test','./internal/api','-run','^TestPollPoolReadMixLocal$','-count=1','-timeout=45m','-v']
|
||||
def run(name,updates):
|
||||
settings={**env,**updates,'POLL_POOL_OUTPUT':str(folder/(name+'.json'))}
|
||||
active['phase']=name
|
||||
started=dt.datetime.now(dt.timezone.utc).isoformat()
|
||||
result=run_command(argv,settings,args.warmup+args.duration+900)
|
||||
proof=write_log(folder,argv,result.stdout+'\n'+result.stderr,'passed' if result.returncode==0 else 'failed')
|
||||
row={'phase':name,'startedAt':started,'endedAt':dt.datetime.now(dt.timezone.utc).isoformat(),'exitCode':result.returncode,**proof,'report':settings['POLL_POOL_OUTPUT']}
|
||||
report['rounds'].append(row)
|
||||
(folder/'comparison.json').write_text(json.dumps(report,indent=2)+'\n')
|
||||
print(json.dumps(row),flush=True)
|
||||
return result.returncode==0
|
||||
active={'phase':'prepare'}
|
||||
if not args.database:
|
||||
if not run('prepare',{'POLL_POOL_FIXTURE_ONLY':'1'}):return 1
|
||||
stopped=threading.Event()
|
||||
def resources():
|
||||
while not stopped.is_set():
|
||||
try:
|
||||
p=subprocess.run(['docker','stats','--no-stream','--format','{{json .}}','fanyu-query-local-mongo-1','fanyu-query-local-valkey-1','fanyu-query-local-typesense-1'],capture_output=True,text=True,timeout=8)
|
||||
if p.returncode==0:report['resourceSamples'].append({'phase':active['phase'],'at':dt.datetime.now(dt.timezone.utc).isoformat(),'containers':[json.loads(line) for line in p.stdout.splitlines()]})
|
||||
except (OSError,ValueError,subprocess.TimeoutExpired):pass
|
||||
(folder/'progress.json').write_text(json.dumps({'phase':active['phase'],'at':dt.datetime.now(dt.timezone.utc).isoformat(),'samples':len(report['resourceSamples'])})+'\n')
|
||||
stopped.wait(2)
|
||||
thread=threading.Thread(target=resources,daemon=True);thread.start()
|
||||
try:
|
||||
for i in range(args.rounds):
|
||||
baseline='baseline' if i==0 else 'baseline-'+str(i+1)
|
||||
run(baseline,{'POLL_POOL_REUSE_FIXTURE':'1','POLL_POOL_ALLOW_BASELINE_FAILURE':'1'})
|
||||
if not run('candidate-'+str(i+1),{'POLL_POOL_REUSE_FIXTURE':'1','POLL_POOL_QUERY_SERVING':'1','POLL_POOL_MAX_SEARCH_P95_MS':'2000'}):break
|
||||
finally:
|
||||
stopped.set();thread.join(timeout=10)
|
||||
report['inputsChangedDuringRun']=source_fingerprint(ROOT)!=before
|
||||
candidates=[r for r in report['rounds'] if r['phase'].startswith('candidate-')]
|
||||
report['comparisons']=[]
|
||||
try:
|
||||
for i,row in enumerate(candidates):
|
||||
basefile='baseline.json' if i==0 else 'baseline-'+str(i+1)+'.json'
|
||||
report['comparisons'].append(compare(json.loads((folder/basefile).read_text()),json.loads(Path(row['report']).read_text())))
|
||||
except (OSError,ValueError,KeyError):report['comparisons'].append({'passed':False,'problems':['missing/invalid comparison report']})
|
||||
report['resourceReviewRequired']=len(report['comparisons'])==3 and any(all(r.get(key,0)>1.1 for r in report['comparisons']) for key in ('cpuRatio','retainedHeapRatio'))
|
||||
report['passed']=not report['resourceReviewRequired'] and len(candidates)==args.rounds and all(r['exitCode']==0 for r in candidates) and not report['inputsChangedDuringRun'] and all(r['passed'] for r in report['comparisons'])
|
||||
(folder/'comparison.json').write_text(json.dumps(report,indent=2)+'\n')
|
||||
print('Comparison: '+str(folder/'comparison.json'),flush=True)
|
||||
return 0 if report['passed'] else 1
|
||||
|
||||
if __name__=='__main__':raise SystemExit(main())
|
||||
@ -883,6 +883,65 @@
|
||||
"warnings": [
|
||||
"工作流门禁仅使用合成媒体和隔离 Mongo;严格模式清单不是校验证书,生产容量、备份恢复和部署需分别验证。"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "query-serving",
|
||||
"paths": [
|
||||
"^internal/queryserving/",
|
||||
"^internal/api/(database_browser|poll_pool_load_test)\\.go$",
|
||||
"^internal/store/query_.*\\.go$",
|
||||
"^internal/store/private_derived_collections\\.go$",
|
||||
"^internal/(api|outputsnapshot)/query_.*\\.go$",
|
||||
"^cmd/query-read-builder/",
|
||||
"^scripts/(test_query_serving.*\\.py|lib/query_serving_evidence\\.py|local/query-serving\\.compose\\.yml)$",
|
||||
"^Dockerfile\\.query-read-builder$",
|
||||
"^tests/release/test_query_serving_evidence\\.py$",
|
||||
"^docs/operations/query-serving\\.md$"
|
||||
],
|
||||
"frontend": [],
|
||||
"go": {
|
||||
"./internal/queryserving": [
|
||||
"^TestQueryServing"
|
||||
],
|
||||
"./internal/store": [
|
||||
"^TestQueryReadModel",
|
||||
"^TestQuerySearch"
|
||||
],
|
||||
"./internal/outputsnapshot": [
|
||||
"^TestQueryServing"
|
||||
],
|
||||
"./internal/api": [
|
||||
"^TestQueryServing"
|
||||
]
|
||||
},
|
||||
"task": [
|
||||
{
|
||||
"argv": [
|
||||
"python3",
|
||||
"scripts/test_query_serving.py",
|
||||
"--start",
|
||||
"--race"
|
||||
],
|
||||
"tests": true,
|
||||
"env": {}
|
||||
},
|
||||
{
|
||||
"argv": [
|
||||
"python3",
|
||||
"-m",
|
||||
"unittest",
|
||||
"discover",
|
||||
"-s",
|
||||
"tests/release",
|
||||
"-p",
|
||||
"test_query_serving_evidence.py"
|
||||
],
|
||||
"tests": true
|
||||
}
|
||||
],
|
||||
"warnings": [
|
||||
"真实三组件仅使用固定本机合成夹具;完整规模三轮、故障恢复容量及生产部署独立验收。"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
29
tests/release/test_query_serving_evidence.py
Normal file
29
tests/release/test_query_serving_evidence.py
Normal file
@ -0,0 +1,29 @@
|
||||
import sys
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
sys.path.insert(0,str(Path(__file__).resolve().parents[2]/'scripts'))
|
||||
from lib.query_serving_evidence import compare
|
||||
|
||||
class QueryEvidenceTests(unittest.TestCase):
|
||||
def fixture(self):
|
||||
sample={'Actor':'reviewer','Key':'acceptance.all.search','Code':200,'MS':20,'Hash':'business-id-claim-counts'}
|
||||
return {'fixture_digest':'same','actors':81,'errors':0,'mongo_write_commands':0,'metrics':{'acceptance.all.search':{'p95_ms':20}},'samples':[sample], 'cold_samples':[], 'query_serving_statistics':dict(report_hits=1,search_hits=1,typesense_reads=1,cache_hits=1)}
|
||||
def test_requires_real_measurements_and_active_components(self):
|
||||
a=self.fixture();b=self.fixture();self.assertTrue(compare(a,b)['passed'])
|
||||
b['query_serving_statistics']={};self.assertFalse(compare(a,b)['passed'])
|
||||
self.assertFalse(compare({}, {})['passed'])
|
||||
def test_results_and_latency_are_both_mandatory(self):
|
||||
a=self.fixture();b=self.fixture();b['samples'][0]['Hash']='lost-task'
|
||||
self.assertFalse(compare(a,b)['passed'])
|
||||
b=self.fixture();b['metrics']['acceptance.all.search']['p95_ms']=2001
|
||||
self.assertFalse(compare(a,b)['passed'])
|
||||
def test_business_write_and_cold_panel_failure_are_not_success(self):
|
||||
a=self.fixture();b=self.fixture();b['mongo_write_commands']=1
|
||||
self.assertFalse(compare(a,b)['passed'])
|
||||
b=self.fixture();b['cold_samples']=[{'Actor':'admin','Key':'admin.production','Code':202,'MS':1,'Hash':'empty'}]
|
||||
self.assertFalse(compare(a,b)['passed'])
|
||||
def test_baseline_timeout_keeps_equivalence_pending(self):
|
||||
a=self.fixture();b=self.fixture();a['samples'][0]['Code']=500
|
||||
r=compare(a,b);self.assertFalse(r['passed']);self.assertTrue(r['baselineUnavailableComparisons'])
|
||||
|
||||
if __name__=='__main__':unittest.main()
|
||||
Loading…
x
Reference in New Issue
Block a user