From ffe186473c2afd52ff5ca2b844ab2c2bef55e60b Mon Sep 17 00:00:00 2001 From: d-robotics Date: Sun, 15 Nov 2026 08:11:09 -0800 Subject: [PATCH] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20Persist=20resumable=20quer?= =?UTF-8?q?y=20build=20generations?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cmd/query-read-builder/main.go | 10 +- docs/adr/0014-independent-query-database.md | 16 + docs/operations/query-build-lifecycle.md | 66 ++++ internal/api/query_serving_test.go | 2 +- internal/store/private_derived_collections.go | 2 +- internal/store/production_stats.go | 3 + internal/store/query_build_apply.go | 321 ++++++++++++++++ internal/store/query_build_capture.go | 362 ++++++++++++++++++ .../store/query_build_capture_local_test.go | 108 ++++++ internal/store/query_build_fact_document.go | 48 +++ internal/store/query_build_jobs.go | 103 +++++ internal/store/query_build_lifecycle.go | 69 ++++ internal/store/query_build_lifecycle_test.go | 46 +++ internal/store/query_build_plan.go | 196 ++++++++++ .../store/query_build_resume_local_test.go | 360 +++++++++++++++++ internal/store/query_build_runner.go | 197 ++++++++++ .../store/query_build_scale_local_test.go | 226 +++++++++++ internal/store/query_model_maintenance.go | 17 + internal/store/query_read_builder.go | 25 +- internal/store/workforce_facts.go | 1 + internal/store/workforce_stats.go | 3 + scripts/test_query_replica.py | 2 + scripts/verification-scopes.json | 8 +- 23 files changed, 2183 insertions(+), 8 deletions(-) create mode 100644 docs/operations/query-build-lifecycle.md create mode 100644 internal/store/query_build_apply.go create mode 100644 internal/store/query_build_capture.go create mode 100644 internal/store/query_build_capture_local_test.go create mode 100644 internal/store/query_build_fact_document.go create mode 100644 internal/store/query_build_jobs.go create mode 100644 internal/store/query_build_lifecycle.go create mode 100644 internal/store/query_build_lifecycle_test.go create mode 100644 internal/store/query_build_plan.go create mode 100644 internal/store/query_build_resume_local_test.go create mode 100644 internal/store/query_build_runner.go create mode 100644 internal/store/query_build_scale_local_test.go diff --git a/cmd/query-read-builder/main.go b/cmd/query-read-builder/main.go index db49830a..29457338 100644 --- a/cmd/query-read-builder/main.go +++ b/cmd/query-read-builder/main.go @@ -4,6 +4,7 @@ package main import ( "context" "droboticflow/internal/store" + "errors" "flag" "go.mongodb.org/mongo-driver/mongo" "go.mongodb.org/mongo-driver/mongo/options" @@ -87,8 +88,13 @@ func main() { return } for workCtx.Err() == nil { - if b.Run(workCtx) != nil && workCtx.Err() == nil { - log.Print("query builder paused; retrying with full reconciliation") + if err := b.Run(workCtx); err != nil && workCtx.Err() == nil { + if errors.Is(err, store.ErrQueryBuildStopped) { + log.Print("query builder stopped; checkpoint retained; explicit recovery required") + <-workCtx.Done() + return + } + log.Print("query builder retrying; durable build checkpoint retained") } select { case <-workCtx.Done(): diff --git a/docs/adr/0014-independent-query-database.md b/docs/adr/0014-independent-query-database.md index e1a8552f..b8b24b27 100644 --- a/docs/adr/0014-independent-query-database.md +++ b/docs/adr/0014-independent-query-database.md @@ -66,3 +66,19 @@ 准备状态通过显式媒体类型协商;旧客户端收到未就绪错误而非成功的非报表JSON。活跃配方的登记触碰合并到15分钟一次,后台统计刷新周期不变。任务池只共享候选身份,个人任务/审查历史维持当前权威查询;权限目录的逐项目重复读合并为请求内批量读,继续复用原授权种子规则,不形成共享权限缓存。 候选身份的冷缓存查询合并仅作用于相同数据库、组织、事实代次和完整筛选,沿用既有共享名称读取的8秒工作上限及独立取消规则;完成后的长效成品仍只在共享缓存/查询库中。每个请求各自取当前任务与运行服务端权限校验,合并结果不带授权结论、正文或可领取标志。 + +## 首次构建与日常刷新分离(2026-11-15,用户批准,实施中) + +首次构建不再以固定5分钟要求全部完成。每轮执行有调度预算,每个读取、持久化批次及提交有独立上限;预算用尽记录可靠断点并让出,属于正常控制流,不触发全流程停副本。窗口到期、无进展、数据库不可用、权限/租约失效、容量及持久化异常仍须暂停并记录分项原因。 + +复用已同步查询事实及e4c4a20d的首代暂存指纹核对。构建采用持久输入/计划、分批应用、检查及原子发布几个阶段;已封存计划在续建、部署或重启后只读取未确认部分,不重新回填权威库,也不重新扫描完整业务输入。输入捕获本身须保留有界读取断点;源快照不可恢复时不得混合前后版本或伪造完成。 + +日常刷新按3分钟后台周期及15分钟展示年龄契约单独安排预算;继续提供上一完整版本。生产统计、人员统计和作业记录相关头只能在同一封存输入代次完成后发布,不能先切部分头。权限、供应商隔离与正式操作仍由原权威路径实时校验。 + +保持多数提交和日志持久性、25Gi容量及现有限额。512KiB/s内部写入限速须计入首次构建的真实数据量及确认耗时;若需调整,参数、共享盘影响和回退另列具体启用计划,不以延长请求超时或降低持久性替代。生产入口仍未切换,本节不是生产验收结论。 + +实现约束:首次构建每轮60秒、日常变化刷新每轮30秒,读取总界限5秒(最多3次瞬时错误重试)、每次提交15秒。到轮次边界仅让出,不取消已开始的持久提交;下轮通过租约和持久游标继续。无进展按累计有效执行时间计算,不计维护停机时间。`RunOnce`在独立周期查询库中也不再强加5分钟全程期限,仍服从调用方取消和维护窗口。 + +输入捕获和计划保存在私有`queryBuildJobs`/`queryBuildParts`。原生输入分段、游标以及写入批次、应用游标分别在同一`majority+journal`事务确认。续建重新核对封存计划指纹和e4暂存事实哈希;未发布的大块结果由清理程序保留。派生事实采用原有版本可见性规则,全部完成及行数核对后才在同一事务发布两个统计头。已追平观察器证明业务输入未变化时,只重新确认完整结果,不重新读取历史、计算或遍历全量事实。 + +输入快照的逻辑时间与展示时间分开保存。已经封存输入后,恢复不依赖原快照仍在Mongo历史窗口内;尚未捕获完的输入若遇到快照历史丢失,必须保留分段并明确停止,不能切换时间点后拼接新旧结果。该异常不自动重新回填权威库。首次上线前须检查查询库快照/oplog保留窗口,不能把任意长停机期间未封存输入的自动恢复视为已经覆盖。 diff --git a/docs/operations/query-build-lifecycle.md b/docs/operations/query-build-lifecycle.md new file mode 100644 index 00000000..07523fb4 --- /dev/null +++ b/docs/operations/query-build-lifecycle.md @@ -0,0 +1,66 @@ +# 查询库首次构建与日常刷新 + +## 范围与状态 + +仅独立周期查询库的构建器采用持久任务。权威Mongo、标注/切片/送检/审核/验收、权限目录和版本竞争路径不改。生产查询入口仍未切换。本地通过、代码部署、Pod就绪分别不代表影子验收或分流降载验收。 + +上一轮e4影子构建已完成:复用125,186条,补写14,820条;事实写入阶段约101秒,后续观察到至少两轮完整刷新。原维护窗口在2026-11-15 14:38:42 UTC到期,两影子作业按约定暂停到0副本并保留断点。本文不延长该窗口。 + +## 执行与持久性 + +| 参数 | 计划值 | 说明 | +| --- | --- | --- | +| 首次构建单轮调度 | 60秒 | 正常让出,不是全程完成期限 | +| 变化刷新单轮调度 | 30秒 | 延续既有3分钟周期、15分钟展示年龄 | +| 原生输入单次逻辑读取 | 5秒 | 最多3次瞬时重试,每次服务端1.5秒 | +| 持久事务 | 15秒 | 多数提交及journal;游标与数据共同确认 | +| 累计无确认进展 | 5分钟有效执行时间 | 不含停机;真正无进展持久记录后停止 | +| 内部写入限速 | 512KiB/s | 暂不调整;不能以延长超时取代限速核对 | +| 应用批次 | 最多200逻辑行,约64KiB写入 | 单条较大的合法事实仍完整写入 | +| 容量/Pod/底层IO | 保持现有配置 | 25Gi、现有内存/CPU/IO边界不变 | + +正常阶段:捕获输入→保存完整计划→应用事实→清理未匹配的未来版本→核对→事务发布。新一代完整之前,读者继续使用原完整代次及原权限校验;超出既有展示年龄仍按原错误/成品回退契约处理,不无限伪装新鲜或集中回源。 + +`queryBuildJobs`记录组织、输入代次、快照、阶段、事实应用游标、清理游标及分项失败;`queryBuildParts`保存带校验和的压缩输入和计划。均为私有集合,禁止从通用数据管理API读写。维护清理只回收不再被当前任务/输入引用的派生分段,绝不清客户标注或同步事实。 + +无变化日常刷新只有在观察器追平证明新鲜且组织、范围、逻辑版本及业务修订相同的情况下,才直接重新确认原完整事实和统计结果。否则新建输入代次并按上述流程计算。变更刷新可以读取查询库必要输入;不会重新启动原Mongo历史回填。 + +## 暂停、恢复与限制 + +- 轮次预算耗尽:保留pending状态和游标,下轮正常续建,不增加失败退避,不因此降为0副本。 +- 进程退出/部署/调用方取消:已确认批次保留;新持有者以新租约恢复,旧持有者不能提交。 +- 数据库/权限/租约/持久化/容量异常:记录阶段、类别及Mongo代码;停止构建,保持旧完整结果,不自动清库重试。 +- 维护截止:按已批准观察器策略停影子作业,保存累计同步预算和断点。启动必须有新的有效窗口。 +- 已封存输入:原快照历史过期不影响应用恢复。未封存输入若历史已丢失,停止并保留分段,需单列恢复方案,不能混入新时间点。 +- 算法/字段语义变更必须更新构建逻辑schema并审核旧任务迁移;封存计划指纹不符时禁止继续混写。 + +## 待确认的实际启用方案 + +1. 通过本任务检查和正式发布门禁,提交候选来源、镜像摘要及最终可执行清单。新代码尚未获得推送/上传或部署确认。 +2. 核对新的维护窗口、当前同步断点/预算、源健康、查询库容量/IO、原有镜像及运行身份;旧窗口不能复用。 +3. 仅为查询库构建器授权私有两集合的必要读写,在查询库`queryWorkforceFacts`增加`workforce_fact_key_v1`:`organizationId,key,fromEpoch,toEpoch`。不在原Mongo建索引。本地14万事实需记录新增索引体积,生产DDL仍列入最终计划。 +4. 构建器保持0副本更新镜像;同步沿原有效授权运行并取得新鲜追平证明后,再启用构建器1副本。保留所有e4确认事实,不能调用旧版`restoreWorkforceStage`清除新任务的未来版本。 +5. 核验首次完整发布、两轮正常刷新、权限/供应商/日期等价、Valkey丢失回退和查询库重启;生产查询入口继续关闭。原Mongo降载需后续经确认切入口后独立验收。 + +回退:先暂停构建器,保留任务/分段/事实、头和同步断点。e4旧构建器不能直接接手尚未完成的新生命周期任务,否则其旧恢复逻辑可能删除未来版本;无损回退优先停构建器并继续读上一完整版本。只有独立核实不存在未完成新任务、已发布完整头可用后,才能另行确认恢复旧镜像。不删表、不回滚客户标注、不重置预算、不降低持久性。 + +## 验证证据 + +证据集中写入本工作树忽略目录`local-data/test-results/query-build-*`,包含读取次数、批次、耗时、数据/索引占用及配置。不得复制客户原文或凭证。正式结果待验证完成后更新,不用短时成功代替完整验收。 + +### 2026-11-15 本地同规模结果 + +固定35,000条合成事件、140,000条事实,查询Mongo为1核/1GiB、WiredTiger 256MiB;生产影子库仍为4核/8GiB,不把本地结果当作生产容量验收。写入限速一直为512KiB/s,持久化一直为majority+journal。 + +| 场景 | 结果 | +| --- | --- | +| 保留e4的125,186条确认事实、补14,814条 | 76.33秒,跨2轮 | +| 同夹具遍历无变化事实作为算法对照 | 42.53秒 | +| 复用已证明无变化的完整结果,两次刷新 | 194.75ms、181.10ms | +| 从零写入140,000条事实 | 318.71秒,跨6轮及进程对象重建 | +| 从零完成后的两次刷新 | 163.63ms、153.75ms | +| 以上构建与刷新对原Mongo的读取 | 0 | + +从零构建确认写入148,610,000 BSON字节,限速理论下界约283秒,实际约319秒;不需要通过增加全程超时或削弱持久性才能完成。采样查询容器CPU最高37.41%,无OOM;Docker VM的IO full avg10最高0.91%,该值不是单库IO等待。新增键索引在14万事实时约14.44MB。内存包含Mongo文件缓存压力,本地1GiB测试不能替代生产8GiB长时验证。 + +快速检查通过。首次全任务执行的浏览器、统计稳定性、权限和共享缓存检查通过;隔离副本组在总编排900秒耗尽后被停止,原报告`verify-1794758125279658000`保留为incomplete,不能写成通过。后续对失败落盘分支完成故障注入后,集中补跑受影响的隔离组;已通过且输入未变的浏览器证据保留。 diff --git a/internal/api/query_serving_test.go b/internal/api/query_serving_test.go index a6585950..bd6174f7 100644 --- a/internal/api/query_serving_test.go +++ b/internal/api/query_serving_test.go @@ -66,7 +66,7 @@ func TestQueryServingProductionResponseChecksCurrentPermissionBeforeCache(t *tes func TestQueryServingPrivateMaterializationsRejectDatabaseBrowser(t *testing.T) { f := newCombinedDraftFixture(t) - for _, name := range []string{"queryReadModelControl", "queryReadModelData"} { + for _, name := range []string{"queryReadModelControl", "queryReadModelData", "queryBuildJobs", "queryBuildParts"} { for _, target := range []string{"platform", "business"} { if r := f.request("GET", "/api/database/collections/"+name+"/documents?target="+target, f.admin, nil); r.Code < 400 { t.Fatalf("private read model leaked: %s %s", name, target) diff --git a/internal/store/private_derived_collections.go b/internal/store/private_derived_collections.go index acb18255..3eabe5da 100644 --- a/internal/store/private_derived_collections.go +++ b/internal/store/private_derived_collections.go @@ -5,5 +5,5 @@ import "droboticflow/internal/outputsnapshot" // Derived generations contain facts from multiple authorization scopes and // must only be served through their permission-aware reporting/search readers. func IsPrivateDerivedCollection(name string) bool { - return name == querySyncDeleteCollection || name == queryReplicaControl || outputsnapshot.IsPrivateCollection(name) || name == "queryReadModelControl" || name == "queryReadModelData" + return name == queryBuildJobsCollection || name == queryBuildPartsCollection || name == querySyncDeleteCollection || name == queryReplicaControl || outputsnapshot.IsPrivateCollection(name) || name == "queryReadModelControl" || name == "queryReadModelData" } diff --git a/internal/store/production_stats.go b/internal/store/production_stats.go index 09a7f302..db8b1929 100644 --- a/internal/store/production_stats.go +++ b/internal/store/production_stats.go @@ -204,6 +204,9 @@ func (s *MongoStore) loadProductionStats(ctx context.Context, org string, projec } } read := func(c mongoCollection, f bson.M, projection bson.M, target any, preferred ...string) error { + if hook, ok := ctx.Value(queryBuildReadKey{}).(queryBuildReadFunc); ok { + return hook(ctx, c, f, projection, target) + } opts := options.Find().SetProjection(projection).SetMaxTime(15 * time.Second) hint := hints[c.Name()] // A project-leading covering index is unsuitable for legacy task-ID lookups. diff --git a/internal/store/query_build_apply.go b/internal/store/query_build_apply.go new file mode 100644 index 00000000..57dc85a2 --- /dev/null +++ b/internal/store/query_build_apply.go @@ -0,0 +1,321 @@ +package store + +import ( + "context" + "errors" + "fmt" + "time" + + "droboticflow/internal/workforcestats" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" +) + +type queryBuildPrevious struct { + ID string `bson:"_id"` + Key string `bson:"key"` + Hash string `bson:"hash"` + From int64 `bson:"fromEpoch"` + To int64 `bson:"toEpoch"` + Retired *time.Time `bson:"retiredAt"` + Expire *time.Time `bson:"expireAt"` +} + +func (s *MongoStore) queryBuildPlanPage(ctx context.Context, j queryBuildJob, n int) ([]workforcestats.Fact, error) { + op, stop := queryBuildOperation(ctx) + defer stop() + var p queryBuildPart + if err := s.db.Collection(queryBuildPartsCollection).FindOne(op, bson.M{"_id": queryBuildPlanPartKey(j, "facts", n), "job": j.Generation}).Decode(&p); err != nil { + return nil, err + } + var facts []workforcestats.Fact + if err := queryBuildJSONDecode(p, &facts); err != nil { + return nil, err + } + if len(facts) == 0 || len(facts) > queryBuildFactPageRows { + return nil, errors.New("invalid sealed fact page") + } + return facts, nil +} + +// A batch checkpoint commits in the very same majority+journal transaction as +// its version writes. A lost acknowledgement can only replay this bounded batch. +func (s *MongoStore) queryBuildApply(ctx context.Context, l queryModelLease, j *queryBuildJob, r *queryBuildRound) error { + c := s.db.Collection(workforceFactCollection) + for j.NextPart < j.PlanParts { + if err := r.check(ctx); err != nil { + return err + } + facts, err := s.queryBuildPlanPage(ctx, *j, j.NextPart) + if err != nil { + return err + } + for j.NextRow < len(facts) { + if err = r.check(ctx); err != nil { + return err + } + models := []mongo.WriteModel{} + bytes, rows, reused := 0, 0, 0 + next := *j + // Keep point lookups bounded; the key index avoids reading large old rows. + keys := []string{} + for _, f := range facts[j.NextRow:] { + keys = append(keys, f.Key) + } + op, stop := queryBuildOperation(ctx) + cur, e := c.Find(op, bson.M{"organizationId": j.Organization, "key": bson.M{"$in": keys}}, options.Find().SetProjection(bson.M{"_id": 1, "key": 1, "hash": 1, "fromEpoch": 1, "toEpoch": 1, "retiredAt": 1, "expireAt": 1}).SetHint("workforce_fact_key_v1")) + previous := map[string][]queryBuildPrevious{} + if e == nil { + var xs []queryBuildPrevious + e = cur.All(op, &xs) + cur.Close(op) + for _, x := range xs { + previous[x.Key] = append(previous[x.Key], x) + } + } + stop() + if e != nil { + return e + } + for next.NextRow < len(facts) { + f := facts[next.NextRow] + doc, hash, size, e := queryBuildFactDocument(j.Organization, j.Epoch, f) + if e != nil { + return e + } + if rows > 0 && bytes+size+512 > queryBuildBatchBytes { + break + } + versions := previous[f.Key] + var current *queryBuildPrevious + var staged *queryBuildPrevious + for i := range versions { + x := &versions[i] + if x.From == j.Epoch && x.ID == doc["_id"] { + staged = x + } + if x.From <= j.OldEpoch && (x.To == 0 || x.To > j.OldEpoch) { + if current != nil { + return errors.New("duplicate committed workforce key") + } + current = x + } + } + unchanged := current != nil && current.Hash == hash + keepID := "" + if unchanged { + keepID = current.ID + if current.To != 0 || current.Retired != nil || current.Expire != nil { + models = append(models, mongo.NewUpdateOneModel().SetFilter(bson.M{"_id": current.ID}).SetUpdate(bson.M{"$unset": bson.M{"toEpoch": "", "retiredAt": "", "expireAt": ""}})) + bytes += 256 + } + reused++ + } else { + next.Changed = true + keepID = doc["_id"].(string) + if current != nil && current.To != j.Epoch { + models = append(models, mongo.NewUpdateOneModel().SetFilter(bson.M{"_id": current.ID}).SetUpdate(bson.M{"$set": bson.M{"toEpoch": j.Epoch, "retiredAt": time.Now().UTC()}})) + bytes += 256 + } + if staged != nil && staged.Hash == hash && staged.To == 0 && staged.Retired == nil && staged.Expire == nil { + reused++ + } else { + models = append(models, mongo.NewReplaceOneModel().SetFilter(bson.M{"_id": doc["_id"]}).SetReplacement(doc).SetUpsert(true)) + bytes += size + } + } + for _, x := range versions { + if x.From > j.OldEpoch && x.ID != keepID { + models = append(models, mongo.NewDeleteOneModel().SetFilter(bson.M{"_id": x.ID, "organizationId": j.Organization, "fromEpoch": bson.M{"$gt": j.OldEpoch}})) + bytes += 256 + } + } + next.NextRow++ + rows++ + } + if rows == 0 { + return errors.New("fact batch made no progress") + } + if next.NextRow == len(facts) { + next.NextPart++ + next.NextRow = 0 + } + next.ConfirmedRows += int64(rows - reused) + next.ReusedRows += int64(reused) + next.ConfirmedBytes += int64(bytes) + next.ProgressAt = time.Now().UTC() + next.ActiveWithoutProgress = 0 + err = s.queryBuildPace(ctx, bytes) + if err != nil { + return err + } + err = s.queryBuildTransaction(ctx, l, func(sc mongo.SessionContext) error { + if len(models) > 0 { + if _, e := c.BulkWrite(sc, models, options.BulkWrite().SetOrdered(true)); e != nil { + return e + } + } + return s.saveQueryBuildJob(sc, next) + }) + if err != nil { + return err + } + *j = next + if j.NextRow == 0 { + break + } + } + } + j.Phase = "cleanup" + return s.queryBuildTransaction(ctx, l, func(sc mongo.SessionContext) error { return s.saveQueryBuildJob(sc, *j) }) +} + +// Merge the sorted sealed plan with one bounded page of old/future versions. +// Persist both cursors, so restart never re-scans all previously compared keys. +func (s *MongoStore) queryBuildCleanup(ctx context.Context, l queryModelLease, j *queryBuildJob, r *queryBuildRound) error { + c := s.db.Collection(workforceFactCollection) + for { + if err := r.check(ctx); err != nil { + return err + } + f := bson.M{"organizationId": j.Organization} + if j.CleanupFuture { + f["fromEpoch"] = bson.M{"$gt": j.OldEpoch} + } else { + f = workforceVersionFilter(j.Organization, j.OldEpoch) + } + if j.CleanupHasAfter { + f["key"] = bson.M{"$gt": j.CleanupAfter} + } + op, stop := queryBuildOperation(ctx) + cur, err := c.Find(op, f, options.Find().SetProjection(bson.M{"_id": 1, "key": 1, "fromEpoch": 1}).SetSort(bson.D{{Key: "key", Value: 1}}).SetHint("workforce_fact_key_v1").SetLimit(200)) + var xs []queryBuildPrevious + if err == nil { + err = cur.All(op, &xs) + cur.Close(op) + } + stop() + if err != nil { + return err + } + if len(xs) == 0 { + if j.CleanupFuture { + j.Phase = "publish" + } else { + j.CleanupFuture = true + j.CleanupAfter = "" + j.CleanupHasAfter = false + j.CleanupPlanPart = 0 + j.CleanupPlanRow = 0 + } + if err = s.queryBuildTransaction(ctx, l, func(sc mongo.SessionContext) error { return s.saveQueryBuildJob(sc, *j) }); err != nil { + return err + } + if j.Phase == "publish" { + return nil + } + continue + } + next := *j + models := []mongo.WriteModel{} + var page []workforcestats.Fact + pageN := -1 + for _, x := range xs { + if x.Key == "" { + return errors.New("invalid workforce fact identity") + } + present := false + for next.CleanupPlanPart < j.PlanParts { + if pageN != next.CleanupPlanPart { + page, err = s.queryBuildPlanPage(ctx, *j, next.CleanupPlanPart) + if err != nil { + return err + } + pageN = next.CleanupPlanPart + } + if next.CleanupPlanRow >= len(page) { + next.CleanupPlanPart++ + next.CleanupPlanRow = 0 + continue + } + key := page[next.CleanupPlanRow].Key + if key < x.Key { + next.CleanupPlanRow++ + continue + } + present = key == x.Key + break + } + if !present { + next.Changed = true + if j.CleanupFuture { + models = append(models, mongo.NewDeleteManyModel().SetFilter(bson.M{"organizationId": j.Organization, "key": x.Key, "fromEpoch": bson.M{"$gt": j.OldEpoch}})) + } else { + models = append(models, mongo.NewUpdateManyModel().SetFilter(bson.M{"$and": bson.A{workforceVersionFilter(j.Organization, j.OldEpoch), bson.M{"key": x.Key}}}).SetUpdate(bson.M{"$set": bson.M{"toEpoch": j.Epoch, "retiredAt": time.Now().UTC()}})) + } + } + next.CleanupAfter = x.Key + next.CleanupHasAfter = true + } + next.ProgressAt = time.Now().UTC() + next.ActiveWithoutProgress = 0 + if err = s.queryBuildPace(ctx, len(models)*256); err != nil { + return err + } + if err = s.queryBuildTransaction(ctx, l, func(sc mongo.SessionContext) error { + if len(models) > 0 { + if _, e := c.BulkWrite(sc, models); e != nil { + return e + } + } + return s.saveQueryBuildJob(sc, next) + }); err != nil { + return err + } + *j = next + } +} + +func (s *MongoStore) queryBuildPublish(ctx context.Context, l queryModelLease, j *queryBuildJob) error { + op, stop := queryBuildOperation(ctx) + defer stop() + epoch := j.Epoch + if !j.Changed && j.OldEpoch > 0 { + epoch = j.OldEpoch + } + count, err := s.db.Collection(workforceFactCollection).CountDocuments(op, workforceVersionFilter(j.Organization, epoch)) + if err != nil { + return err + } + if count != int64(j.PlanRows) { + return fmt.Errorf("sealed workforce row count differs: got=%d want=%d", count, j.PlanRows) + } + next := *j + next.Phase = "complete" + next.ProgressAt = time.Now().UTC() + next.ActiveWithoutProgress = 0 + return s.queryBuildTransaction(ctx, l, func(sc mongo.SessionContext) error { + var state struct { + Heads map[string]QueryModelHead `bson:"heads"` + } + if e := s.queryControl().FindOne(sc, queryLeaseFilter(l)).Decode(&state); e != nil { + return e + } + if state.Heads[workforceFactKey(j.Organization)].FactEpoch != j.OldEpoch { + return errors.New("query build published epoch changed") + } + wf := QueryModelHead{Generation: workforceFactSchema, Collection: workforceFactCollection, FactEpoch: epoch, Rows: j.PlanRows, AsOf: j.AsOf, SourceRevision: j.SourceRevision, GlobalRevision: j.GlobalRevision} + result, e := s.queryControl().UpdateOne(sc, queryLeaseFilter(l), bson.M{"$set": bson.M{"heads." + workforceFactKey(j.Organization): wf, "heads." + queryModelKey("production", j.Organization, j.Projects): j.Production}}) + if e != nil { + return e + } + if result.MatchedCount != 1 { + return errors.New("query build publication fenced") + } + if e = s.saveQueryBuildJob(sc, next); e == nil { + *j = next + } + return e + }) +} diff --git a/internal/store/query_build_capture.go b/internal/store/query_build_capture.go new file mode 100644 index 00000000..25ca6d79 --- /dev/null +++ b/internal/store/query_build_capture.go @@ -0,0 +1,362 @@ +package store + +import ( + "bytes" + "compress/gzip" + "context" + "crypto/sha256" + "encoding/binary" + "encoding/hex" + "errors" + "fmt" + "io" + "reflect" + "sort" + "time" + + "droboticflow/internal/queryserving" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" +) + +type queryBuildReadKey struct{} +type queryBuildReadFunc func(context.Context, mongoCollection, bson.M, bson.M, any) error + +type queryBuildCaptureState struct { + ID string `bson:"_id"` + Job string `bson:"job"` + Parts int `bson:"parts"` + Last bson.RawValue `bson:"last,omitempty"` + HasLast bool `bson:"hasLast"` + Complete bool `bson:"complete"` + Rows int `bson:"rows"` + Bytes int `bson:"bytes"` +} + +type queryBuildPart struct { + ID string `bson:"_id"` + Job string `bson:"job"` + Digest string `bson:"digest"` + Data []byte `bson:"data"` +} + +func queryBuildCanonical(v any) any { + switch d := v.(type) { + case bson.M: + out := map[string]any{} + for k, x := range d { + out[k] = queryBuildCanonical(x) + } + return out + case map[string]any: + out := map[string]any{} + for k, x := range d { + out[k] = queryBuildCanonical(x) + } + return out + case bson.A: + out := make([]any, len(d)) + for i, x := range d { + out[i] = queryBuildCanonical(x) + } + return out + default: + return v + } +} + +func queryBuildQueryKey(generation, name string, f, p bson.M) string { + // JSON map ordering is deterministic. Membership order is semantically + // irrelevant, so normalize caller-generated $in lists before hashing. + var normalize func(any) any + normalize = func(v any) any { + switch d := v.(type) { + case bson.M: + out := map[string]any{} + for k, x := range d { + if k == "$in" { + if ids, ok := x.([]string); ok { + cp := append([]string{}, ids...) + sort.Strings(cp) + out[k] = cp + continue + } + if a, ok := x.(bson.A); ok { + s := make([]string, 0, len(a)) + all := true + for _, id := range a { + t, ok := id.(string) + if !ok { + all = false + break + } + s = append(s, t) + } + if all { + sort.Strings(s) + out[k] = s + continue + } + } + } + out[k] = normalize(x) + } + return out + case bson.A: + a := make([]any, len(d)) + for i, x := range d { + a[i] = normalize(x) + } + return a + default: + return v + } + } + return queryserving.Key("query-build-input/v1", generation, name, normalize(f), queryBuildCanonical(p)) +} + +func queryBuildPack(v any) ([]byte, string, error) { + raw, err := bson.Marshal(v) + if err != nil { + return nil, "", err + } + var out bytes.Buffer + z := gzip.NewWriter(&out) + if _, err = z.Write(raw); err != nil { + return nil, "", err + } + if err = z.Close(); err != nil { + return nil, "", err + } + data := out.Bytes() + h := sha256.Sum256(data) + return data, hex.EncodeToString(h[:]), nil +} +func queryBuildUnpack(data []byte, digest string, out any) error { + h := sha256.Sum256(data) + if hex.EncodeToString(h[:]) != digest { + return errors.New("query build part digest mismatch") + } + z, err := gzip.NewReader(bytes.NewReader(data)) + if err != nil { + return err + } + defer z.Close() + raw, err := io.ReadAll(io.LimitReader(z, (64<<20)+1)) + if err != nil { + return err + } + if len(raw) > 64<<20 { + return errors.New("query build part exceeds decode budget") + } + return bson.Unmarshal(raw, out) +} + +func (s *MongoStore) queryBuildSnapshot(ctx context.Context) (primitive.Timestamp, error) { + op, stop := queryBuildOperation(ctx) + defer stop() + var response struct { + At primitive.Timestamp `bson:"atClusterTime"` + Cursor struct { + At primitive.Timestamp `bson:"atClusterTime"` + } `bson:"cursor"` + } + err := s.db.RunCommand(op, bson.D{{Key: "find", Value: "queryReadModelControl"}, {Key: "filter", Value: bson.M{"_id": "global"}}, {Key: "projection", Value: bson.M{"_id": 1}}, {Key: "limit", Value: 1}, {Key: "singleBatch", Value: true}, {Key: "readConcern", Value: bson.M{"level": "snapshot"}}, {Key: "maxTimeMS", Value: 1500}}).Decode(&response) + if err != nil { + return primitive.Timestamp{}, err + } + if response.At.T == 0 { + response.At = response.Cursor.At + } + if response.At.T == 0 { + return primitive.Timestamp{}, errors.New("query build snapshot timestamp unavailable") + } + return response.At, nil +} + +func (s *MongoStore) queryBuildReadPage(ctx context.Context, c mongoCollection, f, p bson.M, last bson.RawValue, hasLast bool, at primitive.Timestamp) ([]bson.Raw, error) { + filter := any(f) + if hasLast { + filter = bson.M{"$and": bson.A{f, bson.M{"$expr": bson.M{"$gt": bson.A{"$_id", bson.M{"$literal": last}}}}}} + } + projection := bson.M{} + for k, v := range p { + projection[k] = v + } + projection["_id"] = 1 + cmd := bson.D{{Key: "find", Value: c.Name()}, {Key: "filter", Value: filter}, {Key: "projection", Value: projection}, {Key: "sort", Value: bson.D{{Key: "_id", Value: 1}}}, {Key: "hint", Value: "_id_"}, {Key: "limit", Value: 200}, {Key: "batchSize", Value: 200}, {Key: "singleBatch", Value: true}, {Key: "readConcern", Value: bson.M{"level": "snapshot", "atClusterTime": at}}, {Key: "maxTimeMS", Value: 1500}} + op, stop := queryBuildOperation(ctx) + defer stop() + var response struct { + Cursor struct { + Rows []bson.Raw `bson:"firstBatch"` + } `bson:"cursor"` + } + readCtx, readStop := context.WithTimeout(op, 5*time.Second) + defer readStop() + for attempt := 0; attempt < 3; attempt++ { + err := c.Database().RunCommand(readCtx, cmd).Decode(&response) + if err == nil { + return response.Cursor.Rows, nil + } + var command mongo.CommandError + transient := mongo.IsNetworkError(err) || mongo.IsTimeout(err) + if errors.As(err, &command) { + switch command.Code { + case 50, 91, 6, 7, 89, 11600, 11602, 10107: + transient = true + } + } + if !transient || attempt == 2 || readCtx.Err() != nil { + return nil, err + } + } + + return response.Cursor.Rows, nil +} + +func queryBuildAppendRows(ctx context.Context, rows []bson.Raw, target any, projection bson.M) error { + value := reflect.ValueOf(target).Elem() + budget, _ := ctx.Value(workforceBuildBudgetKey{}).(*workforceBuildBudget) + for _, raw := range rows { + if excluded, ok := projection["_id"]; ok && (excluded == 0 || excluded == false) { + elements, e := raw.Elements() + if e != nil { + return e + } + clean := make([]byte, 4, len(raw)) + for _, element := range elements { + if element.Key() != "_id" { + clean = append(clean, element...) + } + } + clean = append(clean, 0) + binary.LittleEndian.PutUint32(clean, uint32(len(clean))) + raw = bson.Raw(clean) + } + if budget != nil { + budget.rows++ + budget.bytes += len(raw) + if budget.rows > 250000 || budget.bytes > 64<<20 { + return errWorkforceBuildBudget + } + } + v := reflect.New(value.Type().Elem()) + if err := bson.Unmarshal(raw, v.Interface()); err != nil { + return err + } + value.Set(reflect.Append(value, v.Elem())) + } + return nil +} + +// This capture reads a native range once. Its payload and cursor commit in the +// same majority+journal transaction. Replaying a completed range reads only the +// immutable captured part, not the native business collections. +func (s *MongoStore) queryBuildCapture(ctx context.Context, l queryModelLease, j *queryBuildJob, round *queryBuildRound, c mongoCollection, f, p bson.M, target any) error { + if !s.queryReplicaFactStore { + return errors.New("query build capture requires independent read store") + } + check, checkStop := queryBuildOperation(ctx) + leaseErr := s.queryControl().FindOne(check, queryLeaseFilter(l), options.FindOne().SetProjection(bson.M{"_id": 1})).Err() + checkStop() + if leaseErr != nil { + return fmt.Errorf("query capture lease check: %w", leaseErr) + } + inputGeneration := j.InputGeneration + if inputGeneration == "" { + inputGeneration = j.Generation + } + key := queryBuildQueryKey(inputGeneration, c.Name(), f, p) + var st queryBuildCaptureState + check, checkStop = queryBuildOperation(ctx) + err := s.db.Collection(queryBuildJobsCollection).FindOne(check, bson.M{"_id": key}).Decode(&st) + checkStop() + if errors.Is(err, mongo.ErrNoDocuments) { + if inputGeneration != j.Generation { + return errors.New("verified input capture missing") + } + st = queryBuildCaptureState{ID: key, Job: inputGeneration} + } else if err != nil { + return err + } + for i := 0; i < st.Parts; i++ { + if err = ctx.Err(); err != nil { + return err + } + var part queryBuildPart + op, stop := queryBuildOperation(ctx) + err = s.db.Collection(queryBuildPartsCollection).FindOne(op, bson.M{"_id": fmt.Sprintf("%s/%09d", key, i)}).Decode(&part) + stop() + if err != nil { + return err + } + var payload struct { + Rows []bson.Raw `bson:"rows"` + } + if err = queryBuildUnpack(part.Data, part.Digest, &payload); err != nil { + return err + } + if err = queryBuildAppendRows(ctx, payload.Rows, target, p); err != nil { + return err + } + } + for !st.Complete { + if err = round.check(ctx); err != nil { + return err + } + rows, e := s.queryBuildReadPage(ctx, c, f, p, st.Last, st.HasLast, j.Snapshot) + if e != nil { + return e + } + next := st + if len(rows) == 0 { + next.Complete = true + } else { + id, e := rows[len(rows)-1].LookupErr("_id") + if e != nil { + return e + } + next.Last = bson.RawValue{Type: id.Type, Value: append([]byte(nil), id.Value...)} + next.HasLast = true + next.Parts++ + next.Rows += len(rows) + } + var part queryBuildPart + if len(rows) > 0 { + data, hash, e := queryBuildPack(bson.M{"rows": rows}) + if e != nil { + return e + } + part = queryBuildPart{ID: fmt.Sprintf("%s/%09d", key, st.Parts), Job: inputGeneration, Digest: hash, Data: data} + next.Bytes += len(data) + if e = s.queryBuildPace(ctx, len(data)); e != nil { + return e + } + } + if err = s.queryBuildTransaction(ctx, l, func(sc mongo.SessionContext) error { + if len(rows) > 0 { + if _, e := s.db.Collection(queryBuildPartsCollection).ReplaceOne(sc, bson.M{"_id": part.ID}, part, options.Replace().SetUpsert(true)); e != nil { + return e + } + } + _, e := s.db.Collection(queryBuildJobsCollection).ReplaceOne(sc, bson.M{"_id": key}, next, options.Replace().SetUpsert(true)) + if e != nil { + return e + } + if j.ID != "" { + _, e = s.db.Collection(queryBuildJobsCollection).UpdateOne(sc, bson.M{"_id": j.ID, "generation": j.Generation}, bson.M{"$set": bson.M{"progressAt": time.Now().UTC(), "activeWithoutProgress": int64(0)}}) + } + return e + }); err != nil { + return err + } + st = next + if err = queryBuildAppendRows(ctx, rows, target, p); err != nil { + return err + } + } + return nil +} diff --git a/internal/store/query_build_capture_local_test.go b/internal/store/query_build_capture_local_test.go new file mode 100644 index 00000000..59a98b8e --- /dev/null +++ b/internal/store/query_build_capture_local_test.go @@ -0,0 +1,108 @@ +package store + +import ( + "context" + "errors" + "fmt" + "reflect" + "sync/atomic" + "testing" + "time" + + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/event" +) + +func TestQueryBuildCaptureResumesNativeCursorWithoutFullRereadLocal(t *testing.T) { + var finds atomic.Int64 + monitor := &event.CommandMonitor{Started: func(_ context.Context, e *event.CommandStartedEvent) { + if e.CommandName == "find" { + if n, ok := e.Command.Lookup("find").StringValueOK(); ok && n == "captureFixture" { + finds.Add(1) + } + } + }} + _, s, _ := newLocalReplica(t, nil, monitor) + ctx := t.Context() + docs := make([]any, 401) + for i := range docs { + docs[i] = bson.M{"_id": fmt.Sprintf("row-%03d", i), "value": i} + } + c := s.db.Collection("captureFixture") + if _, err := c.InsertMany(ctx, docs); err != nil { + t.Fatal(err) + } + lease, ok, err := s.acquireQueryModelLease(ctx) + if err != nil || !ok { + t.Fatal(ok, err) + } + at, err := s.queryBuildSnapshot(ctx) + if err != nil { + t.Fatal(err) + } + j := &queryBuildJob{Generation: "capture-unit", Snapshot: at} + start := time.Now() + checks := 0 + round := &queryBuildRound{until: start.Add(time.Second), now: func() time.Time { + checks++ + if checks > 1 { + return start.Add(2 * time.Second) + } + return start + }} + type row struct { + ID string `bson:"_id"` + Value int `bson:"value"` + } + var first []row + err = s.queryBuildCapture(ctx, lease, j, round, c, bson.M{}, bson.M{"value": 1}, &first) + if !errors.Is(err, ErrQueryBuildYielded) || len(first) != 200 || finds.Load() != 1 { + t.Fatal("first round did not preserve one bounded native page", len(first), finds.Load(), err) + } + // Later native changes cannot be mixed into the already selected snapshot. + if _, err = c.UpdateOne(ctx, bson.M{"_id": "row-000"}, bson.M{"$set": bson.M{"value": 9999}}); err != nil { + t.Fatal(err) + } + restarted := NewMongoStore(s.db) + restarted.queryReplicaFactStore = true + var got []row + if err = restarted.queryBuildCapture(ctx, lease, j, newQueryBuildRound(true), c, bson.M{}, bson.M{"value": 1}, &got); err != nil { + t.Fatal(err) + } + if len(got) != 401 || got[0].Value != 0 || got[400].Value != 400 || finds.Load() != 4 { + t.Fatalf("native prefix re-read or snapshot mixed: rows=%d finds=%d first=%+v", len(got), finds.Load(), got[0]) + } + before := finds.Load() + var repeated []row + if err = restarted.queryBuildCapture(ctx, lease, j, newQueryBuildRound(true), c, bson.M{}, bson.M{"value": 1}, &repeated); err != nil { + t.Fatal(err) + } + if finds.Load() != before || !reflect.DeepEqual(got, repeated) { + t.Fatal("complete input capture repeated native reads") + } +} + +func TestQueryBuildCaptureKeepsLegacyProjectionLocal(t *testing.T) { + _, s, _ := newLocalReplica(t) + ctx := t.Context() + c := s.db.Collection("captureProjectionFixture") + if _, err := c.InsertOne(ctx, bson.M{"_id": "private-native-id", "value": 1}); err != nil { + t.Fatal(err) + } + l, ok, err := s.acquireQueryModelLease(ctx) + if err != nil || !ok { + t.Fatal(err) + } + at, err := s.queryBuildSnapshot(ctx) + if err != nil { + t.Fatal(err) + } + j := &queryBuildJob{Generation: "projection", Snapshot: at} + var rows []bson.M + if err = s.queryBuildCapture(ctx, l, j, newQueryBuildRound(true), c, bson.M{}, bson.M{"_id": 0, "value": 1}, &rows); err != nil { + t.Fatal(err) + } + if len(rows) != 1 || rows[0]["_id"] != nil || rows[0]["value"] != int32(1) { + t.Fatal("native cursor identity changed the loader projection", rows) + } +} diff --git a/internal/store/query_build_fact_document.go b/internal/store/query_build_fact_document.go new file mode 100644 index 00000000..e375cafe --- /dev/null +++ b/internal/store/query_build_fact_document.go @@ -0,0 +1,48 @@ +package store + +import ( + "droboticflow/internal/queryserving" + "droboticflow/internal/workforcestats" + "encoding/json" + "go.mongodb.org/mongo-driver/bson" + "time" +) + +func queryBuildFactDocument(org string, epoch int64, f workforcestats.Fact) (bson.M, string, int, error) { + raw, err := json.Marshal(f) + if err != nil { + return nil, "", 0, err + } + hash := queryserving.Key(workforceFactSchema, string(raw)) + encoded, err := bson.Marshal(f) + if err != nil { + return nil, "", 0, err + } + var doc bson.M + if err = bson.Unmarshal(encoded, &doc); err != nil { + return nil, "", 0, err + } + at, err := time.Parse(time.RFC3339Nano, f.CreatedAt) + if err != nil { + return nil, "", 0, err + } + doc["_id"], doc["organizationId"], doc["fromEpoch"], doc["hash"], doc["at"] = queryserving.Key(workforceFactSchema, org, f.Key, epoch), org, epoch, hash, at + doc["atSeconds"], doc["atNanos"] = at.Unix(), at.Nanosecond() + doc["metric"] = workforcestats.MetricCounterValues(f.Metric) + doc["order"] = "" + if f.Detail != nil { + raw, e := json.Marshal(f.Detail) + if e != nil { + return nil, "", 0, e + } + var detail bson.M + if e = bson.UnmarshalExtJSON(raw, false, &detail); e != nil { + return nil, "", 0, e + } + doc["detail"] = detail + doc["return"] = bson.M{"id": f.Detail.ID, "stage": f.Detail.Stage, "taskIds": f.Detail.TaskIDs, "metrics": detail["metrics"]} + doc["order"] = f.Detail.ID + f.Detail.UserID + f.Detail.SupplierID + } + stored, err := bson.Marshal(doc) + return doc, hash, len(stored), err +} diff --git a/internal/store/query_build_jobs.go b/internal/store/query_build_jobs.go new file mode 100644 index 00000000..22b4aa09 --- /dev/null +++ b/internal/store/query_build_jobs.go @@ -0,0 +1,103 @@ +package store + +import ( + "context" + "errors" + "time" + + "droboticflow/internal/queryserving" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + "go.mongodb.org/mongo-driver/mongo/readconcern" + "go.mongodb.org/mongo-driver/mongo/writeconcern" +) + +const queryBuildJobsCollection = "queryBuildJobs" +const queryBuildPartsCollection = "queryBuildParts" +const queryBuildJobSchema = "query-build-job/v1" + +type queryBuildJob struct { + ID string `bson:"_id"` + Schema string `bson:"schema"` + InputGeneration string `bson:"inputGeneration"` + Generation string `bson:"generation"` + Organization string `bson:"organizationId"` + Projects []string `bson:"projects"` + Cold bool `bson:"cold"` + Phase string `bson:"phase"` + Snapshot primitive.Timestamp `bson:"snapshot"` + AsOf time.Time `bson:"asOf"` + SourceRevision int64 `bson:"sourceRevision"` + GlobalRevision int64 `bson:"globalRevision"` + OldEpoch int64 `bson:"oldEpoch"` + Epoch int64 `bson:"epoch"` + Production QueryModelHead `bson:"production"` + PlanParts int `bson:"planParts"` + PlanRows int `bson:"planRows"` + PlanDigest string `bson:"planDigest"` + PlanSaved int `bson:"planSaved"` + PlanBytes int64 `bson:"planBytes"` + ProductionSaved int `bson:"productionSaved"` + NextPart int `bson:"nextPart"` + NextRow int `bson:"nextRow"` + Changed bool `bson:"changed"` + CleanupHasAfter bool `bson:"cleanupHasAfter"` + CleanupPlanPart int `bson:"cleanupPlanPart"` + CleanupPlanRow int `bson:"cleanupPlanRow"` + CleanupAfter string `bson:"cleanupAfter"` + CleanupFuture bool `bson:"cleanupFuture"` + ConfirmedRows int64 `bson:"confirmedRows"` + ConfirmedBytes int64 `bson:"confirmedBytes"` + ReusedRows int64 `bson:"reusedRows"` + ActiveWithoutProgress time.Duration `bson:"activeWithoutProgress"` + ProgressAt time.Time `bson:"progressAt"` + FailurePhase string `bson:"failurePhase,omitempty"` + FailureCode int32 `bson:"failureCode,omitempty"` + FailureClass string `bson:"failureClass,omitempty"` + UpdatedAt time.Time `bson:"updatedAt"` +} + +func queryBuildJobKey(database, org string) string { + return queryserving.Key(queryBuildJobSchema, database, normalizedOrganizationID(org)) +} + +func (s *MongoStore) queryBuildJob(ctx context.Context, org string) (queryBuildJob, bool, error) { + op, stop := queryBuildOperation(ctx) + defer stop() + var j queryBuildJob + err := s.db.Collection(queryBuildJobsCollection).FindOne(op, bson.M{"_id": queryBuildJobKey(s.db.Name(), org)}).Decode(&j) + if errors.Is(err, mongo.ErrNoDocuments) { + return j, false, nil + } + if err == nil && (j.Schema != queryBuildJobSchema || j.Organization != normalizedOrganizationID(org)) { + return j, false, ErrQueryReplicaUnsupported + } + return j, err == nil, err +} + +// Every checkpoint and the corresponding rows/parts commit together. The same +// global lease serializes publication and fences an old worker after takeover. +func (s *MongoStore) queryBuildTransaction(ctx context.Context, l queryModelLease, fn func(mongo.SessionContext) error) error { + op, stop := queryBuildOperation(ctx) + defer stop() + session, err := s.db.Client().StartSession() + if err != nil { + return err + } + defer session.EndSession(op) + _, err = session.WithTransaction(op, func(sc mongo.SessionContext) (any, error) { + if err := s.renewQueryModelLease(sc, l); err != nil { + return nil, err + } + return nil, fn(sc) + }, options.Transaction().SetReadConcern(readconcern.Snapshot()).SetWriteConcern(writeconcern.New(writeconcern.WMajority(), writeconcern.J(true)))) + return err +} + +func (s *MongoStore) saveQueryBuildJob(sc mongo.SessionContext, j queryBuildJob) error { + j.UpdatedAt = time.Now().UTC() + _, err := s.db.Collection(queryBuildJobsCollection).ReplaceOne(sc, bson.M{"_id": j.ID}, j, options.Replace().SetUpsert(true)) + return err +} diff --git a/internal/store/query_build_lifecycle.go b/internal/store/query_build_lifecycle.go new file mode 100644 index 00000000..283fc390 --- /dev/null +++ b/internal/store/query_build_lifecycle.go @@ -0,0 +1,69 @@ +package store + +import ( + "context" + "errors" + "fmt" + "time" +) + +// Yield is successful bounded progress, not loss of the source safety permit. +// Callers retain pending work and do not stop projector/builder replicas. +var ErrQueryBuildYielded = errors.New("query build round yielded with durable progress retained") +var ErrQueryBuildStopped = errors.New("query build stopped; durable checkpoint retained") +var ErrQueryBuildNoProgress = errors.New("query build made no durable progress") + +const ( + queryColdRoundBudget = time.Minute + queryRefreshRoundBudget = 30 * time.Second + queryBuildOperationBudget = 15 * time.Second + queryBuildNoProgressLimit = 5 * time.Minute +) + +type queryBuildRound struct { + until time.Time + now func() time.Time +} + +func newQueryBuildRound(cold bool) *queryBuildRound { + budget := queryRefreshRoundBudget + if cold { + budget = queryColdRoundBudget + } + return &queryBuildRound{until: time.Now().Add(budget), now: time.Now} +} + +func (r *queryBuildRound) check(ctx context.Context) error { + if err := ctx.Err(); err != nil { + return err + } + if r != nil && !r.now().Before(r.until) { + return ErrQueryBuildYielded + } + return nil +} + +// Operation deadlines bound a single read/commit. They do not shorten the +// lifetime of the initial job or discard any previously confirmed checkpoint. +func queryBuildOperation(ctx context.Context) (context.Context, context.CancelFunc) { + return context.WithTimeout(ctx, queryBuildOperationBudget) +} + +func queryBuildPhaseFailure(phase string, err error) error { + if err == nil || errors.Is(err, ErrQueryBuildYielded) { + return err + } + return fmt.Errorf("query build %s: %w", phase, err) +} + +// Admission/checkpoint backpressure before a write is a resumable yield. A +// failed durable transaction is never converted into success or budget yield. +func (s *MongoStore) queryBuildPace(ctx context.Context, bytes int) error { + op, stop := queryBuildOperation(ctx) + defer stop() + err := s.paceQueryBuild(op, bytes) + if err != nil && ctx.Err() == nil && errors.Is(op.Err(), context.DeadlineExceeded) { + return ErrQueryBuildYielded + } + return err +} diff --git a/internal/store/query_build_lifecycle_test.go b/internal/store/query_build_lifecycle_test.go new file mode 100644 index 00000000..1eeea30b --- /dev/null +++ b/internal/store/query_build_lifecycle_test.go @@ -0,0 +1,46 @@ +package store + +import ( + "context" + "errors" + "testing" + "time" +) + +func TestQueryBuildRoundBudgetIsYieldNotSafetyFailure(t *testing.T) { + now := time.Now() + r := &queryBuildRound{until: now, now: func() time.Time { return now }} + err := r.check(t.Context()) + if !errors.Is(err, ErrQueryBuildYielded) || errors.Is(err, ErrQueryReplicaPaused) { + t.Fatal("round budget became a global safety pause", err) + } + ctx, stop := context.WithCancel(t.Context()) + stop() + if !errors.Is(r.check(ctx), context.Canceled) { + t.Fatal("shutdown must preserve its cause") + } +} + +func TestQueryBuildBudgetsAreIndependent(t *testing.T) { + if queryColdRoundBudget == queryRefreshRoundBudget { + t.Fatal("cold and freshness scheduling budgets coupled") + } + for _, cold := range []bool{false, true} { + r := newQueryBuildRound(cold) + if err := r.check(t.Context()); err != nil { + t.Fatal(err) + } + } + if queryBuildOperationBudget >= queryColdRoundBudget { + t.Fatal("one operation consumed the entire cold round") + } +} + +func TestQueryBuildPacingPreservesCallerCancellation(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + cancel() + s := &MongoStore{} + if err := s.queryBuildPace(ctx, 1024); !errors.Is(err, context.Canceled) || errors.Is(err, ErrQueryBuildYielded) { + t.Fatal(err) + } +} diff --git a/internal/store/query_build_plan.go b/internal/store/query_build_plan.go new file mode 100644 index 00000000..1e60222a --- /dev/null +++ b/internal/store/query_build_plan.go @@ -0,0 +1,196 @@ +package store + +import ( + "bytes" + "compress/gzip" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "sort" + "time" + + "droboticflow/internal/domain" + "droboticflow/internal/productionstats" + "droboticflow/internal/workforcestats" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" +) + +const queryBuildFactPageRows = 200 + +func queryBuildPlanPartKey(j queryBuildJob, kind string, n int) string { + return fmt.Sprintf("%s/%s/%09d", j.Generation, kind, n) +} +func queryBuildJSONPart(j queryBuildJob, kind string, n int, v any) (queryBuildPart, error) { + var b bytes.Buffer + z := gzip.NewWriter(&b) + if err := json.NewEncoder(z).Encode(v); err != nil { + return queryBuildPart{}, err + } + if err := z.Close(); err != nil { + return queryBuildPart{}, err + } + data := b.Bytes() + sum := sha256.Sum256(data) + return queryBuildPart{ID: queryBuildPlanPartKey(j, kind, n), Job: j.Generation, Data: data, Digest: hex.EncodeToString(sum[:])}, nil +} +func queryBuildJSONDecode(p queryBuildPart, out any) error { + sum := sha256.Sum256(p.Data) + if hex.EncodeToString(sum[:]) != p.Digest { + return errors.New("query plan digest mismatch") + } + z, err := gzip.NewReader(bytes.NewReader(p.Data)) + if err != nil { + return err + } + defer z.Close() + raw, err := io.ReadAll(io.LimitReader(z, (64<<20)+1)) + if err != nil { + return err + } + if len(raw) > 64<<20 { + return errors.New("query plan decode budget exceeded") + } + return json.Unmarshal(raw, out) +} + +func (s *MongoStore) queryBuildSavePart(ctx context.Context, l queryModelLease, j *queryBuildJob, p queryBuildPart) error { + if err := s.queryBuildPace(ctx, len(p.Data)); err != nil { + return err + } + return s.queryBuildTransaction(ctx, l, func(sc mongo.SessionContext) error { + if _, err := s.db.Collection(queryBuildPartsCollection).ReplaceOne(sc, bson.M{"_id": p.ID}, p, options.Replace().SetUpsert(true)); err != nil { + return err + } + return s.saveQueryBuildJob(sc, *j) + }) +} + +func (s *MongoStore) queryBuildProductionPlan(ctx context.Context, l queryModelLease, j *queryBuildJob, r *queryBuildRound) error { + readCtx := context.WithValue(ctx, queryBuildReadKey{}, queryBuildReadFunc(func(c context.Context, coll mongoCollection, f, p bson.M, out any) error { + return s.queryBuildCapture(c, l, j, r, coll, f, p, out) + })) + snapshot, err := s.loadProductionStats(readCtx, j.Organization, j.Projects) + if err != nil { + return err + } + snapshot.AsOf = j.AsOf.Format(time.RFC3339Nano) + // Preserve the existing JSON/gzip manifest format for all old readers. + var data bytes.Buffer + z := gzip.NewWriter(&data) + budget := &queryModelBudgetWriter{writer: z} + if err = json.NewEncoder(budget).Encode(productionstats.Build(snapshot)); err != nil { + return err + } + if err = z.Close(); err != nil { + return err + } + if data.Len() > queryModelMaxBytes { + return errors.New("production plan exceeds existing size budget") + } + payload := data.Bytes() + sum := sha256.Sum256(payload) + head := QueryModelHead{Generation: j.Generation + "-production", AsOf: j.AsOf, Bytes: len(payload), ChunkBytes: queryModelBuildChunkBytes, Chunks: (len(payload) + queryModelBuildChunkBytes - 1) / queryModelBuildChunkBytes, Digest: hex.EncodeToString(sum[:]), SourceRevision: j.SourceRevision, GlobalRevision: j.GlobalRevision} + if j.Production.Generation != "" && j.Production.Digest != head.Digest { + return errors.New("sealed production input changed") + } + for n := j.ProductionSaved; n < head.Chunks; n++ { + if err = r.check(ctx); err != nil { + return err + } + p := payload[n*head.ChunkBytes : min((n+1)*head.ChunkBytes, len(payload))] + if err = s.queryBuildPace(ctx, len(p)); err != nil { + return err + } + next := *j + next.ProgressAt = time.Now().UTC() + next.ActiveWithoutProgress = 0 + next.Production = head + next.ProductionSaved = n + 1 + if err = s.queryBuildTransaction(ctx, l, func(sc mongo.SessionContext) error { + key := queryModelBlobKey(queryModelKey("production", j.Organization, j.Projects), head.Generation, n) + _, e := s.queryData().ReplaceOne(sc, bson.M{"_id": key}, bson.M{"_id": key, "generation": head.Generation, "data": p, "createdAt": time.Now().UTC()}, options.Replace().SetUpsert(true)) + if e != nil { + return e + } + return s.saveQueryBuildJob(sc, next) + }); err != nil { + return err + } + *j = next + } + j.Phase = "workforce_plan" + return s.queryBuildTransaction(ctx, l, func(sc mongo.SessionContext) error { return s.saveQueryBuildJob(sc, *j) }) +} + +func (s *MongoStore) queryBuildWorkforcePlan(ctx context.Context, l queryModelLease, j *queryBuildJob, r *queryBuildRound) error { + readCtx := context.WithValue(ctx, queryBuildReadKey{}, queryBuildReadFunc(func(c context.Context, coll mongoCollection, f, p bson.M, out any) error { + return s.queryBuildCapture(c, l, j, r, coll, f, p, out) + })) + readCtx = context.WithValue(readCtx, workforceBuildBudgetKey{}, &workforceBuildBudget{}) + input, err := s.LoadWorkforceStats(readCtx, j.Organization, workforcestats.Query{Mode: domain.RunModeAll}) + if err != nil { + return err + } + facts := workforcestats.Materialize(input) + sort.Slice(facts, func(i, j int) bool { return facts[i].Key < facts[j].Key }) + count := (len(facts) + queryBuildFactPageRows - 1) / queryBuildFactPageRows + if j.PlanRows != 0 && j.PlanRows != len(facts) { + return errors.New("workforce plan input changed") + } + // Recompute only from durable captured input, and verify the sealed prefix + // before adding any later page (including after a code deployment). + prefix := "" + for n := 0; n < j.PlanSaved; n++ { + part, e := queryBuildJSONPart(*j, "facts", n, facts[n*queryBuildFactPageRows:min((n+1)*queryBuildFactPageRows, len(facts))]) + if e != nil { + return e + } + var saved queryBuildPart + op, stop := queryBuildOperation(ctx) + e = s.db.Collection(queryBuildPartsCollection).FindOne(op, bson.M{"_id": part.ID, "job": j.Generation}).Decode(&saved) + stop() + if e != nil { + return e + } + if saved.Digest != part.Digest { + return errors.New("sealed workforce plan fingerprint changed") + } + chain := sha256.Sum256([]byte(prefix + saved.Digest)) + prefix = hex.EncodeToString(chain[:]) + } + if prefix != j.PlanDigest { + return errors.New("sealed workforce plan manifest changed") + } + for n := j.PlanSaved; n < count; n++ { + if err = r.check(ctx); err != nil { + return err + } + part, e := queryBuildJSONPart(*j, "facts", n, facts[n*queryBuildFactPageRows:min((n+1)*queryBuildFactPageRows, len(facts))]) + if e != nil { + return e + } + next := *j + next.ProgressAt = time.Now().UTC() + next.ActiveWithoutProgress = 0 + next.PlanParts = count + next.PlanRows = len(facts) + next.PlanSaved = n + 1 + next.PlanBytes += int64(len(part.Data)) + chain := sha256.Sum256([]byte(next.PlanDigest + part.Digest)) + next.PlanDigest = hex.EncodeToString(chain[:]) + if err = s.queryBuildSavePart(ctx, l, &next, part); err != nil { + return err + } + *j = next + } + j.PlanRows = len(facts) + j.PlanParts = count + j.Phase = "apply" + return s.queryBuildTransaction(ctx, l, func(sc mongo.SessionContext) error { return s.saveQueryBuildJob(sc, *j) }) +} diff --git a/internal/store/query_build_resume_local_test.go b/internal/store/query_build_resume_local_test.go new file mode 100644 index 00000000..afb38fa4 --- /dev/null +++ b/internal/store/query_build_resume_local_test.go @@ -0,0 +1,360 @@ +package store + +import ( + "context" + "errors" + "fmt" + "reflect" + "sync/atomic" + "testing" + "time" + + "droboticflow/internal/domain" + "droboticflow/internal/workforcestats" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/event" + "go.mongodb.org/mongo-driver/mongo" +) + +func TestQueryBuildLifecycleRestartYieldAndTwoRefreshesLocal(t *testing.T) { + var native, sourceReads, weakCommits atomic.Int64 + monitor := &event.CommandMonitor{Started: func(_ context.Context, e *event.CommandStartedEvent) { + if e.CommandName == "find" { + name, _ := e.Command.Lookup("find").StringValueOK() + if querySourceCollection(name) { + native.Add(1) + } + } + if e.CommandName == "commitTransaction" { + var d struct { + WC struct { + W any `bson:"w"` + J bool `bson:"j"` + } `bson:"writeConcern"` + } + _ = bson.Unmarshal(e.Command, &d) + if d.WC.W != "majority" || !d.WC.J { + weakCommits.Add(1) + } + } + }} + _, s, _ := newLocalReplica(t, &event.CommandMonitor{Started: func(context.Context, *event.CommandStartedEvent) { sourceReads.Add(1) }}, monitor) + sourceReads.Store(0) + ctx := t.Context() + org := "workforce-test" + seedWorkforceStats(t, s) + events := []any{} + seconds := 1.0 + for i := 0; i < 250; i++ { + events = append(events, domain.AnnotationWorklogEventRecord{ID: fmt.Sprintf("resume-event-%04d", i), OrganizationID: org, TaskID: fmt.Sprintf("resume-task-%04d", i), ProjectID: "p", SupplierID: "s", UserID: "u", EventType: "annotation_submitted", Mode: domain.RunModeProduction, Revision: 1, VideoSeconds: &seconds, OutputDelta: 1, CreatedAt: time.Date(2026, 9, 28, 13, 0, 0, 0, time.UTC).Add(time.Duration(i) * time.Second).Format(time.RFC3339Nano)}) + } + tasks := []any{} + for i := 0; i < 250; i++ { + id := fmt.Sprintf("resume-task-%04d", i) + tasks = append(tasks, bson.M{"_id": id, "id": id, "organizationId": org, "projectId": "p", "supplierId": "s", "runId": "run"}) + } + if _, e := s.annotationTasks.InsertMany(ctx, tasks); e != nil { + t.Fatal(e) + } + if _, e := s.annotationWorklogEvents.InsertMany(ctx, events); e != nil { + t.Fatal(e) + } + s.queryReplicaFactStore = true + s.queryPeriodic = true + s.queryBuildWrites = nil + if e := s.ensureWorkforceFactIndexes(ctx); e != nil { + t.Fatal(e) + } + l, ok, e := s.acquireQueryModelLease(ctx) + if e != nil || !ok { + t.Fatal(ok, e) + } + // Establish exactly the e4 confirmed, unpublished first epoch. + if e = s.rebuildWorkforceFacts(ctx, l, org, 0); e != nil { + t.Fatal(e) + } + unpublishWorkforceFixture(t, s, org) + weakCommits.Store(0) + j, e := s.queryBuildStart(ctx, l, org, []string{"p"}) + if e != nil { + t.Fatal(e) + } + if e = s.queryBuildProductionPlan(ctx, l, &j, newQueryBuildRound(true)); e != nil { + t.Fatal(e) + } + if e = s.queryBuildWorkforcePlan(ctx, l, &j, newQueryBuildRound(true)); e != nil { + t.Fatal(e) + } + beforeNative := native.Load() + // Allow one committed fact batch, then exhaust this round, not the job. + calls := 0 + start := time.Now() + r := &queryBuildRound{until: start.Add(time.Second), now: func() time.Time { + calls++ + if calls > 2 { + return start.Add(2 * time.Second) + } + return start + }} + if e = s.queryBuildApply(ctx, l, &j, r); !errors.Is(e, ErrQueryBuildYielded) { + t.Fatal("expected successful yield", e) + } + saved, ok, e := s.queryBuildJob(ctx, org) + if e != nil || !ok || saved.NextRow+saved.NextPart == 0 { + t.Fatal("missing confirmed cursor", saved.NextRow, saved.NextPart, e) + } + if _, ok := s.readQueryModelHead(ctx, queryModelKey("production", org, []string{"p"}), time.Hour); ok { + t.Fatal("partial generation published") + } + // Reacquire a new lease and use a fresh process object; old writer is fenced. + if _, e = s.queryControl().UpdateOne(ctx, queryLeaseFilter(l), bson.M{"$set": bson.M{"until": time.Now().Add(-time.Second)}}); e != nil { + t.Fatal(e) + } + restarted := NewMongoStore(s.db) + restarted.queryReplicaFactStore = true + restarted.queryPeriodic = true + l2, ok, e := restarted.acquireQueryModelLease(ctx) + if e != nil || !ok { + t.Fatal(e) + } + if e = s.queryBuildTransaction(ctx, l, func(_ctx mongo.SessionContext) error { return nil }); e == nil { + t.Fatal("old lease committed") + } + if e = restarted.queryBuildRunRound(ctx, l2, org, []string{"p"}, nil); e != nil { + t.Fatal(e) + } + if native.Load() != beforeNative { + t.Fatal("resume repeated raw input reads", beforeNative, native.Load()) + } + completed, _, e := restarted.queryBuildJob(ctx, org) + if e != nil || completed.Phase != "complete" || completed.ReusedRows == 0 || completed.ConfirmedRows != 0 { + t.Fatal("confirmed e4 batches not reused", completed.ReusedRows, completed.ConfirmedRows, e) + } + first, _, e := restarted.workforceHead(ctx, org) + if e != nil { + t.Fatal(e, "head", first, "now", time.Now().UTC()) + } + original, e := restarted.LoadWorkforceStats(ctx, org, workforcestats.Query{Mode: domain.RunModeAll}) + if e != nil { + t.Fatal(e) + } + expected := workforcestats.Build(original, workforcestats.Query{Mode: domain.RunModeAll}) + actual, used, e := restarted.computeWorkforceFactReport(ctx, org, workforcestats.Query{Mode: domain.RunModeAll}, first) + if e != nil || !used || !reflect.DeepEqual(actual.Totals, expected.Totals) { + t.Fatal("fact totals differ", e) + } + // A fresh caught-up observer proves unchanged native input; both daily + // refreshes reuse that complete input instead of re-reading full histories. + for round := 0; round < 2; round++ { + if _, e = s.queryControl().UpdateOne(ctx, queryLeaseFilter(l2), bson.M{"$set": bson.M{"observedAt": time.Now().UTC()}}); e != nil { + t.Fatal(e) + } + before := native.Load() + if e = restarted.queryBuildRunRound(ctx, l2, org, []string{"p"}, nil); e != nil { + t.Fatal(e) + } + if native.Load() != before { + t.Fatal("unchanged refresh reread native histories", round, native.Load()-before) + } + h, ok, e := restarted.workforceHead(ctx, org) + if e != nil || !ok || h.FactEpoch != first.FactEpoch { + t.Fatal("unchanged facts rewritten", h.FactEpoch, e) + } + } + if weakCommits.Load() != 0 { + t.Fatal("majority+journal weakened", weakCommits.Load()) + } + if sourceReads.Load() != 0 { + t.Fatal("build read original Mongo", sourceReads.Load()) + } + if !IsPrivateDerivedCollection(queryBuildJobsCollection) || !IsPrivateDerivedCollection(queryBuildPartsCollection) { + t.Fatal("internal build documents exposed") + } +} + +func TestQueryBuildChangedFingerprintDeletionAndOldGenerationLocal(t *testing.T) { + _, s, _ := newLocalReplica(t) + ctx := t.Context() + org := "workforce-test" + seedWorkforceStats(t, s) + s.queryReplicaFactStore = true + s.queryPeriodic = true + if e := s.ensureWorkforceFactIndexes(ctx); e != nil { + t.Fatal(e) + } + l, ok, e := s.acquireQueryModelLease(ctx) + if e != nil || !ok { + t.Fatal(e) + } + if e = s.queryBuildRunRound(ctx, l, org, []string{"p"}, nil); e != nil { + t.Fatal(e) + } + old, _, e := s.workforceHead(ctx, org) + if e != nil { + t.Fatal(e) + } + // Changed input and a deleted user are handled by the original loader, + // including its historical actor fallback. Neither mutates source records. + if _, e = s.annotationWorklogEvents.UpdateMany(ctx, bson.M{"organizationId": org}, bson.M{"$set": bson.M{"videoSeconds": 999.0}}); e != nil { + t.Fatal(e) + } + if _, e = s.users.DeleteMany(ctx, bson.M{"organizationId": org}); e != nil { + t.Fatal(e) + } + if _, e = s.queryControl().UpdateOne(ctx, queryLeaseFilter(l), bson.M{"$inc": bson.M{"productionRevision": 1}}); e != nil { + t.Fatal(e) + } + j, e := s.queryBuildStart(ctx, l, org, []string{"p"}) + if e != nil { + t.Fatal(e) + } + if e = s.queryBuildProductionPlan(ctx, l, &j, newQueryBuildRound(false)); e != nil { + t.Fatal(e) + } + if e = s.queryBuildWorkforcePlan(ctx, l, &j, newQueryBuildRound(false)); e != nil { + t.Fatal(e) + } + // A deliberately wrong confirmed future fingerprint must be replaced, while + // an orphan from an old attempt must be removed only from the unpublished view. + page, e := s.queryBuildPlanPage(ctx, j, 0) + if e != nil { + t.Fatal(e) + } + doc, _, _, e := queryBuildFactDocument(org, j.Epoch, page[0]) + if e != nil { + t.Fatal(e) + } + doc["hash"] = "outdated" + if _, e = s.db.Collection(workforceFactCollection).InsertOne(ctx, doc); e != nil { + t.Fatal(e) + } + if _, e = s.db.Collection(workforceFactCollection).InsertOne(ctx, bson.M{"_id": "orphan-future", "organizationId": org, "key": "zz-orphan", "hash": "obsolete", "fromEpoch": j.Epoch}); e != nil { + t.Fatal(e) + } + if e = s.queryBuildApply(ctx, l, &j, newQueryBuildRound(false)); e != nil { + t.Fatal(e) + } + // Partial writes have not changed either published manifest. + h, _, e := s.workforceHead(ctx, org) + if e != nil || h.FactEpoch != old.FactEpoch { + t.Fatal("partial generation visible", e) + } + cancelCtx, stop := context.WithCancel(ctx) + stop() + if e = s.queryBuildRunRound(cancelCtx, l, org, []string{"p"}, nil); !errors.Is(e, context.Canceled) { + t.Fatal(e) + } + if e = s.queryBuildRunRound(ctx, l, org, []string{"p"}, nil); e != nil { + t.Fatal(e) + } + h, _, e = s.workforceHead(ctx, org) + if e != nil || h.FactEpoch != old.FactEpoch+1 { + t.Fatal(e) + } + filters := []workforcestats.Query{ + {Mode: domain.RunModeAll}, {Mode: domain.RunModeProduction}, + {Mode: domain.RunModeAll, UserID: "u", MatchUser: true}, + {Mode: domain.RunModeAll, AllowedProjects: map[string]bool{}}, + {Mode: domain.RunModeAll, AllowedSuppliers: map[string]bool{}}, + {Mode: domain.RunModeAll, SupplierID: "s"}, + {Mode: domain.RunModeAll, From: time.Date(2026, 9, 28, 0, 0, 0, 1, time.UTC), To: time.Date(2026, 9, 29, 0, 0, 0, 0, time.UTC)}, + } + for _, q := range filters { + input, e := s.LoadWorkforceStats(ctx, org, q) + if e != nil { + t.Fatal(e) + } + want := workforcestats.Build(input, q) + got, _, e := s.computeWorkforceFactReport(ctx, org, q, h) + if e != nil || !reflect.DeepEqual(got.Totals, want.Totals) { + t.Fatal("permissions/date/legacy parity", q, e, got.Totals, want.Totals) + } + } + n, e := s.db.Collection(workforceFactCollection).CountDocuments(ctx, bson.M{"_id": "orphan-future"}) + if e != nil || n != 0 { + t.Fatal("unmatched staged row survived", n, e) + } +} + +func TestQueryBuildNoProgressStopsButPauseTimeDoesNotLocal(t *testing.T) { + _, s, _ := newLocalReplica(t) + ctx := t.Context() + s.queryReplicaFactStore = true + s.queryPeriodic = true + l, ok, e := s.acquireQueryModelLease(ctx) + if e != nil || !ok { + t.Fatal(e) + } + j, e := s.queryBuildStart(ctx, l, "synthetic-no-progress", nil) + if e != nil { + t.Fatal(e) + } + // An overnight inactive gap is not a five minute active stall. + j.ProgressAt = time.Now().Add(-24 * time.Hour) + if e = s.queryBuildTransaction(ctx, l, func(sc mongo.SessionContext) error { return s.saveQueryBuildJob(sc, j) }); e != nil { + t.Fatal(e) + } + now := time.Now() + expired := func() *queryBuildRound { return &queryBuildRound{until: now, now: func() time.Time { return now }} } + if e = s.queryBuildRunRound(ctx, l, j.Organization, nil, expired()); !errors.Is(e, ErrQueryBuildYielded) || errors.Is(e, ErrQueryBuildStopped) { + t.Fatal("pause treated as stall", e) + } + j, _, e = s.queryBuildJob(ctx, j.Organization) + if e != nil { + t.Fatal(e) + } + j.ActiveWithoutProgress = queryBuildNoProgressLimit + if e = s.queryBuildTransaction(ctx, l, func(sc mongo.SessionContext) error { return s.saveQueryBuildJob(sc, j) }); e != nil { + t.Fatal(e) + } + if e = s.queryBuildRunRound(ctx, l, j.Organization, nil, expired()); !errors.Is(e, ErrQueryBuildStopped) || !errors.Is(e, ErrQueryBuildNoProgress) { + t.Fatal("real active stall ignored", e) + } + if e = s.queryBuildRunRound(ctx, l, j.Organization, nil, nil); !errors.Is(e, ErrQueryBuildStopped) { + t.Fatal("terminal failure automatically resumed", e) + } +} + +func TestQueryBuildSealedInputSurvivesExpiredSnapshotAndOrphanGCLocal(t *testing.T) { + _, s, _ := newLocalReplica(t) + ctx := t.Context() + org := "workforce-test" + seedWorkforceStats(t, s) + s.queryReplicaFactStore = true + s.queryPeriodic = true + if e := s.ensureWorkforceFactIndexes(ctx); e != nil { + t.Fatal(e) + } + l, ok, e := s.acquireQueryModelLease(ctx) + if e != nil || !ok { + t.Fatal(e) + } + j, e := s.queryBuildStart(ctx, l, org, []string{"p"}) + if e != nil { + t.Fatal(e) + } + if e = s.queryBuildProductionPlan(ctx, l, &j, newQueryBuildRound(true)); e != nil { + t.Fatal(e) + } + if e = s.queryBuildWorkforcePlan(ctx, l, &j, newQueryBuildRound(true)); e != nil { + t.Fatal(e) + } + // Once sealed, no original native snapshot is needed for continuation. + j.Snapshot.T = 1 + if e = s.queryBuildTransaction(ctx, l, func(sc mongo.SessionContext) error { return s.saveQueryBuildJob(sc, j) }); e != nil { + t.Fatal(e) + } + if _, e = s.queryData().UpdateMany(ctx, bson.M{"generation": j.Production.Generation}, bson.M{"$set": bson.M{"createdAt": time.Now().Add(-24 * time.Hour)}}); e != nil { + t.Fatal(e) + } + if e = s.cleanQueryModelOrphans(ctx); e != nil { + t.Fatal(e) + } + if e = s.queryBuildRunRound(ctx, l, org, []string{"p"}, nil); e != nil { + t.Fatal("sealed work lost after snapshot expiry", e) + } + n, e := s.queryData().CountDocuments(ctx, bson.M{"generation": j.Production.Generation}) + if e != nil || n != int64(j.Production.Chunks) { + t.Fatal("GC deleted unfinished durable blob", n, e) + } +} diff --git a/internal/store/query_build_runner.go b/internal/store/query_build_runner.go new file mode 100644 index 00000000..cd8c6551 --- /dev/null +++ b/internal/store/query_build_runner.go @@ -0,0 +1,197 @@ +package store + +import ( + "context" + "errors" + "fmt" + "reflect" + "sort" + "time" + + "droboticflow/internal/queryserving" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" +) + +// This lifecycle is exclusive to the independent, periodic read database. +// Authority/legacy callers keep their existing loader and publication behavior. +func (s *MongoStore) queryBuildStart(ctx context.Context, l queryModelLease, org string, projects []string) (queryBuildJob, error) { + org = normalizedOrganizationID(org) + projects = append([]string{}, projects...) + sort.Strings(projects) + old, exists, err := s.queryBuildJob(ctx, org) + if err != nil { + return old, err + } + if exists && old.Phase != "complete" { + if old.FailureClass != "" { + return old, fmt.Errorf("%w: phase=%s category=%s", ErrQueryBuildStopped, old.FailurePhase, old.FailureClass) + } + if !reflect.DeepEqual(old.Projects, projects) { + return old, errors.New("query build scope changed; retained job requires reconciliation") + } + return old, nil + } + observedAt := time.Now().UTC() + at, err := s.queryBuildSnapshot(ctx) + if err != nil { + return old, err + } + var state struct { + Revision int64 `bson:"productionRevision"` + Global int64 `bson:"searchGlobalRevision"` + Observed time.Time `bson:"observedAt"` + Heads map[string]QueryModelHead `bson:"heads"` + } + rows, e := s.queryBuildReadPage(ctx, s.queryControl(), bson.M{"_id": "global"}, bson.M{"productionRevision": 1, "searchGlobalRevision": 1, "heads": 1, "observedAt": 1}, bson.RawValue{}, false, at) + if e != nil { + return old, e + } + if len(rows) != 1 { + return old, errors.New("query build control unavailable at snapshot") + } + if e = bson.Unmarshal(rows[0], &state); e != nil { + return old, e + } + head := state.Heads[workforceFactKey(org)] + j := queryBuildJob{ID: queryBuildJobKey(s.db.Name(), org), Schema: queryBuildJobSchema, Organization: org, Projects: projects, Generation: queryserving.Key(queryBuildJobSchema, org, at.T, at.I, time.Now().UnixNano()), Snapshot: at, AsOf: observedAt, Phase: "production_plan", OldEpoch: head.FactEpoch, Epoch: head.FactEpoch + 1, Cold: head.FactEpoch == 0, SourceRevision: state.Revision, GlobalRevision: state.Global, ProgressAt: time.Now().UTC()} + j.InputGeneration = j.Generation + if exists && old.Phase == "complete" && reflect.DeepEqual(old.Projects, projects) && old.SourceRevision == state.Revision && old.GlobalRevision == state.Global && !state.Observed.Before(old.AsOf) && time.Since(state.Observed) >= 0 && time.Since(state.Observed) < 15*time.Second { + j.AsOf = state.Observed + j.InputGeneration = old.InputGeneration + if j.InputGeneration == "" { + j.InputGeneration = old.Generation + } + // The caught-up stream proves that every business input is unchanged. + // Reconfirm the complete cohort without re-materializing or walking + // 140k unchanged facts and checkpointing thousands of no-op batches. + production, exists := state.Heads[queryModelKey("production", org, projects)] + committed := old.Epoch + if !old.Changed && old.OldEpoch > 0 { + committed = old.OldEpoch + } + if exists && production.Generation != "" && head.Rows == old.PlanRows && head.FactEpoch == committed && production.SourceRevision == old.SourceRevision { + j.Phase = "publish" + j.PlanRows = old.PlanRows + j.ReusedRows = int64(old.PlanRows) + j.Production = production + j.Production.AsOf = j.AsOf + } + } + err = s.queryBuildTransaction(ctx, l, func(sc mongo.SessionContext) error { return s.saveQueryBuildJob(sc, j) }) + return j, err +} + +func (s *MongoStore) queryBuildRunRound(ctx context.Context, l queryModelLease, org string, projects []string, r *queryBuildRound) (err error) { + j, err := s.queryBuildStart(ctx, l, org, projects) + if err != nil { + return err + } + if r == nil { + r = newQueryBuildRound(j.Cold) + } + started := time.Now() + before := j.ProgressAt + defer func() { + if err == nil || ctx.Err() != nil { + return + } + // Maintenance/shutdown time is not counted as active lack of progress. + current, ok, e := s.queryBuildJob(ctx, org) + if e != nil || !ok { + return + } + if !current.ProgressAt.After(before) { + current.ActiveWithoutProgress += time.Since(started) + } else { + current.ActiveWithoutProgress = 0 + } + if errors.Is(err, ErrQueryBuildYielded) && current.ActiveWithoutProgress >= queryBuildNoProgressLimit { + err = ErrQueryBuildNoProgress + } + if !errors.Is(err, ErrQueryBuildYielded) { + current.FailurePhase = current.Phase + diagnostic := QueryReplicaDiagnostic(err) + current.FailureClass = diagnostic.Class + current.FailureCode = diagnostic.MongoCode + if errors.Is(err, ErrQueryBuildNoProgress) { + current.FailureClass = "no_durable_progress" + } + } + if !errors.Is(err, ErrQueryBuildYielded) { + err = fmt.Errorf("%w: %w", ErrQueryBuildStopped, err) + } + if e = s.queryBuildTransaction(ctx, l, func(sc mongo.SessionContext) error { return s.saveQueryBuildJob(sc, current) }); e != nil { + err = fmt.Errorf("query build could not persist outcome: %w", e) + } + }() + for { + if err = r.check(ctx); err != nil { + return err + } + switch j.Phase { + case "production_plan": + err = s.queryBuildProductionPlan(ctx, l, &j, r) + case "workforce_plan": + err = s.queryBuildWorkforcePlan(ctx, l, &j, r) + case "apply": + err = s.queryBuildApply(ctx, l, &j, r) + case "cleanup": + err = s.queryBuildCleanup(ctx, l, &j, r) + case "publish": + err = s.queryBuildPublish(ctx, l, &j) + case "complete": + return nil + default: + return errors.New("unsupported query build phase") + } + if err != nil { + return queryBuildPhaseFailure(j.Phase, err) + } + } +} + +// Explicitly bounded derived-data housekeeping; active jobs are never swept. +func (s *MongoStore) cleanQueryBuildParts(ctx context.Context, l queryModelLease) error { + var jobs []queryBuildJob + cur, err := s.db.Collection(queryBuildJobsCollection).Find(ctx, bson.M{"schema": queryBuildJobSchema}, options.Find().SetProjection(bson.M{"generation": 1, "inputGeneration": 1})) + if err != nil { + return err + } + err = cur.All(ctx, &jobs) + cur.Close(ctx) + if err != nil { + return err + } + keep := []string{} + for _, j := range jobs { + keep = append(keep, j.Generation, j.InputGeneration) + } + for _, collection := range []string{queryBuildPartsCollection, queryBuildJobsCollection} { + f := bson.M{"job": bson.M{"$exists": true, "$nin": keep}} + cur, e := s.db.Collection(collection).Find(ctx, f, options.Find().SetProjection(bson.M{"_id": 1}).SetLimit(1000)) + if e != nil { + return e + } + var rows []bson.M + e = cur.All(ctx, &rows) + cur.Close(ctx) + if e != nil { + return e + } + ids := bson.A{} + for _, row := range rows { + ids = append(ids, row["_id"]) + } + if len(ids) > 0 { + if e = s.queryBuildTransaction(ctx, l, func(sc mongo.SessionContext) error { + _, e := s.db.Collection(collection).DeleteMany(sc, bson.M{"_id": bson.M{"$in": ids}, "job": bson.M{"$nin": keep}}) + return e + }); e != nil { + return e + } + } + } + return nil +} diff --git a/internal/store/query_build_scale_local_test.go b/internal/store/query_build_scale_local_test.go new file mode 100644 index 00000000..b20310ba --- /dev/null +++ b/internal/store/query_build_scale_local_test.go @@ -0,0 +1,226 @@ +package store + +import ( + "context" + "errors" + "fmt" + "os" + "strings" + "sync/atomic" + "testing" + "time" + + "droboticflow/internal/domain" + "droboticflow/internal/workforcestats" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/event" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + "golang.org/x/time/rate" +) + +func TestQueryBuildDurableColdSameScaleLocal(t *testing.T) { + if os.Getenv("QUERY_BUILD_LIFECYCLE_SCALE") != "1" { + t.Skip("explicit 140k fact synthetic comparison") + } + var originalReads, nativeReads atomic.Int64 + _, s, _ := newLocalReplica(t, &event.CommandMonitor{Started: func(context.Context, *event.CommandStartedEvent) { originalReads.Add(1) }}, &event.CommandMonitor{Started: func(_ context.Context, e *event.CommandStartedEvent) { + if e.CommandName == "find" { + n, _ := e.Command.Lookup("find").StringValueOK() + if querySourceCollection(n) { + nativeReads.Add(1) + } + } + }}) + originalReads.Store(0) + ctx := t.Context() + org := "synthetic-durable-scale" + const count = 35000 + existing := 125186 + if os.Getenv("QUERY_BUILD_SCALE_EMPTY") == "1" { + existing = 0 + } + project := strings.Repeat("synthetic-project-", 4) + at := time.Date(2026, 11, 1, 0, 0, 0, 0, time.UTC) + seconds := 1.0 + for start := 0; start < count; start += 1000 { + events, tasks := []any{}, []any{} + for i := start; i < min(start+1000, count); i++ { + id := fmt.Sprintf("task-%06d", i) + events = append(events, domain.AnnotationWorklogEventRecord{ID: fmt.Sprintf("event-%06d", i), OrganizationID: org, ProjectID: project, TaskID: id, UserID: "actor", SupplierID: "supplier", EventType: "annotation_submitted", Mode: domain.RunModeProduction, Revision: 1, TemplateVersion: 1, VideoSeconds: &seconds, OutputDelta: 1, CreatedAt: at.Add(time.Duration(i) * time.Second).Format(time.RFC3339Nano)}) + tasks = append(tasks, bson.M{"_id": id, "id": id, "organizationId": org, "projectId": project, "supplierId": "supplier", "runId": "run"}) + } + if _, e := s.annotationWorklogEvents.InsertMany(ctx, events); e != nil { + t.Fatal(e) + } + if _, e := s.annotationTasks.InsertMany(ctx, tasks); e != nil { + t.Fatal(e) + } + } + for name, doc := range map[string]any{"runs": bson.M{"_id": "run", "id": "run", "organizationId": org, "mode": "production"}, "users": domain.UserRecord{ID: "actor", OrganizationID: org, Name: strings.Repeat("合成用户", 16), SupplierID: "supplier", Role: domain.UserRoleAnnotator}, "suppliers": domain.SupplierRecord{ID: "supplier", OrganizationID: org, Name: "synthetic supplier", Status: "active"}} { + if _, e := s.db.Collection(name).InsertOne(ctx, doc); e != nil { + t.Fatal(e) + } + } + s.queryReplicaFactStore = true + s.queryPeriodic = true + if e := s.ensureWorkforceFactIndexes(ctx); e != nil { + t.Fatal(e) + } + l, ok, e := s.acquireQueryModelLease(ctx) + if e != nil || !ok { + t.Fatal(e) + } + // Generate the exact e4 staging rows; fixture setup alone is unpaced. + if existing > 0 { + if e = s.rebuildWorkforceFacts(ctx, l, org, 0); e != nil { + t.Fatal(e) + } + unpublishWorkforceFixture(t, s, org) + c := s.db.Collection(workforceFactCollection) + cur, e := c.Find(ctx, bson.M{"organizationId": org}, options.Find().SetProjection(bson.M{"_id": 1}).SetLimit(int64(count*4-existing))) + if e != nil { + t.Fatal(e) + } + var ids []bson.M + if e = cur.All(ctx, &ids); e != nil { + t.Fatal(e) + } + cur.Close(ctx) + for start := 0; start < len(ids); start += 1000 { + values := bson.A{} + for _, id := range ids[start:min(start+1000, len(ids))] { + values = append(values, id["_id"]) + } + if _, e = c.DeleteMany(ctx, bson.M{"_id": bson.M{"$in": values}}); e != nil { + t.Fatal(e) + } + } + } + s.queryBuildWrites = s.newQueryBuildWriteGate() + if s.queryBuildWrites.limiter.Limit() != rate.Limit(512<<10) { + t.Fatal("pacing changed") + } + started := time.Now() + rounds := 0 + // Keep renewing the same fenced lease, just as the production process does. + runCtx, cancel := context.WithCancel(ctx) + defer cancel() + done := make(chan struct{}) + leaseStore := s + go func() { + defer close(done) + ticker := time.NewTicker(10 * time.Second) + defer ticker.Stop() + for { + select { + case <-runCtx.Done(): + return + case <-ticker.C: + if leaseStore.renewQueryModelLease(runCtx, l) != nil { + return + } + } + } + }() + defer func() { cancel(); <-done }() + for { + rounds++ + e = s.queryBuildRunRound(ctx, l, org, []string{project}, nil) + if e == nil { + break + } + if !errors.Is(e, ErrQueryBuildYielded) { + t.Fatal("cold round failed", rounds, e) + } + state, _, _ := s.queryBuildJob(ctx, org) + t.Logf("yield round=%d phase=%s confirmed=%d reused=%d nextPart=%d", rounds, state.Phase, state.ConfirmedRows, state.ReusedRows, state.NextPart) + if rounds > 30 { + t.Fatal("synthetic cold build did not progress") + } + fresh := NewMongoStore(s.db) + fresh.queryReplicaFactStore = true + fresh.queryPeriodic = true + fresh.queryBuildWrites = s.queryBuildWrites + s = fresh + } + cold := time.Since(started) + j, ok, e := s.queryBuildJob(ctx, org) + if e != nil || !ok || j.PlanRows != count*4 || j.ReusedRows != int64(existing) || j.ConfirmedRows != int64(count*4-existing) { + t.Fatal("same scale resume parity", j.PlanRows, j.ReusedRows, j.ConfirmedRows, e) + } + h, _, e := s.workforceHead(ctx, org) + if e != nil { + t.Fatal(e) + } + q := workforcestats.Query{Mode: domain.RunModeAll, From: at, To: at.Add(24 * time.Hour)} + report, _, e := s.computeWorkforceFactReport(ctx, org, q, h) + if e != nil || report.Totals.FirstSubmittedTasks != count || report.Totals.FirstVideoSeconds != count { + t.Fatal("same scale report differs", e) + } + baseline := time.Duration(0) + if os.Getenv("QUERY_BUILD_SCALE_BASELINE") == "1" { + if _, e = s.queryControl().UpdateOne(ctx, queryLeaseFilter(l), bson.M{"$set": bson.M{"observedAt": time.Now().UTC()}}); e != nil { + t.Fatal(e) + } + // Explicit algorithm comparison on the identical fixture: retain the + // pre-fast-path walk of unchanged facts, including durable checkpoints. + job, err := s.queryBuildStart(ctx, l, org, []string{project}) + if err != nil { + t.Fatal(err) + } + job.Phase = "production_plan" + job.PlanRows = 0 + job.ReusedRows = 0 + job.Production = QueryModelHead{} + if err = s.queryBuildTransaction(ctx, l, func(sc mongo.SessionContext) error { return s.saveQueryBuildJob(sc, job) }); err != nil { + t.Fatal(err) + } + start := time.Now() + for { + err = s.queryBuildRunRound(ctx, l, org, []string{project}, nil) + if err == nil { + break + } + if !errors.Is(err, ErrQueryBuildYielded) { + t.Fatal(err) + } + } + baseline = time.Since(start) + } + warm := []time.Duration{} + for i := 0; i < 2; i++ { + if _, e = s.queryControl().UpdateOne(ctx, queryLeaseFilter(l), bson.M{"$set": bson.M{"observedAt": time.Now().UTC()}}); e != nil { + t.Fatal(e) + } + before := nativeReads.Load() + start := time.Now() + n := 0 + for { + n++ + e = s.queryBuildRunRound(ctx, l, org, []string{project}, nil) + if e == nil { + break + } + if !errors.Is(e, ErrQueryBuildYielded) || n > 30 { + t.Fatal("refresh failed", i, n, e) + } + } + if nativeReads.Load() != before { + t.Fatal("unchanged refresh rescanned native input", i, nativeReads.Load()-before) + } + warm = append(warm, time.Since(start)) + } + if originalReads.Load() != 0 { + t.Fatal("original Mongo reads", originalReads.Load()) + } + var status struct { + DataSize float64 `bson:"dataSize"` + StorageSize float64 `bson:"storageSize"` + IndexSize float64 `bson:"indexSize"` + } + if e = s.db.RunCommand(ctx, bson.M{"dbStats": 1}).Decode(&status); e != nil { + t.Fatal(e) + } + t.Logf("events=%d facts=%d reused=%d new=%d cold=%s rounds=%d walk_baseline=%s refresh1=%s refresh2=%s write_limit=%d confirmed_bson_bytes=%d original_mongo_reads=%d native_input_reads=%d data_bytes=%.0f storage_bytes=%.0f index_bytes=%.0f production_accepted=false", count, j.PlanRows, j.ReusedRows, j.ConfirmedRows, cold, rounds, baseline, warm[0], warm[1], 512<<10, j.ConfirmedBytes, originalReads.Load(), nativeReads.Load(), status.DataSize, status.StorageSize, status.IndexSize) +} diff --git a/internal/store/query_model_maintenance.go b/internal/store/query_model_maintenance.go index 28e09c42..9fa369d4 100644 --- a/internal/store/query_model_maintenance.go +++ b/internal/store/query_model_maintenance.go @@ -41,6 +41,23 @@ func (s *MongoStore) cleanQueryModelOrphans(ctx context.Context) error { for _, h := range state.Heads { keep = append(keep, h.Generation) } + if s.queryReplicaFactStore && s.periodicQueryStore() { + cur, err := s.db.Collection(queryBuildJobsCollection).Find(ctx, bson.M{"schema": queryBuildJobSchema, "phase": bson.M{"$ne": "complete"}}, options.Find().SetProjection(bson.M{"production.generation": 1})) + if err != nil { + return err + } + var jobs []queryBuildJob + err = cur.All(ctx, &jobs) + cur.Close(ctx) + if err != nil { + return err + } + for _, j := range jobs { + if j.Production.Generation != "" { + keep = append(keep, j.Production.Generation) + } + } + } filter := bson.M{"generation": bson.M{"$nin": keep}, "createdAt": bson.M{"$lt": time.Now().UTC().Add(-15 * time.Minute)}} // Ordinary orphan chunks/rows must still be reclaimed when Typesense is // offline. Retain only its collection intents until remote deletion succeeds. diff --git a/internal/store/query_read_builder.go b/internal/store/query_read_builder.go index 72696f29..5061fabb 100644 --- a/internal/store/query_read_builder.go +++ b/internal/store/query_read_builder.go @@ -118,6 +118,13 @@ func (b *QueryReadBuilder) scopes(ctx context.Context) (map[string][]string, err return out, nil } func (b *QueryReadBuilder) rebuild(ctx context.Context, l queryModelLease, org string, projects []string) (err error) { + if b.store.queryReplicaFactStore && b.store.periodicQueryStore() { + err = b.store.queryBuildRunRound(ctx, l, org, projects, nil) + if errors.Is(err, ErrQueryBuildYielded) { + log.Print("query_model_build result=yielded checkpoint=retained") + } + return err + } started := time.Now() stage := "source" defer func() { @@ -178,7 +185,11 @@ func (b *QueryReadBuilder) rebuild(ctx context.Context, l queryModelLease, org s // RunOnce is for bounded initial prewarming and isolated tests, not a freshness // substitute for Run. After the heartbeat expires search falls back to old reads. func (b *QueryReadBuilder) RunOnce(parent context.Context) error { - ctx, cancel := context.WithTimeout(parent, 5*time.Minute) + ctx, cancel := context.WithCancel(parent) + if !(b.store.queryReplicaFactStore && b.store.periodicQueryStore()) { + cancel() + ctx, cancel = context.WithTimeout(parent, 5*time.Minute) + } defer cancel() started := make(chan struct{}) done := make(chan error, 1) @@ -418,7 +429,9 @@ func (b *QueryReadBuilder) run(ctx context.Context, started chan<- struct{}) err lastScopeRead = time.Now() } for org := range scopes { - if all { + // Even an idle organization needs periodic complete-result + // reconfirmation; freshness must not depend on customer writes. + if all || (b.store.periodicQueryStore() && queryBuildDue(time.Now(), lastStats[org], statsInterval)) { pendingStats[org] = true } if allSearch { @@ -472,7 +485,12 @@ func (b *QueryReadBuilder) run(ctx context.Context, started chan<- struct{}) err delete(statsRetries, org) delete(pendingStats, org) lastStats[org] = time.Now() + } else if errors.Is(e, ErrQueryBuildYielded) { + delete(statsRetries, org) } else { + if b.store.queryReplicaFactStore && b.store.periodicQueryStore() { + return e + } retry := statsRetries[org] retry.fail(30*time.Second, 5*time.Minute) statsRetries[org] = retry @@ -487,6 +505,9 @@ func (b *QueryReadBuilder) run(ctx context.Context, started chan<- struct{}) err if time.Since(lastCleanup) > time.Minute { cleanup, finish := context.WithTimeout(ctx, 5*time.Second) _ = b.store.cleanQueryModelOrphans(cleanup) + if b.store.queryReplicaFactStore && b.store.periodicQueryStore() { + _ = b.store.cleanQueryBuildParts(cleanup, l) + } finish() lastCleanup = time.Now() diff --git a/internal/store/workforce_facts.go b/internal/store/workforce_facts.go index 6aea2028..90470994 100644 --- a/internal/store/workforce_facts.go +++ b/internal/store/workforce_facts.go @@ -47,6 +47,7 @@ func workforceBoundedRead(ctx context.Context, cur *mongo.Cursor, target any, bu func workforceFactKey(org string) string { return queryModelKey("workforce-facts", org, nil) } func (s *MongoStore) ensureWorkforceFactIndexes(ctx context.Context) error { indexes := []mongo.IndexModel{ + {Keys: bson.D{{Key: "organizationId", Value: 1}, {Key: "key", Value: 1}, {Key: "fromEpoch", Value: 1}, {Key: "toEpoch", Value: 1}}, Options: options.Index().SetName("workforce_fact_key_v1")}, {Keys: bson.D{{Key: "organizationId", Value: 1}, {Key: "retiredAt", Value: 1}, {Key: "toEpoch", Value: 1}}, Options: options.Index().SetName("workforce_fact_retirement_v1")}, {Keys: bson.D{{Key: "organizationId", Value: 1}, {Key: "mode", Value: 1}, {Key: "groupTemplate", Value: 1}, {Key: "userId", Value: 1}, {Key: "at", Value: -1}, {Key: "order", Value: 1}}, Options: options.Index().SetName("workforce_fact_user_v1")}, {Keys: bson.D{{Key: "organizationId", Value: 1}, {Key: "mode", Value: 1}, {Key: "groupTemplate", Value: 1}, {Key: "supplierId", Value: 1}, {Key: "projectId", Value: 1}, {Key: "at", Value: -1}}, Options: options.Index().SetName("workforce_fact_scope_v1")}, diff --git a/internal/store/workforce_stats.go b/internal/store/workforce_stats.go index 9e8021ae..c4374800 100644 --- a/internal/store/workforce_stats.go +++ b/internal/store/workforce_stats.go @@ -455,6 +455,9 @@ func (s *MongoStore) LoadWorkforceStats(ctx context.Context, org string, q workf return m } read := func(c mongoCollection, filter, projection bson.M, target any) error { + if hook, ok := ctx.Value(queryBuildReadKey{}).(queryBuildReadFunc); ok { + return hook(ctx, c, filter, projection, target) + } cur, err := c.Find(ctx, filter, options.Find().SetProjection(projection).SetMaxTime(15*time.Second)) if err != nil { return err diff --git a/scripts/test_query_replica.py b/scripts/test_query_replica.py index 9988d521..db907a35 100644 --- a/scripts/test_query_replica.py +++ b/scripts/test_query_replica.py @@ -31,6 +31,8 @@ CASES = ('cmd/query-replica/main_local_test.go', 'internal/store/query_replica_valkey_local_test.go', 'internal/store/workforce_report_cache_local_test.go', 'internal/store/workforce_facts_local_test.go', + 'internal/store/query_build_capture_local_test.go', + 'internal/store/query_build_resume_local_test.go', 'internal/store/workforce_publication_local_test.go', 'internal/store/query_replica_page_local_test.go', 'internal/store/supplier_authorization_batch_local_test.go', diff --git a/scripts/verification-scopes.json b/scripts/verification-scopes.json index 7af13f58..ec9b66e3 100644 --- a/scripts/verification-scopes.json +++ b/scripts/verification-scopes.json @@ -1044,6 +1044,7 @@ "^internal/store/workforce_(report_cache|heap_diagnostic|facts|return_page|publication).*\\.go$", "^internal/store/query_pool_buckets.*\\.go$", "^internal/store/query_cold_write.*\\.go$", + "^internal/store/query_build_.*\\.go$", "^internal/store/query_periodic.*\\.go$", "^internal/store/supplier_authorization_batch_local_test\\.go$", "^internal/api/query_periodic.*\\.go$", @@ -1057,7 +1058,7 @@ "^scripts/local/query-replica\\.query25-local\\.compose\\.yml$", "^scripts/local/query-replica\\.compose\\.yml$", "^Dockerfile\\.query-replica$", - "^docs/(adr/0014-.*|operations/query-replica\\.md)$" + "^docs/(adr/0014-.*|operations/query-replica\\.md|operations/query-build-lifecycle\\.md)$" ], "frontend": [], "go": { @@ -1066,7 +1067,10 @@ "^TestQueryReplicaCountOnly", "^TestQueryReplicaMissingConfiguration", "^TestQueryReplicaRejectsSigned", - "^TestQueryReplicaDiagnostic" + "^TestQueryReplicaDiagnostic", + "^TestQueryBuildRound", + "^TestQueryBuildBudgets", + "^TestQueryBuildPacing" ], "./cmd/query-cdc-probe": ["^TestClassification"], "./cmd/query-replica": ["^TestPassDelay", "^TestPacing"],