♻️ Persist resumable query build generations

This commit is contained in:
d-robotics 2026-11-15 08:11:09 -08:00
parent e4c4a20df5
commit ffe186473c
23 changed files with 2183 additions and 8 deletions

View File

@ -4,6 +4,7 @@ package main
import (
"context"
"droboticflow/internal/store"
"errors"
"flag"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
@ -87,8 +88,13 @@ func main() {
return
}
for workCtx.Err() == nil {
if b.Run(workCtx) != nil && workCtx.Err() == nil {
log.Print("query builder paused; retrying with full reconciliation")
if err := b.Run(workCtx); err != nil && workCtx.Err() == nil {
if errors.Is(err, store.ErrQueryBuildStopped) {
log.Print("query builder stopped; checkpoint retained; explicit recovery required")
<-workCtx.Done()
return
}
log.Print("query builder retrying; durable build checkpoint retained")
}
select {
case <-workCtx.Done():

View File

@ -66,3 +66,19 @@
准备状态通过显式媒体类型协商;旧客户端收到未就绪错误而非成功的非报表JSON。活跃配方的登记触碰合并到15分钟一次,后台统计刷新周期不变。任务池只共享候选身份,个人任务/审查历史维持当前权威查询;权限目录的逐项目重复读合并为请求内批量读,继续复用原授权种子规则,不形成共享权限缓存。
候选身份的冷缓存查询合并仅作用于相同数据库、组织、事实代次和完整筛选,沿用既有共享名称读取的8秒工作上限及独立取消规则;完成后的长效成品仍只在共享缓存/查询库中。每个请求各自取当前任务与运行服务端权限校验,合并结果不带授权结论、正文或可领取标志。
## 首次构建与日常刷新分离(2026-11-15,用户批准,实施中)
首次构建不再以固定5分钟要求全部完成。每轮执行有调度预算,每个读取、持久化批次及提交有独立上限;预算用尽记录可靠断点并让出,属于正常控制流,不触发全流程停副本。窗口到期、无进展、数据库不可用、权限/租约失效、容量及持久化异常仍须暂停并记录分项原因。
复用已同步查询事实及e4c4a20d的首代暂存指纹核对。构建采用持久输入/计划、分批应用、检查及原子发布几个阶段;已封存计划在续建、部署或重启后只读取未确认部分,不重新回填权威库,也不重新扫描完整业务输入。输入捕获本身须保留有界读取断点;源快照不可恢复时不得混合前后版本或伪造完成。
日常刷新按3分钟后台周期及15分钟展示年龄契约单独安排预算;继续提供上一完整版本。生产统计、人员统计和作业记录相关头只能在同一封存输入代次完成后发布,不能先切部分头。权限、供应商隔离与正式操作仍由原权威路径实时校验。
保持多数提交和日志持久性、25Gi容量及现有限额。512KiB/s内部写入限速须计入首次构建的真实数据量及确认耗时;若需调整,参数、共享盘影响和回退另列具体启用计划,不以延长请求超时或降低持久性替代。生产入口仍未切换,本节不是生产验收结论。
实现约束:首次构建每轮60秒、日常变化刷新每轮30秒,读取总界限5秒(最多3次瞬时错误重试)、每次提交15秒。到轮次边界仅让出,不取消已开始的持久提交;下轮通过租约和持久游标继续。无进展按累计有效执行时间计算,不计维护停机时间。`RunOnce`在独立周期查询库中也不再强加5分钟全程期限,仍服从调用方取消和维护窗口。
输入捕获和计划保存在私有`queryBuildJobs`/`queryBuildParts`。原生输入分段、游标以及写入批次、应用游标分别在同一`majority+journal`事务确认。续建重新核对封存计划指纹和e4暂存事实哈希;未发布的大块结果由清理程序保留。派生事实采用原有版本可见性规则,全部完成及行数核对后才在同一事务发布两个统计头。已追平观察器证明业务输入未变化时,只重新确认完整结果,不重新读取历史、计算或遍历全量事实。
输入快照的逻辑时间与展示时间分开保存。已经封存输入后,恢复不依赖原快照仍在Mongo历史窗口内;尚未捕获完的输入若遇到快照历史丢失,必须保留分段并明确停止,不能切换时间点后拼接新旧结果。该异常不自动重新回填权威库。首次上线前须检查查询库快照/oplog保留窗口,不能把任意长停机期间未封存输入的自动恢复视为已经覆盖。

View File

@ -0,0 +1,66 @@
# 查询库首次构建与日常刷新
## 范围与状态
仅独立周期查询库的构建器采用持久任务。权威Mongo、标注/切片/送检/审核/验收、权限目录和版本竞争路径不改。生产查询入口仍未切换。本地通过、代码部署、Pod就绪分别不代表影子验收或分流降载验收。
上一轮e4影子构建已完成:复用125,186条,补写14,820条;事实写入阶段约101秒,后续观察到至少两轮完整刷新。原维护窗口在2026-11-15 14:38:42 UTC到期,两影子作业按约定暂停到0副本并保留断点。本文不延长该窗口。
## 执行与持久性
| 参数 | 计划值 | 说明 |
| --- | --- | --- |
| 首次构建单轮调度 | 60秒 | 正常让出,不是全程完成期限 |
| 变化刷新单轮调度 | 30秒 | 延续既有3分钟周期、15分钟展示年龄 |
| 原生输入单次逻辑读取 | 5秒 | 最多3次瞬时重试,每次服务端1.5秒 |
| 持久事务 | 15秒 | 多数提交及journal;游标与数据共同确认 |
| 累计无确认进展 | 5分钟有效执行时间 | 不含停机;真正无进展持久记录后停止 |
| 内部写入限速 | 512KiB/s | 暂不调整;不能以延长超时取代限速核对 |
| 应用批次 | 最多200逻辑行,约64KiB写入 | 单条较大的合法事实仍完整写入 |
| 容量/Pod/底层IO | 保持现有配置 | 25Gi、现有内存/CPU/IO边界不变 |
正常阶段:捕获输入→保存完整计划→应用事实→清理未匹配的未来版本→核对→事务发布。新一代完整之前,读者继续使用原完整代次及原权限校验;超出既有展示年龄仍按原错误/成品回退契约处理,不无限伪装新鲜或集中回源。
`queryBuildJobs`记录组织、输入代次、快照、阶段、事实应用游标、清理游标及分项失败;`queryBuildParts`保存带校验和的压缩输入和计划。均为私有集合,禁止从通用数据管理API读写。维护清理只回收不再被当前任务/输入引用的派生分段,绝不清客户标注或同步事实。
无变化日常刷新只有在观察器追平证明新鲜且组织、范围、逻辑版本及业务修订相同的情况下,才直接重新确认原完整事实和统计结果。否则新建输入代次并按上述流程计算。变更刷新可以读取查询库必要输入;不会重新启动原Mongo历史回填。
## 暂停、恢复与限制
- 轮次预算耗尽:保留pending状态和游标,下轮正常续建,不增加失败退避,不因此降为0副本。
- 进程退出/部署/调用方取消:已确认批次保留;新持有者以新租约恢复,旧持有者不能提交。
- 数据库/权限/租约/持久化/容量异常:记录阶段、类别及Mongo代码;停止构建,保持旧完整结果,不自动清库重试。
- 维护截止:按已批准观察器策略停影子作业,保存累计同步预算和断点。启动必须有新的有效窗口。
- 已封存输入:原快照历史过期不影响应用恢复。未封存输入若历史已丢失,停止并保留分段,需单列恢复方案,不能混入新时间点。
- 算法/字段语义变更必须更新构建逻辑schema并审核旧任务迁移;封存计划指纹不符时禁止继续混写。
## 待确认的实际启用方案
1. 通过本任务检查和正式发布门禁,提交候选来源、镜像摘要及最终可执行清单。新代码尚未获得推送/上传或部署确认。
2. 核对新的维护窗口、当前同步断点/预算、源健康、查询库容量/IO、原有镜像及运行身份;旧窗口不能复用。
3. 仅为查询库构建器授权私有两集合的必要读写,在查询库`queryWorkforceFacts`增加`workforce_fact_key_v1`:`organizationId,key,fromEpoch,toEpoch`。不在原Mongo建索引。本地14万事实需记录新增索引体积,生产DDL仍列入最终计划。
4. 构建器保持0副本更新镜像;同步沿原有效授权运行并取得新鲜追平证明后,再启用构建器1副本。保留所有e4确认事实,不能调用旧版`restoreWorkforceStage`清除新任务的未来版本。
5. 核验首次完整发布、两轮正常刷新、权限/供应商/日期等价、Valkey丢失回退和查询库重启;生产查询入口继续关闭。原Mongo降载需后续经确认切入口后独立验收。
回退:先暂停构建器,保留任务/分段/事实、头和同步断点。e4旧构建器不能直接接手尚未完成的新生命周期任务,否则其旧恢复逻辑可能删除未来版本;无损回退优先停构建器并继续读上一完整版本。只有独立核实不存在未完成新任务、已发布完整头可用后,才能另行确认恢复旧镜像。不删表、不回滚客户标注、不重置预算、不降低持久性。
## 验证证据
证据集中写入本工作树忽略目录`local-data/test-results/query-build-*`,包含读取次数、批次、耗时、数据/索引占用及配置。不得复制客户原文或凭证。正式结果待验证完成后更新,不用短时成功代替完整验收。
### 2026-11-15 本地同规模结果
固定35,000条合成事件、140,000条事实,查询Mongo为1核/1GiB、WiredTiger 256MiB;生产影子库仍为4核/8GiB,不把本地结果当作生产容量验收。写入限速一直为512KiB/s,持久化一直为majority+journal。
| 场景 | 结果 |
| --- | --- |
| 保留e4的125,186条确认事实、补14,814条 | 76.33秒,跨2轮 |
| 同夹具遍历无变化事实作为算法对照 | 42.53秒 |
| 复用已证明无变化的完整结果,两次刷新 | 194.75ms、181.10ms |
| 从零写入140,000条事实 | 318.71秒,跨6轮及进程对象重建 |
| 从零完成后的两次刷新 | 163.63ms、153.75ms |
| 以上构建与刷新对原Mongo的读取 | 0 |
从零构建确认写入148,610,000 BSON字节,限速理论下界约283秒,实际约319秒;不需要通过增加全程超时或削弱持久性才能完成。采样查询容器CPU最高37.41%,无OOM;Docker VM的IO full avg10最高0.91%,该值不是单库IO等待。新增键索引在14万事实时约14.44MB。内存包含Mongo文件缓存压力,本地1GiB测试不能替代生产8GiB长时验证。
快速检查通过。首次全任务执行的浏览器、统计稳定性、权限和共享缓存检查通过;隔离副本组在总编排900秒耗尽后被停止,原报告`verify-1794758125279658000`保留为incomplete,不能写成通过。后续对失败落盘分支完成故障注入后,集中补跑受影响的隔离组;已通过且输入未变的浏览器证据保留。

View File

@ -66,7 +66,7 @@ func TestQueryServingProductionResponseChecksCurrentPermissionBeforeCache(t *tes
func TestQueryServingPrivateMaterializationsRejectDatabaseBrowser(t *testing.T) {
f := newCombinedDraftFixture(t)
for _, name := range []string{"queryReadModelControl", "queryReadModelData"} {
for _, name := range []string{"queryReadModelControl", "queryReadModelData", "queryBuildJobs", "queryBuildParts"} {
for _, target := range []string{"platform", "business"} {
if r := f.request("GET", "/api/database/collections/"+name+"/documents?target="+target, f.admin, nil); r.Code < 400 {
t.Fatalf("private read model leaked: %s %s", name, target)

View File

@ -5,5 +5,5 @@ import "droboticflow/internal/outputsnapshot"
// Derived generations contain facts from multiple authorization scopes and
// must only be served through their permission-aware reporting/search readers.
func IsPrivateDerivedCollection(name string) bool {
return name == querySyncDeleteCollection || name == queryReplicaControl || outputsnapshot.IsPrivateCollection(name) || name == "queryReadModelControl" || name == "queryReadModelData"
return name == queryBuildJobsCollection || name == queryBuildPartsCollection || name == querySyncDeleteCollection || name == queryReplicaControl || outputsnapshot.IsPrivateCollection(name) || name == "queryReadModelControl" || name == "queryReadModelData"
}

View File

@ -204,6 +204,9 @@ func (s *MongoStore) loadProductionStats(ctx context.Context, org string, projec
}
}
read := func(c mongoCollection, f bson.M, projection bson.M, target any, preferred ...string) error {
if hook, ok := ctx.Value(queryBuildReadKey{}).(queryBuildReadFunc); ok {
return hook(ctx, c, f, projection, target)
}
opts := options.Find().SetProjection(projection).SetMaxTime(15 * time.Second)
hint := hints[c.Name()]
// A project-leading covering index is unsuitable for legacy task-ID lookups.

View File

@ -0,0 +1,321 @@
package store
import (
"context"
"errors"
"fmt"
"time"
"droboticflow/internal/workforcestats"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
)
type queryBuildPrevious struct {
ID string `bson:"_id"`
Key string `bson:"key"`
Hash string `bson:"hash"`
From int64 `bson:"fromEpoch"`
To int64 `bson:"toEpoch"`
Retired *time.Time `bson:"retiredAt"`
Expire *time.Time `bson:"expireAt"`
}
func (s *MongoStore) queryBuildPlanPage(ctx context.Context, j queryBuildJob, n int) ([]workforcestats.Fact, error) {
op, stop := queryBuildOperation(ctx)
defer stop()
var p queryBuildPart
if err := s.db.Collection(queryBuildPartsCollection).FindOne(op, bson.M{"_id": queryBuildPlanPartKey(j, "facts", n), "job": j.Generation}).Decode(&p); err != nil {
return nil, err
}
var facts []workforcestats.Fact
if err := queryBuildJSONDecode(p, &facts); err != nil {
return nil, err
}
if len(facts) == 0 || len(facts) > queryBuildFactPageRows {
return nil, errors.New("invalid sealed fact page")
}
return facts, nil
}
// A batch checkpoint commits in the very same majority+journal transaction as
// its version writes. A lost acknowledgement can only replay this bounded batch.
func (s *MongoStore) queryBuildApply(ctx context.Context, l queryModelLease, j *queryBuildJob, r *queryBuildRound) error {
c := s.db.Collection(workforceFactCollection)
for j.NextPart < j.PlanParts {
if err := r.check(ctx); err != nil {
return err
}
facts, err := s.queryBuildPlanPage(ctx, *j, j.NextPart)
if err != nil {
return err
}
for j.NextRow < len(facts) {
if err = r.check(ctx); err != nil {
return err
}
models := []mongo.WriteModel{}
bytes, rows, reused := 0, 0, 0
next := *j
// Keep point lookups bounded; the key index avoids reading large old rows.
keys := []string{}
for _, f := range facts[j.NextRow:] {
keys = append(keys, f.Key)
}
op, stop := queryBuildOperation(ctx)
cur, e := c.Find(op, bson.M{"organizationId": j.Organization, "key": bson.M{"$in": keys}}, options.Find().SetProjection(bson.M{"_id": 1, "key": 1, "hash": 1, "fromEpoch": 1, "toEpoch": 1, "retiredAt": 1, "expireAt": 1}).SetHint("workforce_fact_key_v1"))
previous := map[string][]queryBuildPrevious{}
if e == nil {
var xs []queryBuildPrevious
e = cur.All(op, &xs)
cur.Close(op)
for _, x := range xs {
previous[x.Key] = append(previous[x.Key], x)
}
}
stop()
if e != nil {
return e
}
for next.NextRow < len(facts) {
f := facts[next.NextRow]
doc, hash, size, e := queryBuildFactDocument(j.Organization, j.Epoch, f)
if e != nil {
return e
}
if rows > 0 && bytes+size+512 > queryBuildBatchBytes {
break
}
versions := previous[f.Key]
var current *queryBuildPrevious
var staged *queryBuildPrevious
for i := range versions {
x := &versions[i]
if x.From == j.Epoch && x.ID == doc["_id"] {
staged = x
}
if x.From <= j.OldEpoch && (x.To == 0 || x.To > j.OldEpoch) {
if current != nil {
return errors.New("duplicate committed workforce key")
}
current = x
}
}
unchanged := current != nil && current.Hash == hash
keepID := ""
if unchanged {
keepID = current.ID
if current.To != 0 || current.Retired != nil || current.Expire != nil {
models = append(models, mongo.NewUpdateOneModel().SetFilter(bson.M{"_id": current.ID}).SetUpdate(bson.M{"$unset": bson.M{"toEpoch": "", "retiredAt": "", "expireAt": ""}}))
bytes += 256
}
reused++
} else {
next.Changed = true
keepID = doc["_id"].(string)
if current != nil && current.To != j.Epoch {
models = append(models, mongo.NewUpdateOneModel().SetFilter(bson.M{"_id": current.ID}).SetUpdate(bson.M{"$set": bson.M{"toEpoch": j.Epoch, "retiredAt": time.Now().UTC()}}))
bytes += 256
}
if staged != nil && staged.Hash == hash && staged.To == 0 && staged.Retired == nil && staged.Expire == nil {
reused++
} else {
models = append(models, mongo.NewReplaceOneModel().SetFilter(bson.M{"_id": doc["_id"]}).SetReplacement(doc).SetUpsert(true))
bytes += size
}
}
for _, x := range versions {
if x.From > j.OldEpoch && x.ID != keepID {
models = append(models, mongo.NewDeleteOneModel().SetFilter(bson.M{"_id": x.ID, "organizationId": j.Organization, "fromEpoch": bson.M{"$gt": j.OldEpoch}}))
bytes += 256
}
}
next.NextRow++
rows++
}
if rows == 0 {
return errors.New("fact batch made no progress")
}
if next.NextRow == len(facts) {
next.NextPart++
next.NextRow = 0
}
next.ConfirmedRows += int64(rows - reused)
next.ReusedRows += int64(reused)
next.ConfirmedBytes += int64(bytes)
next.ProgressAt = time.Now().UTC()
next.ActiveWithoutProgress = 0
err = s.queryBuildPace(ctx, bytes)
if err != nil {
return err
}
err = s.queryBuildTransaction(ctx, l, func(sc mongo.SessionContext) error {
if len(models) > 0 {
if _, e := c.BulkWrite(sc, models, options.BulkWrite().SetOrdered(true)); e != nil {
return e
}
}
return s.saveQueryBuildJob(sc, next)
})
if err != nil {
return err
}
*j = next
if j.NextRow == 0 {
break
}
}
}
j.Phase = "cleanup"
return s.queryBuildTransaction(ctx, l, func(sc mongo.SessionContext) error { return s.saveQueryBuildJob(sc, *j) })
}
// Merge the sorted sealed plan with one bounded page of old/future versions.
// Persist both cursors, so restart never re-scans all previously compared keys.
func (s *MongoStore) queryBuildCleanup(ctx context.Context, l queryModelLease, j *queryBuildJob, r *queryBuildRound) error {
c := s.db.Collection(workforceFactCollection)
for {
if err := r.check(ctx); err != nil {
return err
}
f := bson.M{"organizationId": j.Organization}
if j.CleanupFuture {
f["fromEpoch"] = bson.M{"$gt": j.OldEpoch}
} else {
f = workforceVersionFilter(j.Organization, j.OldEpoch)
}
if j.CleanupHasAfter {
f["key"] = bson.M{"$gt": j.CleanupAfter}
}
op, stop := queryBuildOperation(ctx)
cur, err := c.Find(op, f, options.Find().SetProjection(bson.M{"_id": 1, "key": 1, "fromEpoch": 1}).SetSort(bson.D{{Key: "key", Value: 1}}).SetHint("workforce_fact_key_v1").SetLimit(200))
var xs []queryBuildPrevious
if err == nil {
err = cur.All(op, &xs)
cur.Close(op)
}
stop()
if err != nil {
return err
}
if len(xs) == 0 {
if j.CleanupFuture {
j.Phase = "publish"
} else {
j.CleanupFuture = true
j.CleanupAfter = ""
j.CleanupHasAfter = false
j.CleanupPlanPart = 0
j.CleanupPlanRow = 0
}
if err = s.queryBuildTransaction(ctx, l, func(sc mongo.SessionContext) error { return s.saveQueryBuildJob(sc, *j) }); err != nil {
return err
}
if j.Phase == "publish" {
return nil
}
continue
}
next := *j
models := []mongo.WriteModel{}
var page []workforcestats.Fact
pageN := -1
for _, x := range xs {
if x.Key == "" {
return errors.New("invalid workforce fact identity")
}
present := false
for next.CleanupPlanPart < j.PlanParts {
if pageN != next.CleanupPlanPart {
page, err = s.queryBuildPlanPage(ctx, *j, next.CleanupPlanPart)
if err != nil {
return err
}
pageN = next.CleanupPlanPart
}
if next.CleanupPlanRow >= len(page) {
next.CleanupPlanPart++
next.CleanupPlanRow = 0
continue
}
key := page[next.CleanupPlanRow].Key
if key < x.Key {
next.CleanupPlanRow++
continue
}
present = key == x.Key
break
}
if !present {
next.Changed = true
if j.CleanupFuture {
models = append(models, mongo.NewDeleteManyModel().SetFilter(bson.M{"organizationId": j.Organization, "key": x.Key, "fromEpoch": bson.M{"$gt": j.OldEpoch}}))
} else {
models = append(models, mongo.NewUpdateManyModel().SetFilter(bson.M{"$and": bson.A{workforceVersionFilter(j.Organization, j.OldEpoch), bson.M{"key": x.Key}}}).SetUpdate(bson.M{"$set": bson.M{"toEpoch": j.Epoch, "retiredAt": time.Now().UTC()}}))
}
}
next.CleanupAfter = x.Key
next.CleanupHasAfter = true
}
next.ProgressAt = time.Now().UTC()
next.ActiveWithoutProgress = 0
if err = s.queryBuildPace(ctx, len(models)*256); err != nil {
return err
}
if err = s.queryBuildTransaction(ctx, l, func(sc mongo.SessionContext) error {
if len(models) > 0 {
if _, e := c.BulkWrite(sc, models); e != nil {
return e
}
}
return s.saveQueryBuildJob(sc, next)
}); err != nil {
return err
}
*j = next
}
}
func (s *MongoStore) queryBuildPublish(ctx context.Context, l queryModelLease, j *queryBuildJob) error {
op, stop := queryBuildOperation(ctx)
defer stop()
epoch := j.Epoch
if !j.Changed && j.OldEpoch > 0 {
epoch = j.OldEpoch
}
count, err := s.db.Collection(workforceFactCollection).CountDocuments(op, workforceVersionFilter(j.Organization, epoch))
if err != nil {
return err
}
if count != int64(j.PlanRows) {
return fmt.Errorf("sealed workforce row count differs: got=%d want=%d", count, j.PlanRows)
}
next := *j
next.Phase = "complete"
next.ProgressAt = time.Now().UTC()
next.ActiveWithoutProgress = 0
return s.queryBuildTransaction(ctx, l, func(sc mongo.SessionContext) error {
var state struct {
Heads map[string]QueryModelHead `bson:"heads"`
}
if e := s.queryControl().FindOne(sc, queryLeaseFilter(l)).Decode(&state); e != nil {
return e
}
if state.Heads[workforceFactKey(j.Organization)].FactEpoch != j.OldEpoch {
return errors.New("query build published epoch changed")
}
wf := QueryModelHead{Generation: workforceFactSchema, Collection: workforceFactCollection, FactEpoch: epoch, Rows: j.PlanRows, AsOf: j.AsOf, SourceRevision: j.SourceRevision, GlobalRevision: j.GlobalRevision}
result, e := s.queryControl().UpdateOne(sc, queryLeaseFilter(l), bson.M{"$set": bson.M{"heads." + workforceFactKey(j.Organization): wf, "heads." + queryModelKey("production", j.Organization, j.Projects): j.Production}})
if e != nil {
return e
}
if result.MatchedCount != 1 {
return errors.New("query build publication fenced")
}
if e = s.saveQueryBuildJob(sc, next); e == nil {
*j = next
}
return e
})
}

View File

@ -0,0 +1,362 @@
package store
import (
"bytes"
"compress/gzip"
"context"
"crypto/sha256"
"encoding/binary"
"encoding/hex"
"errors"
"fmt"
"io"
"reflect"
"sort"
"time"
"droboticflow/internal/queryserving"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/bson/primitive"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
)
type queryBuildReadKey struct{}
type queryBuildReadFunc func(context.Context, mongoCollection, bson.M, bson.M, any) error
type queryBuildCaptureState struct {
ID string `bson:"_id"`
Job string `bson:"job"`
Parts int `bson:"parts"`
Last bson.RawValue `bson:"last,omitempty"`
HasLast bool `bson:"hasLast"`
Complete bool `bson:"complete"`
Rows int `bson:"rows"`
Bytes int `bson:"bytes"`
}
type queryBuildPart struct {
ID string `bson:"_id"`
Job string `bson:"job"`
Digest string `bson:"digest"`
Data []byte `bson:"data"`
}
func queryBuildCanonical(v any) any {
switch d := v.(type) {
case bson.M:
out := map[string]any{}
for k, x := range d {
out[k] = queryBuildCanonical(x)
}
return out
case map[string]any:
out := map[string]any{}
for k, x := range d {
out[k] = queryBuildCanonical(x)
}
return out
case bson.A:
out := make([]any, len(d))
for i, x := range d {
out[i] = queryBuildCanonical(x)
}
return out
default:
return v
}
}
func queryBuildQueryKey(generation, name string, f, p bson.M) string {
// JSON map ordering is deterministic. Membership order is semantically
// irrelevant, so normalize caller-generated $in lists before hashing.
var normalize func(any) any
normalize = func(v any) any {
switch d := v.(type) {
case bson.M:
out := map[string]any{}
for k, x := range d {
if k == "$in" {
if ids, ok := x.([]string); ok {
cp := append([]string{}, ids...)
sort.Strings(cp)
out[k] = cp
continue
}
if a, ok := x.(bson.A); ok {
s := make([]string, 0, len(a))
all := true
for _, id := range a {
t, ok := id.(string)
if !ok {
all = false
break
}
s = append(s, t)
}
if all {
sort.Strings(s)
out[k] = s
continue
}
}
}
out[k] = normalize(x)
}
return out
case bson.A:
a := make([]any, len(d))
for i, x := range d {
a[i] = normalize(x)
}
return a
default:
return v
}
}
return queryserving.Key("query-build-input/v1", generation, name, normalize(f), queryBuildCanonical(p))
}
func queryBuildPack(v any) ([]byte, string, error) {
raw, err := bson.Marshal(v)
if err != nil {
return nil, "", err
}
var out bytes.Buffer
z := gzip.NewWriter(&out)
if _, err = z.Write(raw); err != nil {
return nil, "", err
}
if err = z.Close(); err != nil {
return nil, "", err
}
data := out.Bytes()
h := sha256.Sum256(data)
return data, hex.EncodeToString(h[:]), nil
}
func queryBuildUnpack(data []byte, digest string, out any) error {
h := sha256.Sum256(data)
if hex.EncodeToString(h[:]) != digest {
return errors.New("query build part digest mismatch")
}
z, err := gzip.NewReader(bytes.NewReader(data))
if err != nil {
return err
}
defer z.Close()
raw, err := io.ReadAll(io.LimitReader(z, (64<<20)+1))
if err != nil {
return err
}
if len(raw) > 64<<20 {
return errors.New("query build part exceeds decode budget")
}
return bson.Unmarshal(raw, out)
}
func (s *MongoStore) queryBuildSnapshot(ctx context.Context) (primitive.Timestamp, error) {
op, stop := queryBuildOperation(ctx)
defer stop()
var response struct {
At primitive.Timestamp `bson:"atClusterTime"`
Cursor struct {
At primitive.Timestamp `bson:"atClusterTime"`
} `bson:"cursor"`
}
err := s.db.RunCommand(op, bson.D{{Key: "find", Value: "queryReadModelControl"}, {Key: "filter", Value: bson.M{"_id": "global"}}, {Key: "projection", Value: bson.M{"_id": 1}}, {Key: "limit", Value: 1}, {Key: "singleBatch", Value: true}, {Key: "readConcern", Value: bson.M{"level": "snapshot"}}, {Key: "maxTimeMS", Value: 1500}}).Decode(&response)
if err != nil {
return primitive.Timestamp{}, err
}
if response.At.T == 0 {
response.At = response.Cursor.At
}
if response.At.T == 0 {
return primitive.Timestamp{}, errors.New("query build snapshot timestamp unavailable")
}
return response.At, nil
}
func (s *MongoStore) queryBuildReadPage(ctx context.Context, c mongoCollection, f, p bson.M, last bson.RawValue, hasLast bool, at primitive.Timestamp) ([]bson.Raw, error) {
filter := any(f)
if hasLast {
filter = bson.M{"$and": bson.A{f, bson.M{"$expr": bson.M{"$gt": bson.A{"$_id", bson.M{"$literal": last}}}}}}
}
projection := bson.M{}
for k, v := range p {
projection[k] = v
}
projection["_id"] = 1
cmd := bson.D{{Key: "find", Value: c.Name()}, {Key: "filter", Value: filter}, {Key: "projection", Value: projection}, {Key: "sort", Value: bson.D{{Key: "_id", Value: 1}}}, {Key: "hint", Value: "_id_"}, {Key: "limit", Value: 200}, {Key: "batchSize", Value: 200}, {Key: "singleBatch", Value: true}, {Key: "readConcern", Value: bson.M{"level": "snapshot", "atClusterTime": at}}, {Key: "maxTimeMS", Value: 1500}}
op, stop := queryBuildOperation(ctx)
defer stop()
var response struct {
Cursor struct {
Rows []bson.Raw `bson:"firstBatch"`
} `bson:"cursor"`
}
readCtx, readStop := context.WithTimeout(op, 5*time.Second)
defer readStop()
for attempt := 0; attempt < 3; attempt++ {
err := c.Database().RunCommand(readCtx, cmd).Decode(&response)
if err == nil {
return response.Cursor.Rows, nil
}
var command mongo.CommandError
transient := mongo.IsNetworkError(err) || mongo.IsTimeout(err)
if errors.As(err, &command) {
switch command.Code {
case 50, 91, 6, 7, 89, 11600, 11602, 10107:
transient = true
}
}
if !transient || attempt == 2 || readCtx.Err() != nil {
return nil, err
}
}
return response.Cursor.Rows, nil
}
func queryBuildAppendRows(ctx context.Context, rows []bson.Raw, target any, projection bson.M) error {
value := reflect.ValueOf(target).Elem()
budget, _ := ctx.Value(workforceBuildBudgetKey{}).(*workforceBuildBudget)
for _, raw := range rows {
if excluded, ok := projection["_id"]; ok && (excluded == 0 || excluded == false) {
elements, e := raw.Elements()
if e != nil {
return e
}
clean := make([]byte, 4, len(raw))
for _, element := range elements {
if element.Key() != "_id" {
clean = append(clean, element...)
}
}
clean = append(clean, 0)
binary.LittleEndian.PutUint32(clean, uint32(len(clean)))
raw = bson.Raw(clean)
}
if budget != nil {
budget.rows++
budget.bytes += len(raw)
if budget.rows > 250000 || budget.bytes > 64<<20 {
return errWorkforceBuildBudget
}
}
v := reflect.New(value.Type().Elem())
if err := bson.Unmarshal(raw, v.Interface()); err != nil {
return err
}
value.Set(reflect.Append(value, v.Elem()))
}
return nil
}
// This capture reads a native range once. Its payload and cursor commit in the
// same majority+journal transaction. Replaying a completed range reads only the
// immutable captured part, not the native business collections.
func (s *MongoStore) queryBuildCapture(ctx context.Context, l queryModelLease, j *queryBuildJob, round *queryBuildRound, c mongoCollection, f, p bson.M, target any) error {
if !s.queryReplicaFactStore {
return errors.New("query build capture requires independent read store")
}
check, checkStop := queryBuildOperation(ctx)
leaseErr := s.queryControl().FindOne(check, queryLeaseFilter(l), options.FindOne().SetProjection(bson.M{"_id": 1})).Err()
checkStop()
if leaseErr != nil {
return fmt.Errorf("query capture lease check: %w", leaseErr)
}
inputGeneration := j.InputGeneration
if inputGeneration == "" {
inputGeneration = j.Generation
}
key := queryBuildQueryKey(inputGeneration, c.Name(), f, p)
var st queryBuildCaptureState
check, checkStop = queryBuildOperation(ctx)
err := s.db.Collection(queryBuildJobsCollection).FindOne(check, bson.M{"_id": key}).Decode(&st)
checkStop()
if errors.Is(err, mongo.ErrNoDocuments) {
if inputGeneration != j.Generation {
return errors.New("verified input capture missing")
}
st = queryBuildCaptureState{ID: key, Job: inputGeneration}
} else if err != nil {
return err
}
for i := 0; i < st.Parts; i++ {
if err = ctx.Err(); err != nil {
return err
}
var part queryBuildPart
op, stop := queryBuildOperation(ctx)
err = s.db.Collection(queryBuildPartsCollection).FindOne(op, bson.M{"_id": fmt.Sprintf("%s/%09d", key, i)}).Decode(&part)
stop()
if err != nil {
return err
}
var payload struct {
Rows []bson.Raw `bson:"rows"`
}
if err = queryBuildUnpack(part.Data, part.Digest, &payload); err != nil {
return err
}
if err = queryBuildAppendRows(ctx, payload.Rows, target, p); err != nil {
return err
}
}
for !st.Complete {
if err = round.check(ctx); err != nil {
return err
}
rows, e := s.queryBuildReadPage(ctx, c, f, p, st.Last, st.HasLast, j.Snapshot)
if e != nil {
return e
}
next := st
if len(rows) == 0 {
next.Complete = true
} else {
id, e := rows[len(rows)-1].LookupErr("_id")
if e != nil {
return e
}
next.Last = bson.RawValue{Type: id.Type, Value: append([]byte(nil), id.Value...)}
next.HasLast = true
next.Parts++
next.Rows += len(rows)
}
var part queryBuildPart
if len(rows) > 0 {
data, hash, e := queryBuildPack(bson.M{"rows": rows})
if e != nil {
return e
}
part = queryBuildPart{ID: fmt.Sprintf("%s/%09d", key, st.Parts), Job: inputGeneration, Digest: hash, Data: data}
next.Bytes += len(data)
if e = s.queryBuildPace(ctx, len(data)); e != nil {
return e
}
}
if err = s.queryBuildTransaction(ctx, l, func(sc mongo.SessionContext) error {
if len(rows) > 0 {
if _, e := s.db.Collection(queryBuildPartsCollection).ReplaceOne(sc, bson.M{"_id": part.ID}, part, options.Replace().SetUpsert(true)); e != nil {
return e
}
}
_, e := s.db.Collection(queryBuildJobsCollection).ReplaceOne(sc, bson.M{"_id": key}, next, options.Replace().SetUpsert(true))
if e != nil {
return e
}
if j.ID != "" {
_, e = s.db.Collection(queryBuildJobsCollection).UpdateOne(sc, bson.M{"_id": j.ID, "generation": j.Generation}, bson.M{"$set": bson.M{"progressAt": time.Now().UTC(), "activeWithoutProgress": int64(0)}})
}
return e
}); err != nil {
return err
}
st = next
if err = queryBuildAppendRows(ctx, rows, target, p); err != nil {
return err
}
}
return nil
}

View File

@ -0,0 +1,108 @@
package store
import (
"context"
"errors"
"fmt"
"reflect"
"sync/atomic"
"testing"
"time"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/event"
)
func TestQueryBuildCaptureResumesNativeCursorWithoutFullRereadLocal(t *testing.T) {
var finds atomic.Int64
monitor := &event.CommandMonitor{Started: func(_ context.Context, e *event.CommandStartedEvent) {
if e.CommandName == "find" {
if n, ok := e.Command.Lookup("find").StringValueOK(); ok && n == "captureFixture" {
finds.Add(1)
}
}
}}
_, s, _ := newLocalReplica(t, nil, monitor)
ctx := t.Context()
docs := make([]any, 401)
for i := range docs {
docs[i] = bson.M{"_id": fmt.Sprintf("row-%03d", i), "value": i}
}
c := s.db.Collection("captureFixture")
if _, err := c.InsertMany(ctx, docs); err != nil {
t.Fatal(err)
}
lease, ok, err := s.acquireQueryModelLease(ctx)
if err != nil || !ok {
t.Fatal(ok, err)
}
at, err := s.queryBuildSnapshot(ctx)
if err != nil {
t.Fatal(err)
}
j := &queryBuildJob{Generation: "capture-unit", Snapshot: at}
start := time.Now()
checks := 0
round := &queryBuildRound{until: start.Add(time.Second), now: func() time.Time {
checks++
if checks > 1 {
return start.Add(2 * time.Second)
}
return start
}}
type row struct {
ID string `bson:"_id"`
Value int `bson:"value"`
}
var first []row
err = s.queryBuildCapture(ctx, lease, j, round, c, bson.M{}, bson.M{"value": 1}, &first)
if !errors.Is(err, ErrQueryBuildYielded) || len(first) != 200 || finds.Load() != 1 {
t.Fatal("first round did not preserve one bounded native page", len(first), finds.Load(), err)
}
// Later native changes cannot be mixed into the already selected snapshot.
if _, err = c.UpdateOne(ctx, bson.M{"_id": "row-000"}, bson.M{"$set": bson.M{"value": 9999}}); err != nil {
t.Fatal(err)
}
restarted := NewMongoStore(s.db)
restarted.queryReplicaFactStore = true
var got []row
if err = restarted.queryBuildCapture(ctx, lease, j, newQueryBuildRound(true), c, bson.M{}, bson.M{"value": 1}, &got); err != nil {
t.Fatal(err)
}
if len(got) != 401 || got[0].Value != 0 || got[400].Value != 400 || finds.Load() != 4 {
t.Fatalf("native prefix re-read or snapshot mixed: rows=%d finds=%d first=%+v", len(got), finds.Load(), got[0])
}
before := finds.Load()
var repeated []row
if err = restarted.queryBuildCapture(ctx, lease, j, newQueryBuildRound(true), c, bson.M{}, bson.M{"value": 1}, &repeated); err != nil {
t.Fatal(err)
}
if finds.Load() != before || !reflect.DeepEqual(got, repeated) {
t.Fatal("complete input capture repeated native reads")
}
}
func TestQueryBuildCaptureKeepsLegacyProjectionLocal(t *testing.T) {
_, s, _ := newLocalReplica(t)
ctx := t.Context()
c := s.db.Collection("captureProjectionFixture")
if _, err := c.InsertOne(ctx, bson.M{"_id": "private-native-id", "value": 1}); err != nil {
t.Fatal(err)
}
l, ok, err := s.acquireQueryModelLease(ctx)
if err != nil || !ok {
t.Fatal(err)
}
at, err := s.queryBuildSnapshot(ctx)
if err != nil {
t.Fatal(err)
}
j := &queryBuildJob{Generation: "projection", Snapshot: at}
var rows []bson.M
if err = s.queryBuildCapture(ctx, l, j, newQueryBuildRound(true), c, bson.M{}, bson.M{"_id": 0, "value": 1}, &rows); err != nil {
t.Fatal(err)
}
if len(rows) != 1 || rows[0]["_id"] != nil || rows[0]["value"] != int32(1) {
t.Fatal("native cursor identity changed the loader projection", rows)
}
}

View File

@ -0,0 +1,48 @@
package store
import (
"droboticflow/internal/queryserving"
"droboticflow/internal/workforcestats"
"encoding/json"
"go.mongodb.org/mongo-driver/bson"
"time"
)
func queryBuildFactDocument(org string, epoch int64, f workforcestats.Fact) (bson.M, string, int, error) {
raw, err := json.Marshal(f)
if err != nil {
return nil, "", 0, err
}
hash := queryserving.Key(workforceFactSchema, string(raw))
encoded, err := bson.Marshal(f)
if err != nil {
return nil, "", 0, err
}
var doc bson.M
if err = bson.Unmarshal(encoded, &doc); err != nil {
return nil, "", 0, err
}
at, err := time.Parse(time.RFC3339Nano, f.CreatedAt)
if err != nil {
return nil, "", 0, err
}
doc["_id"], doc["organizationId"], doc["fromEpoch"], doc["hash"], doc["at"] = queryserving.Key(workforceFactSchema, org, f.Key, epoch), org, epoch, hash, at
doc["atSeconds"], doc["atNanos"] = at.Unix(), at.Nanosecond()
doc["metric"] = workforcestats.MetricCounterValues(f.Metric)
doc["order"] = ""
if f.Detail != nil {
raw, e := json.Marshal(f.Detail)
if e != nil {
return nil, "", 0, e
}
var detail bson.M
if e = bson.UnmarshalExtJSON(raw, false, &detail); e != nil {
return nil, "", 0, e
}
doc["detail"] = detail
doc["return"] = bson.M{"id": f.Detail.ID, "stage": f.Detail.Stage, "taskIds": f.Detail.TaskIDs, "metrics": detail["metrics"]}
doc["order"] = f.Detail.ID + f.Detail.UserID + f.Detail.SupplierID
}
stored, err := bson.Marshal(doc)
return doc, hash, len(stored), err
}

View File

@ -0,0 +1,103 @@
package store
import (
"context"
"errors"
"time"
"droboticflow/internal/queryserving"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/bson/primitive"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
"go.mongodb.org/mongo-driver/mongo/readconcern"
"go.mongodb.org/mongo-driver/mongo/writeconcern"
)
const queryBuildJobsCollection = "queryBuildJobs"
const queryBuildPartsCollection = "queryBuildParts"
const queryBuildJobSchema = "query-build-job/v1"
type queryBuildJob struct {
ID string `bson:"_id"`
Schema string `bson:"schema"`
InputGeneration string `bson:"inputGeneration"`
Generation string `bson:"generation"`
Organization string `bson:"organizationId"`
Projects []string `bson:"projects"`
Cold bool `bson:"cold"`
Phase string `bson:"phase"`
Snapshot primitive.Timestamp `bson:"snapshot"`
AsOf time.Time `bson:"asOf"`
SourceRevision int64 `bson:"sourceRevision"`
GlobalRevision int64 `bson:"globalRevision"`
OldEpoch int64 `bson:"oldEpoch"`
Epoch int64 `bson:"epoch"`
Production QueryModelHead `bson:"production"`
PlanParts int `bson:"planParts"`
PlanRows int `bson:"planRows"`
PlanDigest string `bson:"planDigest"`
PlanSaved int `bson:"planSaved"`
PlanBytes int64 `bson:"planBytes"`
ProductionSaved int `bson:"productionSaved"`
NextPart int `bson:"nextPart"`
NextRow int `bson:"nextRow"`
Changed bool `bson:"changed"`
CleanupHasAfter bool `bson:"cleanupHasAfter"`
CleanupPlanPart int `bson:"cleanupPlanPart"`
CleanupPlanRow int `bson:"cleanupPlanRow"`
CleanupAfter string `bson:"cleanupAfter"`
CleanupFuture bool `bson:"cleanupFuture"`
ConfirmedRows int64 `bson:"confirmedRows"`
ConfirmedBytes int64 `bson:"confirmedBytes"`
ReusedRows int64 `bson:"reusedRows"`
ActiveWithoutProgress time.Duration `bson:"activeWithoutProgress"`
ProgressAt time.Time `bson:"progressAt"`
FailurePhase string `bson:"failurePhase,omitempty"`
FailureCode int32 `bson:"failureCode,omitempty"`
FailureClass string `bson:"failureClass,omitempty"`
UpdatedAt time.Time `bson:"updatedAt"`
}
func queryBuildJobKey(database, org string) string {
return queryserving.Key(queryBuildJobSchema, database, normalizedOrganizationID(org))
}
func (s *MongoStore) queryBuildJob(ctx context.Context, org string) (queryBuildJob, bool, error) {
op, stop := queryBuildOperation(ctx)
defer stop()
var j queryBuildJob
err := s.db.Collection(queryBuildJobsCollection).FindOne(op, bson.M{"_id": queryBuildJobKey(s.db.Name(), org)}).Decode(&j)
if errors.Is(err, mongo.ErrNoDocuments) {
return j, false, nil
}
if err == nil && (j.Schema != queryBuildJobSchema || j.Organization != normalizedOrganizationID(org)) {
return j, false, ErrQueryReplicaUnsupported
}
return j, err == nil, err
}
// Every checkpoint and the corresponding rows/parts commit together. The same
// global lease serializes publication and fences an old worker after takeover.
func (s *MongoStore) queryBuildTransaction(ctx context.Context, l queryModelLease, fn func(mongo.SessionContext) error) error {
op, stop := queryBuildOperation(ctx)
defer stop()
session, err := s.db.Client().StartSession()
if err != nil {
return err
}
defer session.EndSession(op)
_, err = session.WithTransaction(op, func(sc mongo.SessionContext) (any, error) {
if err := s.renewQueryModelLease(sc, l); err != nil {
return nil, err
}
return nil, fn(sc)
}, options.Transaction().SetReadConcern(readconcern.Snapshot()).SetWriteConcern(writeconcern.New(writeconcern.WMajority(), writeconcern.J(true))))
return err
}
func (s *MongoStore) saveQueryBuildJob(sc mongo.SessionContext, j queryBuildJob) error {
j.UpdatedAt = time.Now().UTC()
_, err := s.db.Collection(queryBuildJobsCollection).ReplaceOne(sc, bson.M{"_id": j.ID}, j, options.Replace().SetUpsert(true))
return err
}

View File

@ -0,0 +1,69 @@
package store
import (
"context"
"errors"
"fmt"
"time"
)
// Yield is successful bounded progress, not loss of the source safety permit.
// Callers retain pending work and do not stop projector/builder replicas.
var ErrQueryBuildYielded = errors.New("query build round yielded with durable progress retained")
var ErrQueryBuildStopped = errors.New("query build stopped; durable checkpoint retained")
var ErrQueryBuildNoProgress = errors.New("query build made no durable progress")
const (
queryColdRoundBudget = time.Minute
queryRefreshRoundBudget = 30 * time.Second
queryBuildOperationBudget = 15 * time.Second
queryBuildNoProgressLimit = 5 * time.Minute
)
type queryBuildRound struct {
until time.Time
now func() time.Time
}
func newQueryBuildRound(cold bool) *queryBuildRound {
budget := queryRefreshRoundBudget
if cold {
budget = queryColdRoundBudget
}
return &queryBuildRound{until: time.Now().Add(budget), now: time.Now}
}
func (r *queryBuildRound) check(ctx context.Context) error {
if err := ctx.Err(); err != nil {
return err
}
if r != nil && !r.now().Before(r.until) {
return ErrQueryBuildYielded
}
return nil
}
// Operation deadlines bound a single read/commit. They do not shorten the
// lifetime of the initial job or discard any previously confirmed checkpoint.
func queryBuildOperation(ctx context.Context) (context.Context, context.CancelFunc) {
return context.WithTimeout(ctx, queryBuildOperationBudget)
}
func queryBuildPhaseFailure(phase string, err error) error {
if err == nil || errors.Is(err, ErrQueryBuildYielded) {
return err
}
return fmt.Errorf("query build %s: %w", phase, err)
}
// Admission/checkpoint backpressure before a write is a resumable yield. A
// failed durable transaction is never converted into success or budget yield.
func (s *MongoStore) queryBuildPace(ctx context.Context, bytes int) error {
op, stop := queryBuildOperation(ctx)
defer stop()
err := s.paceQueryBuild(op, bytes)
if err != nil && ctx.Err() == nil && errors.Is(op.Err(), context.DeadlineExceeded) {
return ErrQueryBuildYielded
}
return err
}

View File

@ -0,0 +1,46 @@
package store
import (
"context"
"errors"
"testing"
"time"
)
func TestQueryBuildRoundBudgetIsYieldNotSafetyFailure(t *testing.T) {
now := time.Now()
r := &queryBuildRound{until: now, now: func() time.Time { return now }}
err := r.check(t.Context())
if !errors.Is(err, ErrQueryBuildYielded) || errors.Is(err, ErrQueryReplicaPaused) {
t.Fatal("round budget became a global safety pause", err)
}
ctx, stop := context.WithCancel(t.Context())
stop()
if !errors.Is(r.check(ctx), context.Canceled) {
t.Fatal("shutdown must preserve its cause")
}
}
func TestQueryBuildBudgetsAreIndependent(t *testing.T) {
if queryColdRoundBudget == queryRefreshRoundBudget {
t.Fatal("cold and freshness scheduling budgets coupled")
}
for _, cold := range []bool{false, true} {
r := newQueryBuildRound(cold)
if err := r.check(t.Context()); err != nil {
t.Fatal(err)
}
}
if queryBuildOperationBudget >= queryColdRoundBudget {
t.Fatal("one operation consumed the entire cold round")
}
}
func TestQueryBuildPacingPreservesCallerCancellation(t *testing.T) {
ctx, cancel := context.WithCancel(t.Context())
cancel()
s := &MongoStore{}
if err := s.queryBuildPace(ctx, 1024); !errors.Is(err, context.Canceled) || errors.Is(err, ErrQueryBuildYielded) {
t.Fatal(err)
}
}

View File

@ -0,0 +1,196 @@
package store
import (
"bytes"
"compress/gzip"
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"sort"
"time"
"droboticflow/internal/domain"
"droboticflow/internal/productionstats"
"droboticflow/internal/workforcestats"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
)
const queryBuildFactPageRows = 200
func queryBuildPlanPartKey(j queryBuildJob, kind string, n int) string {
return fmt.Sprintf("%s/%s/%09d", j.Generation, kind, n)
}
func queryBuildJSONPart(j queryBuildJob, kind string, n int, v any) (queryBuildPart, error) {
var b bytes.Buffer
z := gzip.NewWriter(&b)
if err := json.NewEncoder(z).Encode(v); err != nil {
return queryBuildPart{}, err
}
if err := z.Close(); err != nil {
return queryBuildPart{}, err
}
data := b.Bytes()
sum := sha256.Sum256(data)
return queryBuildPart{ID: queryBuildPlanPartKey(j, kind, n), Job: j.Generation, Data: data, Digest: hex.EncodeToString(sum[:])}, nil
}
func queryBuildJSONDecode(p queryBuildPart, out any) error {
sum := sha256.Sum256(p.Data)
if hex.EncodeToString(sum[:]) != p.Digest {
return errors.New("query plan digest mismatch")
}
z, err := gzip.NewReader(bytes.NewReader(p.Data))
if err != nil {
return err
}
defer z.Close()
raw, err := io.ReadAll(io.LimitReader(z, (64<<20)+1))
if err != nil {
return err
}
if len(raw) > 64<<20 {
return errors.New("query plan decode budget exceeded")
}
return json.Unmarshal(raw, out)
}
func (s *MongoStore) queryBuildSavePart(ctx context.Context, l queryModelLease, j *queryBuildJob, p queryBuildPart) error {
if err := s.queryBuildPace(ctx, len(p.Data)); err != nil {
return err
}
return s.queryBuildTransaction(ctx, l, func(sc mongo.SessionContext) error {
if _, err := s.db.Collection(queryBuildPartsCollection).ReplaceOne(sc, bson.M{"_id": p.ID}, p, options.Replace().SetUpsert(true)); err != nil {
return err
}
return s.saveQueryBuildJob(sc, *j)
})
}
func (s *MongoStore) queryBuildProductionPlan(ctx context.Context, l queryModelLease, j *queryBuildJob, r *queryBuildRound) error {
readCtx := context.WithValue(ctx, queryBuildReadKey{}, queryBuildReadFunc(func(c context.Context, coll mongoCollection, f, p bson.M, out any) error {
return s.queryBuildCapture(c, l, j, r, coll, f, p, out)
}))
snapshot, err := s.loadProductionStats(readCtx, j.Organization, j.Projects)
if err != nil {
return err
}
snapshot.AsOf = j.AsOf.Format(time.RFC3339Nano)
// Preserve the existing JSON/gzip manifest format for all old readers.
var data bytes.Buffer
z := gzip.NewWriter(&data)
budget := &queryModelBudgetWriter{writer: z}
if err = json.NewEncoder(budget).Encode(productionstats.Build(snapshot)); err != nil {
return err
}
if err = z.Close(); err != nil {
return err
}
if data.Len() > queryModelMaxBytes {
return errors.New("production plan exceeds existing size budget")
}
payload := data.Bytes()
sum := sha256.Sum256(payload)
head := QueryModelHead{Generation: j.Generation + "-production", AsOf: j.AsOf, Bytes: len(payload), ChunkBytes: queryModelBuildChunkBytes, Chunks: (len(payload) + queryModelBuildChunkBytes - 1) / queryModelBuildChunkBytes, Digest: hex.EncodeToString(sum[:]), SourceRevision: j.SourceRevision, GlobalRevision: j.GlobalRevision}
if j.Production.Generation != "" && j.Production.Digest != head.Digest {
return errors.New("sealed production input changed")
}
for n := j.ProductionSaved; n < head.Chunks; n++ {
if err = r.check(ctx); err != nil {
return err
}
p := payload[n*head.ChunkBytes : min((n+1)*head.ChunkBytes, len(payload))]
if err = s.queryBuildPace(ctx, len(p)); err != nil {
return err
}
next := *j
next.ProgressAt = time.Now().UTC()
next.ActiveWithoutProgress = 0
next.Production = head
next.ProductionSaved = n + 1
if err = s.queryBuildTransaction(ctx, l, func(sc mongo.SessionContext) error {
key := queryModelBlobKey(queryModelKey("production", j.Organization, j.Projects), head.Generation, n)
_, e := s.queryData().ReplaceOne(sc, bson.M{"_id": key}, bson.M{"_id": key, "generation": head.Generation, "data": p, "createdAt": time.Now().UTC()}, options.Replace().SetUpsert(true))
if e != nil {
return e
}
return s.saveQueryBuildJob(sc, next)
}); err != nil {
return err
}
*j = next
}
j.Phase = "workforce_plan"
return s.queryBuildTransaction(ctx, l, func(sc mongo.SessionContext) error { return s.saveQueryBuildJob(sc, *j) })
}
func (s *MongoStore) queryBuildWorkforcePlan(ctx context.Context, l queryModelLease, j *queryBuildJob, r *queryBuildRound) error {
readCtx := context.WithValue(ctx, queryBuildReadKey{}, queryBuildReadFunc(func(c context.Context, coll mongoCollection, f, p bson.M, out any) error {
return s.queryBuildCapture(c, l, j, r, coll, f, p, out)
}))
readCtx = context.WithValue(readCtx, workforceBuildBudgetKey{}, &workforceBuildBudget{})
input, err := s.LoadWorkforceStats(readCtx, j.Organization, workforcestats.Query{Mode: domain.RunModeAll})
if err != nil {
return err
}
facts := workforcestats.Materialize(input)
sort.Slice(facts, func(i, j int) bool { return facts[i].Key < facts[j].Key })
count := (len(facts) + queryBuildFactPageRows - 1) / queryBuildFactPageRows
if j.PlanRows != 0 && j.PlanRows != len(facts) {
return errors.New("workforce plan input changed")
}
// Recompute only from durable captured input, and verify the sealed prefix
// before adding any later page (including after a code deployment).
prefix := ""
for n := 0; n < j.PlanSaved; n++ {
part, e := queryBuildJSONPart(*j, "facts", n, facts[n*queryBuildFactPageRows:min((n+1)*queryBuildFactPageRows, len(facts))])
if e != nil {
return e
}
var saved queryBuildPart
op, stop := queryBuildOperation(ctx)
e = s.db.Collection(queryBuildPartsCollection).FindOne(op, bson.M{"_id": part.ID, "job": j.Generation}).Decode(&saved)
stop()
if e != nil {
return e
}
if saved.Digest != part.Digest {
return errors.New("sealed workforce plan fingerprint changed")
}
chain := sha256.Sum256([]byte(prefix + saved.Digest))
prefix = hex.EncodeToString(chain[:])
}
if prefix != j.PlanDigest {
return errors.New("sealed workforce plan manifest changed")
}
for n := j.PlanSaved; n < count; n++ {
if err = r.check(ctx); err != nil {
return err
}
part, e := queryBuildJSONPart(*j, "facts", n, facts[n*queryBuildFactPageRows:min((n+1)*queryBuildFactPageRows, len(facts))])
if e != nil {
return e
}
next := *j
next.ProgressAt = time.Now().UTC()
next.ActiveWithoutProgress = 0
next.PlanParts = count
next.PlanRows = len(facts)
next.PlanSaved = n + 1
next.PlanBytes += int64(len(part.Data))
chain := sha256.Sum256([]byte(next.PlanDigest + part.Digest))
next.PlanDigest = hex.EncodeToString(chain[:])
if err = s.queryBuildSavePart(ctx, l, &next, part); err != nil {
return err
}
*j = next
}
j.PlanRows = len(facts)
j.PlanParts = count
j.Phase = "apply"
return s.queryBuildTransaction(ctx, l, func(sc mongo.SessionContext) error { return s.saveQueryBuildJob(sc, *j) })
}

View File

@ -0,0 +1,360 @@
package store
import (
"context"
"errors"
"fmt"
"reflect"
"sync/atomic"
"testing"
"time"
"droboticflow/internal/domain"
"droboticflow/internal/workforcestats"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/event"
"go.mongodb.org/mongo-driver/mongo"
)
func TestQueryBuildLifecycleRestartYieldAndTwoRefreshesLocal(t *testing.T) {
var native, sourceReads, weakCommits atomic.Int64
monitor := &event.CommandMonitor{Started: func(_ context.Context, e *event.CommandStartedEvent) {
if e.CommandName == "find" {
name, _ := e.Command.Lookup("find").StringValueOK()
if querySourceCollection(name) {
native.Add(1)
}
}
if e.CommandName == "commitTransaction" {
var d struct {
WC struct {
W any `bson:"w"`
J bool `bson:"j"`
} `bson:"writeConcern"`
}
_ = bson.Unmarshal(e.Command, &d)
if d.WC.W != "majority" || !d.WC.J {
weakCommits.Add(1)
}
}
}}
_, s, _ := newLocalReplica(t, &event.CommandMonitor{Started: func(context.Context, *event.CommandStartedEvent) { sourceReads.Add(1) }}, monitor)
sourceReads.Store(0)
ctx := t.Context()
org := "workforce-test"
seedWorkforceStats(t, s)
events := []any{}
seconds := 1.0
for i := 0; i < 250; i++ {
events = append(events, domain.AnnotationWorklogEventRecord{ID: fmt.Sprintf("resume-event-%04d", i), OrganizationID: org, TaskID: fmt.Sprintf("resume-task-%04d", i), ProjectID: "p", SupplierID: "s", UserID: "u", EventType: "annotation_submitted", Mode: domain.RunModeProduction, Revision: 1, VideoSeconds: &seconds, OutputDelta: 1, CreatedAt: time.Date(2026, 9, 28, 13, 0, 0, 0, time.UTC).Add(time.Duration(i) * time.Second).Format(time.RFC3339Nano)})
}
tasks := []any{}
for i := 0; i < 250; i++ {
id := fmt.Sprintf("resume-task-%04d", i)
tasks = append(tasks, bson.M{"_id": id, "id": id, "organizationId": org, "projectId": "p", "supplierId": "s", "runId": "run"})
}
if _, e := s.annotationTasks.InsertMany(ctx, tasks); e != nil {
t.Fatal(e)
}
if _, e := s.annotationWorklogEvents.InsertMany(ctx, events); e != nil {
t.Fatal(e)
}
s.queryReplicaFactStore = true
s.queryPeriodic = true
s.queryBuildWrites = nil
if e := s.ensureWorkforceFactIndexes(ctx); e != nil {
t.Fatal(e)
}
l, ok, e := s.acquireQueryModelLease(ctx)
if e != nil || !ok {
t.Fatal(ok, e)
}
// Establish exactly the e4 confirmed, unpublished first epoch.
if e = s.rebuildWorkforceFacts(ctx, l, org, 0); e != nil {
t.Fatal(e)
}
unpublishWorkforceFixture(t, s, org)
weakCommits.Store(0)
j, e := s.queryBuildStart(ctx, l, org, []string{"p"})
if e != nil {
t.Fatal(e)
}
if e = s.queryBuildProductionPlan(ctx, l, &j, newQueryBuildRound(true)); e != nil {
t.Fatal(e)
}
if e = s.queryBuildWorkforcePlan(ctx, l, &j, newQueryBuildRound(true)); e != nil {
t.Fatal(e)
}
beforeNative := native.Load()
// Allow one committed fact batch, then exhaust this round, not the job.
calls := 0
start := time.Now()
r := &queryBuildRound{until: start.Add(time.Second), now: func() time.Time {
calls++
if calls > 2 {
return start.Add(2 * time.Second)
}
return start
}}
if e = s.queryBuildApply(ctx, l, &j, r); !errors.Is(e, ErrQueryBuildYielded) {
t.Fatal("expected successful yield", e)
}
saved, ok, e := s.queryBuildJob(ctx, org)
if e != nil || !ok || saved.NextRow+saved.NextPart == 0 {
t.Fatal("missing confirmed cursor", saved.NextRow, saved.NextPart, e)
}
if _, ok := s.readQueryModelHead(ctx, queryModelKey("production", org, []string{"p"}), time.Hour); ok {
t.Fatal("partial generation published")
}
// Reacquire a new lease and use a fresh process object; old writer is fenced.
if _, e = s.queryControl().UpdateOne(ctx, queryLeaseFilter(l), bson.M{"$set": bson.M{"until": time.Now().Add(-time.Second)}}); e != nil {
t.Fatal(e)
}
restarted := NewMongoStore(s.db)
restarted.queryReplicaFactStore = true
restarted.queryPeriodic = true
l2, ok, e := restarted.acquireQueryModelLease(ctx)
if e != nil || !ok {
t.Fatal(e)
}
if e = s.queryBuildTransaction(ctx, l, func(_ctx mongo.SessionContext) error { return nil }); e == nil {
t.Fatal("old lease committed")
}
if e = restarted.queryBuildRunRound(ctx, l2, org, []string{"p"}, nil); e != nil {
t.Fatal(e)
}
if native.Load() != beforeNative {
t.Fatal("resume repeated raw input reads", beforeNative, native.Load())
}
completed, _, e := restarted.queryBuildJob(ctx, org)
if e != nil || completed.Phase != "complete" || completed.ReusedRows == 0 || completed.ConfirmedRows != 0 {
t.Fatal("confirmed e4 batches not reused", completed.ReusedRows, completed.ConfirmedRows, e)
}
first, _, e := restarted.workforceHead(ctx, org)
if e != nil {
t.Fatal(e, "head", first, "now", time.Now().UTC())
}
original, e := restarted.LoadWorkforceStats(ctx, org, workforcestats.Query{Mode: domain.RunModeAll})
if e != nil {
t.Fatal(e)
}
expected := workforcestats.Build(original, workforcestats.Query{Mode: domain.RunModeAll})
actual, used, e := restarted.computeWorkforceFactReport(ctx, org, workforcestats.Query{Mode: domain.RunModeAll}, first)
if e != nil || !used || !reflect.DeepEqual(actual.Totals, expected.Totals) {
t.Fatal("fact totals differ", e)
}
// A fresh caught-up observer proves unchanged native input; both daily
// refreshes reuse that complete input instead of re-reading full histories.
for round := 0; round < 2; round++ {
if _, e = s.queryControl().UpdateOne(ctx, queryLeaseFilter(l2), bson.M{"$set": bson.M{"observedAt": time.Now().UTC()}}); e != nil {
t.Fatal(e)
}
before := native.Load()
if e = restarted.queryBuildRunRound(ctx, l2, org, []string{"p"}, nil); e != nil {
t.Fatal(e)
}
if native.Load() != before {
t.Fatal("unchanged refresh reread native histories", round, native.Load()-before)
}
h, ok, e := restarted.workforceHead(ctx, org)
if e != nil || !ok || h.FactEpoch != first.FactEpoch {
t.Fatal("unchanged facts rewritten", h.FactEpoch, e)
}
}
if weakCommits.Load() != 0 {
t.Fatal("majority+journal weakened", weakCommits.Load())
}
if sourceReads.Load() != 0 {
t.Fatal("build read original Mongo", sourceReads.Load())
}
if !IsPrivateDerivedCollection(queryBuildJobsCollection) || !IsPrivateDerivedCollection(queryBuildPartsCollection) {
t.Fatal("internal build documents exposed")
}
}
func TestQueryBuildChangedFingerprintDeletionAndOldGenerationLocal(t *testing.T) {
_, s, _ := newLocalReplica(t)
ctx := t.Context()
org := "workforce-test"
seedWorkforceStats(t, s)
s.queryReplicaFactStore = true
s.queryPeriodic = true
if e := s.ensureWorkforceFactIndexes(ctx); e != nil {
t.Fatal(e)
}
l, ok, e := s.acquireQueryModelLease(ctx)
if e != nil || !ok {
t.Fatal(e)
}
if e = s.queryBuildRunRound(ctx, l, org, []string{"p"}, nil); e != nil {
t.Fatal(e)
}
old, _, e := s.workforceHead(ctx, org)
if e != nil {
t.Fatal(e)
}
// Changed input and a deleted user are handled by the original loader,
// including its historical actor fallback. Neither mutates source records.
if _, e = s.annotationWorklogEvents.UpdateMany(ctx, bson.M{"organizationId": org}, bson.M{"$set": bson.M{"videoSeconds": 999.0}}); e != nil {
t.Fatal(e)
}
if _, e = s.users.DeleteMany(ctx, bson.M{"organizationId": org}); e != nil {
t.Fatal(e)
}
if _, e = s.queryControl().UpdateOne(ctx, queryLeaseFilter(l), bson.M{"$inc": bson.M{"productionRevision": 1}}); e != nil {
t.Fatal(e)
}
j, e := s.queryBuildStart(ctx, l, org, []string{"p"})
if e != nil {
t.Fatal(e)
}
if e = s.queryBuildProductionPlan(ctx, l, &j, newQueryBuildRound(false)); e != nil {
t.Fatal(e)
}
if e = s.queryBuildWorkforcePlan(ctx, l, &j, newQueryBuildRound(false)); e != nil {
t.Fatal(e)
}
// A deliberately wrong confirmed future fingerprint must be replaced, while
// an orphan from an old attempt must be removed only from the unpublished view.
page, e := s.queryBuildPlanPage(ctx, j, 0)
if e != nil {
t.Fatal(e)
}
doc, _, _, e := queryBuildFactDocument(org, j.Epoch, page[0])
if e != nil {
t.Fatal(e)
}
doc["hash"] = "outdated"
if _, e = s.db.Collection(workforceFactCollection).InsertOne(ctx, doc); e != nil {
t.Fatal(e)
}
if _, e = s.db.Collection(workforceFactCollection).InsertOne(ctx, bson.M{"_id": "orphan-future", "organizationId": org, "key": "zz-orphan", "hash": "obsolete", "fromEpoch": j.Epoch}); e != nil {
t.Fatal(e)
}
if e = s.queryBuildApply(ctx, l, &j, newQueryBuildRound(false)); e != nil {
t.Fatal(e)
}
// Partial writes have not changed either published manifest.
h, _, e := s.workforceHead(ctx, org)
if e != nil || h.FactEpoch != old.FactEpoch {
t.Fatal("partial generation visible", e)
}
cancelCtx, stop := context.WithCancel(ctx)
stop()
if e = s.queryBuildRunRound(cancelCtx, l, org, []string{"p"}, nil); !errors.Is(e, context.Canceled) {
t.Fatal(e)
}
if e = s.queryBuildRunRound(ctx, l, org, []string{"p"}, nil); e != nil {
t.Fatal(e)
}
h, _, e = s.workforceHead(ctx, org)
if e != nil || h.FactEpoch != old.FactEpoch+1 {
t.Fatal(e)
}
filters := []workforcestats.Query{
{Mode: domain.RunModeAll}, {Mode: domain.RunModeProduction},
{Mode: domain.RunModeAll, UserID: "u", MatchUser: true},
{Mode: domain.RunModeAll, AllowedProjects: map[string]bool{}},
{Mode: domain.RunModeAll, AllowedSuppliers: map[string]bool{}},
{Mode: domain.RunModeAll, SupplierID: "s"},
{Mode: domain.RunModeAll, From: time.Date(2026, 9, 28, 0, 0, 0, 1, time.UTC), To: time.Date(2026, 9, 29, 0, 0, 0, 0, time.UTC)},
}
for _, q := range filters {
input, e := s.LoadWorkforceStats(ctx, org, q)
if e != nil {
t.Fatal(e)
}
want := workforcestats.Build(input, q)
got, _, e := s.computeWorkforceFactReport(ctx, org, q, h)
if e != nil || !reflect.DeepEqual(got.Totals, want.Totals) {
t.Fatal("permissions/date/legacy parity", q, e, got.Totals, want.Totals)
}
}
n, e := s.db.Collection(workforceFactCollection).CountDocuments(ctx, bson.M{"_id": "orphan-future"})
if e != nil || n != 0 {
t.Fatal("unmatched staged row survived", n, e)
}
}
func TestQueryBuildNoProgressStopsButPauseTimeDoesNotLocal(t *testing.T) {
_, s, _ := newLocalReplica(t)
ctx := t.Context()
s.queryReplicaFactStore = true
s.queryPeriodic = true
l, ok, e := s.acquireQueryModelLease(ctx)
if e != nil || !ok {
t.Fatal(e)
}
j, e := s.queryBuildStart(ctx, l, "synthetic-no-progress", nil)
if e != nil {
t.Fatal(e)
}
// An overnight inactive gap is not a five minute active stall.
j.ProgressAt = time.Now().Add(-24 * time.Hour)
if e = s.queryBuildTransaction(ctx, l, func(sc mongo.SessionContext) error { return s.saveQueryBuildJob(sc, j) }); e != nil {
t.Fatal(e)
}
now := time.Now()
expired := func() *queryBuildRound { return &queryBuildRound{until: now, now: func() time.Time { return now }} }
if e = s.queryBuildRunRound(ctx, l, j.Organization, nil, expired()); !errors.Is(e, ErrQueryBuildYielded) || errors.Is(e, ErrQueryBuildStopped) {
t.Fatal("pause treated as stall", e)
}
j, _, e = s.queryBuildJob(ctx, j.Organization)
if e != nil {
t.Fatal(e)
}
j.ActiveWithoutProgress = queryBuildNoProgressLimit
if e = s.queryBuildTransaction(ctx, l, func(sc mongo.SessionContext) error { return s.saveQueryBuildJob(sc, j) }); e != nil {
t.Fatal(e)
}
if e = s.queryBuildRunRound(ctx, l, j.Organization, nil, expired()); !errors.Is(e, ErrQueryBuildStopped) || !errors.Is(e, ErrQueryBuildNoProgress) {
t.Fatal("real active stall ignored", e)
}
if e = s.queryBuildRunRound(ctx, l, j.Organization, nil, nil); !errors.Is(e, ErrQueryBuildStopped) {
t.Fatal("terminal failure automatically resumed", e)
}
}
func TestQueryBuildSealedInputSurvivesExpiredSnapshotAndOrphanGCLocal(t *testing.T) {
_, s, _ := newLocalReplica(t)
ctx := t.Context()
org := "workforce-test"
seedWorkforceStats(t, s)
s.queryReplicaFactStore = true
s.queryPeriodic = true
if e := s.ensureWorkforceFactIndexes(ctx); e != nil {
t.Fatal(e)
}
l, ok, e := s.acquireQueryModelLease(ctx)
if e != nil || !ok {
t.Fatal(e)
}
j, e := s.queryBuildStart(ctx, l, org, []string{"p"})
if e != nil {
t.Fatal(e)
}
if e = s.queryBuildProductionPlan(ctx, l, &j, newQueryBuildRound(true)); e != nil {
t.Fatal(e)
}
if e = s.queryBuildWorkforcePlan(ctx, l, &j, newQueryBuildRound(true)); e != nil {
t.Fatal(e)
}
// Once sealed, no original native snapshot is needed for continuation.
j.Snapshot.T = 1
if e = s.queryBuildTransaction(ctx, l, func(sc mongo.SessionContext) error { return s.saveQueryBuildJob(sc, j) }); e != nil {
t.Fatal(e)
}
if _, e = s.queryData().UpdateMany(ctx, bson.M{"generation": j.Production.Generation}, bson.M{"$set": bson.M{"createdAt": time.Now().Add(-24 * time.Hour)}}); e != nil {
t.Fatal(e)
}
if e = s.cleanQueryModelOrphans(ctx); e != nil {
t.Fatal(e)
}
if e = s.queryBuildRunRound(ctx, l, org, []string{"p"}, nil); e != nil {
t.Fatal("sealed work lost after snapshot expiry", e)
}
n, e := s.queryData().CountDocuments(ctx, bson.M{"generation": j.Production.Generation})
if e != nil || n != int64(j.Production.Chunks) {
t.Fatal("GC deleted unfinished durable blob", n, e)
}
}

View File

@ -0,0 +1,197 @@
package store
import (
"context"
"errors"
"fmt"
"reflect"
"sort"
"time"
"droboticflow/internal/queryserving"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
)
// This lifecycle is exclusive to the independent, periodic read database.
// Authority/legacy callers keep their existing loader and publication behavior.
func (s *MongoStore) queryBuildStart(ctx context.Context, l queryModelLease, org string, projects []string) (queryBuildJob, error) {
org = normalizedOrganizationID(org)
projects = append([]string{}, projects...)
sort.Strings(projects)
old, exists, err := s.queryBuildJob(ctx, org)
if err != nil {
return old, err
}
if exists && old.Phase != "complete" {
if old.FailureClass != "" {
return old, fmt.Errorf("%w: phase=%s category=%s", ErrQueryBuildStopped, old.FailurePhase, old.FailureClass)
}
if !reflect.DeepEqual(old.Projects, projects) {
return old, errors.New("query build scope changed; retained job requires reconciliation")
}
return old, nil
}
observedAt := time.Now().UTC()
at, err := s.queryBuildSnapshot(ctx)
if err != nil {
return old, err
}
var state struct {
Revision int64 `bson:"productionRevision"`
Global int64 `bson:"searchGlobalRevision"`
Observed time.Time `bson:"observedAt"`
Heads map[string]QueryModelHead `bson:"heads"`
}
rows, e := s.queryBuildReadPage(ctx, s.queryControl(), bson.M{"_id": "global"}, bson.M{"productionRevision": 1, "searchGlobalRevision": 1, "heads": 1, "observedAt": 1}, bson.RawValue{}, false, at)
if e != nil {
return old, e
}
if len(rows) != 1 {
return old, errors.New("query build control unavailable at snapshot")
}
if e = bson.Unmarshal(rows[0], &state); e != nil {
return old, e
}
head := state.Heads[workforceFactKey(org)]
j := queryBuildJob{ID: queryBuildJobKey(s.db.Name(), org), Schema: queryBuildJobSchema, Organization: org, Projects: projects, Generation: queryserving.Key(queryBuildJobSchema, org, at.T, at.I, time.Now().UnixNano()), Snapshot: at, AsOf: observedAt, Phase: "production_plan", OldEpoch: head.FactEpoch, Epoch: head.FactEpoch + 1, Cold: head.FactEpoch == 0, SourceRevision: state.Revision, GlobalRevision: state.Global, ProgressAt: time.Now().UTC()}
j.InputGeneration = j.Generation
if exists && old.Phase == "complete" && reflect.DeepEqual(old.Projects, projects) && old.SourceRevision == state.Revision && old.GlobalRevision == state.Global && !state.Observed.Before(old.AsOf) && time.Since(state.Observed) >= 0 && time.Since(state.Observed) < 15*time.Second {
j.AsOf = state.Observed
j.InputGeneration = old.InputGeneration
if j.InputGeneration == "" {
j.InputGeneration = old.Generation
}
// The caught-up stream proves that every business input is unchanged.
// Reconfirm the complete cohort without re-materializing or walking
// 140k unchanged facts and checkpointing thousands of no-op batches.
production, exists := state.Heads[queryModelKey("production", org, projects)]
committed := old.Epoch
if !old.Changed && old.OldEpoch > 0 {
committed = old.OldEpoch
}
if exists && production.Generation != "" && head.Rows == old.PlanRows && head.FactEpoch == committed && production.SourceRevision == old.SourceRevision {
j.Phase = "publish"
j.PlanRows = old.PlanRows
j.ReusedRows = int64(old.PlanRows)
j.Production = production
j.Production.AsOf = j.AsOf
}
}
err = s.queryBuildTransaction(ctx, l, func(sc mongo.SessionContext) error { return s.saveQueryBuildJob(sc, j) })
return j, err
}
func (s *MongoStore) queryBuildRunRound(ctx context.Context, l queryModelLease, org string, projects []string, r *queryBuildRound) (err error) {
j, err := s.queryBuildStart(ctx, l, org, projects)
if err != nil {
return err
}
if r == nil {
r = newQueryBuildRound(j.Cold)
}
started := time.Now()
before := j.ProgressAt
defer func() {
if err == nil || ctx.Err() != nil {
return
}
// Maintenance/shutdown time is not counted as active lack of progress.
current, ok, e := s.queryBuildJob(ctx, org)
if e != nil || !ok {
return
}
if !current.ProgressAt.After(before) {
current.ActiveWithoutProgress += time.Since(started)
} else {
current.ActiveWithoutProgress = 0
}
if errors.Is(err, ErrQueryBuildYielded) && current.ActiveWithoutProgress >= queryBuildNoProgressLimit {
err = ErrQueryBuildNoProgress
}
if !errors.Is(err, ErrQueryBuildYielded) {
current.FailurePhase = current.Phase
diagnostic := QueryReplicaDiagnostic(err)
current.FailureClass = diagnostic.Class
current.FailureCode = diagnostic.MongoCode
if errors.Is(err, ErrQueryBuildNoProgress) {
current.FailureClass = "no_durable_progress"
}
}
if !errors.Is(err, ErrQueryBuildYielded) {
err = fmt.Errorf("%w: %w", ErrQueryBuildStopped, err)
}
if e = s.queryBuildTransaction(ctx, l, func(sc mongo.SessionContext) error { return s.saveQueryBuildJob(sc, current) }); e != nil {
err = fmt.Errorf("query build could not persist outcome: %w", e)
}
}()
for {
if err = r.check(ctx); err != nil {
return err
}
switch j.Phase {
case "production_plan":
err = s.queryBuildProductionPlan(ctx, l, &j, r)
case "workforce_plan":
err = s.queryBuildWorkforcePlan(ctx, l, &j, r)
case "apply":
err = s.queryBuildApply(ctx, l, &j, r)
case "cleanup":
err = s.queryBuildCleanup(ctx, l, &j, r)
case "publish":
err = s.queryBuildPublish(ctx, l, &j)
case "complete":
return nil
default:
return errors.New("unsupported query build phase")
}
if err != nil {
return queryBuildPhaseFailure(j.Phase, err)
}
}
}
// Explicitly bounded derived-data housekeeping; active jobs are never swept.
func (s *MongoStore) cleanQueryBuildParts(ctx context.Context, l queryModelLease) error {
var jobs []queryBuildJob
cur, err := s.db.Collection(queryBuildJobsCollection).Find(ctx, bson.M{"schema": queryBuildJobSchema}, options.Find().SetProjection(bson.M{"generation": 1, "inputGeneration": 1}))
if err != nil {
return err
}
err = cur.All(ctx, &jobs)
cur.Close(ctx)
if err != nil {
return err
}
keep := []string{}
for _, j := range jobs {
keep = append(keep, j.Generation, j.InputGeneration)
}
for _, collection := range []string{queryBuildPartsCollection, queryBuildJobsCollection} {
f := bson.M{"job": bson.M{"$exists": true, "$nin": keep}}
cur, e := s.db.Collection(collection).Find(ctx, f, options.Find().SetProjection(bson.M{"_id": 1}).SetLimit(1000))
if e != nil {
return e
}
var rows []bson.M
e = cur.All(ctx, &rows)
cur.Close(ctx)
if e != nil {
return e
}
ids := bson.A{}
for _, row := range rows {
ids = append(ids, row["_id"])
}
if len(ids) > 0 {
if e = s.queryBuildTransaction(ctx, l, func(sc mongo.SessionContext) error {
_, e := s.db.Collection(collection).DeleteMany(sc, bson.M{"_id": bson.M{"$in": ids}, "job": bson.M{"$nin": keep}})
return e
}); e != nil {
return e
}
}
}
return nil
}

View File

@ -0,0 +1,226 @@
package store
import (
"context"
"errors"
"fmt"
"os"
"strings"
"sync/atomic"
"testing"
"time"
"droboticflow/internal/domain"
"droboticflow/internal/workforcestats"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/event"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
"golang.org/x/time/rate"
)
func TestQueryBuildDurableColdSameScaleLocal(t *testing.T) {
if os.Getenv("QUERY_BUILD_LIFECYCLE_SCALE") != "1" {
t.Skip("explicit 140k fact synthetic comparison")
}
var originalReads, nativeReads atomic.Int64
_, s, _ := newLocalReplica(t, &event.CommandMonitor{Started: func(context.Context, *event.CommandStartedEvent) { originalReads.Add(1) }}, &event.CommandMonitor{Started: func(_ context.Context, e *event.CommandStartedEvent) {
if e.CommandName == "find" {
n, _ := e.Command.Lookup("find").StringValueOK()
if querySourceCollection(n) {
nativeReads.Add(1)
}
}
}})
originalReads.Store(0)
ctx := t.Context()
org := "synthetic-durable-scale"
const count = 35000
existing := 125186
if os.Getenv("QUERY_BUILD_SCALE_EMPTY") == "1" {
existing = 0
}
project := strings.Repeat("synthetic-project-", 4)
at := time.Date(2026, 11, 1, 0, 0, 0, 0, time.UTC)
seconds := 1.0
for start := 0; start < count; start += 1000 {
events, tasks := []any{}, []any{}
for i := start; i < min(start+1000, count); i++ {
id := fmt.Sprintf("task-%06d", i)
events = append(events, domain.AnnotationWorklogEventRecord{ID: fmt.Sprintf("event-%06d", i), OrganizationID: org, ProjectID: project, TaskID: id, UserID: "actor", SupplierID: "supplier", EventType: "annotation_submitted", Mode: domain.RunModeProduction, Revision: 1, TemplateVersion: 1, VideoSeconds: &seconds, OutputDelta: 1, CreatedAt: at.Add(time.Duration(i) * time.Second).Format(time.RFC3339Nano)})
tasks = append(tasks, bson.M{"_id": id, "id": id, "organizationId": org, "projectId": project, "supplierId": "supplier", "runId": "run"})
}
if _, e := s.annotationWorklogEvents.InsertMany(ctx, events); e != nil {
t.Fatal(e)
}
if _, e := s.annotationTasks.InsertMany(ctx, tasks); e != nil {
t.Fatal(e)
}
}
for name, doc := range map[string]any{"runs": bson.M{"_id": "run", "id": "run", "organizationId": org, "mode": "production"}, "users": domain.UserRecord{ID: "actor", OrganizationID: org, Name: strings.Repeat("合成用户", 16), SupplierID: "supplier", Role: domain.UserRoleAnnotator}, "suppliers": domain.SupplierRecord{ID: "supplier", OrganizationID: org, Name: "synthetic supplier", Status: "active"}} {
if _, e := s.db.Collection(name).InsertOne(ctx, doc); e != nil {
t.Fatal(e)
}
}
s.queryReplicaFactStore = true
s.queryPeriodic = true
if e := s.ensureWorkforceFactIndexes(ctx); e != nil {
t.Fatal(e)
}
l, ok, e := s.acquireQueryModelLease(ctx)
if e != nil || !ok {
t.Fatal(e)
}
// Generate the exact e4 staging rows; fixture setup alone is unpaced.
if existing > 0 {
if e = s.rebuildWorkforceFacts(ctx, l, org, 0); e != nil {
t.Fatal(e)
}
unpublishWorkforceFixture(t, s, org)
c := s.db.Collection(workforceFactCollection)
cur, e := c.Find(ctx, bson.M{"organizationId": org}, options.Find().SetProjection(bson.M{"_id": 1}).SetLimit(int64(count*4-existing)))
if e != nil {
t.Fatal(e)
}
var ids []bson.M
if e = cur.All(ctx, &ids); e != nil {
t.Fatal(e)
}
cur.Close(ctx)
for start := 0; start < len(ids); start += 1000 {
values := bson.A{}
for _, id := range ids[start:min(start+1000, len(ids))] {
values = append(values, id["_id"])
}
if _, e = c.DeleteMany(ctx, bson.M{"_id": bson.M{"$in": values}}); e != nil {
t.Fatal(e)
}
}
}
s.queryBuildWrites = s.newQueryBuildWriteGate()
if s.queryBuildWrites.limiter.Limit() != rate.Limit(512<<10) {
t.Fatal("pacing changed")
}
started := time.Now()
rounds := 0
// Keep renewing the same fenced lease, just as the production process does.
runCtx, cancel := context.WithCancel(ctx)
defer cancel()
done := make(chan struct{})
leaseStore := s
go func() {
defer close(done)
ticker := time.NewTicker(10 * time.Second)
defer ticker.Stop()
for {
select {
case <-runCtx.Done():
return
case <-ticker.C:
if leaseStore.renewQueryModelLease(runCtx, l) != nil {
return
}
}
}
}()
defer func() { cancel(); <-done }()
for {
rounds++
e = s.queryBuildRunRound(ctx, l, org, []string{project}, nil)
if e == nil {
break
}
if !errors.Is(e, ErrQueryBuildYielded) {
t.Fatal("cold round failed", rounds, e)
}
state, _, _ := s.queryBuildJob(ctx, org)
t.Logf("yield round=%d phase=%s confirmed=%d reused=%d nextPart=%d", rounds, state.Phase, state.ConfirmedRows, state.ReusedRows, state.NextPart)
if rounds > 30 {
t.Fatal("synthetic cold build did not progress")
}
fresh := NewMongoStore(s.db)
fresh.queryReplicaFactStore = true
fresh.queryPeriodic = true
fresh.queryBuildWrites = s.queryBuildWrites
s = fresh
}
cold := time.Since(started)
j, ok, e := s.queryBuildJob(ctx, org)
if e != nil || !ok || j.PlanRows != count*4 || j.ReusedRows != int64(existing) || j.ConfirmedRows != int64(count*4-existing) {
t.Fatal("same scale resume parity", j.PlanRows, j.ReusedRows, j.ConfirmedRows, e)
}
h, _, e := s.workforceHead(ctx, org)
if e != nil {
t.Fatal(e)
}
q := workforcestats.Query{Mode: domain.RunModeAll, From: at, To: at.Add(24 * time.Hour)}
report, _, e := s.computeWorkforceFactReport(ctx, org, q, h)
if e != nil || report.Totals.FirstSubmittedTasks != count || report.Totals.FirstVideoSeconds != count {
t.Fatal("same scale report differs", e)
}
baseline := time.Duration(0)
if os.Getenv("QUERY_BUILD_SCALE_BASELINE") == "1" {
if _, e = s.queryControl().UpdateOne(ctx, queryLeaseFilter(l), bson.M{"$set": bson.M{"observedAt": time.Now().UTC()}}); e != nil {
t.Fatal(e)
}
// Explicit algorithm comparison on the identical fixture: retain the
// pre-fast-path walk of unchanged facts, including durable checkpoints.
job, err := s.queryBuildStart(ctx, l, org, []string{project})
if err != nil {
t.Fatal(err)
}
job.Phase = "production_plan"
job.PlanRows = 0
job.ReusedRows = 0
job.Production = QueryModelHead{}
if err = s.queryBuildTransaction(ctx, l, func(sc mongo.SessionContext) error { return s.saveQueryBuildJob(sc, job) }); err != nil {
t.Fatal(err)
}
start := time.Now()
for {
err = s.queryBuildRunRound(ctx, l, org, []string{project}, nil)
if err == nil {
break
}
if !errors.Is(err, ErrQueryBuildYielded) {
t.Fatal(err)
}
}
baseline = time.Since(start)
}
warm := []time.Duration{}
for i := 0; i < 2; i++ {
if _, e = s.queryControl().UpdateOne(ctx, queryLeaseFilter(l), bson.M{"$set": bson.M{"observedAt": time.Now().UTC()}}); e != nil {
t.Fatal(e)
}
before := nativeReads.Load()
start := time.Now()
n := 0
for {
n++
e = s.queryBuildRunRound(ctx, l, org, []string{project}, nil)
if e == nil {
break
}
if !errors.Is(e, ErrQueryBuildYielded) || n > 30 {
t.Fatal("refresh failed", i, n, e)
}
}
if nativeReads.Load() != before {
t.Fatal("unchanged refresh rescanned native input", i, nativeReads.Load()-before)
}
warm = append(warm, time.Since(start))
}
if originalReads.Load() != 0 {
t.Fatal("original Mongo reads", originalReads.Load())
}
var status struct {
DataSize float64 `bson:"dataSize"`
StorageSize float64 `bson:"storageSize"`
IndexSize float64 `bson:"indexSize"`
}
if e = s.db.RunCommand(ctx, bson.M{"dbStats": 1}).Decode(&status); e != nil {
t.Fatal(e)
}
t.Logf("events=%d facts=%d reused=%d new=%d cold=%s rounds=%d walk_baseline=%s refresh1=%s refresh2=%s write_limit=%d confirmed_bson_bytes=%d original_mongo_reads=%d native_input_reads=%d data_bytes=%.0f storage_bytes=%.0f index_bytes=%.0f production_accepted=false", count, j.PlanRows, j.ReusedRows, j.ConfirmedRows, cold, rounds, baseline, warm[0], warm[1], 512<<10, j.ConfirmedBytes, originalReads.Load(), nativeReads.Load(), status.DataSize, status.StorageSize, status.IndexSize)
}

View File

@ -41,6 +41,23 @@ func (s *MongoStore) cleanQueryModelOrphans(ctx context.Context) error {
for _, h := range state.Heads {
keep = append(keep, h.Generation)
}
if s.queryReplicaFactStore && s.periodicQueryStore() {
cur, err := s.db.Collection(queryBuildJobsCollection).Find(ctx, bson.M{"schema": queryBuildJobSchema, "phase": bson.M{"$ne": "complete"}}, options.Find().SetProjection(bson.M{"production.generation": 1}))
if err != nil {
return err
}
var jobs []queryBuildJob
err = cur.All(ctx, &jobs)
cur.Close(ctx)
if err != nil {
return err
}
for _, j := range jobs {
if j.Production.Generation != "" {
keep = append(keep, j.Production.Generation)
}
}
}
filter := bson.M{"generation": bson.M{"$nin": keep}, "createdAt": bson.M{"$lt": time.Now().UTC().Add(-15 * time.Minute)}}
// Ordinary orphan chunks/rows must still be reclaimed when Typesense is
// offline. Retain only its collection intents until remote deletion succeeds.

View File

@ -118,6 +118,13 @@ func (b *QueryReadBuilder) scopes(ctx context.Context) (map[string][]string, err
return out, nil
}
func (b *QueryReadBuilder) rebuild(ctx context.Context, l queryModelLease, org string, projects []string) (err error) {
if b.store.queryReplicaFactStore && b.store.periodicQueryStore() {
err = b.store.queryBuildRunRound(ctx, l, org, projects, nil)
if errors.Is(err, ErrQueryBuildYielded) {
log.Print("query_model_build result=yielded checkpoint=retained")
}
return err
}
started := time.Now()
stage := "source"
defer func() {
@ -178,7 +185,11 @@ func (b *QueryReadBuilder) rebuild(ctx context.Context, l queryModelLease, org s
// RunOnce is for bounded initial prewarming and isolated tests, not a freshness
// substitute for Run. After the heartbeat expires search falls back to old reads.
func (b *QueryReadBuilder) RunOnce(parent context.Context) error {
ctx, cancel := context.WithTimeout(parent, 5*time.Minute)
ctx, cancel := context.WithCancel(parent)
if !(b.store.queryReplicaFactStore && b.store.periodicQueryStore()) {
cancel()
ctx, cancel = context.WithTimeout(parent, 5*time.Minute)
}
defer cancel()
started := make(chan struct{})
done := make(chan error, 1)
@ -418,7 +429,9 @@ func (b *QueryReadBuilder) run(ctx context.Context, started chan<- struct{}) err
lastScopeRead = time.Now()
}
for org := range scopes {
if all {
// Even an idle organization needs periodic complete-result
// reconfirmation; freshness must not depend on customer writes.
if all || (b.store.periodicQueryStore() && queryBuildDue(time.Now(), lastStats[org], statsInterval)) {
pendingStats[org] = true
}
if allSearch {
@ -472,7 +485,12 @@ func (b *QueryReadBuilder) run(ctx context.Context, started chan<- struct{}) err
delete(statsRetries, org)
delete(pendingStats, org)
lastStats[org] = time.Now()
} else if errors.Is(e, ErrQueryBuildYielded) {
delete(statsRetries, org)
} else {
if b.store.queryReplicaFactStore && b.store.periodicQueryStore() {
return e
}
retry := statsRetries[org]
retry.fail(30*time.Second, 5*time.Minute)
statsRetries[org] = retry
@ -487,6 +505,9 @@ func (b *QueryReadBuilder) run(ctx context.Context, started chan<- struct{}) err
if time.Since(lastCleanup) > time.Minute {
cleanup, finish := context.WithTimeout(ctx, 5*time.Second)
_ = b.store.cleanQueryModelOrphans(cleanup)
if b.store.queryReplicaFactStore && b.store.periodicQueryStore() {
_ = b.store.cleanQueryBuildParts(cleanup, l)
}
finish()
lastCleanup = time.Now()

View File

@ -47,6 +47,7 @@ func workforceBoundedRead(ctx context.Context, cur *mongo.Cursor, target any, bu
func workforceFactKey(org string) string { return queryModelKey("workforce-facts", org, nil) }
func (s *MongoStore) ensureWorkforceFactIndexes(ctx context.Context) error {
indexes := []mongo.IndexModel{
{Keys: bson.D{{Key: "organizationId", Value: 1}, {Key: "key", Value: 1}, {Key: "fromEpoch", Value: 1}, {Key: "toEpoch", Value: 1}}, Options: options.Index().SetName("workforce_fact_key_v1")},
{Keys: bson.D{{Key: "organizationId", Value: 1}, {Key: "retiredAt", Value: 1}, {Key: "toEpoch", Value: 1}}, Options: options.Index().SetName("workforce_fact_retirement_v1")},
{Keys: bson.D{{Key: "organizationId", Value: 1}, {Key: "mode", Value: 1}, {Key: "groupTemplate", Value: 1}, {Key: "userId", Value: 1}, {Key: "at", Value: -1}, {Key: "order", Value: 1}}, Options: options.Index().SetName("workforce_fact_user_v1")},
{Keys: bson.D{{Key: "organizationId", Value: 1}, {Key: "mode", Value: 1}, {Key: "groupTemplate", Value: 1}, {Key: "supplierId", Value: 1}, {Key: "projectId", Value: 1}, {Key: "at", Value: -1}}, Options: options.Index().SetName("workforce_fact_scope_v1")},

View File

@ -455,6 +455,9 @@ func (s *MongoStore) LoadWorkforceStats(ctx context.Context, org string, q workf
return m
}
read := func(c mongoCollection, filter, projection bson.M, target any) error {
if hook, ok := ctx.Value(queryBuildReadKey{}).(queryBuildReadFunc); ok {
return hook(ctx, c, filter, projection, target)
}
cur, err := c.Find(ctx, filter, options.Find().SetProjection(projection).SetMaxTime(15*time.Second))
if err != nil {
return err

View File

@ -31,6 +31,8 @@ CASES = ('cmd/query-replica/main_local_test.go',
'internal/store/query_replica_valkey_local_test.go',
'internal/store/workforce_report_cache_local_test.go',
'internal/store/workforce_facts_local_test.go',
'internal/store/query_build_capture_local_test.go',
'internal/store/query_build_resume_local_test.go',
'internal/store/workforce_publication_local_test.go',
'internal/store/query_replica_page_local_test.go',
'internal/store/supplier_authorization_batch_local_test.go',

View File

@ -1044,6 +1044,7 @@
"^internal/store/workforce_(report_cache|heap_diagnostic|facts|return_page|publication).*\\.go$",
"^internal/store/query_pool_buckets.*\\.go$",
"^internal/store/query_cold_write.*\\.go$",
"^internal/store/query_build_.*\\.go$",
"^internal/store/query_periodic.*\\.go$",
"^internal/store/supplier_authorization_batch_local_test\\.go$",
"^internal/api/query_periodic.*\\.go$",
@ -1057,7 +1058,7 @@
"^scripts/local/query-replica\\.query25-local\\.compose\\.yml$",
"^scripts/local/query-replica\\.compose\\.yml$",
"^Dockerfile\\.query-replica$",
"^docs/(adr/0014-.*|operations/query-replica\\.md)$"
"^docs/(adr/0014-.*|operations/query-replica\\.md|operations/query-build-lifecycle\\.md)$"
],
"frontend": [],
"go": {
@ -1066,7 +1067,10 @@
"^TestQueryReplicaCountOnly",
"^TestQueryReplicaMissingConfiguration",
"^TestQueryReplicaRejectsSigned",
"^TestQueryReplicaDiagnostic"
"^TestQueryReplicaDiagnostic",
"^TestQueryBuildRound",
"^TestQueryBuildBudgets",
"^TestQueryBuildPacing"
],
"./cmd/query-cdc-probe": ["^TestClassification"],
"./cmd/query-replica": ["^TestPassDelay", "^TestPacing"],