forked from longtao.wu/droboticflow
♻️ Persist resumable query build generations
This commit is contained in:
parent
e4c4a20df5
commit
ffe186473c
@ -4,6 +4,7 @@ package main
|
||||
import (
|
||||
"context"
|
||||
"droboticflow/internal/store"
|
||||
"errors"
|
||||
"flag"
|
||||
"go.mongodb.org/mongo-driver/mongo"
|
||||
"go.mongodb.org/mongo-driver/mongo/options"
|
||||
@ -87,8 +88,13 @@ func main() {
|
||||
return
|
||||
}
|
||||
for workCtx.Err() == nil {
|
||||
if b.Run(workCtx) != nil && workCtx.Err() == nil {
|
||||
log.Print("query builder paused; retrying with full reconciliation")
|
||||
if err := b.Run(workCtx); err != nil && workCtx.Err() == nil {
|
||||
if errors.Is(err, store.ErrQueryBuildStopped) {
|
||||
log.Print("query builder stopped; checkpoint retained; explicit recovery required")
|
||||
<-workCtx.Done()
|
||||
return
|
||||
}
|
||||
log.Print("query builder retrying; durable build checkpoint retained")
|
||||
}
|
||||
select {
|
||||
case <-workCtx.Done():
|
||||
|
||||
@ -66,3 +66,19 @@
|
||||
准备状态通过显式媒体类型协商;旧客户端收到未就绪错误而非成功的非报表JSON。活跃配方的登记触碰合并到15分钟一次,后台统计刷新周期不变。任务池只共享候选身份,个人任务/审查历史维持当前权威查询;权限目录的逐项目重复读合并为请求内批量读,继续复用原授权种子规则,不形成共享权限缓存。
|
||||
|
||||
候选身份的冷缓存查询合并仅作用于相同数据库、组织、事实代次和完整筛选,沿用既有共享名称读取的8秒工作上限及独立取消规则;完成后的长效成品仍只在共享缓存/查询库中。每个请求各自取当前任务与运行服务端权限校验,合并结果不带授权结论、正文或可领取标志。
|
||||
|
||||
## 首次构建与日常刷新分离(2026-11-15,用户批准,实施中)
|
||||
|
||||
首次构建不再以固定5分钟要求全部完成。每轮执行有调度预算,每个读取、持久化批次及提交有独立上限;预算用尽记录可靠断点并让出,属于正常控制流,不触发全流程停副本。窗口到期、无进展、数据库不可用、权限/租约失效、容量及持久化异常仍须暂停并记录分项原因。
|
||||
|
||||
复用已同步查询事实及e4c4a20d的首代暂存指纹核对。构建采用持久输入/计划、分批应用、检查及原子发布几个阶段;已封存计划在续建、部署或重启后只读取未确认部分,不重新回填权威库,也不重新扫描完整业务输入。输入捕获本身须保留有界读取断点;源快照不可恢复时不得混合前后版本或伪造完成。
|
||||
|
||||
日常刷新按3分钟后台周期及15分钟展示年龄契约单独安排预算;继续提供上一完整版本。生产统计、人员统计和作业记录相关头只能在同一封存输入代次完成后发布,不能先切部分头。权限、供应商隔离与正式操作仍由原权威路径实时校验。
|
||||
|
||||
保持多数提交和日志持久性、25Gi容量及现有限额。512KiB/s内部写入限速须计入首次构建的真实数据量及确认耗时;若需调整,参数、共享盘影响和回退另列具体启用计划,不以延长请求超时或降低持久性替代。生产入口仍未切换,本节不是生产验收结论。
|
||||
|
||||
实现约束:首次构建每轮60秒、日常变化刷新每轮30秒,读取总界限5秒(最多3次瞬时错误重试)、每次提交15秒。到轮次边界仅让出,不取消已开始的持久提交;下轮通过租约和持久游标继续。无进展按累计有效执行时间计算,不计维护停机时间。`RunOnce`在独立周期查询库中也不再强加5分钟全程期限,仍服从调用方取消和维护窗口。
|
||||
|
||||
输入捕获和计划保存在私有`queryBuildJobs`/`queryBuildParts`。原生输入分段、游标以及写入批次、应用游标分别在同一`majority+journal`事务确认。续建重新核对封存计划指纹和e4暂存事实哈希;未发布的大块结果由清理程序保留。派生事实采用原有版本可见性规则,全部完成及行数核对后才在同一事务发布两个统计头。已追平观察器证明业务输入未变化时,只重新确认完整结果,不重新读取历史、计算或遍历全量事实。
|
||||
|
||||
输入快照的逻辑时间与展示时间分开保存。已经封存输入后,恢复不依赖原快照仍在Mongo历史窗口内;尚未捕获完的输入若遇到快照历史丢失,必须保留分段并明确停止,不能切换时间点后拼接新旧结果。该异常不自动重新回填权威库。首次上线前须检查查询库快照/oplog保留窗口,不能把任意长停机期间未封存输入的自动恢复视为已经覆盖。
|
||||
|
||||
66
docs/operations/query-build-lifecycle.md
Normal file
66
docs/operations/query-build-lifecycle.md
Normal file
@ -0,0 +1,66 @@
|
||||
# 查询库首次构建与日常刷新
|
||||
|
||||
## 范围与状态
|
||||
|
||||
仅独立周期查询库的构建器采用持久任务。权威Mongo、标注/切片/送检/审核/验收、权限目录和版本竞争路径不改。生产查询入口仍未切换。本地通过、代码部署、Pod就绪分别不代表影子验收或分流降载验收。
|
||||
|
||||
上一轮e4影子构建已完成:复用125,186条,补写14,820条;事实写入阶段约101秒,后续观察到至少两轮完整刷新。原维护窗口在2026-11-15 14:38:42 UTC到期,两影子作业按约定暂停到0副本并保留断点。本文不延长该窗口。
|
||||
|
||||
## 执行与持久性
|
||||
|
||||
| 参数 | 计划值 | 说明 |
|
||||
| --- | --- | --- |
|
||||
| 首次构建单轮调度 | 60秒 | 正常让出,不是全程完成期限 |
|
||||
| 变化刷新单轮调度 | 30秒 | 延续既有3分钟周期、15分钟展示年龄 |
|
||||
| 原生输入单次逻辑读取 | 5秒 | 最多3次瞬时重试,每次服务端1.5秒 |
|
||||
| 持久事务 | 15秒 | 多数提交及journal;游标与数据共同确认 |
|
||||
| 累计无确认进展 | 5分钟有效执行时间 | 不含停机;真正无进展持久记录后停止 |
|
||||
| 内部写入限速 | 512KiB/s | 暂不调整;不能以延长超时取代限速核对 |
|
||||
| 应用批次 | 最多200逻辑行,约64KiB写入 | 单条较大的合法事实仍完整写入 |
|
||||
| 容量/Pod/底层IO | 保持现有配置 | 25Gi、现有内存/CPU/IO边界不变 |
|
||||
|
||||
正常阶段:捕获输入→保存完整计划→应用事实→清理未匹配的未来版本→核对→事务发布。新一代完整之前,读者继续使用原完整代次及原权限校验;超出既有展示年龄仍按原错误/成品回退契约处理,不无限伪装新鲜或集中回源。
|
||||
|
||||
`queryBuildJobs`记录组织、输入代次、快照、阶段、事实应用游标、清理游标及分项失败;`queryBuildParts`保存带校验和的压缩输入和计划。均为私有集合,禁止从通用数据管理API读写。维护清理只回收不再被当前任务/输入引用的派生分段,绝不清客户标注或同步事实。
|
||||
|
||||
无变化日常刷新只有在观察器追平证明新鲜且组织、范围、逻辑版本及业务修订相同的情况下,才直接重新确认原完整事实和统计结果。否则新建输入代次并按上述流程计算。变更刷新可以读取查询库必要输入;不会重新启动原Mongo历史回填。
|
||||
|
||||
## 暂停、恢复与限制
|
||||
|
||||
- 轮次预算耗尽:保留pending状态和游标,下轮正常续建,不增加失败退避,不因此降为0副本。
|
||||
- 进程退出/部署/调用方取消:已确认批次保留;新持有者以新租约恢复,旧持有者不能提交。
|
||||
- 数据库/权限/租约/持久化/容量异常:记录阶段、类别及Mongo代码;停止构建,保持旧完整结果,不自动清库重试。
|
||||
- 维护截止:按已批准观察器策略停影子作业,保存累计同步预算和断点。启动必须有新的有效窗口。
|
||||
- 已封存输入:原快照历史过期不影响应用恢复。未封存输入若历史已丢失,停止并保留分段,需单列恢复方案,不能混入新时间点。
|
||||
- 算法/字段语义变更必须更新构建逻辑schema并审核旧任务迁移;封存计划指纹不符时禁止继续混写。
|
||||
|
||||
## 待确认的实际启用方案
|
||||
|
||||
1. 通过本任务检查和正式发布门禁,提交候选来源、镜像摘要及最终可执行清单。新代码尚未获得推送/上传或部署确认。
|
||||
2. 核对新的维护窗口、当前同步断点/预算、源健康、查询库容量/IO、原有镜像及运行身份;旧窗口不能复用。
|
||||
3. 仅为查询库构建器授权私有两集合的必要读写,在查询库`queryWorkforceFacts`增加`workforce_fact_key_v1`:`organizationId,key,fromEpoch,toEpoch`。不在原Mongo建索引。本地14万事实需记录新增索引体积,生产DDL仍列入最终计划。
|
||||
4. 构建器保持0副本更新镜像;同步沿原有效授权运行并取得新鲜追平证明后,再启用构建器1副本。保留所有e4确认事实,不能调用旧版`restoreWorkforceStage`清除新任务的未来版本。
|
||||
5. 核验首次完整发布、两轮正常刷新、权限/供应商/日期等价、Valkey丢失回退和查询库重启;生产查询入口继续关闭。原Mongo降载需后续经确认切入口后独立验收。
|
||||
|
||||
回退:先暂停构建器,保留任务/分段/事实、头和同步断点。e4旧构建器不能直接接手尚未完成的新生命周期任务,否则其旧恢复逻辑可能删除未来版本;无损回退优先停构建器并继续读上一完整版本。只有独立核实不存在未完成新任务、已发布完整头可用后,才能另行确认恢复旧镜像。不删表、不回滚客户标注、不重置预算、不降低持久性。
|
||||
|
||||
## 验证证据
|
||||
|
||||
证据集中写入本工作树忽略目录`local-data/test-results/query-build-*`,包含读取次数、批次、耗时、数据/索引占用及配置。不得复制客户原文或凭证。正式结果待验证完成后更新,不用短时成功代替完整验收。
|
||||
|
||||
### 2026-11-15 本地同规模结果
|
||||
|
||||
固定35,000条合成事件、140,000条事实,查询Mongo为1核/1GiB、WiredTiger 256MiB;生产影子库仍为4核/8GiB,不把本地结果当作生产容量验收。写入限速一直为512KiB/s,持久化一直为majority+journal。
|
||||
|
||||
| 场景 | 结果 |
|
||||
| --- | --- |
|
||||
| 保留e4的125,186条确认事实、补14,814条 | 76.33秒,跨2轮 |
|
||||
| 同夹具遍历无变化事实作为算法对照 | 42.53秒 |
|
||||
| 复用已证明无变化的完整结果,两次刷新 | 194.75ms、181.10ms |
|
||||
| 从零写入140,000条事实 | 318.71秒,跨6轮及进程对象重建 |
|
||||
| 从零完成后的两次刷新 | 163.63ms、153.75ms |
|
||||
| 以上构建与刷新对原Mongo的读取 | 0 |
|
||||
|
||||
从零构建确认写入148,610,000 BSON字节,限速理论下界约283秒,实际约319秒;不需要通过增加全程超时或削弱持久性才能完成。采样查询容器CPU最高37.41%,无OOM;Docker VM的IO full avg10最高0.91%,该值不是单库IO等待。新增键索引在14万事实时约14.44MB。内存包含Mongo文件缓存压力,本地1GiB测试不能替代生产8GiB长时验证。
|
||||
|
||||
快速检查通过。首次全任务执行的浏览器、统计稳定性、权限和共享缓存检查通过;隔离副本组在总编排900秒耗尽后被停止,原报告`verify-1794758125279658000`保留为incomplete,不能写成通过。后续对失败落盘分支完成故障注入后,集中补跑受影响的隔离组;已通过且输入未变的浏览器证据保留。
|
||||
@ -66,7 +66,7 @@ func TestQueryServingProductionResponseChecksCurrentPermissionBeforeCache(t *tes
|
||||
|
||||
func TestQueryServingPrivateMaterializationsRejectDatabaseBrowser(t *testing.T) {
|
||||
f := newCombinedDraftFixture(t)
|
||||
for _, name := range []string{"queryReadModelControl", "queryReadModelData"} {
|
||||
for _, name := range []string{"queryReadModelControl", "queryReadModelData", "queryBuildJobs", "queryBuildParts"} {
|
||||
for _, target := range []string{"platform", "business"} {
|
||||
if r := f.request("GET", "/api/database/collections/"+name+"/documents?target="+target, f.admin, nil); r.Code < 400 {
|
||||
t.Fatalf("private read model leaked: %s %s", name, target)
|
||||
|
||||
@ -5,5 +5,5 @@ import "droboticflow/internal/outputsnapshot"
|
||||
// Derived generations contain facts from multiple authorization scopes and
|
||||
// must only be served through their permission-aware reporting/search readers.
|
||||
func IsPrivateDerivedCollection(name string) bool {
|
||||
return name == querySyncDeleteCollection || name == queryReplicaControl || outputsnapshot.IsPrivateCollection(name) || name == "queryReadModelControl" || name == "queryReadModelData"
|
||||
return name == queryBuildJobsCollection || name == queryBuildPartsCollection || name == querySyncDeleteCollection || name == queryReplicaControl || outputsnapshot.IsPrivateCollection(name) || name == "queryReadModelControl" || name == "queryReadModelData"
|
||||
}
|
||||
|
||||
@ -204,6 +204,9 @@ func (s *MongoStore) loadProductionStats(ctx context.Context, org string, projec
|
||||
}
|
||||
}
|
||||
read := func(c mongoCollection, f bson.M, projection bson.M, target any, preferred ...string) error {
|
||||
if hook, ok := ctx.Value(queryBuildReadKey{}).(queryBuildReadFunc); ok {
|
||||
return hook(ctx, c, f, projection, target)
|
||||
}
|
||||
opts := options.Find().SetProjection(projection).SetMaxTime(15 * time.Second)
|
||||
hint := hints[c.Name()]
|
||||
// A project-leading covering index is unsuitable for legacy task-ID lookups.
|
||||
|
||||
321
internal/store/query_build_apply.go
Normal file
321
internal/store/query_build_apply.go
Normal file
@ -0,0 +1,321 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"droboticflow/internal/workforcestats"
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
"go.mongodb.org/mongo-driver/mongo"
|
||||
"go.mongodb.org/mongo-driver/mongo/options"
|
||||
)
|
||||
|
||||
type queryBuildPrevious struct {
|
||||
ID string `bson:"_id"`
|
||||
Key string `bson:"key"`
|
||||
Hash string `bson:"hash"`
|
||||
From int64 `bson:"fromEpoch"`
|
||||
To int64 `bson:"toEpoch"`
|
||||
Retired *time.Time `bson:"retiredAt"`
|
||||
Expire *time.Time `bson:"expireAt"`
|
||||
}
|
||||
|
||||
func (s *MongoStore) queryBuildPlanPage(ctx context.Context, j queryBuildJob, n int) ([]workforcestats.Fact, error) {
|
||||
op, stop := queryBuildOperation(ctx)
|
||||
defer stop()
|
||||
var p queryBuildPart
|
||||
if err := s.db.Collection(queryBuildPartsCollection).FindOne(op, bson.M{"_id": queryBuildPlanPartKey(j, "facts", n), "job": j.Generation}).Decode(&p); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var facts []workforcestats.Fact
|
||||
if err := queryBuildJSONDecode(p, &facts); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(facts) == 0 || len(facts) > queryBuildFactPageRows {
|
||||
return nil, errors.New("invalid sealed fact page")
|
||||
}
|
||||
return facts, nil
|
||||
}
|
||||
|
||||
// A batch checkpoint commits in the very same majority+journal transaction as
|
||||
// its version writes. A lost acknowledgement can only replay this bounded batch.
|
||||
func (s *MongoStore) queryBuildApply(ctx context.Context, l queryModelLease, j *queryBuildJob, r *queryBuildRound) error {
|
||||
c := s.db.Collection(workforceFactCollection)
|
||||
for j.NextPart < j.PlanParts {
|
||||
if err := r.check(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
facts, err := s.queryBuildPlanPage(ctx, *j, j.NextPart)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for j.NextRow < len(facts) {
|
||||
if err = r.check(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
models := []mongo.WriteModel{}
|
||||
bytes, rows, reused := 0, 0, 0
|
||||
next := *j
|
||||
// Keep point lookups bounded; the key index avoids reading large old rows.
|
||||
keys := []string{}
|
||||
for _, f := range facts[j.NextRow:] {
|
||||
keys = append(keys, f.Key)
|
||||
}
|
||||
op, stop := queryBuildOperation(ctx)
|
||||
cur, e := c.Find(op, bson.M{"organizationId": j.Organization, "key": bson.M{"$in": keys}}, options.Find().SetProjection(bson.M{"_id": 1, "key": 1, "hash": 1, "fromEpoch": 1, "toEpoch": 1, "retiredAt": 1, "expireAt": 1}).SetHint("workforce_fact_key_v1"))
|
||||
previous := map[string][]queryBuildPrevious{}
|
||||
if e == nil {
|
||||
var xs []queryBuildPrevious
|
||||
e = cur.All(op, &xs)
|
||||
cur.Close(op)
|
||||
for _, x := range xs {
|
||||
previous[x.Key] = append(previous[x.Key], x)
|
||||
}
|
||||
}
|
||||
stop()
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
for next.NextRow < len(facts) {
|
||||
f := facts[next.NextRow]
|
||||
doc, hash, size, e := queryBuildFactDocument(j.Organization, j.Epoch, f)
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
if rows > 0 && bytes+size+512 > queryBuildBatchBytes {
|
||||
break
|
||||
}
|
||||
versions := previous[f.Key]
|
||||
var current *queryBuildPrevious
|
||||
var staged *queryBuildPrevious
|
||||
for i := range versions {
|
||||
x := &versions[i]
|
||||
if x.From == j.Epoch && x.ID == doc["_id"] {
|
||||
staged = x
|
||||
}
|
||||
if x.From <= j.OldEpoch && (x.To == 0 || x.To > j.OldEpoch) {
|
||||
if current != nil {
|
||||
return errors.New("duplicate committed workforce key")
|
||||
}
|
||||
current = x
|
||||
}
|
||||
}
|
||||
unchanged := current != nil && current.Hash == hash
|
||||
keepID := ""
|
||||
if unchanged {
|
||||
keepID = current.ID
|
||||
if current.To != 0 || current.Retired != nil || current.Expire != nil {
|
||||
models = append(models, mongo.NewUpdateOneModel().SetFilter(bson.M{"_id": current.ID}).SetUpdate(bson.M{"$unset": bson.M{"toEpoch": "", "retiredAt": "", "expireAt": ""}}))
|
||||
bytes += 256
|
||||
}
|
||||
reused++
|
||||
} else {
|
||||
next.Changed = true
|
||||
keepID = doc["_id"].(string)
|
||||
if current != nil && current.To != j.Epoch {
|
||||
models = append(models, mongo.NewUpdateOneModel().SetFilter(bson.M{"_id": current.ID}).SetUpdate(bson.M{"$set": bson.M{"toEpoch": j.Epoch, "retiredAt": time.Now().UTC()}}))
|
||||
bytes += 256
|
||||
}
|
||||
if staged != nil && staged.Hash == hash && staged.To == 0 && staged.Retired == nil && staged.Expire == nil {
|
||||
reused++
|
||||
} else {
|
||||
models = append(models, mongo.NewReplaceOneModel().SetFilter(bson.M{"_id": doc["_id"]}).SetReplacement(doc).SetUpsert(true))
|
||||
bytes += size
|
||||
}
|
||||
}
|
||||
for _, x := range versions {
|
||||
if x.From > j.OldEpoch && x.ID != keepID {
|
||||
models = append(models, mongo.NewDeleteOneModel().SetFilter(bson.M{"_id": x.ID, "organizationId": j.Organization, "fromEpoch": bson.M{"$gt": j.OldEpoch}}))
|
||||
bytes += 256
|
||||
}
|
||||
}
|
||||
next.NextRow++
|
||||
rows++
|
||||
}
|
||||
if rows == 0 {
|
||||
return errors.New("fact batch made no progress")
|
||||
}
|
||||
if next.NextRow == len(facts) {
|
||||
next.NextPart++
|
||||
next.NextRow = 0
|
||||
}
|
||||
next.ConfirmedRows += int64(rows - reused)
|
||||
next.ReusedRows += int64(reused)
|
||||
next.ConfirmedBytes += int64(bytes)
|
||||
next.ProgressAt = time.Now().UTC()
|
||||
next.ActiveWithoutProgress = 0
|
||||
err = s.queryBuildPace(ctx, bytes)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = s.queryBuildTransaction(ctx, l, func(sc mongo.SessionContext) error {
|
||||
if len(models) > 0 {
|
||||
if _, e := c.BulkWrite(sc, models, options.BulkWrite().SetOrdered(true)); e != nil {
|
||||
return e
|
||||
}
|
||||
}
|
||||
return s.saveQueryBuildJob(sc, next)
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
*j = next
|
||||
if j.NextRow == 0 {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
j.Phase = "cleanup"
|
||||
return s.queryBuildTransaction(ctx, l, func(sc mongo.SessionContext) error { return s.saveQueryBuildJob(sc, *j) })
|
||||
}
|
||||
|
||||
// Merge the sorted sealed plan with one bounded page of old/future versions.
|
||||
// Persist both cursors, so restart never re-scans all previously compared keys.
|
||||
func (s *MongoStore) queryBuildCleanup(ctx context.Context, l queryModelLease, j *queryBuildJob, r *queryBuildRound) error {
|
||||
c := s.db.Collection(workforceFactCollection)
|
||||
for {
|
||||
if err := r.check(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
f := bson.M{"organizationId": j.Organization}
|
||||
if j.CleanupFuture {
|
||||
f["fromEpoch"] = bson.M{"$gt": j.OldEpoch}
|
||||
} else {
|
||||
f = workforceVersionFilter(j.Organization, j.OldEpoch)
|
||||
}
|
||||
if j.CleanupHasAfter {
|
||||
f["key"] = bson.M{"$gt": j.CleanupAfter}
|
||||
}
|
||||
op, stop := queryBuildOperation(ctx)
|
||||
cur, err := c.Find(op, f, options.Find().SetProjection(bson.M{"_id": 1, "key": 1, "fromEpoch": 1}).SetSort(bson.D{{Key: "key", Value: 1}}).SetHint("workforce_fact_key_v1").SetLimit(200))
|
||||
var xs []queryBuildPrevious
|
||||
if err == nil {
|
||||
err = cur.All(op, &xs)
|
||||
cur.Close(op)
|
||||
}
|
||||
stop()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(xs) == 0 {
|
||||
if j.CleanupFuture {
|
||||
j.Phase = "publish"
|
||||
} else {
|
||||
j.CleanupFuture = true
|
||||
j.CleanupAfter = ""
|
||||
j.CleanupHasAfter = false
|
||||
j.CleanupPlanPart = 0
|
||||
j.CleanupPlanRow = 0
|
||||
}
|
||||
if err = s.queryBuildTransaction(ctx, l, func(sc mongo.SessionContext) error { return s.saveQueryBuildJob(sc, *j) }); err != nil {
|
||||
return err
|
||||
}
|
||||
if j.Phase == "publish" {
|
||||
return nil
|
||||
}
|
||||
continue
|
||||
}
|
||||
next := *j
|
||||
models := []mongo.WriteModel{}
|
||||
var page []workforcestats.Fact
|
||||
pageN := -1
|
||||
for _, x := range xs {
|
||||
if x.Key == "" {
|
||||
return errors.New("invalid workforce fact identity")
|
||||
}
|
||||
present := false
|
||||
for next.CleanupPlanPart < j.PlanParts {
|
||||
if pageN != next.CleanupPlanPart {
|
||||
page, err = s.queryBuildPlanPage(ctx, *j, next.CleanupPlanPart)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
pageN = next.CleanupPlanPart
|
||||
}
|
||||
if next.CleanupPlanRow >= len(page) {
|
||||
next.CleanupPlanPart++
|
||||
next.CleanupPlanRow = 0
|
||||
continue
|
||||
}
|
||||
key := page[next.CleanupPlanRow].Key
|
||||
if key < x.Key {
|
||||
next.CleanupPlanRow++
|
||||
continue
|
||||
}
|
||||
present = key == x.Key
|
||||
break
|
||||
}
|
||||
if !present {
|
||||
next.Changed = true
|
||||
if j.CleanupFuture {
|
||||
models = append(models, mongo.NewDeleteManyModel().SetFilter(bson.M{"organizationId": j.Organization, "key": x.Key, "fromEpoch": bson.M{"$gt": j.OldEpoch}}))
|
||||
} else {
|
||||
models = append(models, mongo.NewUpdateManyModel().SetFilter(bson.M{"$and": bson.A{workforceVersionFilter(j.Organization, j.OldEpoch), bson.M{"key": x.Key}}}).SetUpdate(bson.M{"$set": bson.M{"toEpoch": j.Epoch, "retiredAt": time.Now().UTC()}}))
|
||||
}
|
||||
}
|
||||
next.CleanupAfter = x.Key
|
||||
next.CleanupHasAfter = true
|
||||
}
|
||||
next.ProgressAt = time.Now().UTC()
|
||||
next.ActiveWithoutProgress = 0
|
||||
if err = s.queryBuildPace(ctx, len(models)*256); err != nil {
|
||||
return err
|
||||
}
|
||||
if err = s.queryBuildTransaction(ctx, l, func(sc mongo.SessionContext) error {
|
||||
if len(models) > 0 {
|
||||
if _, e := c.BulkWrite(sc, models); e != nil {
|
||||
return e
|
||||
}
|
||||
}
|
||||
return s.saveQueryBuildJob(sc, next)
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
*j = next
|
||||
}
|
||||
}
|
||||
|
||||
func (s *MongoStore) queryBuildPublish(ctx context.Context, l queryModelLease, j *queryBuildJob) error {
|
||||
op, stop := queryBuildOperation(ctx)
|
||||
defer stop()
|
||||
epoch := j.Epoch
|
||||
if !j.Changed && j.OldEpoch > 0 {
|
||||
epoch = j.OldEpoch
|
||||
}
|
||||
count, err := s.db.Collection(workforceFactCollection).CountDocuments(op, workforceVersionFilter(j.Organization, epoch))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if count != int64(j.PlanRows) {
|
||||
return fmt.Errorf("sealed workforce row count differs: got=%d want=%d", count, j.PlanRows)
|
||||
}
|
||||
next := *j
|
||||
next.Phase = "complete"
|
||||
next.ProgressAt = time.Now().UTC()
|
||||
next.ActiveWithoutProgress = 0
|
||||
return s.queryBuildTransaction(ctx, l, func(sc mongo.SessionContext) error {
|
||||
var state struct {
|
||||
Heads map[string]QueryModelHead `bson:"heads"`
|
||||
}
|
||||
if e := s.queryControl().FindOne(sc, queryLeaseFilter(l)).Decode(&state); e != nil {
|
||||
return e
|
||||
}
|
||||
if state.Heads[workforceFactKey(j.Organization)].FactEpoch != j.OldEpoch {
|
||||
return errors.New("query build published epoch changed")
|
||||
}
|
||||
wf := QueryModelHead{Generation: workforceFactSchema, Collection: workforceFactCollection, FactEpoch: epoch, Rows: j.PlanRows, AsOf: j.AsOf, SourceRevision: j.SourceRevision, GlobalRevision: j.GlobalRevision}
|
||||
result, e := s.queryControl().UpdateOne(sc, queryLeaseFilter(l), bson.M{"$set": bson.M{"heads." + workforceFactKey(j.Organization): wf, "heads." + queryModelKey("production", j.Organization, j.Projects): j.Production}})
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
if result.MatchedCount != 1 {
|
||||
return errors.New("query build publication fenced")
|
||||
}
|
||||
if e = s.saveQueryBuildJob(sc, next); e == nil {
|
||||
*j = next
|
||||
}
|
||||
return e
|
||||
})
|
||||
}
|
||||
362
internal/store/query_build_capture.go
Normal file
362
internal/store/query_build_capture.go
Normal file
@ -0,0 +1,362 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/binary"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"reflect"
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
"droboticflow/internal/queryserving"
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
"go.mongodb.org/mongo-driver/bson/primitive"
|
||||
"go.mongodb.org/mongo-driver/mongo"
|
||||
"go.mongodb.org/mongo-driver/mongo/options"
|
||||
)
|
||||
|
||||
type queryBuildReadKey struct{}
|
||||
type queryBuildReadFunc func(context.Context, mongoCollection, bson.M, bson.M, any) error
|
||||
|
||||
type queryBuildCaptureState struct {
|
||||
ID string `bson:"_id"`
|
||||
Job string `bson:"job"`
|
||||
Parts int `bson:"parts"`
|
||||
Last bson.RawValue `bson:"last,omitempty"`
|
||||
HasLast bool `bson:"hasLast"`
|
||||
Complete bool `bson:"complete"`
|
||||
Rows int `bson:"rows"`
|
||||
Bytes int `bson:"bytes"`
|
||||
}
|
||||
|
||||
type queryBuildPart struct {
|
||||
ID string `bson:"_id"`
|
||||
Job string `bson:"job"`
|
||||
Digest string `bson:"digest"`
|
||||
Data []byte `bson:"data"`
|
||||
}
|
||||
|
||||
func queryBuildCanonical(v any) any {
|
||||
switch d := v.(type) {
|
||||
case bson.M:
|
||||
out := map[string]any{}
|
||||
for k, x := range d {
|
||||
out[k] = queryBuildCanonical(x)
|
||||
}
|
||||
return out
|
||||
case map[string]any:
|
||||
out := map[string]any{}
|
||||
for k, x := range d {
|
||||
out[k] = queryBuildCanonical(x)
|
||||
}
|
||||
return out
|
||||
case bson.A:
|
||||
out := make([]any, len(d))
|
||||
for i, x := range d {
|
||||
out[i] = queryBuildCanonical(x)
|
||||
}
|
||||
return out
|
||||
default:
|
||||
return v
|
||||
}
|
||||
}
|
||||
|
||||
func queryBuildQueryKey(generation, name string, f, p bson.M) string {
|
||||
// JSON map ordering is deterministic. Membership order is semantically
|
||||
// irrelevant, so normalize caller-generated $in lists before hashing.
|
||||
var normalize func(any) any
|
||||
normalize = func(v any) any {
|
||||
switch d := v.(type) {
|
||||
case bson.M:
|
||||
out := map[string]any{}
|
||||
for k, x := range d {
|
||||
if k == "$in" {
|
||||
if ids, ok := x.([]string); ok {
|
||||
cp := append([]string{}, ids...)
|
||||
sort.Strings(cp)
|
||||
out[k] = cp
|
||||
continue
|
||||
}
|
||||
if a, ok := x.(bson.A); ok {
|
||||
s := make([]string, 0, len(a))
|
||||
all := true
|
||||
for _, id := range a {
|
||||
t, ok := id.(string)
|
||||
if !ok {
|
||||
all = false
|
||||
break
|
||||
}
|
||||
s = append(s, t)
|
||||
}
|
||||
if all {
|
||||
sort.Strings(s)
|
||||
out[k] = s
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
out[k] = normalize(x)
|
||||
}
|
||||
return out
|
||||
case bson.A:
|
||||
a := make([]any, len(d))
|
||||
for i, x := range d {
|
||||
a[i] = normalize(x)
|
||||
}
|
||||
return a
|
||||
default:
|
||||
return v
|
||||
}
|
||||
}
|
||||
return queryserving.Key("query-build-input/v1", generation, name, normalize(f), queryBuildCanonical(p))
|
||||
}
|
||||
|
||||
func queryBuildPack(v any) ([]byte, string, error) {
|
||||
raw, err := bson.Marshal(v)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
var out bytes.Buffer
|
||||
z := gzip.NewWriter(&out)
|
||||
if _, err = z.Write(raw); err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
if err = z.Close(); err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
data := out.Bytes()
|
||||
h := sha256.Sum256(data)
|
||||
return data, hex.EncodeToString(h[:]), nil
|
||||
}
|
||||
func queryBuildUnpack(data []byte, digest string, out any) error {
|
||||
h := sha256.Sum256(data)
|
||||
if hex.EncodeToString(h[:]) != digest {
|
||||
return errors.New("query build part digest mismatch")
|
||||
}
|
||||
z, err := gzip.NewReader(bytes.NewReader(data))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer z.Close()
|
||||
raw, err := io.ReadAll(io.LimitReader(z, (64<<20)+1))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(raw) > 64<<20 {
|
||||
return errors.New("query build part exceeds decode budget")
|
||||
}
|
||||
return bson.Unmarshal(raw, out)
|
||||
}
|
||||
|
||||
func (s *MongoStore) queryBuildSnapshot(ctx context.Context) (primitive.Timestamp, error) {
|
||||
op, stop := queryBuildOperation(ctx)
|
||||
defer stop()
|
||||
var response struct {
|
||||
At primitive.Timestamp `bson:"atClusterTime"`
|
||||
Cursor struct {
|
||||
At primitive.Timestamp `bson:"atClusterTime"`
|
||||
} `bson:"cursor"`
|
||||
}
|
||||
err := s.db.RunCommand(op, bson.D{{Key: "find", Value: "queryReadModelControl"}, {Key: "filter", Value: bson.M{"_id": "global"}}, {Key: "projection", Value: bson.M{"_id": 1}}, {Key: "limit", Value: 1}, {Key: "singleBatch", Value: true}, {Key: "readConcern", Value: bson.M{"level": "snapshot"}}, {Key: "maxTimeMS", Value: 1500}}).Decode(&response)
|
||||
if err != nil {
|
||||
return primitive.Timestamp{}, err
|
||||
}
|
||||
if response.At.T == 0 {
|
||||
response.At = response.Cursor.At
|
||||
}
|
||||
if response.At.T == 0 {
|
||||
return primitive.Timestamp{}, errors.New("query build snapshot timestamp unavailable")
|
||||
}
|
||||
return response.At, nil
|
||||
}
|
||||
|
||||
func (s *MongoStore) queryBuildReadPage(ctx context.Context, c mongoCollection, f, p bson.M, last bson.RawValue, hasLast bool, at primitive.Timestamp) ([]bson.Raw, error) {
|
||||
filter := any(f)
|
||||
if hasLast {
|
||||
filter = bson.M{"$and": bson.A{f, bson.M{"$expr": bson.M{"$gt": bson.A{"$_id", bson.M{"$literal": last}}}}}}
|
||||
}
|
||||
projection := bson.M{}
|
||||
for k, v := range p {
|
||||
projection[k] = v
|
||||
}
|
||||
projection["_id"] = 1
|
||||
cmd := bson.D{{Key: "find", Value: c.Name()}, {Key: "filter", Value: filter}, {Key: "projection", Value: projection}, {Key: "sort", Value: bson.D{{Key: "_id", Value: 1}}}, {Key: "hint", Value: "_id_"}, {Key: "limit", Value: 200}, {Key: "batchSize", Value: 200}, {Key: "singleBatch", Value: true}, {Key: "readConcern", Value: bson.M{"level": "snapshot", "atClusterTime": at}}, {Key: "maxTimeMS", Value: 1500}}
|
||||
op, stop := queryBuildOperation(ctx)
|
||||
defer stop()
|
||||
var response struct {
|
||||
Cursor struct {
|
||||
Rows []bson.Raw `bson:"firstBatch"`
|
||||
} `bson:"cursor"`
|
||||
}
|
||||
readCtx, readStop := context.WithTimeout(op, 5*time.Second)
|
||||
defer readStop()
|
||||
for attempt := 0; attempt < 3; attempt++ {
|
||||
err := c.Database().RunCommand(readCtx, cmd).Decode(&response)
|
||||
if err == nil {
|
||||
return response.Cursor.Rows, nil
|
||||
}
|
||||
var command mongo.CommandError
|
||||
transient := mongo.IsNetworkError(err) || mongo.IsTimeout(err)
|
||||
if errors.As(err, &command) {
|
||||
switch command.Code {
|
||||
case 50, 91, 6, 7, 89, 11600, 11602, 10107:
|
||||
transient = true
|
||||
}
|
||||
}
|
||||
if !transient || attempt == 2 || readCtx.Err() != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
return response.Cursor.Rows, nil
|
||||
}
|
||||
|
||||
func queryBuildAppendRows(ctx context.Context, rows []bson.Raw, target any, projection bson.M) error {
|
||||
value := reflect.ValueOf(target).Elem()
|
||||
budget, _ := ctx.Value(workforceBuildBudgetKey{}).(*workforceBuildBudget)
|
||||
for _, raw := range rows {
|
||||
if excluded, ok := projection["_id"]; ok && (excluded == 0 || excluded == false) {
|
||||
elements, e := raw.Elements()
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
clean := make([]byte, 4, len(raw))
|
||||
for _, element := range elements {
|
||||
if element.Key() != "_id" {
|
||||
clean = append(clean, element...)
|
||||
}
|
||||
}
|
||||
clean = append(clean, 0)
|
||||
binary.LittleEndian.PutUint32(clean, uint32(len(clean)))
|
||||
raw = bson.Raw(clean)
|
||||
}
|
||||
if budget != nil {
|
||||
budget.rows++
|
||||
budget.bytes += len(raw)
|
||||
if budget.rows > 250000 || budget.bytes > 64<<20 {
|
||||
return errWorkforceBuildBudget
|
||||
}
|
||||
}
|
||||
v := reflect.New(value.Type().Elem())
|
||||
if err := bson.Unmarshal(raw, v.Interface()); err != nil {
|
||||
return err
|
||||
}
|
||||
value.Set(reflect.Append(value, v.Elem()))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// This capture reads a native range once. Its payload and cursor commit in the
|
||||
// same majority+journal transaction. Replaying a completed range reads only the
|
||||
// immutable captured part, not the native business collections.
|
||||
func (s *MongoStore) queryBuildCapture(ctx context.Context, l queryModelLease, j *queryBuildJob, round *queryBuildRound, c mongoCollection, f, p bson.M, target any) error {
|
||||
if !s.queryReplicaFactStore {
|
||||
return errors.New("query build capture requires independent read store")
|
||||
}
|
||||
check, checkStop := queryBuildOperation(ctx)
|
||||
leaseErr := s.queryControl().FindOne(check, queryLeaseFilter(l), options.FindOne().SetProjection(bson.M{"_id": 1})).Err()
|
||||
checkStop()
|
||||
if leaseErr != nil {
|
||||
return fmt.Errorf("query capture lease check: %w", leaseErr)
|
||||
}
|
||||
inputGeneration := j.InputGeneration
|
||||
if inputGeneration == "" {
|
||||
inputGeneration = j.Generation
|
||||
}
|
||||
key := queryBuildQueryKey(inputGeneration, c.Name(), f, p)
|
||||
var st queryBuildCaptureState
|
||||
check, checkStop = queryBuildOperation(ctx)
|
||||
err := s.db.Collection(queryBuildJobsCollection).FindOne(check, bson.M{"_id": key}).Decode(&st)
|
||||
checkStop()
|
||||
if errors.Is(err, mongo.ErrNoDocuments) {
|
||||
if inputGeneration != j.Generation {
|
||||
return errors.New("verified input capture missing")
|
||||
}
|
||||
st = queryBuildCaptureState{ID: key, Job: inputGeneration}
|
||||
} else if err != nil {
|
||||
return err
|
||||
}
|
||||
for i := 0; i < st.Parts; i++ {
|
||||
if err = ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
var part queryBuildPart
|
||||
op, stop := queryBuildOperation(ctx)
|
||||
err = s.db.Collection(queryBuildPartsCollection).FindOne(op, bson.M{"_id": fmt.Sprintf("%s/%09d", key, i)}).Decode(&part)
|
||||
stop()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var payload struct {
|
||||
Rows []bson.Raw `bson:"rows"`
|
||||
}
|
||||
if err = queryBuildUnpack(part.Data, part.Digest, &payload); err != nil {
|
||||
return err
|
||||
}
|
||||
if err = queryBuildAppendRows(ctx, payload.Rows, target, p); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
for !st.Complete {
|
||||
if err = round.check(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
rows, e := s.queryBuildReadPage(ctx, c, f, p, st.Last, st.HasLast, j.Snapshot)
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
next := st
|
||||
if len(rows) == 0 {
|
||||
next.Complete = true
|
||||
} else {
|
||||
id, e := rows[len(rows)-1].LookupErr("_id")
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
next.Last = bson.RawValue{Type: id.Type, Value: append([]byte(nil), id.Value...)}
|
||||
next.HasLast = true
|
||||
next.Parts++
|
||||
next.Rows += len(rows)
|
||||
}
|
||||
var part queryBuildPart
|
||||
if len(rows) > 0 {
|
||||
data, hash, e := queryBuildPack(bson.M{"rows": rows})
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
part = queryBuildPart{ID: fmt.Sprintf("%s/%09d", key, st.Parts), Job: inputGeneration, Digest: hash, Data: data}
|
||||
next.Bytes += len(data)
|
||||
if e = s.queryBuildPace(ctx, len(data)); e != nil {
|
||||
return e
|
||||
}
|
||||
}
|
||||
if err = s.queryBuildTransaction(ctx, l, func(sc mongo.SessionContext) error {
|
||||
if len(rows) > 0 {
|
||||
if _, e := s.db.Collection(queryBuildPartsCollection).ReplaceOne(sc, bson.M{"_id": part.ID}, part, options.Replace().SetUpsert(true)); e != nil {
|
||||
return e
|
||||
}
|
||||
}
|
||||
_, e := s.db.Collection(queryBuildJobsCollection).ReplaceOne(sc, bson.M{"_id": key}, next, options.Replace().SetUpsert(true))
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
if j.ID != "" {
|
||||
_, e = s.db.Collection(queryBuildJobsCollection).UpdateOne(sc, bson.M{"_id": j.ID, "generation": j.Generation}, bson.M{"$set": bson.M{"progressAt": time.Now().UTC(), "activeWithoutProgress": int64(0)}})
|
||||
}
|
||||
return e
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
st = next
|
||||
if err = queryBuildAppendRows(ctx, rows, target, p); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
108
internal/store/query_build_capture_local_test.go
Normal file
108
internal/store/query_build_capture_local_test.go
Normal file
@ -0,0 +1,108 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
"go.mongodb.org/mongo-driver/event"
|
||||
)
|
||||
|
||||
func TestQueryBuildCaptureResumesNativeCursorWithoutFullRereadLocal(t *testing.T) {
|
||||
var finds atomic.Int64
|
||||
monitor := &event.CommandMonitor{Started: func(_ context.Context, e *event.CommandStartedEvent) {
|
||||
if e.CommandName == "find" {
|
||||
if n, ok := e.Command.Lookup("find").StringValueOK(); ok && n == "captureFixture" {
|
||||
finds.Add(1)
|
||||
}
|
||||
}
|
||||
}}
|
||||
_, s, _ := newLocalReplica(t, nil, monitor)
|
||||
ctx := t.Context()
|
||||
docs := make([]any, 401)
|
||||
for i := range docs {
|
||||
docs[i] = bson.M{"_id": fmt.Sprintf("row-%03d", i), "value": i}
|
||||
}
|
||||
c := s.db.Collection("captureFixture")
|
||||
if _, err := c.InsertMany(ctx, docs); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
lease, ok, err := s.acquireQueryModelLease(ctx)
|
||||
if err != nil || !ok {
|
||||
t.Fatal(ok, err)
|
||||
}
|
||||
at, err := s.queryBuildSnapshot(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
j := &queryBuildJob{Generation: "capture-unit", Snapshot: at}
|
||||
start := time.Now()
|
||||
checks := 0
|
||||
round := &queryBuildRound{until: start.Add(time.Second), now: func() time.Time {
|
||||
checks++
|
||||
if checks > 1 {
|
||||
return start.Add(2 * time.Second)
|
||||
}
|
||||
return start
|
||||
}}
|
||||
type row struct {
|
||||
ID string `bson:"_id"`
|
||||
Value int `bson:"value"`
|
||||
}
|
||||
var first []row
|
||||
err = s.queryBuildCapture(ctx, lease, j, round, c, bson.M{}, bson.M{"value": 1}, &first)
|
||||
if !errors.Is(err, ErrQueryBuildYielded) || len(first) != 200 || finds.Load() != 1 {
|
||||
t.Fatal("first round did not preserve one bounded native page", len(first), finds.Load(), err)
|
||||
}
|
||||
// Later native changes cannot be mixed into the already selected snapshot.
|
||||
if _, err = c.UpdateOne(ctx, bson.M{"_id": "row-000"}, bson.M{"$set": bson.M{"value": 9999}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
restarted := NewMongoStore(s.db)
|
||||
restarted.queryReplicaFactStore = true
|
||||
var got []row
|
||||
if err = restarted.queryBuildCapture(ctx, lease, j, newQueryBuildRound(true), c, bson.M{}, bson.M{"value": 1}, &got); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(got) != 401 || got[0].Value != 0 || got[400].Value != 400 || finds.Load() != 4 {
|
||||
t.Fatalf("native prefix re-read or snapshot mixed: rows=%d finds=%d first=%+v", len(got), finds.Load(), got[0])
|
||||
}
|
||||
before := finds.Load()
|
||||
var repeated []row
|
||||
if err = restarted.queryBuildCapture(ctx, lease, j, newQueryBuildRound(true), c, bson.M{}, bson.M{"value": 1}, &repeated); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if finds.Load() != before || !reflect.DeepEqual(got, repeated) {
|
||||
t.Fatal("complete input capture repeated native reads")
|
||||
}
|
||||
}
|
||||
|
||||
func TestQueryBuildCaptureKeepsLegacyProjectionLocal(t *testing.T) {
|
||||
_, s, _ := newLocalReplica(t)
|
||||
ctx := t.Context()
|
||||
c := s.db.Collection("captureProjectionFixture")
|
||||
if _, err := c.InsertOne(ctx, bson.M{"_id": "private-native-id", "value": 1}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
l, ok, err := s.acquireQueryModelLease(ctx)
|
||||
if err != nil || !ok {
|
||||
t.Fatal(err)
|
||||
}
|
||||
at, err := s.queryBuildSnapshot(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
j := &queryBuildJob{Generation: "projection", Snapshot: at}
|
||||
var rows []bson.M
|
||||
if err = s.queryBuildCapture(ctx, l, j, newQueryBuildRound(true), c, bson.M{}, bson.M{"_id": 0, "value": 1}, &rows); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(rows) != 1 || rows[0]["_id"] != nil || rows[0]["value"] != int32(1) {
|
||||
t.Fatal("native cursor identity changed the loader projection", rows)
|
||||
}
|
||||
}
|
||||
48
internal/store/query_build_fact_document.go
Normal file
48
internal/store/query_build_fact_document.go
Normal file
@ -0,0 +1,48 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"droboticflow/internal/queryserving"
|
||||
"droboticflow/internal/workforcestats"
|
||||
"encoding/json"
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
"time"
|
||||
)
|
||||
|
||||
func queryBuildFactDocument(org string, epoch int64, f workforcestats.Fact) (bson.M, string, int, error) {
|
||||
raw, err := json.Marshal(f)
|
||||
if err != nil {
|
||||
return nil, "", 0, err
|
||||
}
|
||||
hash := queryserving.Key(workforceFactSchema, string(raw))
|
||||
encoded, err := bson.Marshal(f)
|
||||
if err != nil {
|
||||
return nil, "", 0, err
|
||||
}
|
||||
var doc bson.M
|
||||
if err = bson.Unmarshal(encoded, &doc); err != nil {
|
||||
return nil, "", 0, err
|
||||
}
|
||||
at, err := time.Parse(time.RFC3339Nano, f.CreatedAt)
|
||||
if err != nil {
|
||||
return nil, "", 0, err
|
||||
}
|
||||
doc["_id"], doc["organizationId"], doc["fromEpoch"], doc["hash"], doc["at"] = queryserving.Key(workforceFactSchema, org, f.Key, epoch), org, epoch, hash, at
|
||||
doc["atSeconds"], doc["atNanos"] = at.Unix(), at.Nanosecond()
|
||||
doc["metric"] = workforcestats.MetricCounterValues(f.Metric)
|
||||
doc["order"] = ""
|
||||
if f.Detail != nil {
|
||||
raw, e := json.Marshal(f.Detail)
|
||||
if e != nil {
|
||||
return nil, "", 0, e
|
||||
}
|
||||
var detail bson.M
|
||||
if e = bson.UnmarshalExtJSON(raw, false, &detail); e != nil {
|
||||
return nil, "", 0, e
|
||||
}
|
||||
doc["detail"] = detail
|
||||
doc["return"] = bson.M{"id": f.Detail.ID, "stage": f.Detail.Stage, "taskIds": f.Detail.TaskIDs, "metrics": detail["metrics"]}
|
||||
doc["order"] = f.Detail.ID + f.Detail.UserID + f.Detail.SupplierID
|
||||
}
|
||||
stored, err := bson.Marshal(doc)
|
||||
return doc, hash, len(stored), err
|
||||
}
|
||||
103
internal/store/query_build_jobs.go
Normal file
103
internal/store/query_build_jobs.go
Normal file
@ -0,0 +1,103 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
"droboticflow/internal/queryserving"
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
"go.mongodb.org/mongo-driver/bson/primitive"
|
||||
"go.mongodb.org/mongo-driver/mongo"
|
||||
"go.mongodb.org/mongo-driver/mongo/options"
|
||||
"go.mongodb.org/mongo-driver/mongo/readconcern"
|
||||
"go.mongodb.org/mongo-driver/mongo/writeconcern"
|
||||
)
|
||||
|
||||
const queryBuildJobsCollection = "queryBuildJobs"
|
||||
const queryBuildPartsCollection = "queryBuildParts"
|
||||
const queryBuildJobSchema = "query-build-job/v1"
|
||||
|
||||
type queryBuildJob struct {
|
||||
ID string `bson:"_id"`
|
||||
Schema string `bson:"schema"`
|
||||
InputGeneration string `bson:"inputGeneration"`
|
||||
Generation string `bson:"generation"`
|
||||
Organization string `bson:"organizationId"`
|
||||
Projects []string `bson:"projects"`
|
||||
Cold bool `bson:"cold"`
|
||||
Phase string `bson:"phase"`
|
||||
Snapshot primitive.Timestamp `bson:"snapshot"`
|
||||
AsOf time.Time `bson:"asOf"`
|
||||
SourceRevision int64 `bson:"sourceRevision"`
|
||||
GlobalRevision int64 `bson:"globalRevision"`
|
||||
OldEpoch int64 `bson:"oldEpoch"`
|
||||
Epoch int64 `bson:"epoch"`
|
||||
Production QueryModelHead `bson:"production"`
|
||||
PlanParts int `bson:"planParts"`
|
||||
PlanRows int `bson:"planRows"`
|
||||
PlanDigest string `bson:"planDigest"`
|
||||
PlanSaved int `bson:"planSaved"`
|
||||
PlanBytes int64 `bson:"planBytes"`
|
||||
ProductionSaved int `bson:"productionSaved"`
|
||||
NextPart int `bson:"nextPart"`
|
||||
NextRow int `bson:"nextRow"`
|
||||
Changed bool `bson:"changed"`
|
||||
CleanupHasAfter bool `bson:"cleanupHasAfter"`
|
||||
CleanupPlanPart int `bson:"cleanupPlanPart"`
|
||||
CleanupPlanRow int `bson:"cleanupPlanRow"`
|
||||
CleanupAfter string `bson:"cleanupAfter"`
|
||||
CleanupFuture bool `bson:"cleanupFuture"`
|
||||
ConfirmedRows int64 `bson:"confirmedRows"`
|
||||
ConfirmedBytes int64 `bson:"confirmedBytes"`
|
||||
ReusedRows int64 `bson:"reusedRows"`
|
||||
ActiveWithoutProgress time.Duration `bson:"activeWithoutProgress"`
|
||||
ProgressAt time.Time `bson:"progressAt"`
|
||||
FailurePhase string `bson:"failurePhase,omitempty"`
|
||||
FailureCode int32 `bson:"failureCode,omitempty"`
|
||||
FailureClass string `bson:"failureClass,omitempty"`
|
||||
UpdatedAt time.Time `bson:"updatedAt"`
|
||||
}
|
||||
|
||||
func queryBuildJobKey(database, org string) string {
|
||||
return queryserving.Key(queryBuildJobSchema, database, normalizedOrganizationID(org))
|
||||
}
|
||||
|
||||
func (s *MongoStore) queryBuildJob(ctx context.Context, org string) (queryBuildJob, bool, error) {
|
||||
op, stop := queryBuildOperation(ctx)
|
||||
defer stop()
|
||||
var j queryBuildJob
|
||||
err := s.db.Collection(queryBuildJobsCollection).FindOne(op, bson.M{"_id": queryBuildJobKey(s.db.Name(), org)}).Decode(&j)
|
||||
if errors.Is(err, mongo.ErrNoDocuments) {
|
||||
return j, false, nil
|
||||
}
|
||||
if err == nil && (j.Schema != queryBuildJobSchema || j.Organization != normalizedOrganizationID(org)) {
|
||||
return j, false, ErrQueryReplicaUnsupported
|
||||
}
|
||||
return j, err == nil, err
|
||||
}
|
||||
|
||||
// Every checkpoint and the corresponding rows/parts commit together. The same
|
||||
// global lease serializes publication and fences an old worker after takeover.
|
||||
func (s *MongoStore) queryBuildTransaction(ctx context.Context, l queryModelLease, fn func(mongo.SessionContext) error) error {
|
||||
op, stop := queryBuildOperation(ctx)
|
||||
defer stop()
|
||||
session, err := s.db.Client().StartSession()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer session.EndSession(op)
|
||||
_, err = session.WithTransaction(op, func(sc mongo.SessionContext) (any, error) {
|
||||
if err := s.renewQueryModelLease(sc, l); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return nil, fn(sc)
|
||||
}, options.Transaction().SetReadConcern(readconcern.Snapshot()).SetWriteConcern(writeconcern.New(writeconcern.WMajority(), writeconcern.J(true))))
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *MongoStore) saveQueryBuildJob(sc mongo.SessionContext, j queryBuildJob) error {
|
||||
j.UpdatedAt = time.Now().UTC()
|
||||
_, err := s.db.Collection(queryBuildJobsCollection).ReplaceOne(sc, bson.M{"_id": j.ID}, j, options.Replace().SetUpsert(true))
|
||||
return err
|
||||
}
|
||||
69
internal/store/query_build_lifecycle.go
Normal file
69
internal/store/query_build_lifecycle.go
Normal file
@ -0,0 +1,69 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Yield is successful bounded progress, not loss of the source safety permit.
|
||||
// Callers retain pending work and do not stop projector/builder replicas.
|
||||
var ErrQueryBuildYielded = errors.New("query build round yielded with durable progress retained")
|
||||
var ErrQueryBuildStopped = errors.New("query build stopped; durable checkpoint retained")
|
||||
var ErrQueryBuildNoProgress = errors.New("query build made no durable progress")
|
||||
|
||||
const (
|
||||
queryColdRoundBudget = time.Minute
|
||||
queryRefreshRoundBudget = 30 * time.Second
|
||||
queryBuildOperationBudget = 15 * time.Second
|
||||
queryBuildNoProgressLimit = 5 * time.Minute
|
||||
)
|
||||
|
||||
type queryBuildRound struct {
|
||||
until time.Time
|
||||
now func() time.Time
|
||||
}
|
||||
|
||||
func newQueryBuildRound(cold bool) *queryBuildRound {
|
||||
budget := queryRefreshRoundBudget
|
||||
if cold {
|
||||
budget = queryColdRoundBudget
|
||||
}
|
||||
return &queryBuildRound{until: time.Now().Add(budget), now: time.Now}
|
||||
}
|
||||
|
||||
func (r *queryBuildRound) check(ctx context.Context) error {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if r != nil && !r.now().Before(r.until) {
|
||||
return ErrQueryBuildYielded
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Operation deadlines bound a single read/commit. They do not shorten the
|
||||
// lifetime of the initial job or discard any previously confirmed checkpoint.
|
||||
func queryBuildOperation(ctx context.Context) (context.Context, context.CancelFunc) {
|
||||
return context.WithTimeout(ctx, queryBuildOperationBudget)
|
||||
}
|
||||
|
||||
func queryBuildPhaseFailure(phase string, err error) error {
|
||||
if err == nil || errors.Is(err, ErrQueryBuildYielded) {
|
||||
return err
|
||||
}
|
||||
return fmt.Errorf("query build %s: %w", phase, err)
|
||||
}
|
||||
|
||||
// Admission/checkpoint backpressure before a write is a resumable yield. A
|
||||
// failed durable transaction is never converted into success or budget yield.
|
||||
func (s *MongoStore) queryBuildPace(ctx context.Context, bytes int) error {
|
||||
op, stop := queryBuildOperation(ctx)
|
||||
defer stop()
|
||||
err := s.paceQueryBuild(op, bytes)
|
||||
if err != nil && ctx.Err() == nil && errors.Is(op.Err(), context.DeadlineExceeded) {
|
||||
return ErrQueryBuildYielded
|
||||
}
|
||||
return err
|
||||
}
|
||||
46
internal/store/query_build_lifecycle_test.go
Normal file
46
internal/store/query_build_lifecycle_test.go
Normal file
@ -0,0 +1,46 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestQueryBuildRoundBudgetIsYieldNotSafetyFailure(t *testing.T) {
|
||||
now := time.Now()
|
||||
r := &queryBuildRound{until: now, now: func() time.Time { return now }}
|
||||
err := r.check(t.Context())
|
||||
if !errors.Is(err, ErrQueryBuildYielded) || errors.Is(err, ErrQueryReplicaPaused) {
|
||||
t.Fatal("round budget became a global safety pause", err)
|
||||
}
|
||||
ctx, stop := context.WithCancel(t.Context())
|
||||
stop()
|
||||
if !errors.Is(r.check(ctx), context.Canceled) {
|
||||
t.Fatal("shutdown must preserve its cause")
|
||||
}
|
||||
}
|
||||
|
||||
func TestQueryBuildBudgetsAreIndependent(t *testing.T) {
|
||||
if queryColdRoundBudget == queryRefreshRoundBudget {
|
||||
t.Fatal("cold and freshness scheduling budgets coupled")
|
||||
}
|
||||
for _, cold := range []bool{false, true} {
|
||||
r := newQueryBuildRound(cold)
|
||||
if err := r.check(t.Context()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if queryBuildOperationBudget >= queryColdRoundBudget {
|
||||
t.Fatal("one operation consumed the entire cold round")
|
||||
}
|
||||
}
|
||||
|
||||
func TestQueryBuildPacingPreservesCallerCancellation(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
cancel()
|
||||
s := &MongoStore{}
|
||||
if err := s.queryBuildPace(ctx, 1024); !errors.Is(err, context.Canceled) || errors.Is(err, ErrQueryBuildYielded) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
196
internal/store/query_build_plan.go
Normal file
196
internal/store/query_build_plan.go
Normal file
@ -0,0 +1,196 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
"droboticflow/internal/domain"
|
||||
"droboticflow/internal/productionstats"
|
||||
"droboticflow/internal/workforcestats"
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
"go.mongodb.org/mongo-driver/mongo"
|
||||
"go.mongodb.org/mongo-driver/mongo/options"
|
||||
)
|
||||
|
||||
const queryBuildFactPageRows = 200
|
||||
|
||||
func queryBuildPlanPartKey(j queryBuildJob, kind string, n int) string {
|
||||
return fmt.Sprintf("%s/%s/%09d", j.Generation, kind, n)
|
||||
}
|
||||
func queryBuildJSONPart(j queryBuildJob, kind string, n int, v any) (queryBuildPart, error) {
|
||||
var b bytes.Buffer
|
||||
z := gzip.NewWriter(&b)
|
||||
if err := json.NewEncoder(z).Encode(v); err != nil {
|
||||
return queryBuildPart{}, err
|
||||
}
|
||||
if err := z.Close(); err != nil {
|
||||
return queryBuildPart{}, err
|
||||
}
|
||||
data := b.Bytes()
|
||||
sum := sha256.Sum256(data)
|
||||
return queryBuildPart{ID: queryBuildPlanPartKey(j, kind, n), Job: j.Generation, Data: data, Digest: hex.EncodeToString(sum[:])}, nil
|
||||
}
|
||||
func queryBuildJSONDecode(p queryBuildPart, out any) error {
|
||||
sum := sha256.Sum256(p.Data)
|
||||
if hex.EncodeToString(sum[:]) != p.Digest {
|
||||
return errors.New("query plan digest mismatch")
|
||||
}
|
||||
z, err := gzip.NewReader(bytes.NewReader(p.Data))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer z.Close()
|
||||
raw, err := io.ReadAll(io.LimitReader(z, (64<<20)+1))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(raw) > 64<<20 {
|
||||
return errors.New("query plan decode budget exceeded")
|
||||
}
|
||||
return json.Unmarshal(raw, out)
|
||||
}
|
||||
|
||||
func (s *MongoStore) queryBuildSavePart(ctx context.Context, l queryModelLease, j *queryBuildJob, p queryBuildPart) error {
|
||||
if err := s.queryBuildPace(ctx, len(p.Data)); err != nil {
|
||||
return err
|
||||
}
|
||||
return s.queryBuildTransaction(ctx, l, func(sc mongo.SessionContext) error {
|
||||
if _, err := s.db.Collection(queryBuildPartsCollection).ReplaceOne(sc, bson.M{"_id": p.ID}, p, options.Replace().SetUpsert(true)); err != nil {
|
||||
return err
|
||||
}
|
||||
return s.saveQueryBuildJob(sc, *j)
|
||||
})
|
||||
}
|
||||
|
||||
func (s *MongoStore) queryBuildProductionPlan(ctx context.Context, l queryModelLease, j *queryBuildJob, r *queryBuildRound) error {
|
||||
readCtx := context.WithValue(ctx, queryBuildReadKey{}, queryBuildReadFunc(func(c context.Context, coll mongoCollection, f, p bson.M, out any) error {
|
||||
return s.queryBuildCapture(c, l, j, r, coll, f, p, out)
|
||||
}))
|
||||
snapshot, err := s.loadProductionStats(readCtx, j.Organization, j.Projects)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
snapshot.AsOf = j.AsOf.Format(time.RFC3339Nano)
|
||||
// Preserve the existing JSON/gzip manifest format for all old readers.
|
||||
var data bytes.Buffer
|
||||
z := gzip.NewWriter(&data)
|
||||
budget := &queryModelBudgetWriter{writer: z}
|
||||
if err = json.NewEncoder(budget).Encode(productionstats.Build(snapshot)); err != nil {
|
||||
return err
|
||||
}
|
||||
if err = z.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
if data.Len() > queryModelMaxBytes {
|
||||
return errors.New("production plan exceeds existing size budget")
|
||||
}
|
||||
payload := data.Bytes()
|
||||
sum := sha256.Sum256(payload)
|
||||
head := QueryModelHead{Generation: j.Generation + "-production", AsOf: j.AsOf, Bytes: len(payload), ChunkBytes: queryModelBuildChunkBytes, Chunks: (len(payload) + queryModelBuildChunkBytes - 1) / queryModelBuildChunkBytes, Digest: hex.EncodeToString(sum[:]), SourceRevision: j.SourceRevision, GlobalRevision: j.GlobalRevision}
|
||||
if j.Production.Generation != "" && j.Production.Digest != head.Digest {
|
||||
return errors.New("sealed production input changed")
|
||||
}
|
||||
for n := j.ProductionSaved; n < head.Chunks; n++ {
|
||||
if err = r.check(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
p := payload[n*head.ChunkBytes : min((n+1)*head.ChunkBytes, len(payload))]
|
||||
if err = s.queryBuildPace(ctx, len(p)); err != nil {
|
||||
return err
|
||||
}
|
||||
next := *j
|
||||
next.ProgressAt = time.Now().UTC()
|
||||
next.ActiveWithoutProgress = 0
|
||||
next.Production = head
|
||||
next.ProductionSaved = n + 1
|
||||
if err = s.queryBuildTransaction(ctx, l, func(sc mongo.SessionContext) error {
|
||||
key := queryModelBlobKey(queryModelKey("production", j.Organization, j.Projects), head.Generation, n)
|
||||
_, e := s.queryData().ReplaceOne(sc, bson.M{"_id": key}, bson.M{"_id": key, "generation": head.Generation, "data": p, "createdAt": time.Now().UTC()}, options.Replace().SetUpsert(true))
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
return s.saveQueryBuildJob(sc, next)
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
*j = next
|
||||
}
|
||||
j.Phase = "workforce_plan"
|
||||
return s.queryBuildTransaction(ctx, l, func(sc mongo.SessionContext) error { return s.saveQueryBuildJob(sc, *j) })
|
||||
}
|
||||
|
||||
func (s *MongoStore) queryBuildWorkforcePlan(ctx context.Context, l queryModelLease, j *queryBuildJob, r *queryBuildRound) error {
|
||||
readCtx := context.WithValue(ctx, queryBuildReadKey{}, queryBuildReadFunc(func(c context.Context, coll mongoCollection, f, p bson.M, out any) error {
|
||||
return s.queryBuildCapture(c, l, j, r, coll, f, p, out)
|
||||
}))
|
||||
readCtx = context.WithValue(readCtx, workforceBuildBudgetKey{}, &workforceBuildBudget{})
|
||||
input, err := s.LoadWorkforceStats(readCtx, j.Organization, workforcestats.Query{Mode: domain.RunModeAll})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
facts := workforcestats.Materialize(input)
|
||||
sort.Slice(facts, func(i, j int) bool { return facts[i].Key < facts[j].Key })
|
||||
count := (len(facts) + queryBuildFactPageRows - 1) / queryBuildFactPageRows
|
||||
if j.PlanRows != 0 && j.PlanRows != len(facts) {
|
||||
return errors.New("workforce plan input changed")
|
||||
}
|
||||
// Recompute only from durable captured input, and verify the sealed prefix
|
||||
// before adding any later page (including after a code deployment).
|
||||
prefix := ""
|
||||
for n := 0; n < j.PlanSaved; n++ {
|
||||
part, e := queryBuildJSONPart(*j, "facts", n, facts[n*queryBuildFactPageRows:min((n+1)*queryBuildFactPageRows, len(facts))])
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
var saved queryBuildPart
|
||||
op, stop := queryBuildOperation(ctx)
|
||||
e = s.db.Collection(queryBuildPartsCollection).FindOne(op, bson.M{"_id": part.ID, "job": j.Generation}).Decode(&saved)
|
||||
stop()
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
if saved.Digest != part.Digest {
|
||||
return errors.New("sealed workforce plan fingerprint changed")
|
||||
}
|
||||
chain := sha256.Sum256([]byte(prefix + saved.Digest))
|
||||
prefix = hex.EncodeToString(chain[:])
|
||||
}
|
||||
if prefix != j.PlanDigest {
|
||||
return errors.New("sealed workforce plan manifest changed")
|
||||
}
|
||||
for n := j.PlanSaved; n < count; n++ {
|
||||
if err = r.check(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
part, e := queryBuildJSONPart(*j, "facts", n, facts[n*queryBuildFactPageRows:min((n+1)*queryBuildFactPageRows, len(facts))])
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
next := *j
|
||||
next.ProgressAt = time.Now().UTC()
|
||||
next.ActiveWithoutProgress = 0
|
||||
next.PlanParts = count
|
||||
next.PlanRows = len(facts)
|
||||
next.PlanSaved = n + 1
|
||||
next.PlanBytes += int64(len(part.Data))
|
||||
chain := sha256.Sum256([]byte(next.PlanDigest + part.Digest))
|
||||
next.PlanDigest = hex.EncodeToString(chain[:])
|
||||
if err = s.queryBuildSavePart(ctx, l, &next, part); err != nil {
|
||||
return err
|
||||
}
|
||||
*j = next
|
||||
}
|
||||
j.PlanRows = len(facts)
|
||||
j.PlanParts = count
|
||||
j.Phase = "apply"
|
||||
return s.queryBuildTransaction(ctx, l, func(sc mongo.SessionContext) error { return s.saveQueryBuildJob(sc, *j) })
|
||||
}
|
||||
360
internal/store/query_build_resume_local_test.go
Normal file
360
internal/store/query_build_resume_local_test.go
Normal file
@ -0,0 +1,360 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"droboticflow/internal/domain"
|
||||
"droboticflow/internal/workforcestats"
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
"go.mongodb.org/mongo-driver/event"
|
||||
"go.mongodb.org/mongo-driver/mongo"
|
||||
)
|
||||
|
||||
func TestQueryBuildLifecycleRestartYieldAndTwoRefreshesLocal(t *testing.T) {
|
||||
var native, sourceReads, weakCommits atomic.Int64
|
||||
monitor := &event.CommandMonitor{Started: func(_ context.Context, e *event.CommandStartedEvent) {
|
||||
if e.CommandName == "find" {
|
||||
name, _ := e.Command.Lookup("find").StringValueOK()
|
||||
if querySourceCollection(name) {
|
||||
native.Add(1)
|
||||
}
|
||||
}
|
||||
if e.CommandName == "commitTransaction" {
|
||||
var d struct {
|
||||
WC struct {
|
||||
W any `bson:"w"`
|
||||
J bool `bson:"j"`
|
||||
} `bson:"writeConcern"`
|
||||
}
|
||||
_ = bson.Unmarshal(e.Command, &d)
|
||||
if d.WC.W != "majority" || !d.WC.J {
|
||||
weakCommits.Add(1)
|
||||
}
|
||||
}
|
||||
}}
|
||||
_, s, _ := newLocalReplica(t, &event.CommandMonitor{Started: func(context.Context, *event.CommandStartedEvent) { sourceReads.Add(1) }}, monitor)
|
||||
sourceReads.Store(0)
|
||||
ctx := t.Context()
|
||||
org := "workforce-test"
|
||||
seedWorkforceStats(t, s)
|
||||
events := []any{}
|
||||
seconds := 1.0
|
||||
for i := 0; i < 250; i++ {
|
||||
events = append(events, domain.AnnotationWorklogEventRecord{ID: fmt.Sprintf("resume-event-%04d", i), OrganizationID: org, TaskID: fmt.Sprintf("resume-task-%04d", i), ProjectID: "p", SupplierID: "s", UserID: "u", EventType: "annotation_submitted", Mode: domain.RunModeProduction, Revision: 1, VideoSeconds: &seconds, OutputDelta: 1, CreatedAt: time.Date(2026, 9, 28, 13, 0, 0, 0, time.UTC).Add(time.Duration(i) * time.Second).Format(time.RFC3339Nano)})
|
||||
}
|
||||
tasks := []any{}
|
||||
for i := 0; i < 250; i++ {
|
||||
id := fmt.Sprintf("resume-task-%04d", i)
|
||||
tasks = append(tasks, bson.M{"_id": id, "id": id, "organizationId": org, "projectId": "p", "supplierId": "s", "runId": "run"})
|
||||
}
|
||||
if _, e := s.annotationTasks.InsertMany(ctx, tasks); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if _, e := s.annotationWorklogEvents.InsertMany(ctx, events); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
s.queryReplicaFactStore = true
|
||||
s.queryPeriodic = true
|
||||
s.queryBuildWrites = nil
|
||||
if e := s.ensureWorkforceFactIndexes(ctx); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
l, ok, e := s.acquireQueryModelLease(ctx)
|
||||
if e != nil || !ok {
|
||||
t.Fatal(ok, e)
|
||||
}
|
||||
// Establish exactly the e4 confirmed, unpublished first epoch.
|
||||
if e = s.rebuildWorkforceFacts(ctx, l, org, 0); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
unpublishWorkforceFixture(t, s, org)
|
||||
weakCommits.Store(0)
|
||||
j, e := s.queryBuildStart(ctx, l, org, []string{"p"})
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if e = s.queryBuildProductionPlan(ctx, l, &j, newQueryBuildRound(true)); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if e = s.queryBuildWorkforcePlan(ctx, l, &j, newQueryBuildRound(true)); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
beforeNative := native.Load()
|
||||
// Allow one committed fact batch, then exhaust this round, not the job.
|
||||
calls := 0
|
||||
start := time.Now()
|
||||
r := &queryBuildRound{until: start.Add(time.Second), now: func() time.Time {
|
||||
calls++
|
||||
if calls > 2 {
|
||||
return start.Add(2 * time.Second)
|
||||
}
|
||||
return start
|
||||
}}
|
||||
if e = s.queryBuildApply(ctx, l, &j, r); !errors.Is(e, ErrQueryBuildYielded) {
|
||||
t.Fatal("expected successful yield", e)
|
||||
}
|
||||
saved, ok, e := s.queryBuildJob(ctx, org)
|
||||
if e != nil || !ok || saved.NextRow+saved.NextPart == 0 {
|
||||
t.Fatal("missing confirmed cursor", saved.NextRow, saved.NextPart, e)
|
||||
}
|
||||
if _, ok := s.readQueryModelHead(ctx, queryModelKey("production", org, []string{"p"}), time.Hour); ok {
|
||||
t.Fatal("partial generation published")
|
||||
}
|
||||
// Reacquire a new lease and use a fresh process object; old writer is fenced.
|
||||
if _, e = s.queryControl().UpdateOne(ctx, queryLeaseFilter(l), bson.M{"$set": bson.M{"until": time.Now().Add(-time.Second)}}); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
restarted := NewMongoStore(s.db)
|
||||
restarted.queryReplicaFactStore = true
|
||||
restarted.queryPeriodic = true
|
||||
l2, ok, e := restarted.acquireQueryModelLease(ctx)
|
||||
if e != nil || !ok {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if e = s.queryBuildTransaction(ctx, l, func(_ctx mongo.SessionContext) error { return nil }); e == nil {
|
||||
t.Fatal("old lease committed")
|
||||
}
|
||||
if e = restarted.queryBuildRunRound(ctx, l2, org, []string{"p"}, nil); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if native.Load() != beforeNative {
|
||||
t.Fatal("resume repeated raw input reads", beforeNative, native.Load())
|
||||
}
|
||||
completed, _, e := restarted.queryBuildJob(ctx, org)
|
||||
if e != nil || completed.Phase != "complete" || completed.ReusedRows == 0 || completed.ConfirmedRows != 0 {
|
||||
t.Fatal("confirmed e4 batches not reused", completed.ReusedRows, completed.ConfirmedRows, e)
|
||||
}
|
||||
first, _, e := restarted.workforceHead(ctx, org)
|
||||
if e != nil {
|
||||
t.Fatal(e, "head", first, "now", time.Now().UTC())
|
||||
}
|
||||
original, e := restarted.LoadWorkforceStats(ctx, org, workforcestats.Query{Mode: domain.RunModeAll})
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
expected := workforcestats.Build(original, workforcestats.Query{Mode: domain.RunModeAll})
|
||||
actual, used, e := restarted.computeWorkforceFactReport(ctx, org, workforcestats.Query{Mode: domain.RunModeAll}, first)
|
||||
if e != nil || !used || !reflect.DeepEqual(actual.Totals, expected.Totals) {
|
||||
t.Fatal("fact totals differ", e)
|
||||
}
|
||||
// A fresh caught-up observer proves unchanged native input; both daily
|
||||
// refreshes reuse that complete input instead of re-reading full histories.
|
||||
for round := 0; round < 2; round++ {
|
||||
if _, e = s.queryControl().UpdateOne(ctx, queryLeaseFilter(l2), bson.M{"$set": bson.M{"observedAt": time.Now().UTC()}}); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
before := native.Load()
|
||||
if e = restarted.queryBuildRunRound(ctx, l2, org, []string{"p"}, nil); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if native.Load() != before {
|
||||
t.Fatal("unchanged refresh reread native histories", round, native.Load()-before)
|
||||
}
|
||||
h, ok, e := restarted.workforceHead(ctx, org)
|
||||
if e != nil || !ok || h.FactEpoch != first.FactEpoch {
|
||||
t.Fatal("unchanged facts rewritten", h.FactEpoch, e)
|
||||
}
|
||||
}
|
||||
if weakCommits.Load() != 0 {
|
||||
t.Fatal("majority+journal weakened", weakCommits.Load())
|
||||
}
|
||||
if sourceReads.Load() != 0 {
|
||||
t.Fatal("build read original Mongo", sourceReads.Load())
|
||||
}
|
||||
if !IsPrivateDerivedCollection(queryBuildJobsCollection) || !IsPrivateDerivedCollection(queryBuildPartsCollection) {
|
||||
t.Fatal("internal build documents exposed")
|
||||
}
|
||||
}
|
||||
|
||||
func TestQueryBuildChangedFingerprintDeletionAndOldGenerationLocal(t *testing.T) {
|
||||
_, s, _ := newLocalReplica(t)
|
||||
ctx := t.Context()
|
||||
org := "workforce-test"
|
||||
seedWorkforceStats(t, s)
|
||||
s.queryReplicaFactStore = true
|
||||
s.queryPeriodic = true
|
||||
if e := s.ensureWorkforceFactIndexes(ctx); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
l, ok, e := s.acquireQueryModelLease(ctx)
|
||||
if e != nil || !ok {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if e = s.queryBuildRunRound(ctx, l, org, []string{"p"}, nil); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
old, _, e := s.workforceHead(ctx, org)
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
// Changed input and a deleted user are handled by the original loader,
|
||||
// including its historical actor fallback. Neither mutates source records.
|
||||
if _, e = s.annotationWorklogEvents.UpdateMany(ctx, bson.M{"organizationId": org}, bson.M{"$set": bson.M{"videoSeconds": 999.0}}); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if _, e = s.users.DeleteMany(ctx, bson.M{"organizationId": org}); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if _, e = s.queryControl().UpdateOne(ctx, queryLeaseFilter(l), bson.M{"$inc": bson.M{"productionRevision": 1}}); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
j, e := s.queryBuildStart(ctx, l, org, []string{"p"})
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if e = s.queryBuildProductionPlan(ctx, l, &j, newQueryBuildRound(false)); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if e = s.queryBuildWorkforcePlan(ctx, l, &j, newQueryBuildRound(false)); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
// A deliberately wrong confirmed future fingerprint must be replaced, while
|
||||
// an orphan from an old attempt must be removed only from the unpublished view.
|
||||
page, e := s.queryBuildPlanPage(ctx, j, 0)
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
doc, _, _, e := queryBuildFactDocument(org, j.Epoch, page[0])
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
doc["hash"] = "outdated"
|
||||
if _, e = s.db.Collection(workforceFactCollection).InsertOne(ctx, doc); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if _, e = s.db.Collection(workforceFactCollection).InsertOne(ctx, bson.M{"_id": "orphan-future", "organizationId": org, "key": "zz-orphan", "hash": "obsolete", "fromEpoch": j.Epoch}); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if e = s.queryBuildApply(ctx, l, &j, newQueryBuildRound(false)); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
// Partial writes have not changed either published manifest.
|
||||
h, _, e := s.workforceHead(ctx, org)
|
||||
if e != nil || h.FactEpoch != old.FactEpoch {
|
||||
t.Fatal("partial generation visible", e)
|
||||
}
|
||||
cancelCtx, stop := context.WithCancel(ctx)
|
||||
stop()
|
||||
if e = s.queryBuildRunRound(cancelCtx, l, org, []string{"p"}, nil); !errors.Is(e, context.Canceled) {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if e = s.queryBuildRunRound(ctx, l, org, []string{"p"}, nil); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
h, _, e = s.workforceHead(ctx, org)
|
||||
if e != nil || h.FactEpoch != old.FactEpoch+1 {
|
||||
t.Fatal(e)
|
||||
}
|
||||
filters := []workforcestats.Query{
|
||||
{Mode: domain.RunModeAll}, {Mode: domain.RunModeProduction},
|
||||
{Mode: domain.RunModeAll, UserID: "u", MatchUser: true},
|
||||
{Mode: domain.RunModeAll, AllowedProjects: map[string]bool{}},
|
||||
{Mode: domain.RunModeAll, AllowedSuppliers: map[string]bool{}},
|
||||
{Mode: domain.RunModeAll, SupplierID: "s"},
|
||||
{Mode: domain.RunModeAll, From: time.Date(2026, 9, 28, 0, 0, 0, 1, time.UTC), To: time.Date(2026, 9, 29, 0, 0, 0, 0, time.UTC)},
|
||||
}
|
||||
for _, q := range filters {
|
||||
input, e := s.LoadWorkforceStats(ctx, org, q)
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
want := workforcestats.Build(input, q)
|
||||
got, _, e := s.computeWorkforceFactReport(ctx, org, q, h)
|
||||
if e != nil || !reflect.DeepEqual(got.Totals, want.Totals) {
|
||||
t.Fatal("permissions/date/legacy parity", q, e, got.Totals, want.Totals)
|
||||
}
|
||||
}
|
||||
n, e := s.db.Collection(workforceFactCollection).CountDocuments(ctx, bson.M{"_id": "orphan-future"})
|
||||
if e != nil || n != 0 {
|
||||
t.Fatal("unmatched staged row survived", n, e)
|
||||
}
|
||||
}
|
||||
|
||||
func TestQueryBuildNoProgressStopsButPauseTimeDoesNotLocal(t *testing.T) {
|
||||
_, s, _ := newLocalReplica(t)
|
||||
ctx := t.Context()
|
||||
s.queryReplicaFactStore = true
|
||||
s.queryPeriodic = true
|
||||
l, ok, e := s.acquireQueryModelLease(ctx)
|
||||
if e != nil || !ok {
|
||||
t.Fatal(e)
|
||||
}
|
||||
j, e := s.queryBuildStart(ctx, l, "synthetic-no-progress", nil)
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
// An overnight inactive gap is not a five minute active stall.
|
||||
j.ProgressAt = time.Now().Add(-24 * time.Hour)
|
||||
if e = s.queryBuildTransaction(ctx, l, func(sc mongo.SessionContext) error { return s.saveQueryBuildJob(sc, j) }); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
now := time.Now()
|
||||
expired := func() *queryBuildRound { return &queryBuildRound{until: now, now: func() time.Time { return now }} }
|
||||
if e = s.queryBuildRunRound(ctx, l, j.Organization, nil, expired()); !errors.Is(e, ErrQueryBuildYielded) || errors.Is(e, ErrQueryBuildStopped) {
|
||||
t.Fatal("pause treated as stall", e)
|
||||
}
|
||||
j, _, e = s.queryBuildJob(ctx, j.Organization)
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
j.ActiveWithoutProgress = queryBuildNoProgressLimit
|
||||
if e = s.queryBuildTransaction(ctx, l, func(sc mongo.SessionContext) error { return s.saveQueryBuildJob(sc, j) }); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if e = s.queryBuildRunRound(ctx, l, j.Organization, nil, expired()); !errors.Is(e, ErrQueryBuildStopped) || !errors.Is(e, ErrQueryBuildNoProgress) {
|
||||
t.Fatal("real active stall ignored", e)
|
||||
}
|
||||
if e = s.queryBuildRunRound(ctx, l, j.Organization, nil, nil); !errors.Is(e, ErrQueryBuildStopped) {
|
||||
t.Fatal("terminal failure automatically resumed", e)
|
||||
}
|
||||
}
|
||||
|
||||
func TestQueryBuildSealedInputSurvivesExpiredSnapshotAndOrphanGCLocal(t *testing.T) {
|
||||
_, s, _ := newLocalReplica(t)
|
||||
ctx := t.Context()
|
||||
org := "workforce-test"
|
||||
seedWorkforceStats(t, s)
|
||||
s.queryReplicaFactStore = true
|
||||
s.queryPeriodic = true
|
||||
if e := s.ensureWorkforceFactIndexes(ctx); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
l, ok, e := s.acquireQueryModelLease(ctx)
|
||||
if e != nil || !ok {
|
||||
t.Fatal(e)
|
||||
}
|
||||
j, e := s.queryBuildStart(ctx, l, org, []string{"p"})
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if e = s.queryBuildProductionPlan(ctx, l, &j, newQueryBuildRound(true)); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if e = s.queryBuildWorkforcePlan(ctx, l, &j, newQueryBuildRound(true)); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
// Once sealed, no original native snapshot is needed for continuation.
|
||||
j.Snapshot.T = 1
|
||||
if e = s.queryBuildTransaction(ctx, l, func(sc mongo.SessionContext) error { return s.saveQueryBuildJob(sc, j) }); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if _, e = s.queryData().UpdateMany(ctx, bson.M{"generation": j.Production.Generation}, bson.M{"$set": bson.M{"createdAt": time.Now().Add(-24 * time.Hour)}}); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if e = s.cleanQueryModelOrphans(ctx); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if e = s.queryBuildRunRound(ctx, l, org, []string{"p"}, nil); e != nil {
|
||||
t.Fatal("sealed work lost after snapshot expiry", e)
|
||||
}
|
||||
n, e := s.queryData().CountDocuments(ctx, bson.M{"generation": j.Production.Generation})
|
||||
if e != nil || n != int64(j.Production.Chunks) {
|
||||
t.Fatal("GC deleted unfinished durable blob", n, e)
|
||||
}
|
||||
}
|
||||
197
internal/store/query_build_runner.go
Normal file
197
internal/store/query_build_runner.go
Normal file
@ -0,0 +1,197 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
"droboticflow/internal/queryserving"
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
"go.mongodb.org/mongo-driver/mongo"
|
||||
"go.mongodb.org/mongo-driver/mongo/options"
|
||||
)
|
||||
|
||||
// This lifecycle is exclusive to the independent, periodic read database.
|
||||
// Authority/legacy callers keep their existing loader and publication behavior.
|
||||
func (s *MongoStore) queryBuildStart(ctx context.Context, l queryModelLease, org string, projects []string) (queryBuildJob, error) {
|
||||
org = normalizedOrganizationID(org)
|
||||
projects = append([]string{}, projects...)
|
||||
sort.Strings(projects)
|
||||
old, exists, err := s.queryBuildJob(ctx, org)
|
||||
if err != nil {
|
||||
return old, err
|
||||
}
|
||||
if exists && old.Phase != "complete" {
|
||||
if old.FailureClass != "" {
|
||||
return old, fmt.Errorf("%w: phase=%s category=%s", ErrQueryBuildStopped, old.FailurePhase, old.FailureClass)
|
||||
}
|
||||
if !reflect.DeepEqual(old.Projects, projects) {
|
||||
return old, errors.New("query build scope changed; retained job requires reconciliation")
|
||||
}
|
||||
return old, nil
|
||||
}
|
||||
observedAt := time.Now().UTC()
|
||||
at, err := s.queryBuildSnapshot(ctx)
|
||||
if err != nil {
|
||||
return old, err
|
||||
}
|
||||
var state struct {
|
||||
Revision int64 `bson:"productionRevision"`
|
||||
Global int64 `bson:"searchGlobalRevision"`
|
||||
Observed time.Time `bson:"observedAt"`
|
||||
Heads map[string]QueryModelHead `bson:"heads"`
|
||||
}
|
||||
rows, e := s.queryBuildReadPage(ctx, s.queryControl(), bson.M{"_id": "global"}, bson.M{"productionRevision": 1, "searchGlobalRevision": 1, "heads": 1, "observedAt": 1}, bson.RawValue{}, false, at)
|
||||
if e != nil {
|
||||
return old, e
|
||||
}
|
||||
if len(rows) != 1 {
|
||||
return old, errors.New("query build control unavailable at snapshot")
|
||||
}
|
||||
if e = bson.Unmarshal(rows[0], &state); e != nil {
|
||||
return old, e
|
||||
}
|
||||
head := state.Heads[workforceFactKey(org)]
|
||||
j := queryBuildJob{ID: queryBuildJobKey(s.db.Name(), org), Schema: queryBuildJobSchema, Organization: org, Projects: projects, Generation: queryserving.Key(queryBuildJobSchema, org, at.T, at.I, time.Now().UnixNano()), Snapshot: at, AsOf: observedAt, Phase: "production_plan", OldEpoch: head.FactEpoch, Epoch: head.FactEpoch + 1, Cold: head.FactEpoch == 0, SourceRevision: state.Revision, GlobalRevision: state.Global, ProgressAt: time.Now().UTC()}
|
||||
j.InputGeneration = j.Generation
|
||||
if exists && old.Phase == "complete" && reflect.DeepEqual(old.Projects, projects) && old.SourceRevision == state.Revision && old.GlobalRevision == state.Global && !state.Observed.Before(old.AsOf) && time.Since(state.Observed) >= 0 && time.Since(state.Observed) < 15*time.Second {
|
||||
j.AsOf = state.Observed
|
||||
j.InputGeneration = old.InputGeneration
|
||||
if j.InputGeneration == "" {
|
||||
j.InputGeneration = old.Generation
|
||||
}
|
||||
// The caught-up stream proves that every business input is unchanged.
|
||||
// Reconfirm the complete cohort without re-materializing or walking
|
||||
// 140k unchanged facts and checkpointing thousands of no-op batches.
|
||||
production, exists := state.Heads[queryModelKey("production", org, projects)]
|
||||
committed := old.Epoch
|
||||
if !old.Changed && old.OldEpoch > 0 {
|
||||
committed = old.OldEpoch
|
||||
}
|
||||
if exists && production.Generation != "" && head.Rows == old.PlanRows && head.FactEpoch == committed && production.SourceRevision == old.SourceRevision {
|
||||
j.Phase = "publish"
|
||||
j.PlanRows = old.PlanRows
|
||||
j.ReusedRows = int64(old.PlanRows)
|
||||
j.Production = production
|
||||
j.Production.AsOf = j.AsOf
|
||||
}
|
||||
}
|
||||
err = s.queryBuildTransaction(ctx, l, func(sc mongo.SessionContext) error { return s.saveQueryBuildJob(sc, j) })
|
||||
return j, err
|
||||
}
|
||||
|
||||
func (s *MongoStore) queryBuildRunRound(ctx context.Context, l queryModelLease, org string, projects []string, r *queryBuildRound) (err error) {
|
||||
j, err := s.queryBuildStart(ctx, l, org, projects)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if r == nil {
|
||||
r = newQueryBuildRound(j.Cold)
|
||||
}
|
||||
started := time.Now()
|
||||
before := j.ProgressAt
|
||||
defer func() {
|
||||
if err == nil || ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
// Maintenance/shutdown time is not counted as active lack of progress.
|
||||
current, ok, e := s.queryBuildJob(ctx, org)
|
||||
if e != nil || !ok {
|
||||
return
|
||||
}
|
||||
if !current.ProgressAt.After(before) {
|
||||
current.ActiveWithoutProgress += time.Since(started)
|
||||
} else {
|
||||
current.ActiveWithoutProgress = 0
|
||||
}
|
||||
if errors.Is(err, ErrQueryBuildYielded) && current.ActiveWithoutProgress >= queryBuildNoProgressLimit {
|
||||
err = ErrQueryBuildNoProgress
|
||||
}
|
||||
if !errors.Is(err, ErrQueryBuildYielded) {
|
||||
current.FailurePhase = current.Phase
|
||||
diagnostic := QueryReplicaDiagnostic(err)
|
||||
current.FailureClass = diagnostic.Class
|
||||
current.FailureCode = diagnostic.MongoCode
|
||||
if errors.Is(err, ErrQueryBuildNoProgress) {
|
||||
current.FailureClass = "no_durable_progress"
|
||||
}
|
||||
}
|
||||
if !errors.Is(err, ErrQueryBuildYielded) {
|
||||
err = fmt.Errorf("%w: %w", ErrQueryBuildStopped, err)
|
||||
}
|
||||
if e = s.queryBuildTransaction(ctx, l, func(sc mongo.SessionContext) error { return s.saveQueryBuildJob(sc, current) }); e != nil {
|
||||
err = fmt.Errorf("query build could not persist outcome: %w", e)
|
||||
}
|
||||
}()
|
||||
for {
|
||||
if err = r.check(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
switch j.Phase {
|
||||
case "production_plan":
|
||||
err = s.queryBuildProductionPlan(ctx, l, &j, r)
|
||||
case "workforce_plan":
|
||||
err = s.queryBuildWorkforcePlan(ctx, l, &j, r)
|
||||
case "apply":
|
||||
err = s.queryBuildApply(ctx, l, &j, r)
|
||||
case "cleanup":
|
||||
err = s.queryBuildCleanup(ctx, l, &j, r)
|
||||
case "publish":
|
||||
err = s.queryBuildPublish(ctx, l, &j)
|
||||
case "complete":
|
||||
return nil
|
||||
default:
|
||||
return errors.New("unsupported query build phase")
|
||||
}
|
||||
if err != nil {
|
||||
return queryBuildPhaseFailure(j.Phase, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Explicitly bounded derived-data housekeeping; active jobs are never swept.
|
||||
func (s *MongoStore) cleanQueryBuildParts(ctx context.Context, l queryModelLease) error {
|
||||
var jobs []queryBuildJob
|
||||
cur, err := s.db.Collection(queryBuildJobsCollection).Find(ctx, bson.M{"schema": queryBuildJobSchema}, options.Find().SetProjection(bson.M{"generation": 1, "inputGeneration": 1}))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = cur.All(ctx, &jobs)
|
||||
cur.Close(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
keep := []string{}
|
||||
for _, j := range jobs {
|
||||
keep = append(keep, j.Generation, j.InputGeneration)
|
||||
}
|
||||
for _, collection := range []string{queryBuildPartsCollection, queryBuildJobsCollection} {
|
||||
f := bson.M{"job": bson.M{"$exists": true, "$nin": keep}}
|
||||
cur, e := s.db.Collection(collection).Find(ctx, f, options.Find().SetProjection(bson.M{"_id": 1}).SetLimit(1000))
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
var rows []bson.M
|
||||
e = cur.All(ctx, &rows)
|
||||
cur.Close(ctx)
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
ids := bson.A{}
|
||||
for _, row := range rows {
|
||||
ids = append(ids, row["_id"])
|
||||
}
|
||||
if len(ids) > 0 {
|
||||
if e = s.queryBuildTransaction(ctx, l, func(sc mongo.SessionContext) error {
|
||||
_, e := s.db.Collection(collection).DeleteMany(sc, bson.M{"_id": bson.M{"$in": ids}, "job": bson.M{"$nin": keep}})
|
||||
return e
|
||||
}); e != nil {
|
||||
return e
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
226
internal/store/query_build_scale_local_test.go
Normal file
226
internal/store/query_build_scale_local_test.go
Normal file
@ -0,0 +1,226 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"droboticflow/internal/domain"
|
||||
"droboticflow/internal/workforcestats"
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
"go.mongodb.org/mongo-driver/event"
|
||||
"go.mongodb.org/mongo-driver/mongo"
|
||||
"go.mongodb.org/mongo-driver/mongo/options"
|
||||
"golang.org/x/time/rate"
|
||||
)
|
||||
|
||||
func TestQueryBuildDurableColdSameScaleLocal(t *testing.T) {
|
||||
if os.Getenv("QUERY_BUILD_LIFECYCLE_SCALE") != "1" {
|
||||
t.Skip("explicit 140k fact synthetic comparison")
|
||||
}
|
||||
var originalReads, nativeReads atomic.Int64
|
||||
_, s, _ := newLocalReplica(t, &event.CommandMonitor{Started: func(context.Context, *event.CommandStartedEvent) { originalReads.Add(1) }}, &event.CommandMonitor{Started: func(_ context.Context, e *event.CommandStartedEvent) {
|
||||
if e.CommandName == "find" {
|
||||
n, _ := e.Command.Lookup("find").StringValueOK()
|
||||
if querySourceCollection(n) {
|
||||
nativeReads.Add(1)
|
||||
}
|
||||
}
|
||||
}})
|
||||
originalReads.Store(0)
|
||||
ctx := t.Context()
|
||||
org := "synthetic-durable-scale"
|
||||
const count = 35000
|
||||
existing := 125186
|
||||
if os.Getenv("QUERY_BUILD_SCALE_EMPTY") == "1" {
|
||||
existing = 0
|
||||
}
|
||||
project := strings.Repeat("synthetic-project-", 4)
|
||||
at := time.Date(2026, 11, 1, 0, 0, 0, 0, time.UTC)
|
||||
seconds := 1.0
|
||||
for start := 0; start < count; start += 1000 {
|
||||
events, tasks := []any{}, []any{}
|
||||
for i := start; i < min(start+1000, count); i++ {
|
||||
id := fmt.Sprintf("task-%06d", i)
|
||||
events = append(events, domain.AnnotationWorklogEventRecord{ID: fmt.Sprintf("event-%06d", i), OrganizationID: org, ProjectID: project, TaskID: id, UserID: "actor", SupplierID: "supplier", EventType: "annotation_submitted", Mode: domain.RunModeProduction, Revision: 1, TemplateVersion: 1, VideoSeconds: &seconds, OutputDelta: 1, CreatedAt: at.Add(time.Duration(i) * time.Second).Format(time.RFC3339Nano)})
|
||||
tasks = append(tasks, bson.M{"_id": id, "id": id, "organizationId": org, "projectId": project, "supplierId": "supplier", "runId": "run"})
|
||||
}
|
||||
if _, e := s.annotationWorklogEvents.InsertMany(ctx, events); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if _, e := s.annotationTasks.InsertMany(ctx, tasks); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
}
|
||||
for name, doc := range map[string]any{"runs": bson.M{"_id": "run", "id": "run", "organizationId": org, "mode": "production"}, "users": domain.UserRecord{ID: "actor", OrganizationID: org, Name: strings.Repeat("合成用户", 16), SupplierID: "supplier", Role: domain.UserRoleAnnotator}, "suppliers": domain.SupplierRecord{ID: "supplier", OrganizationID: org, Name: "synthetic supplier", Status: "active"}} {
|
||||
if _, e := s.db.Collection(name).InsertOne(ctx, doc); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
}
|
||||
s.queryReplicaFactStore = true
|
||||
s.queryPeriodic = true
|
||||
if e := s.ensureWorkforceFactIndexes(ctx); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
l, ok, e := s.acquireQueryModelLease(ctx)
|
||||
if e != nil || !ok {
|
||||
t.Fatal(e)
|
||||
}
|
||||
// Generate the exact e4 staging rows; fixture setup alone is unpaced.
|
||||
if existing > 0 {
|
||||
if e = s.rebuildWorkforceFacts(ctx, l, org, 0); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
unpublishWorkforceFixture(t, s, org)
|
||||
c := s.db.Collection(workforceFactCollection)
|
||||
cur, e := c.Find(ctx, bson.M{"organizationId": org}, options.Find().SetProjection(bson.M{"_id": 1}).SetLimit(int64(count*4-existing)))
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
var ids []bson.M
|
||||
if e = cur.All(ctx, &ids); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
cur.Close(ctx)
|
||||
for start := 0; start < len(ids); start += 1000 {
|
||||
values := bson.A{}
|
||||
for _, id := range ids[start:min(start+1000, len(ids))] {
|
||||
values = append(values, id["_id"])
|
||||
}
|
||||
if _, e = c.DeleteMany(ctx, bson.M{"_id": bson.M{"$in": values}}); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
}
|
||||
}
|
||||
s.queryBuildWrites = s.newQueryBuildWriteGate()
|
||||
if s.queryBuildWrites.limiter.Limit() != rate.Limit(512<<10) {
|
||||
t.Fatal("pacing changed")
|
||||
}
|
||||
started := time.Now()
|
||||
rounds := 0
|
||||
// Keep renewing the same fenced lease, just as the production process does.
|
||||
runCtx, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
done := make(chan struct{})
|
||||
leaseStore := s
|
||||
go func() {
|
||||
defer close(done)
|
||||
ticker := time.NewTicker(10 * time.Second)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-runCtx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
if leaseStore.renewQueryModelLease(runCtx, l) != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
defer func() { cancel(); <-done }()
|
||||
for {
|
||||
rounds++
|
||||
e = s.queryBuildRunRound(ctx, l, org, []string{project}, nil)
|
||||
if e == nil {
|
||||
break
|
||||
}
|
||||
if !errors.Is(e, ErrQueryBuildYielded) {
|
||||
t.Fatal("cold round failed", rounds, e)
|
||||
}
|
||||
state, _, _ := s.queryBuildJob(ctx, org)
|
||||
t.Logf("yield round=%d phase=%s confirmed=%d reused=%d nextPart=%d", rounds, state.Phase, state.ConfirmedRows, state.ReusedRows, state.NextPart)
|
||||
if rounds > 30 {
|
||||
t.Fatal("synthetic cold build did not progress")
|
||||
}
|
||||
fresh := NewMongoStore(s.db)
|
||||
fresh.queryReplicaFactStore = true
|
||||
fresh.queryPeriodic = true
|
||||
fresh.queryBuildWrites = s.queryBuildWrites
|
||||
s = fresh
|
||||
}
|
||||
cold := time.Since(started)
|
||||
j, ok, e := s.queryBuildJob(ctx, org)
|
||||
if e != nil || !ok || j.PlanRows != count*4 || j.ReusedRows != int64(existing) || j.ConfirmedRows != int64(count*4-existing) {
|
||||
t.Fatal("same scale resume parity", j.PlanRows, j.ReusedRows, j.ConfirmedRows, e)
|
||||
}
|
||||
h, _, e := s.workforceHead(ctx, org)
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
q := workforcestats.Query{Mode: domain.RunModeAll, From: at, To: at.Add(24 * time.Hour)}
|
||||
report, _, e := s.computeWorkforceFactReport(ctx, org, q, h)
|
||||
if e != nil || report.Totals.FirstSubmittedTasks != count || report.Totals.FirstVideoSeconds != count {
|
||||
t.Fatal("same scale report differs", e)
|
||||
}
|
||||
baseline := time.Duration(0)
|
||||
if os.Getenv("QUERY_BUILD_SCALE_BASELINE") == "1" {
|
||||
if _, e = s.queryControl().UpdateOne(ctx, queryLeaseFilter(l), bson.M{"$set": bson.M{"observedAt": time.Now().UTC()}}); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
// Explicit algorithm comparison on the identical fixture: retain the
|
||||
// pre-fast-path walk of unchanged facts, including durable checkpoints.
|
||||
job, err := s.queryBuildStart(ctx, l, org, []string{project})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
job.Phase = "production_plan"
|
||||
job.PlanRows = 0
|
||||
job.ReusedRows = 0
|
||||
job.Production = QueryModelHead{}
|
||||
if err = s.queryBuildTransaction(ctx, l, func(sc mongo.SessionContext) error { return s.saveQueryBuildJob(sc, job) }); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
start := time.Now()
|
||||
for {
|
||||
err = s.queryBuildRunRound(ctx, l, org, []string{project}, nil)
|
||||
if err == nil {
|
||||
break
|
||||
}
|
||||
if !errors.Is(err, ErrQueryBuildYielded) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
baseline = time.Since(start)
|
||||
}
|
||||
warm := []time.Duration{}
|
||||
for i := 0; i < 2; i++ {
|
||||
if _, e = s.queryControl().UpdateOne(ctx, queryLeaseFilter(l), bson.M{"$set": bson.M{"observedAt": time.Now().UTC()}}); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
before := nativeReads.Load()
|
||||
start := time.Now()
|
||||
n := 0
|
||||
for {
|
||||
n++
|
||||
e = s.queryBuildRunRound(ctx, l, org, []string{project}, nil)
|
||||
if e == nil {
|
||||
break
|
||||
}
|
||||
if !errors.Is(e, ErrQueryBuildYielded) || n > 30 {
|
||||
t.Fatal("refresh failed", i, n, e)
|
||||
}
|
||||
}
|
||||
if nativeReads.Load() != before {
|
||||
t.Fatal("unchanged refresh rescanned native input", i, nativeReads.Load()-before)
|
||||
}
|
||||
warm = append(warm, time.Since(start))
|
||||
}
|
||||
if originalReads.Load() != 0 {
|
||||
t.Fatal("original Mongo reads", originalReads.Load())
|
||||
}
|
||||
var status struct {
|
||||
DataSize float64 `bson:"dataSize"`
|
||||
StorageSize float64 `bson:"storageSize"`
|
||||
IndexSize float64 `bson:"indexSize"`
|
||||
}
|
||||
if e = s.db.RunCommand(ctx, bson.M{"dbStats": 1}).Decode(&status); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
t.Logf("events=%d facts=%d reused=%d new=%d cold=%s rounds=%d walk_baseline=%s refresh1=%s refresh2=%s write_limit=%d confirmed_bson_bytes=%d original_mongo_reads=%d native_input_reads=%d data_bytes=%.0f storage_bytes=%.0f index_bytes=%.0f production_accepted=false", count, j.PlanRows, j.ReusedRows, j.ConfirmedRows, cold, rounds, baseline, warm[0], warm[1], 512<<10, j.ConfirmedBytes, originalReads.Load(), nativeReads.Load(), status.DataSize, status.StorageSize, status.IndexSize)
|
||||
}
|
||||
@ -41,6 +41,23 @@ func (s *MongoStore) cleanQueryModelOrphans(ctx context.Context) error {
|
||||
for _, h := range state.Heads {
|
||||
keep = append(keep, h.Generation)
|
||||
}
|
||||
if s.queryReplicaFactStore && s.periodicQueryStore() {
|
||||
cur, err := s.db.Collection(queryBuildJobsCollection).Find(ctx, bson.M{"schema": queryBuildJobSchema, "phase": bson.M{"$ne": "complete"}}, options.Find().SetProjection(bson.M{"production.generation": 1}))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var jobs []queryBuildJob
|
||||
err = cur.All(ctx, &jobs)
|
||||
cur.Close(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, j := range jobs {
|
||||
if j.Production.Generation != "" {
|
||||
keep = append(keep, j.Production.Generation)
|
||||
}
|
||||
}
|
||||
}
|
||||
filter := bson.M{"generation": bson.M{"$nin": keep}, "createdAt": bson.M{"$lt": time.Now().UTC().Add(-15 * time.Minute)}}
|
||||
// Ordinary orphan chunks/rows must still be reclaimed when Typesense is
|
||||
// offline. Retain only its collection intents until remote deletion succeeds.
|
||||
|
||||
@ -118,6 +118,13 @@ func (b *QueryReadBuilder) scopes(ctx context.Context) (map[string][]string, err
|
||||
return out, nil
|
||||
}
|
||||
func (b *QueryReadBuilder) rebuild(ctx context.Context, l queryModelLease, org string, projects []string) (err error) {
|
||||
if b.store.queryReplicaFactStore && b.store.periodicQueryStore() {
|
||||
err = b.store.queryBuildRunRound(ctx, l, org, projects, nil)
|
||||
if errors.Is(err, ErrQueryBuildYielded) {
|
||||
log.Print("query_model_build result=yielded checkpoint=retained")
|
||||
}
|
||||
return err
|
||||
}
|
||||
started := time.Now()
|
||||
stage := "source"
|
||||
defer func() {
|
||||
@ -178,7 +185,11 @@ func (b *QueryReadBuilder) rebuild(ctx context.Context, l queryModelLease, org s
|
||||
// RunOnce is for bounded initial prewarming and isolated tests, not a freshness
|
||||
// substitute for Run. After the heartbeat expires search falls back to old reads.
|
||||
func (b *QueryReadBuilder) RunOnce(parent context.Context) error {
|
||||
ctx, cancel := context.WithTimeout(parent, 5*time.Minute)
|
||||
ctx, cancel := context.WithCancel(parent)
|
||||
if !(b.store.queryReplicaFactStore && b.store.periodicQueryStore()) {
|
||||
cancel()
|
||||
ctx, cancel = context.WithTimeout(parent, 5*time.Minute)
|
||||
}
|
||||
defer cancel()
|
||||
started := make(chan struct{})
|
||||
done := make(chan error, 1)
|
||||
@ -418,7 +429,9 @@ func (b *QueryReadBuilder) run(ctx context.Context, started chan<- struct{}) err
|
||||
lastScopeRead = time.Now()
|
||||
}
|
||||
for org := range scopes {
|
||||
if all {
|
||||
// Even an idle organization needs periodic complete-result
|
||||
// reconfirmation; freshness must not depend on customer writes.
|
||||
if all || (b.store.periodicQueryStore() && queryBuildDue(time.Now(), lastStats[org], statsInterval)) {
|
||||
pendingStats[org] = true
|
||||
}
|
||||
if allSearch {
|
||||
@ -472,7 +485,12 @@ func (b *QueryReadBuilder) run(ctx context.Context, started chan<- struct{}) err
|
||||
delete(statsRetries, org)
|
||||
delete(pendingStats, org)
|
||||
lastStats[org] = time.Now()
|
||||
} else if errors.Is(e, ErrQueryBuildYielded) {
|
||||
delete(statsRetries, org)
|
||||
} else {
|
||||
if b.store.queryReplicaFactStore && b.store.periodicQueryStore() {
|
||||
return e
|
||||
}
|
||||
retry := statsRetries[org]
|
||||
retry.fail(30*time.Second, 5*time.Minute)
|
||||
statsRetries[org] = retry
|
||||
@ -487,6 +505,9 @@ func (b *QueryReadBuilder) run(ctx context.Context, started chan<- struct{}) err
|
||||
if time.Since(lastCleanup) > time.Minute {
|
||||
cleanup, finish := context.WithTimeout(ctx, 5*time.Second)
|
||||
_ = b.store.cleanQueryModelOrphans(cleanup)
|
||||
if b.store.queryReplicaFactStore && b.store.periodicQueryStore() {
|
||||
_ = b.store.cleanQueryBuildParts(cleanup, l)
|
||||
}
|
||||
|
||||
finish()
|
||||
lastCleanup = time.Now()
|
||||
|
||||
@ -47,6 +47,7 @@ func workforceBoundedRead(ctx context.Context, cur *mongo.Cursor, target any, bu
|
||||
func workforceFactKey(org string) string { return queryModelKey("workforce-facts", org, nil) }
|
||||
func (s *MongoStore) ensureWorkforceFactIndexes(ctx context.Context) error {
|
||||
indexes := []mongo.IndexModel{
|
||||
{Keys: bson.D{{Key: "organizationId", Value: 1}, {Key: "key", Value: 1}, {Key: "fromEpoch", Value: 1}, {Key: "toEpoch", Value: 1}}, Options: options.Index().SetName("workforce_fact_key_v1")},
|
||||
{Keys: bson.D{{Key: "organizationId", Value: 1}, {Key: "retiredAt", Value: 1}, {Key: "toEpoch", Value: 1}}, Options: options.Index().SetName("workforce_fact_retirement_v1")},
|
||||
{Keys: bson.D{{Key: "organizationId", Value: 1}, {Key: "mode", Value: 1}, {Key: "groupTemplate", Value: 1}, {Key: "userId", Value: 1}, {Key: "at", Value: -1}, {Key: "order", Value: 1}}, Options: options.Index().SetName("workforce_fact_user_v1")},
|
||||
{Keys: bson.D{{Key: "organizationId", Value: 1}, {Key: "mode", Value: 1}, {Key: "groupTemplate", Value: 1}, {Key: "supplierId", Value: 1}, {Key: "projectId", Value: 1}, {Key: "at", Value: -1}}, Options: options.Index().SetName("workforce_fact_scope_v1")},
|
||||
|
||||
@ -455,6 +455,9 @@ func (s *MongoStore) LoadWorkforceStats(ctx context.Context, org string, q workf
|
||||
return m
|
||||
}
|
||||
read := func(c mongoCollection, filter, projection bson.M, target any) error {
|
||||
if hook, ok := ctx.Value(queryBuildReadKey{}).(queryBuildReadFunc); ok {
|
||||
return hook(ctx, c, filter, projection, target)
|
||||
}
|
||||
cur, err := c.Find(ctx, filter, options.Find().SetProjection(projection).SetMaxTime(15*time.Second))
|
||||
if err != nil {
|
||||
return err
|
||||
|
||||
@ -31,6 +31,8 @@ CASES = ('cmd/query-replica/main_local_test.go',
|
||||
'internal/store/query_replica_valkey_local_test.go',
|
||||
'internal/store/workforce_report_cache_local_test.go',
|
||||
'internal/store/workforce_facts_local_test.go',
|
||||
'internal/store/query_build_capture_local_test.go',
|
||||
'internal/store/query_build_resume_local_test.go',
|
||||
'internal/store/workforce_publication_local_test.go',
|
||||
'internal/store/query_replica_page_local_test.go',
|
||||
'internal/store/supplier_authorization_batch_local_test.go',
|
||||
|
||||
@ -1044,6 +1044,7 @@
|
||||
"^internal/store/workforce_(report_cache|heap_diagnostic|facts|return_page|publication).*\\.go$",
|
||||
"^internal/store/query_pool_buckets.*\\.go$",
|
||||
"^internal/store/query_cold_write.*\\.go$",
|
||||
"^internal/store/query_build_.*\\.go$",
|
||||
"^internal/store/query_periodic.*\\.go$",
|
||||
"^internal/store/supplier_authorization_batch_local_test\\.go$",
|
||||
"^internal/api/query_periodic.*\\.go$",
|
||||
@ -1057,7 +1058,7 @@
|
||||
"^scripts/local/query-replica\\.query25-local\\.compose\\.yml$",
|
||||
"^scripts/local/query-replica\\.compose\\.yml$",
|
||||
"^Dockerfile\\.query-replica$",
|
||||
"^docs/(adr/0014-.*|operations/query-replica\\.md)$"
|
||||
"^docs/(adr/0014-.*|operations/query-replica\\.md|operations/query-build-lifecycle\\.md)$"
|
||||
],
|
||||
"frontend": [],
|
||||
"go": {
|
||||
@ -1066,7 +1067,10 @@
|
||||
"^TestQueryReplicaCountOnly",
|
||||
"^TestQueryReplicaMissingConfiguration",
|
||||
"^TestQueryReplicaRejectsSigned",
|
||||
"^TestQueryReplicaDiagnostic"
|
||||
"^TestQueryReplicaDiagnostic",
|
||||
"^TestQueryBuildRound",
|
||||
"^TestQueryBuildBudgets",
|
||||
"^TestQueryBuildPacing"
|
||||
],
|
||||
"./cmd/query-cdc-probe": ["^TestClassification"],
|
||||
"./cmd/query-replica": ["^TestPassDelay", "^TestPacing"],
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user