forked from longtao.wu/droboticflow
🐛 Bound review journal snapshot reads without extending budgets
Replace complementary journal scans with bounded seek pages and scoped primary-key problem refill. Preserve legacy/in-flight evidence, memory boundaries and original query/build deadlines. Cover scoped plans, source parity, cancellation, real refresh cadence and restart.
This commit is contained in:
parent
fc7786190f
commit
c0512c8bf6
27
docs/operations/review-operation-history-read.md
Normal file
27
docs/operations/review-operation-history-read.md
Normal file
@ -0,0 +1,27 @@
|
||||
# 统计审核操作历史读取超时
|
||||
|
||||
## 现场与口径
|
||||
|
||||
2026-11-14,线上 fc778619 的统计快照后台报告 `annotation output read reviewOperations ... MaxTimeMSExpired ... getMore`;快照健康出现 `partitioned_build / timeout`,旧完整代次仍能在原15分钟窗口内响应。HTTP 200 和 Pod 就绪不能证明统计刷新成功。
|
||||
|
||||
只读元数据约3.03万条审核操作、583MB原始文档;现有统计读取先排除正式验收退回,再读取正式验收退回,两条互补条件均需要 Fetch 操作日志正文。小投影没有消除 Mongo 大文档读取,单个全历史游标可能在 getMore 时耗尽15秒查询预算。现有 lookup 索引不能覆盖统计所需的 effects 和来源数组。
|
||||
|
||||
## 修复范围
|
||||
|
||||
- 所有操作先按 `_id` 有界顺序读取一次元数据,每页最多1024条,并按8MiB投影字节提前续页;大问题事实补读同样按字节分批并逐批消费,避免在流式构建中积累整页大正文。用一次只读主键上界限制本轮增长,每页保留原组织兼容和项目范围,不丢弃缺字段、旧任务或未完成操作。
|
||||
- 仅为元数据中已经识别出的正式验收退回,按主键每批最多256条补充问题 ID、作用域和来源 ID;不再为这些记录第二次遍历全历史。正文、草稿文字、视频及凭证不进入事实投影。
|
||||
- 使用 BSON 总排序的 seek 条件,避免普通 `$gt` 的类型边界遗漏旧主键;主扫描让规划器在现有组织索引与主键范围间选路,避免小范围被强制跨组织扫描;仅精确ID补读使用主键hint。不新增生产索引或启动迁移。
|
||||
- 原每次查询15秒、整体构建90秒、15分钟展示窗口、并发和源数据预算保持不变。源读取字节/条数包含必要的补读开销,不通过少计数据达标。
|
||||
- 补读缺失或决定变化视为本轮构建失败,不能发布混合或不完整代次。保留上一完整快照;取消、重启及下一轮读取不改写源任务。
|
||||
|
||||
只改变统计源读取。权限、供应商隔离、任务版本、领取、保存、切片、送检、质检和验收正式决定路径不变,查询库与共享缓存开关保持原状态。
|
||||
|
||||
## 验证与发布
|
||||
|
||||
本机唯一合成数据库覆盖旧投影结果等价、正式退回与未完成记录、正文排除、分页取消/重试、混合 BSON 主键和分页索引扫描;显式规模夹具约3.03万条大载荷操作。刷新测试采用真实120秒周期,验证旧完整代次继续服务、新代次完整发布以及重启读取,不伪造快照时间或延长产品预算。
|
||||
|
||||
专项已映射到 verification-scopes.json 的 review-operation-history;快速检查不代替显式 Mongo 和真实周期检查。首个只分页方案的性能不佳,对照记录保留,最终实现采用普通元数据投影和按 ID 补读。
|
||||
|
||||
生产只做有界只读诊断,不执行全历史压力对照;首128条及第二页128条的诊断证实主键 seek 不重复扫描前缀。该诊断不等于完整构建或并发容量验收。本地稳定场景也不能代替原Mongo高负载下的端到端刷新验证。
|
||||
|
||||
发布须另行同步dev、准备镜像与正式计划并确认。上线后必须观察实际源码、快照时间晚于新构建、失败计数归零及下一轮继续刷新;返回旧快照的200不能算成功。回退仅切回旧应用,不回滚数据库、不覆盖已有标注,不切换生产查询入口。
|
||||
@ -159,7 +159,7 @@ func (s *MongoStore) loadAnnotationOutputTo(ctx context.Context, org string, pro
|
||||
var firstReadErr error
|
||||
var firstReadFailure sync.Once
|
||||
totalBytes, totalDocs := 0, 0
|
||||
read := func(c mongoCollection, filter, projection bson.M, out any) (err error) {
|
||||
read := func(c mongoCollection, filter, projection bson.M, out any, pages ...*outputOperationPage) (err error) {
|
||||
select {
|
||||
case sem <- struct{}{}:
|
||||
case <-readCtx.Done():
|
||||
@ -178,13 +178,28 @@ func (s *MongoStore) loadAnnotationOutputTo(ctx context.Context, org string, pro
|
||||
cancel()
|
||||
}
|
||||
}()
|
||||
cursor, err := c.Find(readCtx, filter, options.Find().SetProjection(projection).SetMaxTime(15*time.Second))
|
||||
find := options.Find().SetProjection(projection).SetMaxTime(15 * time.Second)
|
||||
var page *outputOperationPage
|
||||
if len(pages) > 0 {
|
||||
page = pages[0]
|
||||
find.SetSort(bson.D{{Key: "_id", Value: 1}}).SetLimit(outputOperationPageSize).SetBatchSize(outputOperationPageSize)
|
||||
if page.byID {
|
||||
find.SetHint("_id_")
|
||||
}
|
||||
}
|
||||
cursor, err := c.Find(readCtx, filter, find)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer cursor.Close(readCtx)
|
||||
target := reflect.ValueOf(out).Elem()
|
||||
for cursor.Next(readCtx) {
|
||||
if page != nil {
|
||||
if err = cursor.Current.Lookup("_id").Unmarshal(&page.last); err != nil {
|
||||
return err
|
||||
}
|
||||
page.count++
|
||||
}
|
||||
budgetMu.Lock()
|
||||
totalBytes += len(cursor.Current)
|
||||
totalDocs++
|
||||
@ -194,6 +209,15 @@ func (s *MongoStore) loadAnnotationOutputTo(ctx context.Context, org string, pro
|
||||
if exceeded {
|
||||
return fmt.Errorf("%w: bytes=%d/%d documents=%d/%d", workforcestats.ErrOutputSourceBudget, bytes, budget.bytes, documents, budget.documents)
|
||||
}
|
||||
if page != nil {
|
||||
page.rows = append(page.rows, append(bson.Raw(nil), cursor.Current...))
|
||||
page.bytes += len(cursor.Current)
|
||||
if page.bytes >= outputOperationPageBytes {
|
||||
page.byteLimited = true
|
||||
break
|
||||
}
|
||||
continue
|
||||
}
|
||||
if spool != nil {
|
||||
if err = spool.add(c.Name(), cursor.Current); err != nil {
|
||||
return err
|
||||
@ -250,19 +274,17 @@ func (s *MongoStore) loadAnnotationOutputTo(ctx context.Context, org string, pro
|
||||
return read(s.annotationWorklogEvents, bson.M{"$and": bson.A{f, bson.M{"eventType": bson.M{"$in": workforcestats.OutputSubmissionEventTypes()}}}}, fields("id", "taskId", "userId", "supplierId", "eventType", "submissionId", "submissionKind", "outputDelta", "revision", "createdAt", "mode", "temporary"), &d.Events)
|
||||
})
|
||||
g.Go(func() error {
|
||||
// Only completed acceptance rejections consume draft problem facts.
|
||||
// Keep every operation's status and effect links for in-flight fencing
|
||||
// and legacy attribution, without loading unused quality drafts twice
|
||||
// (their formal problem facts already come from reworkBaselines).
|
||||
draftNeeded := bson.M{"stageId": bson.M{"$in": workforcestats.OutputAcceptanceStageIDs()}, "status": "completed", "decision": "rejected"}
|
||||
projection := fields("_id", "operationId", "projectId", "userId", "itemId", "taskId", "stageId", "decision", "status", "sourceItemIds", "reviewerSupplierId", "createdAt", "completedAt", "effects.review.id", "effects.review.taskId", "effects.review.revision", "effects.beforeTask.id", "effects.beforeTask.supplierId", "effects.worklog.id")
|
||||
if err := read(s.queryCollection("reviewOperations"), bson.M{"$and": bson.A{f, bson.M{"$nor": bson.A{draftNeeded}}}}, projection, &d.Operations); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, name := range []string{"draft.problems.id", "draft.problems.scope", "draft.problems.sourceItemId"} {
|
||||
projection[name] = 1
|
||||
}
|
||||
return read(s.queryCollection("reviewOperations"), bson.M{"$and": bson.A{f, draftNeeded}}, projection, &d.Operations)
|
||||
return readOutputOperations(readCtx, s.queryCollection("reviewOperations"), f, &d.Operations, read, func(raw bson.Raw) error {
|
||||
if spool != nil {
|
||||
return spool.add("reviewOperations", raw)
|
||||
}
|
||||
var operation domain.ReviewOperation
|
||||
if err := bson.Unmarshal(raw, &operation); err != nil {
|
||||
return err
|
||||
}
|
||||
d.Operations = append(d.Operations, operation)
|
||||
return nil
|
||||
})
|
||||
})
|
||||
g.Go(func() error {
|
||||
return read(s.deliveryPackages, f, fields("id", "itemIds", "status", "manifestDigest", "releasePartitionId", "parentRunId", "mode", "temporary", "qualitySummary.prePackageAcceptance", "acceptanceReviews.id", "acceptanceReviews.decision", "acceptanceReviews.reviewerUserId", "acceptanceReviews.createdAt"), &d.Packages)
|
||||
|
||||
140
internal/store/annotation_output_operations.go
Normal file
140
internal/store/annotation_output_operations.go
Normal file
@ -0,0 +1,140 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
"droboticflow/internal/workforcestats"
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
"go.mongodb.org/mongo-driver/mongo"
|
||||
"go.mongodb.org/mongo-driver/mongo/options"
|
||||
)
|
||||
|
||||
const outputOperationPageSize = 1024
|
||||
const outputOperationProblemBatch = 256
|
||||
const outputOperationPageBytes = 8 << 20
|
||||
|
||||
type outputOperationPage struct {
|
||||
byID bool
|
||||
bytes int
|
||||
byteLimited bool
|
||||
last any
|
||||
count int
|
||||
rows []bson.Raw
|
||||
}
|
||||
type outputOperationRead func(mongoCollection, bson.M, bson.M, any, ...*outputOperationPage) error
|
||||
|
||||
// Read all operation metadata once, preserving legacy and unfinished plans.
|
||||
// Only completed acceptance rejections need problem identities; fetch those
|
||||
// by exact primary keys rather than scanning the full journal a second time.
|
||||
// Caller deadlines and cumulative source budgets are unchanged.
|
||||
func readOutputOperations(ctx context.Context, collection mongoCollection, scope bson.M, out any, read outputOperationRead, emit func(bson.Raw) error) error {
|
||||
var bound struct {
|
||||
ID any `bson:"_id"`
|
||||
}
|
||||
// Covered global _id bound reads no tenant body. All pages and point lookups
|
||||
// retain the original organization/project scope.
|
||||
err := collection.FindOne(ctx, bson.M{}, options.FindOne().SetProjection(bson.M{"_id": 1}).SetSort(bson.D{{Key: "_id", Value: -1}}).SetHint("_id_").SetMaxTime(15*time.Second)).Decode(&bound)
|
||||
if errors.Is(err, mongo.ErrNoDocuments) {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var last any
|
||||
for {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
comparisons := bson.A{bson.M{"$lte": bson.A{"$_id", bson.M{"$literal": bound.ID}}}}
|
||||
if last != nil {
|
||||
comparisons = append(comparisons, bson.M{"$gt": bson.A{"$_id", bson.M{"$literal": last}}})
|
||||
}
|
||||
// $expr retains total BSON ordering across mixed legacy _id types.
|
||||
filter := bson.M{"$and": bson.A{scope, bson.M{"$expr": bson.M{"$and": comparisons}}}}
|
||||
page := &outputOperationPage{}
|
||||
if err := read(collection, filter, outputOperationProjection(), out, page); err != nil {
|
||||
return err
|
||||
}
|
||||
ids, keys := bson.A{}, []string{}
|
||||
for _, raw := range page.rows {
|
||||
if !outputOperationNeedsProblems(raw) {
|
||||
if err := emit(raw); err != nil {
|
||||
return err
|
||||
}
|
||||
continue
|
||||
}
|
||||
var id any
|
||||
if err := raw.Lookup("_id").Unmarshal(&id); err != nil {
|
||||
return err
|
||||
}
|
||||
ids = append(ids, id)
|
||||
keys = append(keys, outputOperationIdentity(raw))
|
||||
}
|
||||
for start := 0; start < len(ids); start += outputOperationProblemBatch {
|
||||
end := min(start+outputOperationProblemBatch, len(ids))
|
||||
pending := map[string]bool{}
|
||||
for _, key := range keys[start:end] {
|
||||
pending[key] = true
|
||||
}
|
||||
for len(pending) > 0 {
|
||||
remaining := bson.A{}
|
||||
for index := start; index < end; index++ {
|
||||
if pending[keys[index]] {
|
||||
remaining = append(remaining, ids[index])
|
||||
}
|
||||
}
|
||||
point := &outputOperationPage{byID: true}
|
||||
filter := bson.M{"$and": bson.A{scope, bson.M{"_id": bson.M{"$in": remaining}}}}
|
||||
if err := read(collection, filter, outputOperationProblemProjection(), out, point); err != nil {
|
||||
return err
|
||||
}
|
||||
if point.count == 0 || point.count < len(remaining) && !point.byteLimited {
|
||||
return errors.New("review operation history changed during statistics read")
|
||||
}
|
||||
for _, raw := range point.rows {
|
||||
key := outputOperationIdentity(raw)
|
||||
if !pending[key] || !outputOperationNeedsProblems(raw) {
|
||||
return errors.New("review operation decision changed during statistics read")
|
||||
}
|
||||
if err := emit(raw); err != nil {
|
||||
return err
|
||||
}
|
||||
delete(pending, key)
|
||||
}
|
||||
}
|
||||
}
|
||||
if page.count < outputOperationPageSize && !page.byteLimited {
|
||||
return nil
|
||||
}
|
||||
if page.last == nil {
|
||||
return errors.New("review operation page has no continuation identity")
|
||||
}
|
||||
last = page.last
|
||||
}
|
||||
}
|
||||
func outputOperationIdentity(raw bson.Raw) string {
|
||||
v := raw.Lookup("_id")
|
||||
return string([]byte{byte(v.Type)}) + string(v.Value)
|
||||
}
|
||||
func outputOperationNeedsProblems(raw bson.Raw) bool {
|
||||
stage, _ := raw.Lookup("stageId").StringValueOK()
|
||||
status, _ := raw.Lookup("status").StringValueOK()
|
||||
decision, _ := raw.Lookup("decision").StringValueOK()
|
||||
return workforcestats.IsOutputAcceptanceStage(stage) && status == "completed" && decision == "rejected"
|
||||
}
|
||||
func outputOperationProjection() bson.M {
|
||||
p := bson.M{}
|
||||
for _, field := range []string{"_id", "operationId", "projectId", "userId", "itemId", "taskId", "stageId", "decision", "status", "sourceItemIds", "reviewerSupplierId", "createdAt", "completedAt", "effects.review.id", "effects.review.taskId", "effects.review.revision", "effects.beforeTask.id", "effects.beforeTask.supplierId", "effects.worklog.id"} {
|
||||
p[field] = 1
|
||||
}
|
||||
return p
|
||||
}
|
||||
func outputOperationProblemProjection() bson.M {
|
||||
p := outputOperationProjection()
|
||||
for _, field := range []string{"draft.problems.id", "draft.problems.scope", "draft.problems.sourceItemId"} {
|
||||
p[field] = 1
|
||||
}
|
||||
return p
|
||||
}
|
||||
206
internal/store/annotation_output_operations_recovery_test.go
Normal file
206
internal/store/annotation_output_operations_recovery_test.go
Normal file
@ -0,0 +1,206 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"reflect"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"droboticflow/internal/outputsnapshot"
|
||||
"droboticflow/internal/workforcestats"
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
"go.mongodb.org/mongo-driver/bson/primitive"
|
||||
"go.mongodb.org/mongo-driver/event"
|
||||
"go.mongodb.org/mongo-driver/mongo"
|
||||
"go.mongodb.org/mongo-driver/mongo/options"
|
||||
)
|
||||
|
||||
func TestAnnotationOutputOperationMongoPageCancellationAndRestart(t *testing.T) {
|
||||
db := localOutputMongo(t)
|
||||
var docs []any
|
||||
for i := 0; i < outputOperationPageSize+44; i++ {
|
||||
docs = append(docs, bson.M{"_id": fmt.Sprintf("op-%04d", i), "organizationId": "out", "projectId": "p", "status": "prepared"})
|
||||
}
|
||||
if _, e := db.Collection("reviewOperations").InsertMany(t.Context(), docs); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
defer cancel()
|
||||
var pages atomic.Int32
|
||||
var cancelled atomic.Bool
|
||||
monitor := &event.CommandMonitor{Started: func(_ context.Context, e *event.CommandStartedEvent) {
|
||||
if e.CommandName != "find" {
|
||||
return
|
||||
}
|
||||
var m bson.M
|
||||
_ = bson.Unmarshal(e.Command, &m)
|
||||
if m["find"] == "reviewOperations" && m["limit"] == int64(outputOperationPageSize) && pages.Add(1) == 2 {
|
||||
cancelled.Store(true)
|
||||
cancel()
|
||||
}
|
||||
}}
|
||||
c, e := mongo.Connect(t.Context(), options.Client().ApplyURI("mongodb://127.0.0.1:27018").SetMonitor(monitor))
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
defer c.Disconnect(context.Background())
|
||||
s := NewMongoStore(c.Database(db.Name()))
|
||||
partial, e := s.LoadAnnotationOutput(ctx, "out", []string{"p"})
|
||||
if !cancelled.Load() || !errors.Is(e, context.Canceled) {
|
||||
t.Fatalf("cancel returned partial success: cancelled=%t operations=%d error=%v", cancelled.Load(), len(partial.Operations), e)
|
||||
}
|
||||
recovered, e := NewMongoStore(db).LoadAnnotationOutput(t.Context(), "out", []string{"p"})
|
||||
if e != nil || len(recovered.Operations) != outputOperationPageSize+44 {
|
||||
t.Fatalf("restart lost operation history: operations=%d error=%v", len(recovered.Operations), e)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnnotationOutputOperationMongoSeekPreservesMixedLegacyIdentities(t *testing.T) {
|
||||
db := localOutputMongo(t)
|
||||
var docs []any
|
||||
for i := 0; i < outputOperationPageSize/2+10; i++ {
|
||||
docs = append(docs, bson.M{"_id": i, "organizationId": "out", "projectId": "p"})
|
||||
}
|
||||
for i := 0; i < outputOperationPageSize/2+10; i++ {
|
||||
docs = append(docs, bson.M{"_id": fmt.Sprintf("legacy-%04d", i), "organizationId": "out", "projectId": "p"})
|
||||
}
|
||||
for i := 0; i < 3; i++ {
|
||||
docs = append(docs, bson.M{"_id": primitive.NewObjectID(), "organizationId": "out", "projectId": "p"})
|
||||
}
|
||||
docs = append(docs, bson.M{"_id": "foreign", "organizationId": "other", "projectId": "p"})
|
||||
if _, e := db.Collection("reviewOperations").InsertMany(t.Context(), docs); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
scope := organizationScopedFilter("out", bson.M{"projectId": "p"})
|
||||
var want []bson.M
|
||||
cur, e := db.Collection("reviewOperations").Find(t.Context(), scope, options.Find().SetSort(bson.D{{Key: "_id", Value: 1}}).SetProjection(bson.M{"_id": 1}))
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if e = cur.All(t.Context(), &want); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
var got []bson.M
|
||||
reader := func(c mongoCollection, f, p bson.M, _ any, pages ...*outputOperationPage) error {
|
||||
cur, e := c.Find(t.Context(), f, options.Find().SetProjection(p).SetSort(bson.D{{Key: "_id", Value: 1}}).SetHint("_id_").SetLimit(outputOperationPageSize).SetBatchSize(outputOperationPageSize))
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
defer cur.Close(t.Context())
|
||||
for cur.Next(t.Context()) {
|
||||
var row bson.M
|
||||
if e = cur.Decode(&row); e != nil {
|
||||
return e
|
||||
}
|
||||
pages[0].last = row["_id"]
|
||||
pages[0].count++
|
||||
pages[0].rows = append(pages[0].rows, append(bson.Raw(nil), cur.Current...))
|
||||
}
|
||||
return cur.Err()
|
||||
}
|
||||
if e = readOutputOperations(t.Context(), NewMongoStore(db).queryCollection("reviewOperations"), scope, nil, reader, func(raw bson.Raw) error {
|
||||
var row bson.M
|
||||
if e := bson.Unmarshal(raw, &row); e != nil {
|
||||
return e
|
||||
}
|
||||
got = append(got, bson.M{"_id": row["_id"]})
|
||||
return nil
|
||||
}); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("mixed BSON identities skipped or duplicated: got=%d want=%d", len(got), len(want))
|
||||
}
|
||||
}
|
||||
|
||||
func TestReviewOperationActualSnapshotMongoRefreshCycle(t *testing.T) {
|
||||
if os.Getenv("RUN_OUTPUT_REFRESH_CYCLE") != "1" {
|
||||
t.Skip("explicit real 120-second refresh cadence test")
|
||||
}
|
||||
db := localOutputMongo(t)
|
||||
s := NewMongoStore(db)
|
||||
seedAnnotationOutput(t, s)
|
||||
beforeTask, e := s.GetAnnotationTask(t.Context(), "t")
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
var builds atomic.Int32
|
||||
refreshing, release := make(chan struct{}), make(chan struct{})
|
||||
source := func(c context.Context, o string, ids []string, yield func(workforcestats.OutputSnapshot) error) (*workforcestats.OutputSourceStats, error) {
|
||||
if builds.Add(1) == 2 {
|
||||
close(refreshing)
|
||||
select {
|
||||
case <-release:
|
||||
case <-c.Done():
|
||||
return nil, c.Err()
|
||||
}
|
||||
}
|
||||
return s.StreamAnnotationOutput(c, o, ids, yield)
|
||||
}
|
||||
r := outputsnapshot.NewStreamReader(s, source)
|
||||
defer r.Close()
|
||||
q := workforcestats.OutputQuery{From: time.Date(2026, 10, 1, 0, 0, 0, 0, time.UTC), To: time.Date(2026, 10, 20, 0, 0, 0, 0, time.UTC)}
|
||||
wait := func(reader *outputsnapshot.Reader, previous string) (*workforcestats.OutputReport, outputsnapshot.Info) {
|
||||
t.Helper()
|
||||
until := time.Now().Add(5 * time.Second)
|
||||
for {
|
||||
report, info, e := reader.Query(t.Context(), "out", []string{"p"}, "", q)
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if report != nil && info.ID != previous {
|
||||
return report, info
|
||||
}
|
||||
if time.Now().After(until) {
|
||||
t.Fatal("complete generation did not refresh")
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
first, info := wait(r, "")
|
||||
if first.Totals.Acceptance.Count != 1 {
|
||||
t.Fatalf("first report: %+v", first.Totals)
|
||||
}
|
||||
op, e := s.GetReviewOperation(t.Context(), "ac")
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
revision := op.Revision
|
||||
op.Decision = "rejected"
|
||||
op.CompletedAt = "2026-10-05T00:00:00Z"
|
||||
if ok, e := s.CompareAndSwapReviewOperation(t.Context(), *op, revision); e != nil || !ok {
|
||||
t.Fatalf("synthetic decision update: %t %v", ok, e)
|
||||
}
|
||||
// Exercise the production 120-second cadence. No forged generation dates,
|
||||
// altered TTLs or longer build deadlines are used to make refresh pass.
|
||||
time.Sleep(outputsnapshot.RefreshInterval + 100*time.Millisecond)
|
||||
|
||||
old, stale, e := r.Query(t.Context(), "out", []string{"p"}, "", q)
|
||||
if e != nil || old == nil || !stale.Stale || old.Totals.Acceptance.Count != 1 {
|
||||
t.Fatalf("old complete generation lost while refreshing: %+v %v", stale, e)
|
||||
}
|
||||
select {
|
||||
case <-refreshing:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("background refresh did not start")
|
||||
}
|
||||
close(release)
|
||||
next, nextInfo := wait(r, info.ID)
|
||||
if next.Totals.Acceptance.Count != 0 || nextInfo.Stale {
|
||||
t.Fatalf("new complete report was not published: %+v %+v", next.Totals, nextInfo)
|
||||
}
|
||||
afterTask, e := s.GetAnnotationTask(t.Context(), "t")
|
||||
if e != nil || !reflect.DeepEqual(beforeTask, afterTask) {
|
||||
t.Fatal("report refresh mutated annotation task")
|
||||
}
|
||||
restarted := outputsnapshot.NewStreamReader(NewMongoStore(db), s.StreamAnnotationOutput)
|
||||
defer restarted.Close()
|
||||
persisted, persistedInfo := wait(restarted, "")
|
||||
if persistedInfo.ID != nextInfo.ID || persisted.Totals.Acceptance.Count != 0 {
|
||||
t.Fatal("restart failed to recover the published report")
|
||||
}
|
||||
}
|
||||
311
internal/store/annotation_output_operations_test.go
Normal file
311
internal/store/annotation_output_operations_test.go
Normal file
@ -0,0 +1,311 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"reflect"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"droboticflow/internal/domain"
|
||||
"droboticflow/internal/workforcestats"
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
"go.mongodb.org/mongo-driver/event"
|
||||
"go.mongodb.org/mongo-driver/mongo"
|
||||
"go.mongodb.org/mongo-driver/mongo/options"
|
||||
)
|
||||
|
||||
func TestAnnotationOutputReviewOperationsMongoBoundedSinglePass(t *testing.T) {
|
||||
db := localOutputMongo(t)
|
||||
var mu sync.Mutex
|
||||
var finds []bson.M
|
||||
var getMores atomic.Int32
|
||||
monitor := &event.CommandMonitor{Started: func(_ context.Context, e *event.CommandStartedEvent) {
|
||||
if e.CommandName == "getMore" && e.Command.Lookup("collection").StringValue() == "reviewOperations" {
|
||||
getMores.Add(1)
|
||||
}
|
||||
if e.CommandName != "find" {
|
||||
return
|
||||
}
|
||||
var m bson.M
|
||||
if bson.Unmarshal(e.Command, &m) == nil && m["find"] == "reviewOperations" {
|
||||
mu.Lock()
|
||||
finds = append(finds, m)
|
||||
mu.Unlock()
|
||||
}
|
||||
}}
|
||||
client, err := mongo.Connect(t.Context(), options.Client().ApplyURI("mongodb://127.0.0.1:27018").SetMonitor(monitor))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer client.Disconnect(context.Background())
|
||||
s := NewMongoStore(client.Database(db.Name()))
|
||||
seedAnnotationOutput(t, s)
|
||||
n := 2*outputOperationPageSize + 44
|
||||
if os.Getenv("RUN_OUTPUT_REVIEW_SCALE") == "1" {
|
||||
n = 30310
|
||||
}
|
||||
if _, err := db.Collection("reviewOperations").Indexes().CreateOne(t.Context(), PlatformReadIndexes()["reviewOperations"][0]); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var pending []any
|
||||
for i := 0; i < n; i++ {
|
||||
op := domain.ReviewOperation{ID: fmt.Sprintf("op-%04d", i), OperationID: fmt.Sprintf("client-%d", i), OrganizationID: "out", ProjectID: "p", UserID: "qc", ItemID: "i", TaskID: "t", StageID: "quality_review", Decision: "approved", Status: "completed", SourceItemIDs: []string{"i"}, CompletedAt: "2026-10-04T00:00:00Z", Effects: []domain.ReviewDecisionEffect{{Review: &domain.AnnotationReviewRecord{ID: fmt.Sprintf("r-%d", i), TaskID: "t", Revision: 1}, BeforeTask: &domain.AnnotationTaskRecord{ID: "t", SupplierID: "v"}, BeforeItem: &domain.HumanItemRecord{Subject: domain.HumanSubjectRef{Payload: map[string]any{"annotationDraft": strings.Repeat("PRIVATE-NOT-REPORT", 2048)}}}}}}
|
||||
switch i % 6 {
|
||||
case 0:
|
||||
op.StageID = "prepackage_acceptance"
|
||||
op.Decision = "rejected"
|
||||
op.Draft.Problems = []domain.ReviewProblem{{ID: "problem", Scope: "slice", SourceItemID: "i", Text: "PRIVATE-NOT-REPORT"}}
|
||||
case 1:
|
||||
op.StageID = "final_acceptance"
|
||||
op.Status = "applying"
|
||||
case 2:
|
||||
op.Kind = "quality_draft"
|
||||
op.Decision = ""
|
||||
op.SourceItemIDs = nil
|
||||
op.Effects = nil
|
||||
case 3:
|
||||
op.Kind = ""
|
||||
op.Status = "prepared"
|
||||
}
|
||||
pending = append(pending, op)
|
||||
if len(pending) == 256 || i == n-1 {
|
||||
if _, err := db.Collection("reviewOperations").InsertMany(t.Context(), pending); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
pending = nil
|
||||
}
|
||||
}
|
||||
// Nonmatching records must not leak even when the seek uses the _id index.
|
||||
if _, err := db.Collection("reviewOperations").InsertOne(t.Context(), bson.M{"_id": "foreign-tail", "organizationId": "other", "projectId": "p"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Independent old two-query projection is the semantic oracle, including
|
||||
// missing kind, unfinished decisions and acceptance problem attribution.
|
||||
projection := bson.M{}
|
||||
for _, field := range []string{"_id", "operationId", "projectId", "userId", "itemId", "taskId", "stageId", "decision", "status", "sourceItemIds", "reviewerSupplierId", "createdAt", "completedAt", "effects.review.id", "effects.review.taskId", "effects.review.revision", "effects.beforeTask.id", "effects.beforeTask.supplierId", "effects.worklog.id"} {
|
||||
projection[field] = 1
|
||||
}
|
||||
needed := bson.M{"stageId": bson.M{"$in": bson.A{"acceptance", "prepackage_acceptance", "final_acceptance"}}, "status": "completed", "decision": "rejected"}
|
||||
scope := organizationScopedFilter("out", bson.M{"projectId": bson.M{"$in": bson.A{"p"}}})
|
||||
legacyStarted := time.Now()
|
||||
var want []domain.ReviewOperation
|
||||
for i, condition := range []bson.M{bson.M{"$nor": bson.A{needed}}, needed} {
|
||||
if i == 1 {
|
||||
for _, f := range []string{"draft.problems.id", "draft.problems.scope", "draft.problems.sourceItemId"} {
|
||||
projection[f] = 1
|
||||
}
|
||||
}
|
||||
cur, e := db.Collection("reviewOperations").Find(t.Context(), bson.M{"$and": bson.A{scope, condition}}, options.Find().SetProjection(projection))
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
var rows []domain.ReviewOperation
|
||||
if e = cur.All(t.Context(), &rows); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
want = append(want, rows...)
|
||||
}
|
||||
legacyElapsed := time.Since(legacyStarted)
|
||||
start := time.Now()
|
||||
got, e := s.LoadAnnotationOutput(t.Context(), "out", []string{"p"})
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
loaderElapsed := time.Since(start)
|
||||
t.Logf("source_operations=%d source_bytes=%d legacy_review_only=%s whole_loader=%s", len(got.Operations), got.SourceStats.Bytes, legacyElapsed, loaderElapsed)
|
||||
sort.Slice(want, func(i, j int) bool { return want[i].ID < want[j].ID })
|
||||
sort.Slice(got.Operations, func(i, j int) bool { return got.Operations[i].ID < got.Operations[j].ID })
|
||||
if !reflect.DeepEqual(got.Operations, want) {
|
||||
a, _ := json.Marshal(got.Operations)
|
||||
b, _ := json.Marshal(want)
|
||||
t.Fatalf("operation projection differs: got=%s want=%s", a, b)
|
||||
}
|
||||
encoded, _ := json.Marshal(got.Operations)
|
||||
if strings.Contains(string(encoded), "PRIVATE-NOT-REPORT") {
|
||||
t.Fatal("body/draft text escaped the metadata projection")
|
||||
}
|
||||
if getMores.Load() != 0 {
|
||||
t.Fatalf("bounded fixture unexpectedly used getMore for journal history: %d", getMores.Load())
|
||||
}
|
||||
mu.Lock()
|
||||
queries := append([]bson.M(nil), finds...)
|
||||
mu.Unlock()
|
||||
var pages []bson.M
|
||||
for _, q := range queries {
|
||||
limit, _ := q["limit"].(int64)
|
||||
if limit < 0 {
|
||||
limit = -limit
|
||||
}
|
||||
if limit == 1 {
|
||||
continue
|
||||
}
|
||||
if limit == 0 || limit > outputOperationPageSize {
|
||||
t.Fatalf("unbounded reviewOperations cursor: limit=%d; must not spend one maxTimeMS on all journal bodies", limit)
|
||||
}
|
||||
encodedFilter, _ := bson.MarshalExtJSON(q["filter"], false, false)
|
||||
if strings.Contains(string(encodedFilter), "\"$expr\"") {
|
||||
pages = append(pages, q)
|
||||
}
|
||||
}
|
||||
expectedPages := (n+1)/outputOperationPageSize + 1
|
||||
if len(pages) != expectedPages {
|
||||
t.Fatalf("expected %d disjoint source pages, got %d", expectedPages, len(pages))
|
||||
}
|
||||
// A bounded query is insufficient if page N rescans pages 1..N-1.
|
||||
second := pages[1]
|
||||
command := bson.D{{Key: "find", Value: "reviewOperations"}, {Key: "filter", Value: second["filter"]}, {Key: "projection", Value: second["projection"]}, {Key: "sort", Value: second["sort"]}, {Key: "limit", Value: second["limit"]}}
|
||||
var explain struct {
|
||||
Stats struct {
|
||||
Returned int `bson:"nReturned"`
|
||||
Keys int `bson:"totalKeysExamined"`
|
||||
} `bson:"executionStats"`
|
||||
}
|
||||
if e := db.RunCommand(t.Context(), bson.D{{Key: "explain", Value: command}, {Key: "verbosity", Value: "executionStats"}}).Decode(&explain); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if explain.Stats.Returned != outputOperationPageSize || explain.Stats.Keys > outputOperationPageSize+2 {
|
||||
t.Fatalf("seek page rescans prefix: %+v", explain.Stats)
|
||||
}
|
||||
t.Logf("complete operations=%d source_pages=%d legacy_two_scan=%s bounded_loader=%s page2_keys=%d", len(got.Operations), len(pages), legacyElapsed, time.Since(start), explain.Stats.Keys)
|
||||
streamContext, cancel := context.WithTimeout(t.Context(), 90*time.Second)
|
||||
defer cancel()
|
||||
streamed := 0
|
||||
streamStarted := time.Now()
|
||||
_, err = s.StreamAnnotationOutput(streamContext, "out", []string{"p"}, func(batch workforcestats.OutputSnapshot) error { streamed += len(batch.Operations); return nil })
|
||||
if err != nil || streamed != len(want) {
|
||||
t.Fatalf("spooled history lost operations: got=%d want=%d error=%v", streamed, len(want), err)
|
||||
}
|
||||
t.Logf("spooled operations=%d complete_stream=%s", streamed, time.Since(streamStarted))
|
||||
}
|
||||
|
||||
func TestAnnotationOutputOperationMongoSelectiveScopeAvoidsForeignScan(t *testing.T) {
|
||||
db := localOutputMongo(t)
|
||||
if _, e := db.Collection("reviewOperations").Indexes().CreateOne(t.Context(), PlatformReadIndexes()["reviewOperations"][0]); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
var docs []any
|
||||
for i := 0; i < 4000; i++ {
|
||||
docs = append(docs, bson.M{"_id": fmt.Sprintf("foreign-%04d", i), "organizationId": "other", "projectId": "p", "unused": strings.Repeat("body", 4096)})
|
||||
}
|
||||
for i := 0; i < 5; i++ {
|
||||
docs = append(docs, bson.M{"_id": fmt.Sprintf("target-%04d", i), "organizationId": "out", "projectId": "p"})
|
||||
}
|
||||
if _, e := db.Collection("reviewOperations").InsertMany(t.Context(), docs); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
var query bson.M
|
||||
monitor := &event.CommandMonitor{Started: func(_ context.Context, e *event.CommandStartedEvent) {
|
||||
if e.CommandName != "find" {
|
||||
return
|
||||
}
|
||||
var m bson.M
|
||||
_ = bson.Unmarshal(e.Command, &m)
|
||||
if m["find"] == "reviewOperations" && m["limit"] == int64(outputOperationPageSize) {
|
||||
query = m
|
||||
}
|
||||
}}
|
||||
c, e := mongo.Connect(t.Context(), options.Client().ApplyURI("mongodb://127.0.0.1:27018").SetMonitor(monitor))
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
defer c.Disconnect(context.Background())
|
||||
result, e := NewMongoStore(c.Database(db.Name())).LoadAnnotationOutput(t.Context(), "out", []string{"p"})
|
||||
if e != nil || len(result.Operations) != 5 {
|
||||
t.Fatalf("scoped history mismatch: %d %v", len(result.Operations), e)
|
||||
}
|
||||
command := bson.D{{Key: "find", Value: "reviewOperations"}, {Key: "filter", Value: query["filter"]}, {Key: "projection", Value: query["projection"]}, {Key: "sort", Value: query["sort"]}, {Key: "limit", Value: query["limit"]}}
|
||||
if h := query["hint"]; h != nil {
|
||||
command = append(command, bson.E{Key: "hint", Value: h})
|
||||
}
|
||||
var explanation struct {
|
||||
Stats struct {
|
||||
Keys int `bson:"totalKeysExamined"`
|
||||
Documents int `bson:"totalDocsExamined"`
|
||||
} `bson:"executionStats"`
|
||||
}
|
||||
if e = db.RunCommand(t.Context(), bson.D{{Key: "explain", Value: command}, {Key: "verbosity", Value: "executionStats"}}).Decode(&explanation); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if explanation.Stats.Documents > 5 || explanation.Stats.Keys > 6 {
|
||||
t.Fatalf("small scope scanned unrelated organization: %+v", explanation.Stats)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnnotationOutputOperationMongoLargeMetadataAndProblemsRemainBounded(t *testing.T) {
|
||||
for _, mode := range []string{"wide-metadata", "large-problem-refill"} {
|
||||
t.Run(mode, func(t *testing.T) {
|
||||
db := localOutputMongo(t)
|
||||
var seeks, points atomic.Int32
|
||||
monitor := &event.CommandMonitor{Started: func(_ context.Context, e *event.CommandStartedEvent) {
|
||||
if e.CommandName != "find" {
|
||||
return
|
||||
}
|
||||
var m bson.M
|
||||
_ = bson.Unmarshal(e.Command, &m)
|
||||
if m["find"] != "reviewOperations" || m["limit"] != int64(outputOperationPageSize) {
|
||||
return
|
||||
}
|
||||
f, _ := bson.MarshalExtJSON(m["filter"], false, false)
|
||||
if strings.Contains(string(f), "\"$expr\"") {
|
||||
seeks.Add(1)
|
||||
} else {
|
||||
points.Add(1)
|
||||
}
|
||||
}}
|
||||
c, e := mongo.Connect(t.Context(), options.Client().ApplyURI("mongodb://127.0.0.1:27018").SetMonitor(monitor))
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
defer c.Disconnect(context.Background())
|
||||
n := 400
|
||||
if mode == "large-problem-refill" {
|
||||
n = 24
|
||||
}
|
||||
for i := 0; i < n; i++ {
|
||||
d := bson.M{"_id": fmt.Sprintf("op-%04d", i), "organizationId": "out", "projectId": "p", "stageId": "quality_review", "status": "prepared"}
|
||||
if mode == "wide-metadata" {
|
||||
ids := bson.A{}
|
||||
for j := 0; j < 300; j++ {
|
||||
ids = append(ids, fmt.Sprintf("source-%04d-%s", j, strings.Repeat("s", 96)))
|
||||
}
|
||||
d["sourceItemIds"] = ids
|
||||
} else {
|
||||
d["stageId"], d["status"], d["decision"] = "acceptance", "completed", "rejected"
|
||||
problems := bson.A{}
|
||||
for j := 0; j < 2500; j++ {
|
||||
problems = append(problems, bson.M{"id": fmt.Sprintf("problem-%04d-%s", j, strings.Repeat("p", 192)), "scope": "slice", "sourceItemId": "i", "text": strings.Repeat("private text", 32)})
|
||||
}
|
||||
d["draft"] = bson.M{"problems": problems}
|
||||
}
|
||||
if _, e := db.Collection("reviewOperations").InsertOne(t.Context(), d); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
}
|
||||
result, e := NewMongoStore(c.Database(db.Name())).LoadAnnotationOutput(t.Context(), "out", []string{"p"})
|
||||
if e != nil || len(result.Operations) != n {
|
||||
t.Fatalf("byte-split lost records: %d/%d %v", len(result.Operations), n, e)
|
||||
}
|
||||
if mode == "wide-metadata" && seeks.Load() < 2 {
|
||||
t.Fatal("metadata page ignored the byte boundary")
|
||||
}
|
||||
if mode == "large-problem-refill" && points.Load() < 2 {
|
||||
t.Fatal("problem refill accumulated an unbounded page")
|
||||
}
|
||||
for _, op := range result.Operations {
|
||||
if mode == "wide-metadata" && len(op.SourceItemIDs) != 300 {
|
||||
t.Fatal("lost source identities")
|
||||
}
|
||||
if mode == "large-problem-refill" && (len(op.Draft.Problems) != 2500 || op.Draft.Problems[0].Text != "") {
|
||||
t.Fatal("lost problem facts or exposed opinion text")
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@ -1077,6 +1077,38 @@
|
||||
"warnings": [
|
||||
"独立源库/查询库仅使用合成数据;生产写者、47个源索引、迁移及集群资源须另行审批。native测试没有CPU配额,不能证明4核容量。"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "review-operation-history",
|
||||
"paths": [
|
||||
"^internal/store/annotation_output_operations(_recovery)?(_test)?\\.go$",
|
||||
"^internal/store/annotation_output\\.go$"
|
||||
],
|
||||
"frontend": [],
|
||||
"go": {},
|
||||
"task": [
|
||||
{
|
||||
"argv": [
|
||||
"go",
|
||||
"test",
|
||||
"./internal/store",
|
||||
"-run",
|
||||
"^TestAnnotationOutput.*Operation.*Mongo|^TestReviewOperationActualSnapshotMongoRefreshCycle$",
|
||||
"-count=1",
|
||||
"-timeout=180s",
|
||||
"-v"
|
||||
],
|
||||
"env": {
|
||||
"TEST_WORKFORCE_MONGO_URI": "mongodb://127.0.0.1:27018",
|
||||
"RUN_OUTPUT_REVIEW_SCALE": "1",
|
||||
"RUN_OUTPUT_REFRESH_CYCLE": "1"
|
||||
},
|
||||
"tests": true
|
||||
}
|
||||
],
|
||||
"warnings": [
|
||||
"审核操作历史读取保持完整、原15秒查询和90秒构建预算;显式本机合成夹具覆盖同规模、取消、旧ID及真实120秒刷新,不代表生产已刷新。"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user