From c0512c8bf678d0c96a44eaedb3c762a3c19c5d62 Mon Sep 17 00:00:00 2001 From: d-robotics Date: Sat, 14 Nov 2026 01:03:23 -0800 Subject: [PATCH] =?UTF-8?q?=F0=9F=90=9B=20Bound=20review=20journal=20snaps?= =?UTF-8?q?hot=20reads=20without=20extending=20budgets?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace complementary journal scans with bounded seek pages and scoped primary-key problem refill. Preserve legacy/in-flight evidence, memory boundaries and original query/build deadlines. Cover scoped plans, source parity, cancellation, real refresh cadence and restart. --- .../review-operation-history-read.md | 27 ++ internal/store/annotation_output.go | 52 ++- .../store/annotation_output_operations.go | 140 ++++++++ ...otation_output_operations_recovery_test.go | 206 ++++++++++++ .../annotation_output_operations_test.go | 311 ++++++++++++++++++ scripts/verification-scopes.json | 32 ++ 6 files changed, 753 insertions(+), 15 deletions(-) create mode 100644 docs/operations/review-operation-history-read.md create mode 100644 internal/store/annotation_output_operations.go create mode 100644 internal/store/annotation_output_operations_recovery_test.go create mode 100644 internal/store/annotation_output_operations_test.go diff --git a/docs/operations/review-operation-history-read.md b/docs/operations/review-operation-history-read.md new file mode 100644 index 00000000..bc807d4f --- /dev/null +++ b/docs/operations/review-operation-history-read.md @@ -0,0 +1,27 @@ +# 统计审核操作历史读取超时 + +## 现场与口径 + +2026-11-14,线上 fc778619 的统计快照后台报告 `annotation output read reviewOperations ... MaxTimeMSExpired ... getMore`;快照健康出现 `partitioned_build / timeout`,旧完整代次仍能在原15分钟窗口内响应。HTTP 200 和 Pod 就绪不能证明统计刷新成功。 + +只读元数据约3.03万条审核操作、583MB原始文档;现有统计读取先排除正式验收退回,再读取正式验收退回,两条互补条件均需要 Fetch 操作日志正文。小投影没有消除 Mongo 大文档读取,单个全历史游标可能在 getMore 时耗尽15秒查询预算。现有 lookup 索引不能覆盖统计所需的 effects 和来源数组。 + +## 修复范围 + +- 所有操作先按 `_id` 有界顺序读取一次元数据,每页最多1024条,并按8MiB投影字节提前续页;大问题事实补读同样按字节分批并逐批消费,避免在流式构建中积累整页大正文。用一次只读主键上界限制本轮增长,每页保留原组织兼容和项目范围,不丢弃缺字段、旧任务或未完成操作。 +- 仅为元数据中已经识别出的正式验收退回,按主键每批最多256条补充问题 ID、作用域和来源 ID;不再为这些记录第二次遍历全历史。正文、草稿文字、视频及凭证不进入事实投影。 +- 使用 BSON 总排序的 seek 条件,避免普通 `$gt` 的类型边界遗漏旧主键;主扫描让规划器在现有组织索引与主键范围间选路,避免小范围被强制跨组织扫描;仅精确ID补读使用主键hint。不新增生产索引或启动迁移。 +- 原每次查询15秒、整体构建90秒、15分钟展示窗口、并发和源数据预算保持不变。源读取字节/条数包含必要的补读开销,不通过少计数据达标。 +- 补读缺失或决定变化视为本轮构建失败,不能发布混合或不完整代次。保留上一完整快照;取消、重启及下一轮读取不改写源任务。 + +只改变统计源读取。权限、供应商隔离、任务版本、领取、保存、切片、送检、质检和验收正式决定路径不变,查询库与共享缓存开关保持原状态。 + +## 验证与发布 + +本机唯一合成数据库覆盖旧投影结果等价、正式退回与未完成记录、正文排除、分页取消/重试、混合 BSON 主键和分页索引扫描;显式规模夹具约3.03万条大载荷操作。刷新测试采用真实120秒周期,验证旧完整代次继续服务、新代次完整发布以及重启读取,不伪造快照时间或延长产品预算。 + +专项已映射到 verification-scopes.json 的 review-operation-history;快速检查不代替显式 Mongo 和真实周期检查。首个只分页方案的性能不佳,对照记录保留,最终实现采用普通元数据投影和按 ID 补读。 + +生产只做有界只读诊断,不执行全历史压力对照;首128条及第二页128条的诊断证实主键 seek 不重复扫描前缀。该诊断不等于完整构建或并发容量验收。本地稳定场景也不能代替原Mongo高负载下的端到端刷新验证。 + +发布须另行同步dev、准备镜像与正式计划并确认。上线后必须观察实际源码、快照时间晚于新构建、失败计数归零及下一轮继续刷新;返回旧快照的200不能算成功。回退仅切回旧应用,不回滚数据库、不覆盖已有标注,不切换生产查询入口。 diff --git a/internal/store/annotation_output.go b/internal/store/annotation_output.go index 7e95051b..dd0e00b4 100644 --- a/internal/store/annotation_output.go +++ b/internal/store/annotation_output.go @@ -159,7 +159,7 @@ func (s *MongoStore) loadAnnotationOutputTo(ctx context.Context, org string, pro var firstReadErr error var firstReadFailure sync.Once totalBytes, totalDocs := 0, 0 - read := func(c mongoCollection, filter, projection bson.M, out any) (err error) { + read := func(c mongoCollection, filter, projection bson.M, out any, pages ...*outputOperationPage) (err error) { select { case sem <- struct{}{}: case <-readCtx.Done(): @@ -178,13 +178,28 @@ func (s *MongoStore) loadAnnotationOutputTo(ctx context.Context, org string, pro cancel() } }() - cursor, err := c.Find(readCtx, filter, options.Find().SetProjection(projection).SetMaxTime(15*time.Second)) + find := options.Find().SetProjection(projection).SetMaxTime(15 * time.Second) + var page *outputOperationPage + if len(pages) > 0 { + page = pages[0] + find.SetSort(bson.D{{Key: "_id", Value: 1}}).SetLimit(outputOperationPageSize).SetBatchSize(outputOperationPageSize) + if page.byID { + find.SetHint("_id_") + } + } + cursor, err := c.Find(readCtx, filter, find) if err != nil { return err } defer cursor.Close(readCtx) target := reflect.ValueOf(out).Elem() for cursor.Next(readCtx) { + if page != nil { + if err = cursor.Current.Lookup("_id").Unmarshal(&page.last); err != nil { + return err + } + page.count++ + } budgetMu.Lock() totalBytes += len(cursor.Current) totalDocs++ @@ -194,6 +209,15 @@ func (s *MongoStore) loadAnnotationOutputTo(ctx context.Context, org string, pro if exceeded { return fmt.Errorf("%w: bytes=%d/%d documents=%d/%d", workforcestats.ErrOutputSourceBudget, bytes, budget.bytes, documents, budget.documents) } + if page != nil { + page.rows = append(page.rows, append(bson.Raw(nil), cursor.Current...)) + page.bytes += len(cursor.Current) + if page.bytes >= outputOperationPageBytes { + page.byteLimited = true + break + } + continue + } if spool != nil { if err = spool.add(c.Name(), cursor.Current); err != nil { return err @@ -250,19 +274,17 @@ func (s *MongoStore) loadAnnotationOutputTo(ctx context.Context, org string, pro return read(s.annotationWorklogEvents, bson.M{"$and": bson.A{f, bson.M{"eventType": bson.M{"$in": workforcestats.OutputSubmissionEventTypes()}}}}, fields("id", "taskId", "userId", "supplierId", "eventType", "submissionId", "submissionKind", "outputDelta", "revision", "createdAt", "mode", "temporary"), &d.Events) }) g.Go(func() error { - // Only completed acceptance rejections consume draft problem facts. - // Keep every operation's status and effect links for in-flight fencing - // and legacy attribution, without loading unused quality drafts twice - // (their formal problem facts already come from reworkBaselines). - draftNeeded := bson.M{"stageId": bson.M{"$in": workforcestats.OutputAcceptanceStageIDs()}, "status": "completed", "decision": "rejected"} - projection := fields("_id", "operationId", "projectId", "userId", "itemId", "taskId", "stageId", "decision", "status", "sourceItemIds", "reviewerSupplierId", "createdAt", "completedAt", "effects.review.id", "effects.review.taskId", "effects.review.revision", "effects.beforeTask.id", "effects.beforeTask.supplierId", "effects.worklog.id") - if err := read(s.queryCollection("reviewOperations"), bson.M{"$and": bson.A{f, bson.M{"$nor": bson.A{draftNeeded}}}}, projection, &d.Operations); err != nil { - return err - } - for _, name := range []string{"draft.problems.id", "draft.problems.scope", "draft.problems.sourceItemId"} { - projection[name] = 1 - } - return read(s.queryCollection("reviewOperations"), bson.M{"$and": bson.A{f, draftNeeded}}, projection, &d.Operations) + return readOutputOperations(readCtx, s.queryCollection("reviewOperations"), f, &d.Operations, read, func(raw bson.Raw) error { + if spool != nil { + return spool.add("reviewOperations", raw) + } + var operation domain.ReviewOperation + if err := bson.Unmarshal(raw, &operation); err != nil { + return err + } + d.Operations = append(d.Operations, operation) + return nil + }) }) g.Go(func() error { return read(s.deliveryPackages, f, fields("id", "itemIds", "status", "manifestDigest", "releasePartitionId", "parentRunId", "mode", "temporary", "qualitySummary.prePackageAcceptance", "acceptanceReviews.id", "acceptanceReviews.decision", "acceptanceReviews.reviewerUserId", "acceptanceReviews.createdAt"), &d.Packages) diff --git a/internal/store/annotation_output_operations.go b/internal/store/annotation_output_operations.go new file mode 100644 index 00000000..18139f45 --- /dev/null +++ b/internal/store/annotation_output_operations.go @@ -0,0 +1,140 @@ +package store + +import ( + "context" + "errors" + "time" + + "droboticflow/internal/workforcestats" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" +) + +const outputOperationPageSize = 1024 +const outputOperationProblemBatch = 256 +const outputOperationPageBytes = 8 << 20 + +type outputOperationPage struct { + byID bool + bytes int + byteLimited bool + last any + count int + rows []bson.Raw +} +type outputOperationRead func(mongoCollection, bson.M, bson.M, any, ...*outputOperationPage) error + +// Read all operation metadata once, preserving legacy and unfinished plans. +// Only completed acceptance rejections need problem identities; fetch those +// by exact primary keys rather than scanning the full journal a second time. +// Caller deadlines and cumulative source budgets are unchanged. +func readOutputOperations(ctx context.Context, collection mongoCollection, scope bson.M, out any, read outputOperationRead, emit func(bson.Raw) error) error { + var bound struct { + ID any `bson:"_id"` + } + // Covered global _id bound reads no tenant body. All pages and point lookups + // retain the original organization/project scope. + err := collection.FindOne(ctx, bson.M{}, options.FindOne().SetProjection(bson.M{"_id": 1}).SetSort(bson.D{{Key: "_id", Value: -1}}).SetHint("_id_").SetMaxTime(15*time.Second)).Decode(&bound) + if errors.Is(err, mongo.ErrNoDocuments) { + return nil + } + if err != nil { + return err + } + var last any + for { + if err := ctx.Err(); err != nil { + return err + } + comparisons := bson.A{bson.M{"$lte": bson.A{"$_id", bson.M{"$literal": bound.ID}}}} + if last != nil { + comparisons = append(comparisons, bson.M{"$gt": bson.A{"$_id", bson.M{"$literal": last}}}) + } + // $expr retains total BSON ordering across mixed legacy _id types. + filter := bson.M{"$and": bson.A{scope, bson.M{"$expr": bson.M{"$and": comparisons}}}} + page := &outputOperationPage{} + if err := read(collection, filter, outputOperationProjection(), out, page); err != nil { + return err + } + ids, keys := bson.A{}, []string{} + for _, raw := range page.rows { + if !outputOperationNeedsProblems(raw) { + if err := emit(raw); err != nil { + return err + } + continue + } + var id any + if err := raw.Lookup("_id").Unmarshal(&id); err != nil { + return err + } + ids = append(ids, id) + keys = append(keys, outputOperationIdentity(raw)) + } + for start := 0; start < len(ids); start += outputOperationProblemBatch { + end := min(start+outputOperationProblemBatch, len(ids)) + pending := map[string]bool{} + for _, key := range keys[start:end] { + pending[key] = true + } + for len(pending) > 0 { + remaining := bson.A{} + for index := start; index < end; index++ { + if pending[keys[index]] { + remaining = append(remaining, ids[index]) + } + } + point := &outputOperationPage{byID: true} + filter := bson.M{"$and": bson.A{scope, bson.M{"_id": bson.M{"$in": remaining}}}} + if err := read(collection, filter, outputOperationProblemProjection(), out, point); err != nil { + return err + } + if point.count == 0 || point.count < len(remaining) && !point.byteLimited { + return errors.New("review operation history changed during statistics read") + } + for _, raw := range point.rows { + key := outputOperationIdentity(raw) + if !pending[key] || !outputOperationNeedsProblems(raw) { + return errors.New("review operation decision changed during statistics read") + } + if err := emit(raw); err != nil { + return err + } + delete(pending, key) + } + } + } + if page.count < outputOperationPageSize && !page.byteLimited { + return nil + } + if page.last == nil { + return errors.New("review operation page has no continuation identity") + } + last = page.last + } +} +func outputOperationIdentity(raw bson.Raw) string { + v := raw.Lookup("_id") + return string([]byte{byte(v.Type)}) + string(v.Value) +} +func outputOperationNeedsProblems(raw bson.Raw) bool { + stage, _ := raw.Lookup("stageId").StringValueOK() + status, _ := raw.Lookup("status").StringValueOK() + decision, _ := raw.Lookup("decision").StringValueOK() + return workforcestats.IsOutputAcceptanceStage(stage) && status == "completed" && decision == "rejected" +} +func outputOperationProjection() bson.M { + p := bson.M{} + for _, field := range []string{"_id", "operationId", "projectId", "userId", "itemId", "taskId", "stageId", "decision", "status", "sourceItemIds", "reviewerSupplierId", "createdAt", "completedAt", "effects.review.id", "effects.review.taskId", "effects.review.revision", "effects.beforeTask.id", "effects.beforeTask.supplierId", "effects.worklog.id"} { + p[field] = 1 + } + return p +} +func outputOperationProblemProjection() bson.M { + p := outputOperationProjection() + for _, field := range []string{"draft.problems.id", "draft.problems.scope", "draft.problems.sourceItemId"} { + p[field] = 1 + } + return p +} diff --git a/internal/store/annotation_output_operations_recovery_test.go b/internal/store/annotation_output_operations_recovery_test.go new file mode 100644 index 00000000..70cdb5a0 --- /dev/null +++ b/internal/store/annotation_output_operations_recovery_test.go @@ -0,0 +1,206 @@ +package store + +import ( + "context" + "errors" + "fmt" + "os" + "reflect" + "sync/atomic" + "testing" + "time" + + "droboticflow/internal/outputsnapshot" + "droboticflow/internal/workforcestats" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" + "go.mongodb.org/mongo-driver/event" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" +) + +func TestAnnotationOutputOperationMongoPageCancellationAndRestart(t *testing.T) { + db := localOutputMongo(t) + var docs []any + for i := 0; i < outputOperationPageSize+44; i++ { + docs = append(docs, bson.M{"_id": fmt.Sprintf("op-%04d", i), "organizationId": "out", "projectId": "p", "status": "prepared"}) + } + if _, e := db.Collection("reviewOperations").InsertMany(t.Context(), docs); e != nil { + t.Fatal(e) + } + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + var pages atomic.Int32 + var cancelled atomic.Bool + monitor := &event.CommandMonitor{Started: func(_ context.Context, e *event.CommandStartedEvent) { + if e.CommandName != "find" { + return + } + var m bson.M + _ = bson.Unmarshal(e.Command, &m) + if m["find"] == "reviewOperations" && m["limit"] == int64(outputOperationPageSize) && pages.Add(1) == 2 { + cancelled.Store(true) + cancel() + } + }} + c, e := mongo.Connect(t.Context(), options.Client().ApplyURI("mongodb://127.0.0.1:27018").SetMonitor(monitor)) + if e != nil { + t.Fatal(e) + } + defer c.Disconnect(context.Background()) + s := NewMongoStore(c.Database(db.Name())) + partial, e := s.LoadAnnotationOutput(ctx, "out", []string{"p"}) + if !cancelled.Load() || !errors.Is(e, context.Canceled) { + t.Fatalf("cancel returned partial success: cancelled=%t operations=%d error=%v", cancelled.Load(), len(partial.Operations), e) + } + recovered, e := NewMongoStore(db).LoadAnnotationOutput(t.Context(), "out", []string{"p"}) + if e != nil || len(recovered.Operations) != outputOperationPageSize+44 { + t.Fatalf("restart lost operation history: operations=%d error=%v", len(recovered.Operations), e) + } +} + +func TestAnnotationOutputOperationMongoSeekPreservesMixedLegacyIdentities(t *testing.T) { + db := localOutputMongo(t) + var docs []any + for i := 0; i < outputOperationPageSize/2+10; i++ { + docs = append(docs, bson.M{"_id": i, "organizationId": "out", "projectId": "p"}) + } + for i := 0; i < outputOperationPageSize/2+10; i++ { + docs = append(docs, bson.M{"_id": fmt.Sprintf("legacy-%04d", i), "organizationId": "out", "projectId": "p"}) + } + for i := 0; i < 3; i++ { + docs = append(docs, bson.M{"_id": primitive.NewObjectID(), "organizationId": "out", "projectId": "p"}) + } + docs = append(docs, bson.M{"_id": "foreign", "organizationId": "other", "projectId": "p"}) + if _, e := db.Collection("reviewOperations").InsertMany(t.Context(), docs); e != nil { + t.Fatal(e) + } + scope := organizationScopedFilter("out", bson.M{"projectId": "p"}) + var want []bson.M + cur, e := db.Collection("reviewOperations").Find(t.Context(), scope, options.Find().SetSort(bson.D{{Key: "_id", Value: 1}}).SetProjection(bson.M{"_id": 1})) + if e != nil { + t.Fatal(e) + } + if e = cur.All(t.Context(), &want); e != nil { + t.Fatal(e) + } + var got []bson.M + reader := func(c mongoCollection, f, p bson.M, _ any, pages ...*outputOperationPage) error { + cur, e := c.Find(t.Context(), f, options.Find().SetProjection(p).SetSort(bson.D{{Key: "_id", Value: 1}}).SetHint("_id_").SetLimit(outputOperationPageSize).SetBatchSize(outputOperationPageSize)) + if e != nil { + return e + } + defer cur.Close(t.Context()) + for cur.Next(t.Context()) { + var row bson.M + if e = cur.Decode(&row); e != nil { + return e + } + pages[0].last = row["_id"] + pages[0].count++ + pages[0].rows = append(pages[0].rows, append(bson.Raw(nil), cur.Current...)) + } + return cur.Err() + } + if e = readOutputOperations(t.Context(), NewMongoStore(db).queryCollection("reviewOperations"), scope, nil, reader, func(raw bson.Raw) error { + var row bson.M + if e := bson.Unmarshal(raw, &row); e != nil { + return e + } + got = append(got, bson.M{"_id": row["_id"]}) + return nil + }); e != nil { + t.Fatal(e) + } + if !reflect.DeepEqual(got, want) { + t.Fatalf("mixed BSON identities skipped or duplicated: got=%d want=%d", len(got), len(want)) + } +} + +func TestReviewOperationActualSnapshotMongoRefreshCycle(t *testing.T) { + if os.Getenv("RUN_OUTPUT_REFRESH_CYCLE") != "1" { + t.Skip("explicit real 120-second refresh cadence test") + } + db := localOutputMongo(t) + s := NewMongoStore(db) + seedAnnotationOutput(t, s) + beforeTask, e := s.GetAnnotationTask(t.Context(), "t") + if e != nil { + t.Fatal(e) + } + var builds atomic.Int32 + refreshing, release := make(chan struct{}), make(chan struct{}) + source := func(c context.Context, o string, ids []string, yield func(workforcestats.OutputSnapshot) error) (*workforcestats.OutputSourceStats, error) { + if builds.Add(1) == 2 { + close(refreshing) + select { + case <-release: + case <-c.Done(): + return nil, c.Err() + } + } + return s.StreamAnnotationOutput(c, o, ids, yield) + } + r := outputsnapshot.NewStreamReader(s, source) + defer r.Close() + q := workforcestats.OutputQuery{From: time.Date(2026, 10, 1, 0, 0, 0, 0, time.UTC), To: time.Date(2026, 10, 20, 0, 0, 0, 0, time.UTC)} + wait := func(reader *outputsnapshot.Reader, previous string) (*workforcestats.OutputReport, outputsnapshot.Info) { + t.Helper() + until := time.Now().Add(5 * time.Second) + for { + report, info, e := reader.Query(t.Context(), "out", []string{"p"}, "", q) + if e != nil { + t.Fatal(e) + } + if report != nil && info.ID != previous { + return report, info + } + if time.Now().After(until) { + t.Fatal("complete generation did not refresh") + } + time.Sleep(10 * time.Millisecond) + } + } + first, info := wait(r, "") + if first.Totals.Acceptance.Count != 1 { + t.Fatalf("first report: %+v", first.Totals) + } + op, e := s.GetReviewOperation(t.Context(), "ac") + if e != nil { + t.Fatal(e) + } + revision := op.Revision + op.Decision = "rejected" + op.CompletedAt = "2026-10-05T00:00:00Z" + if ok, e := s.CompareAndSwapReviewOperation(t.Context(), *op, revision); e != nil || !ok { + t.Fatalf("synthetic decision update: %t %v", ok, e) + } + // Exercise the production 120-second cadence. No forged generation dates, + // altered TTLs or longer build deadlines are used to make refresh pass. + time.Sleep(outputsnapshot.RefreshInterval + 100*time.Millisecond) + + old, stale, e := r.Query(t.Context(), "out", []string{"p"}, "", q) + if e != nil || old == nil || !stale.Stale || old.Totals.Acceptance.Count != 1 { + t.Fatalf("old complete generation lost while refreshing: %+v %v", stale, e) + } + select { + case <-refreshing: + case <-time.After(time.Second): + t.Fatal("background refresh did not start") + } + close(release) + next, nextInfo := wait(r, info.ID) + if next.Totals.Acceptance.Count != 0 || nextInfo.Stale { + t.Fatalf("new complete report was not published: %+v %+v", next.Totals, nextInfo) + } + afterTask, e := s.GetAnnotationTask(t.Context(), "t") + if e != nil || !reflect.DeepEqual(beforeTask, afterTask) { + t.Fatal("report refresh mutated annotation task") + } + restarted := outputsnapshot.NewStreamReader(NewMongoStore(db), s.StreamAnnotationOutput) + defer restarted.Close() + persisted, persistedInfo := wait(restarted, "") + if persistedInfo.ID != nextInfo.ID || persisted.Totals.Acceptance.Count != 0 { + t.Fatal("restart failed to recover the published report") + } +} diff --git a/internal/store/annotation_output_operations_test.go b/internal/store/annotation_output_operations_test.go new file mode 100644 index 00000000..e6f4b710 --- /dev/null +++ b/internal/store/annotation_output_operations_test.go @@ -0,0 +1,311 @@ +package store + +import ( + "context" + "encoding/json" + "fmt" + "os" + "reflect" + "sort" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "droboticflow/internal/domain" + "droboticflow/internal/workforcestats" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/event" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" +) + +func TestAnnotationOutputReviewOperationsMongoBoundedSinglePass(t *testing.T) { + db := localOutputMongo(t) + var mu sync.Mutex + var finds []bson.M + var getMores atomic.Int32 + monitor := &event.CommandMonitor{Started: func(_ context.Context, e *event.CommandStartedEvent) { + if e.CommandName == "getMore" && e.Command.Lookup("collection").StringValue() == "reviewOperations" { + getMores.Add(1) + } + if e.CommandName != "find" { + return + } + var m bson.M + if bson.Unmarshal(e.Command, &m) == nil && m["find"] == "reviewOperations" { + mu.Lock() + finds = append(finds, m) + mu.Unlock() + } + }} + client, err := mongo.Connect(t.Context(), options.Client().ApplyURI("mongodb://127.0.0.1:27018").SetMonitor(monitor)) + if err != nil { + t.Fatal(err) + } + defer client.Disconnect(context.Background()) + s := NewMongoStore(client.Database(db.Name())) + seedAnnotationOutput(t, s) + n := 2*outputOperationPageSize + 44 + if os.Getenv("RUN_OUTPUT_REVIEW_SCALE") == "1" { + n = 30310 + } + if _, err := db.Collection("reviewOperations").Indexes().CreateOne(t.Context(), PlatformReadIndexes()["reviewOperations"][0]); err != nil { + t.Fatal(err) + } + var pending []any + for i := 0; i < n; i++ { + op := domain.ReviewOperation{ID: fmt.Sprintf("op-%04d", i), OperationID: fmt.Sprintf("client-%d", i), OrganizationID: "out", ProjectID: "p", UserID: "qc", ItemID: "i", TaskID: "t", StageID: "quality_review", Decision: "approved", Status: "completed", SourceItemIDs: []string{"i"}, CompletedAt: "2026-10-04T00:00:00Z", Effects: []domain.ReviewDecisionEffect{{Review: &domain.AnnotationReviewRecord{ID: fmt.Sprintf("r-%d", i), TaskID: "t", Revision: 1}, BeforeTask: &domain.AnnotationTaskRecord{ID: "t", SupplierID: "v"}, BeforeItem: &domain.HumanItemRecord{Subject: domain.HumanSubjectRef{Payload: map[string]any{"annotationDraft": strings.Repeat("PRIVATE-NOT-REPORT", 2048)}}}}}} + switch i % 6 { + case 0: + op.StageID = "prepackage_acceptance" + op.Decision = "rejected" + op.Draft.Problems = []domain.ReviewProblem{{ID: "problem", Scope: "slice", SourceItemID: "i", Text: "PRIVATE-NOT-REPORT"}} + case 1: + op.StageID = "final_acceptance" + op.Status = "applying" + case 2: + op.Kind = "quality_draft" + op.Decision = "" + op.SourceItemIDs = nil + op.Effects = nil + case 3: + op.Kind = "" + op.Status = "prepared" + } + pending = append(pending, op) + if len(pending) == 256 || i == n-1 { + if _, err := db.Collection("reviewOperations").InsertMany(t.Context(), pending); err != nil { + t.Fatal(err) + } + pending = nil + } + } + // Nonmatching records must not leak even when the seek uses the _id index. + if _, err := db.Collection("reviewOperations").InsertOne(t.Context(), bson.M{"_id": "foreign-tail", "organizationId": "other", "projectId": "p"}); err != nil { + t.Fatal(err) + } + // Independent old two-query projection is the semantic oracle, including + // missing kind, unfinished decisions and acceptance problem attribution. + projection := bson.M{} + for _, field := range []string{"_id", "operationId", "projectId", "userId", "itemId", "taskId", "stageId", "decision", "status", "sourceItemIds", "reviewerSupplierId", "createdAt", "completedAt", "effects.review.id", "effects.review.taskId", "effects.review.revision", "effects.beforeTask.id", "effects.beforeTask.supplierId", "effects.worklog.id"} { + projection[field] = 1 + } + needed := bson.M{"stageId": bson.M{"$in": bson.A{"acceptance", "prepackage_acceptance", "final_acceptance"}}, "status": "completed", "decision": "rejected"} + scope := organizationScopedFilter("out", bson.M{"projectId": bson.M{"$in": bson.A{"p"}}}) + legacyStarted := time.Now() + var want []domain.ReviewOperation + for i, condition := range []bson.M{bson.M{"$nor": bson.A{needed}}, needed} { + if i == 1 { + for _, f := range []string{"draft.problems.id", "draft.problems.scope", "draft.problems.sourceItemId"} { + projection[f] = 1 + } + } + cur, e := db.Collection("reviewOperations").Find(t.Context(), bson.M{"$and": bson.A{scope, condition}}, options.Find().SetProjection(projection)) + if e != nil { + t.Fatal(e) + } + var rows []domain.ReviewOperation + if e = cur.All(t.Context(), &rows); e != nil { + t.Fatal(e) + } + want = append(want, rows...) + } + legacyElapsed := time.Since(legacyStarted) + start := time.Now() + got, e := s.LoadAnnotationOutput(t.Context(), "out", []string{"p"}) + if e != nil { + t.Fatal(e) + } + loaderElapsed := time.Since(start) + t.Logf("source_operations=%d source_bytes=%d legacy_review_only=%s whole_loader=%s", len(got.Operations), got.SourceStats.Bytes, legacyElapsed, loaderElapsed) + sort.Slice(want, func(i, j int) bool { return want[i].ID < want[j].ID }) + sort.Slice(got.Operations, func(i, j int) bool { return got.Operations[i].ID < got.Operations[j].ID }) + if !reflect.DeepEqual(got.Operations, want) { + a, _ := json.Marshal(got.Operations) + b, _ := json.Marshal(want) + t.Fatalf("operation projection differs: got=%s want=%s", a, b) + } + encoded, _ := json.Marshal(got.Operations) + if strings.Contains(string(encoded), "PRIVATE-NOT-REPORT") { + t.Fatal("body/draft text escaped the metadata projection") + } + if getMores.Load() != 0 { + t.Fatalf("bounded fixture unexpectedly used getMore for journal history: %d", getMores.Load()) + } + mu.Lock() + queries := append([]bson.M(nil), finds...) + mu.Unlock() + var pages []bson.M + for _, q := range queries { + limit, _ := q["limit"].(int64) + if limit < 0 { + limit = -limit + } + if limit == 1 { + continue + } + if limit == 0 || limit > outputOperationPageSize { + t.Fatalf("unbounded reviewOperations cursor: limit=%d; must not spend one maxTimeMS on all journal bodies", limit) + } + encodedFilter, _ := bson.MarshalExtJSON(q["filter"], false, false) + if strings.Contains(string(encodedFilter), "\"$expr\"") { + pages = append(pages, q) + } + } + expectedPages := (n+1)/outputOperationPageSize + 1 + if len(pages) != expectedPages { + t.Fatalf("expected %d disjoint source pages, got %d", expectedPages, len(pages)) + } + // A bounded query is insufficient if page N rescans pages 1..N-1. + second := pages[1] + command := bson.D{{Key: "find", Value: "reviewOperations"}, {Key: "filter", Value: second["filter"]}, {Key: "projection", Value: second["projection"]}, {Key: "sort", Value: second["sort"]}, {Key: "limit", Value: second["limit"]}} + var explain struct { + Stats struct { + Returned int `bson:"nReturned"` + Keys int `bson:"totalKeysExamined"` + } `bson:"executionStats"` + } + if e := db.RunCommand(t.Context(), bson.D{{Key: "explain", Value: command}, {Key: "verbosity", Value: "executionStats"}}).Decode(&explain); e != nil { + t.Fatal(e) + } + if explain.Stats.Returned != outputOperationPageSize || explain.Stats.Keys > outputOperationPageSize+2 { + t.Fatalf("seek page rescans prefix: %+v", explain.Stats) + } + t.Logf("complete operations=%d source_pages=%d legacy_two_scan=%s bounded_loader=%s page2_keys=%d", len(got.Operations), len(pages), legacyElapsed, time.Since(start), explain.Stats.Keys) + streamContext, cancel := context.WithTimeout(t.Context(), 90*time.Second) + defer cancel() + streamed := 0 + streamStarted := time.Now() + _, err = s.StreamAnnotationOutput(streamContext, "out", []string{"p"}, func(batch workforcestats.OutputSnapshot) error { streamed += len(batch.Operations); return nil }) + if err != nil || streamed != len(want) { + t.Fatalf("spooled history lost operations: got=%d want=%d error=%v", streamed, len(want), err) + } + t.Logf("spooled operations=%d complete_stream=%s", streamed, time.Since(streamStarted)) +} + +func TestAnnotationOutputOperationMongoSelectiveScopeAvoidsForeignScan(t *testing.T) { + db := localOutputMongo(t) + if _, e := db.Collection("reviewOperations").Indexes().CreateOne(t.Context(), PlatformReadIndexes()["reviewOperations"][0]); e != nil { + t.Fatal(e) + } + var docs []any + for i := 0; i < 4000; i++ { + docs = append(docs, bson.M{"_id": fmt.Sprintf("foreign-%04d", i), "organizationId": "other", "projectId": "p", "unused": strings.Repeat("body", 4096)}) + } + for i := 0; i < 5; i++ { + docs = append(docs, bson.M{"_id": fmt.Sprintf("target-%04d", i), "organizationId": "out", "projectId": "p"}) + } + if _, e := db.Collection("reviewOperations").InsertMany(t.Context(), docs); e != nil { + t.Fatal(e) + } + var query bson.M + monitor := &event.CommandMonitor{Started: func(_ context.Context, e *event.CommandStartedEvent) { + if e.CommandName != "find" { + return + } + var m bson.M + _ = bson.Unmarshal(e.Command, &m) + if m["find"] == "reviewOperations" && m["limit"] == int64(outputOperationPageSize) { + query = m + } + }} + c, e := mongo.Connect(t.Context(), options.Client().ApplyURI("mongodb://127.0.0.1:27018").SetMonitor(monitor)) + if e != nil { + t.Fatal(e) + } + defer c.Disconnect(context.Background()) + result, e := NewMongoStore(c.Database(db.Name())).LoadAnnotationOutput(t.Context(), "out", []string{"p"}) + if e != nil || len(result.Operations) != 5 { + t.Fatalf("scoped history mismatch: %d %v", len(result.Operations), e) + } + command := bson.D{{Key: "find", Value: "reviewOperations"}, {Key: "filter", Value: query["filter"]}, {Key: "projection", Value: query["projection"]}, {Key: "sort", Value: query["sort"]}, {Key: "limit", Value: query["limit"]}} + if h := query["hint"]; h != nil { + command = append(command, bson.E{Key: "hint", Value: h}) + } + var explanation struct { + Stats struct { + Keys int `bson:"totalKeysExamined"` + Documents int `bson:"totalDocsExamined"` + } `bson:"executionStats"` + } + if e = db.RunCommand(t.Context(), bson.D{{Key: "explain", Value: command}, {Key: "verbosity", Value: "executionStats"}}).Decode(&explanation); e != nil { + t.Fatal(e) + } + if explanation.Stats.Documents > 5 || explanation.Stats.Keys > 6 { + t.Fatalf("small scope scanned unrelated organization: %+v", explanation.Stats) + } +} + +func TestAnnotationOutputOperationMongoLargeMetadataAndProblemsRemainBounded(t *testing.T) { + for _, mode := range []string{"wide-metadata", "large-problem-refill"} { + t.Run(mode, func(t *testing.T) { + db := localOutputMongo(t) + var seeks, points atomic.Int32 + monitor := &event.CommandMonitor{Started: func(_ context.Context, e *event.CommandStartedEvent) { + if e.CommandName != "find" { + return + } + var m bson.M + _ = bson.Unmarshal(e.Command, &m) + if m["find"] != "reviewOperations" || m["limit"] != int64(outputOperationPageSize) { + return + } + f, _ := bson.MarshalExtJSON(m["filter"], false, false) + if strings.Contains(string(f), "\"$expr\"") { + seeks.Add(1) + } else { + points.Add(1) + } + }} + c, e := mongo.Connect(t.Context(), options.Client().ApplyURI("mongodb://127.0.0.1:27018").SetMonitor(monitor)) + if e != nil { + t.Fatal(e) + } + defer c.Disconnect(context.Background()) + n := 400 + if mode == "large-problem-refill" { + n = 24 + } + for i := 0; i < n; i++ { + d := bson.M{"_id": fmt.Sprintf("op-%04d", i), "organizationId": "out", "projectId": "p", "stageId": "quality_review", "status": "prepared"} + if mode == "wide-metadata" { + ids := bson.A{} + for j := 0; j < 300; j++ { + ids = append(ids, fmt.Sprintf("source-%04d-%s", j, strings.Repeat("s", 96))) + } + d["sourceItemIds"] = ids + } else { + d["stageId"], d["status"], d["decision"] = "acceptance", "completed", "rejected" + problems := bson.A{} + for j := 0; j < 2500; j++ { + problems = append(problems, bson.M{"id": fmt.Sprintf("problem-%04d-%s", j, strings.Repeat("p", 192)), "scope": "slice", "sourceItemId": "i", "text": strings.Repeat("private text", 32)}) + } + d["draft"] = bson.M{"problems": problems} + } + if _, e := db.Collection("reviewOperations").InsertOne(t.Context(), d); e != nil { + t.Fatal(e) + } + } + result, e := NewMongoStore(c.Database(db.Name())).LoadAnnotationOutput(t.Context(), "out", []string{"p"}) + if e != nil || len(result.Operations) != n { + t.Fatalf("byte-split lost records: %d/%d %v", len(result.Operations), n, e) + } + if mode == "wide-metadata" && seeks.Load() < 2 { + t.Fatal("metadata page ignored the byte boundary") + } + if mode == "large-problem-refill" && points.Load() < 2 { + t.Fatal("problem refill accumulated an unbounded page") + } + for _, op := range result.Operations { + if mode == "wide-metadata" && len(op.SourceItemIDs) != 300 { + t.Fatal("lost source identities") + } + if mode == "large-problem-refill" && (len(op.Draft.Problems) != 2500 || op.Draft.Problems[0].Text != "") { + t.Fatal("lost problem facts or exposed opinion text") + } + } + }) + } +} diff --git a/scripts/verification-scopes.json b/scripts/verification-scopes.json index 8fc9ab9e..c83328d4 100644 --- a/scripts/verification-scopes.json +++ b/scripts/verification-scopes.json @@ -1077,6 +1077,38 @@ "warnings": [ "独立源库/查询库仅使用合成数据;生产写者、47个源索引、迁移及集群资源须另行审批。native测试没有CPU配额,不能证明4核容量。" ] + }, + { + "name": "review-operation-history", + "paths": [ + "^internal/store/annotation_output_operations(_recovery)?(_test)?\\.go$", + "^internal/store/annotation_output\\.go$" + ], + "frontend": [], + "go": {}, + "task": [ + { + "argv": [ + "go", + "test", + "./internal/store", + "-run", + "^TestAnnotationOutput.*Operation.*Mongo|^TestReviewOperationActualSnapshotMongoRefreshCycle$", + "-count=1", + "-timeout=180s", + "-v" + ], + "env": { + "TEST_WORKFORCE_MONGO_URI": "mongodb://127.0.0.1:27018", + "RUN_OUTPUT_REVIEW_SCALE": "1", + "RUN_OUTPUT_REFRESH_CYCLE": "1" + }, + "tests": true + } + ], + "warnings": [ + "审核操作历史读取保持完整、原15秒查询和90秒构建预算;显式本机合成夹具覆盖同规模、取消、旧ID及真实120秒刷新,不代表生产已刷新。" + ] } ] }