🐛 Bound review journal snapshot reads without extending budgets

Replace complementary journal scans with bounded seek pages and scoped primary-key problem refill. Preserve legacy/in-flight evidence, memory boundaries and original query/build deadlines. Cover scoped plans, source parity, cancellation, real refresh cadence and restart.
This commit is contained in:
d-robotics 2026-11-14 01:03:23 -08:00
parent fc7786190f
commit c0512c8bf6
6 changed files with 753 additions and 15 deletions

View File

@ -0,0 +1,27 @@
# 统计审核操作历史读取超时
## 现场与口径
2026-11-14,线上 fc778619 的统计快照后台报告 `annotation output read reviewOperations ... MaxTimeMSExpired ... getMore`;快照健康出现 `partitioned_build / timeout`,旧完整代次仍能在原15分钟窗口内响应。HTTP 200 和 Pod 就绪不能证明统计刷新成功。
只读元数据约3.03万条审核操作、583MB原始文档;现有统计读取先排除正式验收退回,再读取正式验收退回,两条互补条件均需要 Fetch 操作日志正文。小投影没有消除 Mongo 大文档读取,单个全历史游标可能在 getMore 时耗尽15秒查询预算。现有 lookup 索引不能覆盖统计所需的 effects 和来源数组。
## 修复范围
- 所有操作先按 `_id` 有界顺序读取一次元数据,每页最多1024条,并按8MiB投影字节提前续页;大问题事实补读同样按字节分批并逐批消费,避免在流式构建中积累整页大正文。用一次只读主键上界限制本轮增长,每页保留原组织兼容和项目范围,不丢弃缺字段、旧任务或未完成操作。
- 仅为元数据中已经识别出的正式验收退回,按主键每批最多256条补充问题 ID、作用域和来源 ID;不再为这些记录第二次遍历全历史。正文、草稿文字、视频及凭证不进入事实投影。
- 使用 BSON 总排序的 seek 条件,避免普通 `$gt` 的类型边界遗漏旧主键;主扫描让规划器在现有组织索引与主键范围间选路,避免小范围被强制跨组织扫描;仅精确ID补读使用主键hint。不新增生产索引或启动迁移。
- 原每次查询15秒、整体构建90秒、15分钟展示窗口、并发和源数据预算保持不变。源读取字节/条数包含必要的补读开销,不通过少计数据达标。
- 补读缺失或决定变化视为本轮构建失败,不能发布混合或不完整代次。保留上一完整快照;取消、重启及下一轮读取不改写源任务。
只改变统计源读取。权限、供应商隔离、任务版本、领取、保存、切片、送检、质检和验收正式决定路径不变,查询库与共享缓存开关保持原状态。
## 验证与发布
本机唯一合成数据库覆盖旧投影结果等价、正式退回与未完成记录、正文排除、分页取消/重试、混合 BSON 主键和分页索引扫描;显式规模夹具约3.03万条大载荷操作。刷新测试采用真实120秒周期,验证旧完整代次继续服务、新代次完整发布以及重启读取,不伪造快照时间或延长产品预算。
专项已映射到 verification-scopes.json 的 review-operation-history;快速检查不代替显式 Mongo 和真实周期检查。首个只分页方案的性能不佳,对照记录保留,最终实现采用普通元数据投影和按 ID 补读。
生产只做有界只读诊断,不执行全历史压力对照;首128条及第二页128条的诊断证实主键 seek 不重复扫描前缀。该诊断不等于完整构建或并发容量验收。本地稳定场景也不能代替原Mongo高负载下的端到端刷新验证。
发布须另行同步dev、准备镜像与正式计划并确认。上线后必须观察实际源码、快照时间晚于新构建、失败计数归零及下一轮继续刷新;返回旧快照的200不能算成功。回退仅切回旧应用,不回滚数据库、不覆盖已有标注,不切换生产查询入口。

View File

@ -159,7 +159,7 @@ func (s *MongoStore) loadAnnotationOutputTo(ctx context.Context, org string, pro
var firstReadErr error
var firstReadFailure sync.Once
totalBytes, totalDocs := 0, 0
read := func(c mongoCollection, filter, projection bson.M, out any) (err error) {
read := func(c mongoCollection, filter, projection bson.M, out any, pages ...*outputOperationPage) (err error) {
select {
case sem <- struct{}{}:
case <-readCtx.Done():
@ -178,13 +178,28 @@ func (s *MongoStore) loadAnnotationOutputTo(ctx context.Context, org string, pro
cancel()
}
}()
cursor, err := c.Find(readCtx, filter, options.Find().SetProjection(projection).SetMaxTime(15*time.Second))
find := options.Find().SetProjection(projection).SetMaxTime(15 * time.Second)
var page *outputOperationPage
if len(pages) > 0 {
page = pages[0]
find.SetSort(bson.D{{Key: "_id", Value: 1}}).SetLimit(outputOperationPageSize).SetBatchSize(outputOperationPageSize)
if page.byID {
find.SetHint("_id_")
}
}
cursor, err := c.Find(readCtx, filter, find)
if err != nil {
return err
}
defer cursor.Close(readCtx)
target := reflect.ValueOf(out).Elem()
for cursor.Next(readCtx) {
if page != nil {
if err = cursor.Current.Lookup("_id").Unmarshal(&page.last); err != nil {
return err
}
page.count++
}
budgetMu.Lock()
totalBytes += len(cursor.Current)
totalDocs++
@ -194,6 +209,15 @@ func (s *MongoStore) loadAnnotationOutputTo(ctx context.Context, org string, pro
if exceeded {
return fmt.Errorf("%w: bytes=%d/%d documents=%d/%d", workforcestats.ErrOutputSourceBudget, bytes, budget.bytes, documents, budget.documents)
}
if page != nil {
page.rows = append(page.rows, append(bson.Raw(nil), cursor.Current...))
page.bytes += len(cursor.Current)
if page.bytes >= outputOperationPageBytes {
page.byteLimited = true
break
}
continue
}
if spool != nil {
if err = spool.add(c.Name(), cursor.Current); err != nil {
return err
@ -250,19 +274,17 @@ func (s *MongoStore) loadAnnotationOutputTo(ctx context.Context, org string, pro
return read(s.annotationWorklogEvents, bson.M{"$and": bson.A{f, bson.M{"eventType": bson.M{"$in": workforcestats.OutputSubmissionEventTypes()}}}}, fields("id", "taskId", "userId", "supplierId", "eventType", "submissionId", "submissionKind", "outputDelta", "revision", "createdAt", "mode", "temporary"), &d.Events)
})
g.Go(func() error {
// Only completed acceptance rejections consume draft problem facts.
// Keep every operation's status and effect links for in-flight fencing
// and legacy attribution, without loading unused quality drafts twice
// (their formal problem facts already come from reworkBaselines).
draftNeeded := bson.M{"stageId": bson.M{"$in": workforcestats.OutputAcceptanceStageIDs()}, "status": "completed", "decision": "rejected"}
projection := fields("_id", "operationId", "projectId", "userId", "itemId", "taskId", "stageId", "decision", "status", "sourceItemIds", "reviewerSupplierId", "createdAt", "completedAt", "effects.review.id", "effects.review.taskId", "effects.review.revision", "effects.beforeTask.id", "effects.beforeTask.supplierId", "effects.worklog.id")
if err := read(s.queryCollection("reviewOperations"), bson.M{"$and": bson.A{f, bson.M{"$nor": bson.A{draftNeeded}}}}, projection, &d.Operations); err != nil {
return err
}
for _, name := range []string{"draft.problems.id", "draft.problems.scope", "draft.problems.sourceItemId"} {
projection[name] = 1
}
return read(s.queryCollection("reviewOperations"), bson.M{"$and": bson.A{f, draftNeeded}}, projection, &d.Operations)
return readOutputOperations(readCtx, s.queryCollection("reviewOperations"), f, &d.Operations, read, func(raw bson.Raw) error {
if spool != nil {
return spool.add("reviewOperations", raw)
}
var operation domain.ReviewOperation
if err := bson.Unmarshal(raw, &operation); err != nil {
return err
}
d.Operations = append(d.Operations, operation)
return nil
})
})
g.Go(func() error {
return read(s.deliveryPackages, f, fields("id", "itemIds", "status", "manifestDigest", "releasePartitionId", "parentRunId", "mode", "temporary", "qualitySummary.prePackageAcceptance", "acceptanceReviews.id", "acceptanceReviews.decision", "acceptanceReviews.reviewerUserId", "acceptanceReviews.createdAt"), &d.Packages)

View File

@ -0,0 +1,140 @@
package store
import (
"context"
"errors"
"time"
"droboticflow/internal/workforcestats"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
)
const outputOperationPageSize = 1024
const outputOperationProblemBatch = 256
const outputOperationPageBytes = 8 << 20
type outputOperationPage struct {
byID bool
bytes int
byteLimited bool
last any
count int
rows []bson.Raw
}
type outputOperationRead func(mongoCollection, bson.M, bson.M, any, ...*outputOperationPage) error
// Read all operation metadata once, preserving legacy and unfinished plans.
// Only completed acceptance rejections need problem identities; fetch those
// by exact primary keys rather than scanning the full journal a second time.
// Caller deadlines and cumulative source budgets are unchanged.
func readOutputOperations(ctx context.Context, collection mongoCollection, scope bson.M, out any, read outputOperationRead, emit func(bson.Raw) error) error {
var bound struct {
ID any `bson:"_id"`
}
// Covered global _id bound reads no tenant body. All pages and point lookups
// retain the original organization/project scope.
err := collection.FindOne(ctx, bson.M{}, options.FindOne().SetProjection(bson.M{"_id": 1}).SetSort(bson.D{{Key: "_id", Value: -1}}).SetHint("_id_").SetMaxTime(15*time.Second)).Decode(&bound)
if errors.Is(err, mongo.ErrNoDocuments) {
return nil
}
if err != nil {
return err
}
var last any
for {
if err := ctx.Err(); err != nil {
return err
}
comparisons := bson.A{bson.M{"$lte": bson.A{"$_id", bson.M{"$literal": bound.ID}}}}
if last != nil {
comparisons = append(comparisons, bson.M{"$gt": bson.A{"$_id", bson.M{"$literal": last}}})
}
// $expr retains total BSON ordering across mixed legacy _id types.
filter := bson.M{"$and": bson.A{scope, bson.M{"$expr": bson.M{"$and": comparisons}}}}
page := &outputOperationPage{}
if err := read(collection, filter, outputOperationProjection(), out, page); err != nil {
return err
}
ids, keys := bson.A{}, []string{}
for _, raw := range page.rows {
if !outputOperationNeedsProblems(raw) {
if err := emit(raw); err != nil {
return err
}
continue
}
var id any
if err := raw.Lookup("_id").Unmarshal(&id); err != nil {
return err
}
ids = append(ids, id)
keys = append(keys, outputOperationIdentity(raw))
}
for start := 0; start < len(ids); start += outputOperationProblemBatch {
end := min(start+outputOperationProblemBatch, len(ids))
pending := map[string]bool{}
for _, key := range keys[start:end] {
pending[key] = true
}
for len(pending) > 0 {
remaining := bson.A{}
for index := start; index < end; index++ {
if pending[keys[index]] {
remaining = append(remaining, ids[index])
}
}
point := &outputOperationPage{byID: true}
filter := bson.M{"$and": bson.A{scope, bson.M{"_id": bson.M{"$in": remaining}}}}
if err := read(collection, filter, outputOperationProblemProjection(), out, point); err != nil {
return err
}
if point.count == 0 || point.count < len(remaining) && !point.byteLimited {
return errors.New("review operation history changed during statistics read")
}
for _, raw := range point.rows {
key := outputOperationIdentity(raw)
if !pending[key] || !outputOperationNeedsProblems(raw) {
return errors.New("review operation decision changed during statistics read")
}
if err := emit(raw); err != nil {
return err
}
delete(pending, key)
}
}
}
if page.count < outputOperationPageSize && !page.byteLimited {
return nil
}
if page.last == nil {
return errors.New("review operation page has no continuation identity")
}
last = page.last
}
}
func outputOperationIdentity(raw bson.Raw) string {
v := raw.Lookup("_id")
return string([]byte{byte(v.Type)}) + string(v.Value)
}
func outputOperationNeedsProblems(raw bson.Raw) bool {
stage, _ := raw.Lookup("stageId").StringValueOK()
status, _ := raw.Lookup("status").StringValueOK()
decision, _ := raw.Lookup("decision").StringValueOK()
return workforcestats.IsOutputAcceptanceStage(stage) && status == "completed" && decision == "rejected"
}
func outputOperationProjection() bson.M {
p := bson.M{}
for _, field := range []string{"_id", "operationId", "projectId", "userId", "itemId", "taskId", "stageId", "decision", "status", "sourceItemIds", "reviewerSupplierId", "createdAt", "completedAt", "effects.review.id", "effects.review.taskId", "effects.review.revision", "effects.beforeTask.id", "effects.beforeTask.supplierId", "effects.worklog.id"} {
p[field] = 1
}
return p
}
func outputOperationProblemProjection() bson.M {
p := outputOperationProjection()
for _, field := range []string{"draft.problems.id", "draft.problems.scope", "draft.problems.sourceItemId"} {
p[field] = 1
}
return p
}

View File

@ -0,0 +1,206 @@
package store
import (
"context"
"errors"
"fmt"
"os"
"reflect"
"sync/atomic"
"testing"
"time"
"droboticflow/internal/outputsnapshot"
"droboticflow/internal/workforcestats"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/bson/primitive"
"go.mongodb.org/mongo-driver/event"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
)
func TestAnnotationOutputOperationMongoPageCancellationAndRestart(t *testing.T) {
db := localOutputMongo(t)
var docs []any
for i := 0; i < outputOperationPageSize+44; i++ {
docs = append(docs, bson.M{"_id": fmt.Sprintf("op-%04d", i), "organizationId": "out", "projectId": "p", "status": "prepared"})
}
if _, e := db.Collection("reviewOperations").InsertMany(t.Context(), docs); e != nil {
t.Fatal(e)
}
ctx, cancel := context.WithCancel(t.Context())
defer cancel()
var pages atomic.Int32
var cancelled atomic.Bool
monitor := &event.CommandMonitor{Started: func(_ context.Context, e *event.CommandStartedEvent) {
if e.CommandName != "find" {
return
}
var m bson.M
_ = bson.Unmarshal(e.Command, &m)
if m["find"] == "reviewOperations" && m["limit"] == int64(outputOperationPageSize) && pages.Add(1) == 2 {
cancelled.Store(true)
cancel()
}
}}
c, e := mongo.Connect(t.Context(), options.Client().ApplyURI("mongodb://127.0.0.1:27018").SetMonitor(monitor))
if e != nil {
t.Fatal(e)
}
defer c.Disconnect(context.Background())
s := NewMongoStore(c.Database(db.Name()))
partial, e := s.LoadAnnotationOutput(ctx, "out", []string{"p"})
if !cancelled.Load() || !errors.Is(e, context.Canceled) {
t.Fatalf("cancel returned partial success: cancelled=%t operations=%d error=%v", cancelled.Load(), len(partial.Operations), e)
}
recovered, e := NewMongoStore(db).LoadAnnotationOutput(t.Context(), "out", []string{"p"})
if e != nil || len(recovered.Operations) != outputOperationPageSize+44 {
t.Fatalf("restart lost operation history: operations=%d error=%v", len(recovered.Operations), e)
}
}
func TestAnnotationOutputOperationMongoSeekPreservesMixedLegacyIdentities(t *testing.T) {
db := localOutputMongo(t)
var docs []any
for i := 0; i < outputOperationPageSize/2+10; i++ {
docs = append(docs, bson.M{"_id": i, "organizationId": "out", "projectId": "p"})
}
for i := 0; i < outputOperationPageSize/2+10; i++ {
docs = append(docs, bson.M{"_id": fmt.Sprintf("legacy-%04d", i), "organizationId": "out", "projectId": "p"})
}
for i := 0; i < 3; i++ {
docs = append(docs, bson.M{"_id": primitive.NewObjectID(), "organizationId": "out", "projectId": "p"})
}
docs = append(docs, bson.M{"_id": "foreign", "organizationId": "other", "projectId": "p"})
if _, e := db.Collection("reviewOperations").InsertMany(t.Context(), docs); e != nil {
t.Fatal(e)
}
scope := organizationScopedFilter("out", bson.M{"projectId": "p"})
var want []bson.M
cur, e := db.Collection("reviewOperations").Find(t.Context(), scope, options.Find().SetSort(bson.D{{Key: "_id", Value: 1}}).SetProjection(bson.M{"_id": 1}))
if e != nil {
t.Fatal(e)
}
if e = cur.All(t.Context(), &want); e != nil {
t.Fatal(e)
}
var got []bson.M
reader := func(c mongoCollection, f, p bson.M, _ any, pages ...*outputOperationPage) error {
cur, e := c.Find(t.Context(), f, options.Find().SetProjection(p).SetSort(bson.D{{Key: "_id", Value: 1}}).SetHint("_id_").SetLimit(outputOperationPageSize).SetBatchSize(outputOperationPageSize))
if e != nil {
return e
}
defer cur.Close(t.Context())
for cur.Next(t.Context()) {
var row bson.M
if e = cur.Decode(&row); e != nil {
return e
}
pages[0].last = row["_id"]
pages[0].count++
pages[0].rows = append(pages[0].rows, append(bson.Raw(nil), cur.Current...))
}
return cur.Err()
}
if e = readOutputOperations(t.Context(), NewMongoStore(db).queryCollection("reviewOperations"), scope, nil, reader, func(raw bson.Raw) error {
var row bson.M
if e := bson.Unmarshal(raw, &row); e != nil {
return e
}
got = append(got, bson.M{"_id": row["_id"]})
return nil
}); e != nil {
t.Fatal(e)
}
if !reflect.DeepEqual(got, want) {
t.Fatalf("mixed BSON identities skipped or duplicated: got=%d want=%d", len(got), len(want))
}
}
func TestReviewOperationActualSnapshotMongoRefreshCycle(t *testing.T) {
if os.Getenv("RUN_OUTPUT_REFRESH_CYCLE") != "1" {
t.Skip("explicit real 120-second refresh cadence test")
}
db := localOutputMongo(t)
s := NewMongoStore(db)
seedAnnotationOutput(t, s)
beforeTask, e := s.GetAnnotationTask(t.Context(), "t")
if e != nil {
t.Fatal(e)
}
var builds atomic.Int32
refreshing, release := make(chan struct{}), make(chan struct{})
source := func(c context.Context, o string, ids []string, yield func(workforcestats.OutputSnapshot) error) (*workforcestats.OutputSourceStats, error) {
if builds.Add(1) == 2 {
close(refreshing)
select {
case <-release:
case <-c.Done():
return nil, c.Err()
}
}
return s.StreamAnnotationOutput(c, o, ids, yield)
}
r := outputsnapshot.NewStreamReader(s, source)
defer r.Close()
q := workforcestats.OutputQuery{From: time.Date(2026, 10, 1, 0, 0, 0, 0, time.UTC), To: time.Date(2026, 10, 20, 0, 0, 0, 0, time.UTC)}
wait := func(reader *outputsnapshot.Reader, previous string) (*workforcestats.OutputReport, outputsnapshot.Info) {
t.Helper()
until := time.Now().Add(5 * time.Second)
for {
report, info, e := reader.Query(t.Context(), "out", []string{"p"}, "", q)
if e != nil {
t.Fatal(e)
}
if report != nil && info.ID != previous {
return report, info
}
if time.Now().After(until) {
t.Fatal("complete generation did not refresh")
}
time.Sleep(10 * time.Millisecond)
}
}
first, info := wait(r, "")
if first.Totals.Acceptance.Count != 1 {
t.Fatalf("first report: %+v", first.Totals)
}
op, e := s.GetReviewOperation(t.Context(), "ac")
if e != nil {
t.Fatal(e)
}
revision := op.Revision
op.Decision = "rejected"
op.CompletedAt = "2026-10-05T00:00:00Z"
if ok, e := s.CompareAndSwapReviewOperation(t.Context(), *op, revision); e != nil || !ok {
t.Fatalf("synthetic decision update: %t %v", ok, e)
}
// Exercise the production 120-second cadence. No forged generation dates,
// altered TTLs or longer build deadlines are used to make refresh pass.
time.Sleep(outputsnapshot.RefreshInterval + 100*time.Millisecond)
old, stale, e := r.Query(t.Context(), "out", []string{"p"}, "", q)
if e != nil || old == nil || !stale.Stale || old.Totals.Acceptance.Count != 1 {
t.Fatalf("old complete generation lost while refreshing: %+v %v", stale, e)
}
select {
case <-refreshing:
case <-time.After(time.Second):
t.Fatal("background refresh did not start")
}
close(release)
next, nextInfo := wait(r, info.ID)
if next.Totals.Acceptance.Count != 0 || nextInfo.Stale {
t.Fatalf("new complete report was not published: %+v %+v", next.Totals, nextInfo)
}
afterTask, e := s.GetAnnotationTask(t.Context(), "t")
if e != nil || !reflect.DeepEqual(beforeTask, afterTask) {
t.Fatal("report refresh mutated annotation task")
}
restarted := outputsnapshot.NewStreamReader(NewMongoStore(db), s.StreamAnnotationOutput)
defer restarted.Close()
persisted, persistedInfo := wait(restarted, "")
if persistedInfo.ID != nextInfo.ID || persisted.Totals.Acceptance.Count != 0 {
t.Fatal("restart failed to recover the published report")
}
}

View File

@ -0,0 +1,311 @@
package store
import (
"context"
"encoding/json"
"fmt"
"os"
"reflect"
"sort"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
"droboticflow/internal/domain"
"droboticflow/internal/workforcestats"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/event"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
)
func TestAnnotationOutputReviewOperationsMongoBoundedSinglePass(t *testing.T) {
db := localOutputMongo(t)
var mu sync.Mutex
var finds []bson.M
var getMores atomic.Int32
monitor := &event.CommandMonitor{Started: func(_ context.Context, e *event.CommandStartedEvent) {
if e.CommandName == "getMore" && e.Command.Lookup("collection").StringValue() == "reviewOperations" {
getMores.Add(1)
}
if e.CommandName != "find" {
return
}
var m bson.M
if bson.Unmarshal(e.Command, &m) == nil && m["find"] == "reviewOperations" {
mu.Lock()
finds = append(finds, m)
mu.Unlock()
}
}}
client, err := mongo.Connect(t.Context(), options.Client().ApplyURI("mongodb://127.0.0.1:27018").SetMonitor(monitor))
if err != nil {
t.Fatal(err)
}
defer client.Disconnect(context.Background())
s := NewMongoStore(client.Database(db.Name()))
seedAnnotationOutput(t, s)
n := 2*outputOperationPageSize + 44
if os.Getenv("RUN_OUTPUT_REVIEW_SCALE") == "1" {
n = 30310
}
if _, err := db.Collection("reviewOperations").Indexes().CreateOne(t.Context(), PlatformReadIndexes()["reviewOperations"][0]); err != nil {
t.Fatal(err)
}
var pending []any
for i := 0; i < n; i++ {
op := domain.ReviewOperation{ID: fmt.Sprintf("op-%04d", i), OperationID: fmt.Sprintf("client-%d", i), OrganizationID: "out", ProjectID: "p", UserID: "qc", ItemID: "i", TaskID: "t", StageID: "quality_review", Decision: "approved", Status: "completed", SourceItemIDs: []string{"i"}, CompletedAt: "2026-10-04T00:00:00Z", Effects: []domain.ReviewDecisionEffect{{Review: &domain.AnnotationReviewRecord{ID: fmt.Sprintf("r-%d", i), TaskID: "t", Revision: 1}, BeforeTask: &domain.AnnotationTaskRecord{ID: "t", SupplierID: "v"}, BeforeItem: &domain.HumanItemRecord{Subject: domain.HumanSubjectRef{Payload: map[string]any{"annotationDraft": strings.Repeat("PRIVATE-NOT-REPORT", 2048)}}}}}}
switch i % 6 {
case 0:
op.StageID = "prepackage_acceptance"
op.Decision = "rejected"
op.Draft.Problems = []domain.ReviewProblem{{ID: "problem", Scope: "slice", SourceItemID: "i", Text: "PRIVATE-NOT-REPORT"}}
case 1:
op.StageID = "final_acceptance"
op.Status = "applying"
case 2:
op.Kind = "quality_draft"
op.Decision = ""
op.SourceItemIDs = nil
op.Effects = nil
case 3:
op.Kind = ""
op.Status = "prepared"
}
pending = append(pending, op)
if len(pending) == 256 || i == n-1 {
if _, err := db.Collection("reviewOperations").InsertMany(t.Context(), pending); err != nil {
t.Fatal(err)
}
pending = nil
}
}
// Nonmatching records must not leak even when the seek uses the _id index.
if _, err := db.Collection("reviewOperations").InsertOne(t.Context(), bson.M{"_id": "foreign-tail", "organizationId": "other", "projectId": "p"}); err != nil {
t.Fatal(err)
}
// Independent old two-query projection is the semantic oracle, including
// missing kind, unfinished decisions and acceptance problem attribution.
projection := bson.M{}
for _, field := range []string{"_id", "operationId", "projectId", "userId", "itemId", "taskId", "stageId", "decision", "status", "sourceItemIds", "reviewerSupplierId", "createdAt", "completedAt", "effects.review.id", "effects.review.taskId", "effects.review.revision", "effects.beforeTask.id", "effects.beforeTask.supplierId", "effects.worklog.id"} {
projection[field] = 1
}
needed := bson.M{"stageId": bson.M{"$in": bson.A{"acceptance", "prepackage_acceptance", "final_acceptance"}}, "status": "completed", "decision": "rejected"}
scope := organizationScopedFilter("out", bson.M{"projectId": bson.M{"$in": bson.A{"p"}}})
legacyStarted := time.Now()
var want []domain.ReviewOperation
for i, condition := range []bson.M{bson.M{"$nor": bson.A{needed}}, needed} {
if i == 1 {
for _, f := range []string{"draft.problems.id", "draft.problems.scope", "draft.problems.sourceItemId"} {
projection[f] = 1
}
}
cur, e := db.Collection("reviewOperations").Find(t.Context(), bson.M{"$and": bson.A{scope, condition}}, options.Find().SetProjection(projection))
if e != nil {
t.Fatal(e)
}
var rows []domain.ReviewOperation
if e = cur.All(t.Context(), &rows); e != nil {
t.Fatal(e)
}
want = append(want, rows...)
}
legacyElapsed := time.Since(legacyStarted)
start := time.Now()
got, e := s.LoadAnnotationOutput(t.Context(), "out", []string{"p"})
if e != nil {
t.Fatal(e)
}
loaderElapsed := time.Since(start)
t.Logf("source_operations=%d source_bytes=%d legacy_review_only=%s whole_loader=%s", len(got.Operations), got.SourceStats.Bytes, legacyElapsed, loaderElapsed)
sort.Slice(want, func(i, j int) bool { return want[i].ID < want[j].ID })
sort.Slice(got.Operations, func(i, j int) bool { return got.Operations[i].ID < got.Operations[j].ID })
if !reflect.DeepEqual(got.Operations, want) {
a, _ := json.Marshal(got.Operations)
b, _ := json.Marshal(want)
t.Fatalf("operation projection differs: got=%s want=%s", a, b)
}
encoded, _ := json.Marshal(got.Operations)
if strings.Contains(string(encoded), "PRIVATE-NOT-REPORT") {
t.Fatal("body/draft text escaped the metadata projection")
}
if getMores.Load() != 0 {
t.Fatalf("bounded fixture unexpectedly used getMore for journal history: %d", getMores.Load())
}
mu.Lock()
queries := append([]bson.M(nil), finds...)
mu.Unlock()
var pages []bson.M
for _, q := range queries {
limit, _ := q["limit"].(int64)
if limit < 0 {
limit = -limit
}
if limit == 1 {
continue
}
if limit == 0 || limit > outputOperationPageSize {
t.Fatalf("unbounded reviewOperations cursor: limit=%d; must not spend one maxTimeMS on all journal bodies", limit)
}
encodedFilter, _ := bson.MarshalExtJSON(q["filter"], false, false)
if strings.Contains(string(encodedFilter), "\"$expr\"") {
pages = append(pages, q)
}
}
expectedPages := (n+1)/outputOperationPageSize + 1
if len(pages) != expectedPages {
t.Fatalf("expected %d disjoint source pages, got %d", expectedPages, len(pages))
}
// A bounded query is insufficient if page N rescans pages 1..N-1.
second := pages[1]
command := bson.D{{Key: "find", Value: "reviewOperations"}, {Key: "filter", Value: second["filter"]}, {Key: "projection", Value: second["projection"]}, {Key: "sort", Value: second["sort"]}, {Key: "limit", Value: second["limit"]}}
var explain struct {
Stats struct {
Returned int `bson:"nReturned"`
Keys int `bson:"totalKeysExamined"`
} `bson:"executionStats"`
}
if e := db.RunCommand(t.Context(), bson.D{{Key: "explain", Value: command}, {Key: "verbosity", Value: "executionStats"}}).Decode(&explain); e != nil {
t.Fatal(e)
}
if explain.Stats.Returned != outputOperationPageSize || explain.Stats.Keys > outputOperationPageSize+2 {
t.Fatalf("seek page rescans prefix: %+v", explain.Stats)
}
t.Logf("complete operations=%d source_pages=%d legacy_two_scan=%s bounded_loader=%s page2_keys=%d", len(got.Operations), len(pages), legacyElapsed, time.Since(start), explain.Stats.Keys)
streamContext, cancel := context.WithTimeout(t.Context(), 90*time.Second)
defer cancel()
streamed := 0
streamStarted := time.Now()
_, err = s.StreamAnnotationOutput(streamContext, "out", []string{"p"}, func(batch workforcestats.OutputSnapshot) error { streamed += len(batch.Operations); return nil })
if err != nil || streamed != len(want) {
t.Fatalf("spooled history lost operations: got=%d want=%d error=%v", streamed, len(want), err)
}
t.Logf("spooled operations=%d complete_stream=%s", streamed, time.Since(streamStarted))
}
func TestAnnotationOutputOperationMongoSelectiveScopeAvoidsForeignScan(t *testing.T) {
db := localOutputMongo(t)
if _, e := db.Collection("reviewOperations").Indexes().CreateOne(t.Context(), PlatformReadIndexes()["reviewOperations"][0]); e != nil {
t.Fatal(e)
}
var docs []any
for i := 0; i < 4000; i++ {
docs = append(docs, bson.M{"_id": fmt.Sprintf("foreign-%04d", i), "organizationId": "other", "projectId": "p", "unused": strings.Repeat("body", 4096)})
}
for i := 0; i < 5; i++ {
docs = append(docs, bson.M{"_id": fmt.Sprintf("target-%04d", i), "organizationId": "out", "projectId": "p"})
}
if _, e := db.Collection("reviewOperations").InsertMany(t.Context(), docs); e != nil {
t.Fatal(e)
}
var query bson.M
monitor := &event.CommandMonitor{Started: func(_ context.Context, e *event.CommandStartedEvent) {
if e.CommandName != "find" {
return
}
var m bson.M
_ = bson.Unmarshal(e.Command, &m)
if m["find"] == "reviewOperations" && m["limit"] == int64(outputOperationPageSize) {
query = m
}
}}
c, e := mongo.Connect(t.Context(), options.Client().ApplyURI("mongodb://127.0.0.1:27018").SetMonitor(monitor))
if e != nil {
t.Fatal(e)
}
defer c.Disconnect(context.Background())
result, e := NewMongoStore(c.Database(db.Name())).LoadAnnotationOutput(t.Context(), "out", []string{"p"})
if e != nil || len(result.Operations) != 5 {
t.Fatalf("scoped history mismatch: %d %v", len(result.Operations), e)
}
command := bson.D{{Key: "find", Value: "reviewOperations"}, {Key: "filter", Value: query["filter"]}, {Key: "projection", Value: query["projection"]}, {Key: "sort", Value: query["sort"]}, {Key: "limit", Value: query["limit"]}}
if h := query["hint"]; h != nil {
command = append(command, bson.E{Key: "hint", Value: h})
}
var explanation struct {
Stats struct {
Keys int `bson:"totalKeysExamined"`
Documents int `bson:"totalDocsExamined"`
} `bson:"executionStats"`
}
if e = db.RunCommand(t.Context(), bson.D{{Key: "explain", Value: command}, {Key: "verbosity", Value: "executionStats"}}).Decode(&explanation); e != nil {
t.Fatal(e)
}
if explanation.Stats.Documents > 5 || explanation.Stats.Keys > 6 {
t.Fatalf("small scope scanned unrelated organization: %+v", explanation.Stats)
}
}
func TestAnnotationOutputOperationMongoLargeMetadataAndProblemsRemainBounded(t *testing.T) {
for _, mode := range []string{"wide-metadata", "large-problem-refill"} {
t.Run(mode, func(t *testing.T) {
db := localOutputMongo(t)
var seeks, points atomic.Int32
monitor := &event.CommandMonitor{Started: func(_ context.Context, e *event.CommandStartedEvent) {
if e.CommandName != "find" {
return
}
var m bson.M
_ = bson.Unmarshal(e.Command, &m)
if m["find"] != "reviewOperations" || m["limit"] != int64(outputOperationPageSize) {
return
}
f, _ := bson.MarshalExtJSON(m["filter"], false, false)
if strings.Contains(string(f), "\"$expr\"") {
seeks.Add(1)
} else {
points.Add(1)
}
}}
c, e := mongo.Connect(t.Context(), options.Client().ApplyURI("mongodb://127.0.0.1:27018").SetMonitor(monitor))
if e != nil {
t.Fatal(e)
}
defer c.Disconnect(context.Background())
n := 400
if mode == "large-problem-refill" {
n = 24
}
for i := 0; i < n; i++ {
d := bson.M{"_id": fmt.Sprintf("op-%04d", i), "organizationId": "out", "projectId": "p", "stageId": "quality_review", "status": "prepared"}
if mode == "wide-metadata" {
ids := bson.A{}
for j := 0; j < 300; j++ {
ids = append(ids, fmt.Sprintf("source-%04d-%s", j, strings.Repeat("s", 96)))
}
d["sourceItemIds"] = ids
} else {
d["stageId"], d["status"], d["decision"] = "acceptance", "completed", "rejected"
problems := bson.A{}
for j := 0; j < 2500; j++ {
problems = append(problems, bson.M{"id": fmt.Sprintf("problem-%04d-%s", j, strings.Repeat("p", 192)), "scope": "slice", "sourceItemId": "i", "text": strings.Repeat("private text", 32)})
}
d["draft"] = bson.M{"problems": problems}
}
if _, e := db.Collection("reviewOperations").InsertOne(t.Context(), d); e != nil {
t.Fatal(e)
}
}
result, e := NewMongoStore(c.Database(db.Name())).LoadAnnotationOutput(t.Context(), "out", []string{"p"})
if e != nil || len(result.Operations) != n {
t.Fatalf("byte-split lost records: %d/%d %v", len(result.Operations), n, e)
}
if mode == "wide-metadata" && seeks.Load() < 2 {
t.Fatal("metadata page ignored the byte boundary")
}
if mode == "large-problem-refill" && points.Load() < 2 {
t.Fatal("problem refill accumulated an unbounded page")
}
for _, op := range result.Operations {
if mode == "wide-metadata" && len(op.SourceItemIDs) != 300 {
t.Fatal("lost source identities")
}
if mode == "large-problem-refill" && (len(op.Draft.Problems) != 2500 || op.Draft.Problems[0].Text != "") {
t.Fatal("lost problem facts or exposed opinion text")
}
}
})
}
}

View File

@ -1077,6 +1077,38 @@
"warnings": [
"独立源库/查询库仅使用合成数据;生产写者、47个源索引、迁移及集群资源须另行审批。native测试没有CPU配额,不能证明4核容量。"
]
},
{
"name": "review-operation-history",
"paths": [
"^internal/store/annotation_output_operations(_recovery)?(_test)?\\.go$",
"^internal/store/annotation_output\\.go$"
],
"frontend": [],
"go": {},
"task": [
{
"argv": [
"go",
"test",
"./internal/store",
"-run",
"^TestAnnotationOutput.*Operation.*Mongo|^TestReviewOperationActualSnapshotMongoRefreshCycle$",
"-count=1",
"-timeout=180s",
"-v"
],
"env": {
"TEST_WORKFORCE_MONGO_URI": "mongodb://127.0.0.1:27018",
"RUN_OUTPUT_REVIEW_SCALE": "1",
"RUN_OUTPUT_REFRESH_CYCLE": "1"
},
"tests": true
}
],
"warnings": [
"审核操作历史读取保持完整、原15秒查询和90秒构建预算;显式本机合成夹具覆盖同规模、取消、旧ID及真实120秒刷新,不代表生产已刷新。"
]
}
]
}