From 2071b0a2eb9ddbeef73b959a9880a79b024a2919 Mon Sep 17 00:00:00 2001 From: d-robotics Date: Thu, 15 Oct 2026 19:58:22 +0800 Subject: [PATCH] =?UTF-8?q?=E2=9A=A1=20Scope=20claim=20counts=20and=20list?= =?UTF-8?q?=20metadata=20to=20candidate=20batches?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../claim-count-production-20261015/spec.md | 41 +++ docs/annotation-operation-latency.md | 37 +- .../claim_count_regression_mongo_test.go | 327 ++++++++++++++++++ internal/store/human_claim.go | 230 ++++++------ internal/store/human_loop.go | 2 +- internal/store/pool_count_performance_test.go | 56 +-- 6 files changed, 559 insertions(+), 134 deletions(-) create mode 100644 internal/store/claim_count_regression_mongo_test.go diff --git a/.scratch/claim-count-production-20261015/spec.md b/.scratch/claim-count-production-20261015/spec.md index d7687078..f5598fbb 100644 --- a/.scratch/claim-count-production-20261015/spec.md +++ b/.scratch/claim-count-production-20261015/spec.md @@ -44,3 +44,44 @@ the pre-visibility item filter, restrict metadata to candidate batch IDs and reu ListHumanBatchAccessRecords. Preserve own/author and non-annotation plugin visibility, first stage-ID precedence and cross-project legacy batch metadata. No index, hint, permission cache or business write is added. + +## Local implementation and remaining acceptance + +Removed the batches-first correlated count lookup. Mongo now groups compact +legacy/pool membership counters for current candidates, then the existing +projected batch-access reader and authoritative domain classifiers compute the +scalar total. Blind list metadata is restricted to matching item batch IDs; +own/author visibility, non-QA items and legacy cross-project metadata remain. +First duplicate stage ID still wins. No startup, persisted schema, index, +permission cache, dependency or frontend change is included. + +Mongo8 focused tests cover 60 actual Count+List stage/role combinations, +membership/drain, malformed IDs and exceptional/missing metadata. Dense 12k +count: 168ms locally. Sparse 1,200-history regression: candidate and metadata reads +each examine one document, versus the previous all-history metadata walk. +Mongo7 sparse count/list checks also pass. These are isolated local fixtures, +not production latency measurements. + +Production sampled lookup uses an existing index and no collection scan; a +missing-index hypothesis was not supported. The old projected-read control and +candidate both timed out during bulk getMore through the diagnostic kubectl TCP +tunnel, so those method probes cannot establish real API latency. The actual +deployed d8 HTTP count timeout remains a confirmed regression. + +Status remains claimed: local repair is ready for a separately approved immutable +image release, but real in-cluster HTTP count/list latency and exact count still +require read-only verification after that rollout. The first release record +retains its failed/unverified business evidence. No new deployment, rollback or +customer write is authorized by this local repair. + +Full Go gate reproduced the 70 known failures. A fresh uncached run inside the +restricted sandbox failed at httptest listen/bind (operation not permitted), +before timeline assertions; this is an environment limitation, not an accepted +new baseline. The final gate is rerun with local network permission. + +Final network-enabled full Go gate: 70 exact known failures, no new failures. +`go build ./...`, task-file gofmt and `git diff --check` passed. Local commit uses +the established HUSKY=0 exception for two unrelated pre-existing formatting +failures in internal/domain/review_decision.go and review_draft.go; neither +baseline file nor tests/known-failures.json was changed. Release-required +checks are recorded separately with the immutable candidate. diff --git a/docs/annotation-operation-latency.md b/docs/annotation-operation-latency.md index ae0d755a..0fc13948 100644 --- a/docs/annotation-operation-latency.md +++ b/docs/annotation-operation-latency.md @@ -18,8 +18,9 @@ Owned annotation tasks load independently of claim-count. Failed counts remain unknown instead of blocking or emptying the owned pool; account/project/load changes fence late responses. Blind annotator visibility is pushed into storage before pagination, preserving author history and exceptional legacy stage roles. -Final server permission checks remain authoritative. Claim-count uses a database -aggregate with the existing supplier, membership and batch predicates. +Final server permission checks remain authoritative. Claim-count groups compact +candidate counters in the database and applies current batch rules through the +existing projected access reader, preserving supplier and membership predicates. Review opinion time navigation selects the owning slice and seeks together. Rapid navigation back to the current render's slice also replaces an earlier pending @@ -42,13 +43,39 @@ indexes, and zero documents/keys for the blind actor query. Full scoped listing including stage classification took 38ms; scalar count took 885ms. These are local observations, not production or concurrent P95 results. -Prepared indexes have only been created in isolated local databases. Production -index creation requires a separately approved plan. Retry-capable API/frontend +The three prepared indexes were created in Fanyu on October 15 under the user's +separately approved index plan; existing indexes were retained. Further index +changes require another approved plan. Retry-capable API/frontend require the matching processor: update and verify the processor first, then the API/frontend. Older processors do not understand retained-workspace retry data. For rollback, disable the new API entry first and finish or reconcile outstanding retry jobs before replacing the processor with an older version. This is release -preparation, not authorization for deployment, indexes or customer retries. +compatibility guidance; it does not authorize further releases or customer retries. + +### Production count regression and follow-up + +The first d8a8530c rollout passed infrastructure and three supplied-role login +checks, but actual claim-count HTTP timed out at 45 seconds versus a prior +4.179-second observation. Owned tasks remained usable at 261ms; available-list +read took 7.954 seconds. The first release is not business-accepted. Its local +885ms aggregate result above did not predict production behavior. + +The follow-up removes correlated per-batch counting and full-history metadata +classification. MongoDB groups current candidates by batch/stage with separate +legacy and supplier-pool membership counts. The store reuses +ListHumanBatchAccessRecords for only those groups and the authoritative Go stage +and workflow classifiers to return one count. Blind lists also restrict access +metadata to candidate batch IDs before pagination; source batches outside the +item's project remain supported. No permission cache, new index, task migration +or business write is added. Separate request-scoped reads are not a transactional +snapshot; claim mutation still checks current authority and candidate state. + +A sparse 1,200-batch fixture reproduced the historical metadata scan and now +budgets candidate and metadata reads independently. Dense 12k and legacy/pool/ +revoked-drain/role/duplicate-stage scenarios run against isolated local Mongo7/8. +Actual hotfix HTTP latency must be measured after its separately approved rollout. +Direct method probes through the diagnostic kubectl TCP tunnel are not equivalent: +the old projected-read control also timed out during bulk getMore over that path. ## Unchanged contracts diff --git a/internal/store/claim_count_regression_mongo_test.go b/internal/store/claim_count_regression_mongo_test.go new file mode 100644 index 00000000..d9af2a97 --- /dev/null +++ b/internal/store/claim_count_regression_mongo_test.go @@ -0,0 +1,327 @@ +package store + +import ( + "context" + "fmt" + "net/url" + "os" + "sync" + "testing" + "time" + + "droboticflow/internal/domain" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/event" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" +) + +// A project can retain thousands of migrated, one-item batches after most items +// have been claimed. Eligibility metadata must only read candidate batches, +// and the scalar count must avoid foreign lookups across the retained backlog. +func TestClaimCountMongoSparseBacklogPlan(t *testing.T) { + uri := os.Getenv("POOL_LOCAL_MONGO_URI") + if uri == "" { + t.Skip("POOL_LOCAL_MONGO_URI required") + } + parsed, err := url.Parse(uri) + if err != nil || parsed.Scheme != "mongodb" || (parsed.Host != "127.0.0.1:27018" && parsed.Host != "127.0.0.1:27017") { + t.Fatal("isolated local Mongo only") + } + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + var mu sync.Mutex + var aggregates []bson.D + monitor := &event.CommandMonitor{Started: func(_ context.Context, e *event.CommandStartedEvent) { + if e.CommandName != "aggregate" && e.CommandName != "find" && e.CommandName != "distinct" { + return + } + var command bson.D + if bson.Unmarshal(e.Command, &command) != nil { + return + } + mu.Lock() + aggregates = append(aggregates, command) + mu.Unlock() + }} + client, err := mongo.Connect(ctx, options.Client().ApplyURI(uri).SetMonitor(monitor)) + if err != nil { + t.Fatal("local Mongo connect failed") + } + db := client.Database(fmt.Sprintf("claim_count_regression_test_%d", time.Now().UnixNano())) + defer client.Disconnect(context.Background()) + defer func() { + if err := db.Drop(context.Background()); err != nil { + t.Error("isolated database cleanup failed") + } + }() + s := NewMongoStore(db) + if err := s.EnsureIndexes(ctx); err != nil { + t.Fatal(err) + } + const backlog = 1200 + batches, items := make([]interface{}, backlog), make([]interface{}, backlog) + for i := 0; i < backlog; i++ { + id := fmt.Sprintf("batch-%04d", i) + batches[i] = domain.HumanBatchRecord{ID: id, OrganizationID: "regression-org", ProjectID: "regression-project", RunID: id, PluginID: "annotation-qa", Status: domain.HumanBatchActive, Stages: domain.DefaultFanyuV58HumanStages()} + item := domain.HumanItemRecord{ID: id, BatchID: id, OrganizationID: "regression-org", ProjectID: "regression-project", PluginID: "annotation-qa", LegacyTaskID: id, CurrentStageID: domain.FanyuV58StageAnnotationCombined, Status: domain.HumanItemClaimed, AssignedUserID: "another-actor", Subject: domain.HumanSubjectRef{ItemKey: id}} + if i == 0 { + item.Status = domain.HumanItemReady + item.AssignedUserID = "" + } + items[i] = item + } + if _, err := s.humanBatches.InsertMany(ctx, batches); err != nil { + t.Fatal(err) + } + if _, err := s.humanItems.InsertMany(ctx, items); err != nil { + t.Fatal(err) + } + query := AnnotationClaimCountQuery{ProjectID: "regression-project", ActiveSupplierIDs: []string{"vendor"}, Memberships: []AnnotationClaimMembership{{SupplierID: "vendor", Legacy: true}}} + started := time.Now() + got, err := s.CountAnnotationClaimCandidates(ctx, "regression-org", query) + if err != nil || got != 1 { + t.Fatalf("count=%d want=1 err=%v", got, err) + } + duration := time.Since(started) + mu.Lock() + commands := aggregates + mu.Unlock() + if len(commands) == 0 { + t.Fatal("count did not execute a Mongo read command") + } + // Explain each read command, including grouped counts and access metadata. + // Budgeting only the final item query misses an all-project metadata walk. + for _, command := range commands { + explain := claimCountExplainRead(t, ctx, s.db, command) + examined := claimCountMaxExamined(explain) + t.Logf("collection=%v backlog=%d count=%d countDuration=%s maxDocumentsExamined=%d", command[0].Value, backlog, got, duration, examined) + if examined < 1 || examined > 20 { + t.Fatalf("one candidate examined %d documents across %d retained batches; metadata and scalar count must stay within candidate scope", examined, backlog) + } + } + t.Run("blind-list-candidate-metadata", func(t *testing.T) { + mu.Lock() + aggregates = nil + mu.Unlock() + q := domain.HumanItemQuery{PluginID: "annotation-qa", ProjectID: "regression-project", UnassignedOnly: true, Statuses: []domain.HumanItemStatus{domain.HumanItemReady}, BlindAnnotationActorID: "actor", ListProjection: true, Limit: 1} + items, err := s.ListHumanItems(ctx, "regression-org", q) + if err != nil || len(items) != 0 { + t.Fatalf("blind list returned %d entries err=%v", len(items), err) + } + mu.Lock() + commands := aggregates + mu.Unlock() + for _, command := range commands { + explain := claimCountExplainRead(t, ctx, s.db, command) + examined := claimCountMaxExamined(explain) + t.Logf("blind list command=%s collection=%v examined=%d", command[0].Key, command[0].Value, examined) + if examined > 20 { + t.Fatalf("blind list read %d documents for one candidate among %d retained batches", examined, backlog) + } + } + }) + // No remaining candidates must not trigger an eligibility metadata walk. + if _, err := s.humanItems.UpdateOne(ctx, bson.M{"id": "batch-0000"}, bson.M{"$set": bson.M{"assignedUserId": "another-actor"}}); err != nil { + t.Fatal(err) + } + mu.Lock() + aggregates = nil + mu.Unlock() + if got, err := s.CountAnnotationClaimCandidates(ctx, "regression-org", query); err != nil || got != 0 { + t.Fatalf("empty candidate count=%d err=%v", got, err) + } + mu.Lock() + emptyCommands := aggregates + mu.Unlock() + if len(emptyCommands) != 1 || emptyCommands[0][0].Key != "aggregate" { + t.Fatalf("empty candidates executed %d commands instead of only grouped counts", len(emptyCommands)) + } + query.Memberships = nil + mu.Lock() + aggregates = nil + mu.Unlock() + if got, err := s.CountAnnotationClaimCandidates(ctx, "regression-org", query); err != nil || got != 0 { + t.Fatalf("empty membership count=%d err=%v", got, err) + } + mu.Lock() + noMembershipCommands := len(aggregates) + mu.Unlock() + if noMembershipCommands != 0 { + t.Fatalf("empty memberships executed %d commands", noMembershipCommands) + } +} + +func claimCountExplainRead(t *testing.T, ctx context.Context, db *mongo.Database, command bson.D) bson.M { + t.Helper() + explainCommand := bson.D{} + for _, field := range command { + if field.Key != "$db" && field.Key != "lsid" && field.Key != "$clusterTime" { + explainCommand = append(explainCommand, field) + } + } + var explain bson.M + if err := db.RunCommand(ctx, bson.D{{Key: "explain", Value: explainCommand}, {Key: "verbosity", Value: "executionStats"}}).Decode(&explain); err != nil { + t.Fatal(err) + } + return explain +} + +func claimCountMaxExamined(value interface{}) int64 { + var max int64 + switch v := value.(type) { + case bson.M: + for key, child := range v { + var n int64 + if key == "totalDocsExamined" { + switch x := child.(type) { + case int32: + n = int64(x) + case int64: + n = x + case float64: + n = int64(x) + } + } else { + n = claimCountMaxExamined(child) + } + if n > max { + max = n + } + } + case bson.A: + for _, child := range v { + if n := claimCountMaxExamined(child); n > max { + max = n + } + } + } + return max +} + +func TestClaimCountMongoBatchScopeAndFirstStage(t *testing.T) { + ctx, s := poolLocalMongo(t) + for _, tc := range []struct { + name string + edit func(*domain.HumanBatchRecord) + want int + }{ + {"first-duplicate-annotator", func(b *domain.HumanBatchRecord) { + b.Stages = []domain.HumanStageDefinition{{ID: "duplicate", Kind: domain.HumanStageAnnotation, RequiredRole: "annotator"}, {ID: "duplicate", Kind: domain.HumanStageReview, RequiredRole: "quality_reviewer"}} + }, 1}, + {"first-duplicate-reviewer", func(b *domain.HumanBatchRecord) { + b.Stages = []domain.HumanStageDefinition{{ID: "duplicate", Kind: domain.HumanStageReview, RequiredRole: "quality_reviewer"}, {ID: "duplicate", Kind: domain.HumanStageAnnotation, RequiredRole: "annotator"}} + }, 0}, + {"foreign-batch-organization", func(b *domain.HumanBatchRecord) { b.OrganizationID = "foreign-org" }, 0}, + {"foreign-batch-project", func(b *domain.HumanBatchRecord) { b.ProjectID = "foreign-project" }, 0}, + {"missing-stages", func(b *domain.HumanBatchRecord) { b.Stages = nil }, 0}, + } { + t.Run(tc.name, func(t *testing.T) { + batch := domain.HumanBatchRecord{ID: tc.name, OrganizationID: "scope-org", ProjectID: tc.name, Status: domain.HumanBatchActive, Stages: domain.DefaultFanyuV58HumanStages()} + item := domain.HumanItemRecord{ID: tc.name, BatchID: batch.ID, OrganizationID: batch.OrganizationID, ProjectID: batch.ProjectID, PluginID: "annotation-qa", LegacyTaskID: tc.name, CurrentStageID: domain.FanyuV58StageAnnotationCombined, Status: domain.HumanItemReady} + tc.edit(&batch) + if tc.name == "first-duplicate-annotator" || tc.name == "first-duplicate-reviewer" { + item.CurrentStageID = "duplicate" + } + if err := s.UpsertHumanBatch(ctx, batch); err != nil { + t.Fatal(err) + } + if err := s.UpsertHumanItem(ctx, item); err != nil { + t.Fatal(err) + } + q := AnnotationClaimCountQuery{ProjectID: item.ProjectID, ActiveSupplierIDs: []string{"vendor"}, Memberships: []AnnotationClaimMembership{{SupplierID: "vendor", Legacy: true, SupplierPool: true}}} + got, err := s.CountAnnotationClaimCandidates(ctx, item.OrganizationID, q) + if err != nil || got != tc.want { + t.Fatalf("count=%d want=%d err=%v", got, tc.want, err) + } + }) + } +} + +func TestClaimCountMongoMissingBatchIDsDoNotMatchMissingBatch(t *testing.T) { + ctx, s := poolLocalMongo(t) + for _, name := range []string{"missing", "null", "empty"} { + t.Run(name, func(t *testing.T) { + batch := bson.M{"organizationId": "missing-id-org", "projectId": name, "status": domain.HumanBatchActive, "stages": domain.DefaultFanyuV58HumanStages(), "processingProfile": ""} + if _, err := s.humanBatches.InsertOne(ctx, batch); err != nil { + t.Fatal(err) + } + item := bson.M{"id": name, "organizationId": "missing-id-org", "projectId": name, "pluginId": "annotation-qa", "legacyTaskId": name, "currentStageId": domain.FanyuV58StageAnnotationCombined, "status": domain.HumanItemReady} + if name == "null" { + item["batchId"] = nil + } else if name == "empty" { + item["batchId"] = "" + } + if _, err := s.humanItems.InsertOne(ctx, item); err != nil { + t.Fatal(err) + } + q := AnnotationClaimCountQuery{ProjectID: name, ActiveSupplierIDs: []string{"vendor"}, Memberships: []AnnotationClaimMembership{{SupplierID: "vendor", Legacy: true}}} + got, err := s.CountAnnotationClaimCandidates(ctx, "missing-id-org", q) + if err != nil || got != 0 { + t.Fatalf("malformed batch association count=%d err=%v", got, err) + } + }) + } +} + +func TestBlindListMongoPreservesLegacyAndMissingMetadata(t *testing.T) { + ctx, s := poolLocalMongo(t) + const org = "blind-legacy-org" + const project = "blind-items-project" + review := domain.HumanStageDefinition{ID: "exception", Kind: domain.HumanStageReview, RequiredRole: "cutter"} + annotation := review + annotation.Kind = domain.HumanStageAnnotation + for _, tc := range []struct { + id, batchOrg, batchProject string + stages []domain.HumanStageDefinition + }{ + {"cross-project", org, "legacy-other-project", []domain.HumanStageDefinition{review}}, + {"foreign-org", "other-org", project, []domain.HumanStageDefinition{review}}, + {"first-review", org, project, []domain.HumanStageDefinition{review, annotation}}, + {"first-annotation", org, project, []domain.HumanStageDefinition{annotation, review}}, + } { + if err := s.UpsertHumanBatch(ctx, domain.HumanBatchRecord{ID: tc.id, OrganizationID: tc.batchOrg, ProjectID: tc.batchProject, Stages: tc.stages}); err != nil { + t.Fatal(err) + } + if err := s.UpsertHumanItem(ctx, domain.HumanItemRecord{ID: tc.id, BatchID: tc.id, OrganizationID: org, ProjectID: project, PluginID: "annotation-qa", CurrentStageID: "exception", Status: domain.HumanItemReady}); err != nil { + t.Fatal(err) + } + } + // Missing routing metadata cannot remove own/author history or another plugin. + for _, tc := range []struct{ id, plugin, owner, author string }{ + {"own-no-batch", "annotation-qa", "actor", ""}, + {"author-no-batch", "annotation-qa", "", "actor"}, + {"other-plugin-no-batch", "other-plugin", "", ""}, + } { + if _, err := s.humanItems.InsertOne(ctx, bson.M{"id": tc.id, "organizationId": org, "projectId": "no-batch-ids", "pluginId": tc.plugin, "assignedUserId": tc.owner, "latestAuthorUserId": tc.author, "status": domain.HumanItemReady}); err != nil { + t.Fatal(err) + } + } + for _, tc := range []struct { + name, project, plugin string + want []string + }{ + {"legacy-batch-scope", project, "annotation-qa", []string{"cross-project", "first-review"}}, + {"own-history-no-batch", "no-batch-ids", "annotation-qa", []string{"own-no-batch", "author-no-batch"}}, + {"mixed-plugins-no-batch", "no-batch-ids", "", []string{"own-no-batch", "author-no-batch", "other-plugin-no-batch"}}, + } { + t.Run(tc.name, func(t *testing.T) { + items, err := s.ListHumanItems(ctx, org, domain.HumanItemQuery{ProjectID: tc.project, PluginID: tc.plugin, BlindAnnotationActorID: "actor", ListProjection: true}) + if err != nil { + t.Fatal(err) + } + got := map[string]bool{} + for _, item := range items { + got[item.ID] = true + } + if len(got) != len(tc.want) { + t.Fatalf("visible=%v want=%v", got, tc.want) + } + for _, id := range tc.want { + if !got[id] { + t.Fatalf("missing visible item %s", id) + } + } + }) + } +} diff --git a/internal/store/human_claim.go b/internal/store/human_claim.go index 37764d8a..6414de83 100644 --- a/internal/store/human_claim.go +++ b/internal/store/human_claim.go @@ -351,17 +351,10 @@ type AnnotationClaimCountQuery struct { } func annotationClaimCountMatches(item domain.HumanItemRecord, batch domain.HumanBatchRecord, org string, q AnnotationClaimCountQuery) bool { - if !humanOrganizationMatches(item.OrganizationID, org) || item.ProjectID != q.ProjectID || !annotationClaimCandidate(item) || !humanOrganizationMatches(batch.OrganizationID, org) || batch.ProjectID != q.ProjectID || batch.Status == domain.HumanBatchCompleted { + if !humanOrganizationMatches(item.OrganizationID, org) || item.ProjectID != q.ProjectID || !annotationClaimCandidate(item) || !annotationClaimCountStageMatches(batch, org, q.ProjectID, item.CurrentStageID) { return false } - stageFound := false - for _, stage := range batch.Stages { - if stage.ID == item.CurrentStageID { - stageFound = stage.Kind == domain.HumanStageAnnotation && domain.AnnotationRoleForHumanStage(stage) == domain.AnnotationRoleAnnotator - break - } - } - if !stageFound || (item.SupplierID != "" && !containsHumanItemID(q.ActiveSupplierIDs, item.SupplierID)) || (item.ProcessingScope == "internal" && item.SupplierID == "") { + if (item.SupplierID != "" && !containsHumanItemID(q.ActiveSupplierIDs, item.SupplierID)) || (item.ProcessingScope == "internal" && item.SupplierID == "") { return false } pool := batch.ProcessingProfile == domain.ProcessingProfileRawHuman && domain.IsFanyuV58ThreeRoleHumanLoop(batch.Stages) @@ -375,6 +368,18 @@ func annotationClaimCountMatches(item domain.HumanItemRecord, batch domain.Human } return false } +func annotationClaimCountStageMatches(batch domain.HumanBatchRecord, org, project, stageID string) bool { + if !humanOrganizationMatches(batch.OrganizationID, org) || batch.ProjectID != project || batch.Status == domain.HumanBatchCompleted { + return false + } + for _, stage := range batch.Stages { + if stage.ID == stageID { + return stage.Kind == domain.HumanStageAnnotation && domain.AnnotationRoleForHumanStage(stage) == domain.AnnotationRoleAnnotator + } + } + return false +} + func (s *MemoryStore) CountAnnotationClaimCandidates(_ context.Context, org string, q AnnotationClaimCountQuery) (int, error) { s.mu.Lock() defer s.mu.Unlock() @@ -399,122 +404,141 @@ func blindAnnotationVisible(item domain.HumanItemRecord, batch domain.HumanBatch return false } -// Mongo counterpart of AnnotationRoleForHumanStage's annotator result. Stage -// aliases override requiredRole, including older cutter/review combinations. -func annotatorStageMongoExpression(stage string) bson.M { - return bson.M{"$switch": bson.M{"branches": bson.A{ - bson.M{"case": bson.M{"$in": bson.A{stage + ".id", bson.A{"split_annotation", domain.FanyuV5StageCutAdjustment, "split_quality_review", domain.FanyuV5StageCutReview, domain.FanyuV5StageSemanticAnnotation}}}, "then": true}, - bson.M{"case": bson.M{"$in": bson.A{stage + ".id", bson.A{"quality_review", domain.FanyuV5StageSemanticReview, "acceptance", "prepackage_acceptance"}}}, "then": false}, - }, "default": bson.M{"$not": bson.A{bson.M{"$in": bson.A{bson.M{"$ifNull": bson.A{stage + ".requiredRole", ""}}, bson.A{"quality_reviewer", "acceptance_reviewer"}}}}}}} -} - -// humanStage selects the first matching ID even in malformed legacy batches. -// Keep the same selection when aggregating metadata or counting candidates. -func firstHumanStagesMongoExpression() bson.M { - return bson.M{"$reduce": bson.M{ - "input": bson.M{"$ifNull": bson.A{"$stages", bson.A{}}}, "initialValue": bson.A{}, - "in": bson.M{"$cond": bson.A{ - bson.M{"$in": bson.A{"$$this.id", bson.M{"$map": bson.M{"input": "$$value", "as": "stage", "in": "$$stage.id"}}}}, - "$$value", bson.M{"$concatArrays": bson.A{"$$value", bson.A{"$$this"}}}, - }}, - }} -} - -func (s *MongoStore) blindAnnotationMongoVisibility(ctx context.Context, org string, q domain.HumanItemQuery) (bson.M, error) { - batchFilter := bson.M{"organizationId": normalizedOrganizationID(org)} - if q.BatchID != "" { - batchFilter["id"] = q.BatchID - } - // Only unusual, non-annotation stages can be visible outside actor history. - // The normal Fanyu backlog returns no batch metadata to the application. - pipeline := mongo.Pipeline{ - {{Key: "$match", Value: batchFilter}}, - {{Key: "$project", Value: bson.M{"id": 1, "stages": firstHumanStagesMongoExpression()}}}, - {{Key: "$unwind", Value: "$stages"}}, - {{Key: "$match", Value: bson.M{"stages.kind": bson.M{"$ne": domain.HumanStageAnnotation}, "$expr": annotatorStageMongoExpression("$stages")}}}, - {{Key: "$group", Value: bson.M{"_id": "$stages.id", "batchIds": bson.M{"$addToSet": "$id"}}}}, - } - cursor, err := s.humanBatches.Aggregate(ctx, pipeline) - if err != nil { - return nil, err - } - var scopes []struct { - StageID string `bson:"_id"` - BatchIDs []string `bson:"batchIds"` - } - if err := cursor.All(ctx, &scopes); err != nil { - return nil, err - } +func (s *MongoStore) blindAnnotationMongoVisibility(ctx context.Context, org string, q domain.HumanItemQuery, baseFilter bson.M) (bson.M, error) { own := bson.A{bson.M{"assignedUserId": q.BlindAnnotationActorID}, bson.M{"latestAuthorUserId": q.BlindAnnotationActorID}} if q.PluginID != "annotation-qa" { own = append(own, bson.M{"pluginId": bson.M{"$ne": "annotation-qa"}}) } - for _, scope := range scopes { - own = append(own, bson.M{"batchId": bson.M{"$in": scope.BatchIDs}, "currentStageId": scope.StageID}) + values, err := s.humanItems.Distinct(ctx, "batchId", baseFilter) + if err != nil { + return nil, err + } + batchIDs := make([]string, 0, len(values)) + for _, value := range values { + if id, ok := value.(string); ok { + batchIDs = append(batchIDs, id) + } + } + // Restrict access metadata to matching items. Do not filter batch project: + // legacy cross-project metadata still determines exceptional-stage visibility. + batches, err := s.ListHumanBatchAccessRecords(ctx, org, batchIDs) + if err != nil { + return nil, err + } + scopes := map[string][]string{} + for _, batch := range batches { + seen := map[string]bool{} + for _, stage := range batch.Stages { + if seen[stage.ID] { + continue + } + seen[stage.ID] = true + if stage.Kind != domain.HumanStageAnnotation && domain.AnnotationRoleForHumanStage(stage) == domain.AnnotationRoleAnnotator { + scopes[stage.ID] = append(scopes[stage.ID], batch.ID) + } + } + } + for stageID, ids := range scopes { + own = append(own, bson.M{"batchId": bson.M{"$in": ids}, "currentStageId": stageID}) } return bson.M{"$or": own}, nil } -func supplierPoolMongoExpression() bson.M { - // Keep the strict persisted two-stage shape used by IsFanyuV58ThreeRoleHumanLoop. - first := bson.M{"$arrayElemAt": bson.A{"$stages", 0}} - second := bson.M{"$arrayElemAt": bson.A{"$stages", 1}} - return bson.M{"$let": bson.M{"vars": bson.M{"first": first, "second": second}, "in": bson.M{"$and": bson.A{ - bson.M{"$eq": bson.A{"$processingProfile", domain.ProcessingProfileRawHuman}}, - bson.M{"$eq": bson.A{bson.M{"$size": bson.M{"$ifNull": bson.A{"$stages", bson.A{}}}}, 2}}, - bson.M{"$eq": bson.A{"$$first.id", domain.FanyuV58StageAnnotationCombined}}, - bson.M{"$eq": bson.A{"$$first.kind", domain.HumanStageAnnotation}}, - bson.M{"$eq": bson.A{"$$first.requiredRole", "annotator"}}, - bson.M{"$eq": bson.A{"$$second.id", domain.FanyuV58StageQualityReview}}, - bson.M{"$eq": bson.A{"$$second.kind", domain.HumanStageReview}}, - bson.M{"$eq": bson.A{"$$second.requiredRole", "quality_reviewer"}}, - bson.M{"$eq": bson.A{"$$second.rejectTargets", bson.A{domain.FanyuV58StageAnnotationCombined}}}, - }}}} +// Counts are grouped in MongoDB before compact access metadata is fetched. No +// candidate records, batch payloads, or cached authority reach the count handler. +type annotationClaimCandidateCount struct { + ID struct { + BatchID string `bson:"batchId"` + StageID string `bson:"stageId"` + } `bson:"_id"` + LegacyCount int `bson:"legacyCount"` + PoolCount int `bson:"poolCount"` } -func annotationClaimCountPipeline(org string, q AnnotationClaimCountQuery) mongo.Pipeline { - scopes := bson.A{} - for _, m := range q.Memberships { - supplier := bson.M{"supplierId": bson.M{"$in": bson.A{"", nil, m.SupplierID}}} - if m.Legacy { - scopes = append(scopes, bson.M{"$and": bson.A{supplier, bson.M{"$expr": bson.M{"$eq": bson.A{"$$supplierPool", false}}}}}) - } - if m.SupplierPool { - scopes = append(scopes, bson.M{"$and": bson.A{supplier, bson.M{"$expr": bson.M{"$eq": bson.A{"$$supplierPool", true}}}}}) - } else if len(m.DrainTaskIDs) > 0 { - scopes = append(scopes, bson.M{"supplierId": m.SupplierID, "legacyTaskId": bson.M{"$in": m.DrainTaskIDs}, "$expr": bson.M{"$eq": bson.A{"$$supplierPool", true}}}) + +func annotationClaimMembershipMongoExpression(memberships []AnnotationClaimMembership, pool bool) bson.M { + choices := bson.A{} + for _, m := range memberships { + if !pool && m.Legacy || pool && m.SupplierPool { + choices = append(choices, bson.M{"$in": bson.A{bson.M{"$ifNull": bson.A{"$supplierId", ""}}, bson.A{"", m.SupplierID}}}) + } else if pool && len(m.DrainTaskIDs) > 0 { + choices = append(choices, bson.M{"$and": bson.A{bson.M{"$eq": bson.A{"$supplierId", m.SupplierID}}, bson.M{"$in": bson.A{"$legacyTaskId", m.DrainTaskIDs}}}}) } } - // An empty $or is invalid; match no records without changing the pipeline shape. - if len(scopes) == 0 { - scopes = append(scopes, bson.M{"$expr": false}) + if len(choices) == 0 { + return bson.M{"$literal": false} } + return bson.M{"$or": choices} +} + +func annotationClaimCandidateCountPipeline(org string, q AnnotationClaimCountQuery) mongo.Pipeline { + // Keep the indexed candidate match; compute only compact per-batch/stage + // counters. Scalar string routing IDs avoid null equality matching missing IDs. + return mongo.Pipeline{ + {{Key: "$match", Value: annotationClaimCountCandidateFilter(org, q)}}, + {{Key: "$project", Value: bson.M{ + "batchId": 1, "currentStageId": 1, + "validIDs": bson.M{"$and": bson.A{bson.M{"$eq": bson.A{bson.M{"$type": "$batchId"}, "string"}}, bson.M{"$eq": bson.A{bson.M{"$type": "$currentStageId"}, "string"}}}}, + "legacy": annotationClaimMembershipMongoExpression(q.Memberships, false), + "pool": annotationClaimMembershipMongoExpression(q.Memberships, true), + }}}, + {{Key: "$match", Value: bson.M{"validIDs": true, "$or": bson.A{bson.M{"legacy": true}, bson.M{"pool": true}}}}}, + {{Key: "$group", Value: bson.M{ + "_id": bson.M{"batchId": "$batchId", "stageId": "$currentStageId"}, + "legacyCount": bson.M{"$sum": bson.M{"$cond": bson.A{"$legacy", 1, 0}}}, + "poolCount": bson.M{"$sum": bson.M{"$cond": bson.A{"$pool", 1, 0}}}, + }}}, + } +} + +func annotationClaimCountCandidateFilter(org string, q AnnotationClaimCountQuery) bson.M { suppliers := bson.A{"", nil} for _, id := range q.ActiveSupplierIDs { suppliers = append(suppliers, id) } - itemFilter := bson.M{"organizationId": normalizedOrganizationID(org), "projectId": q.ProjectID, "pluginId": "annotation-qa", "legacyTaskId": bson.M{"$type": "string", "$ne": ""}, "assignedUserId": bson.M{"$in": bson.A{"", nil}}, "status": bson.M{"$in": bson.A{domain.HumanItemReady, domain.HumanItemRework, domain.HumanItemSubmitted}}, "supplierId": bson.M{"$in": suppliers}, "$nor": bson.A{bson.M{"processingScope": "internal", "supplierId": bson.M{"$in": bson.A{"", nil}}}}, "$or": scopes, "$expr": bson.M{"$and": bson.A{bson.M{"$eq": bson.A{"$batchId", "$$batchId"}}, bson.M{"$eq": bson.A{"$currentStageId", "$$stageId"}}}}} - return mongo.Pipeline{ - {{Key: "$match", Value: bson.M{"organizationId": normalizedOrganizationID(org), "projectId": q.ProjectID, "status": bson.M{"$ne": domain.HumanBatchCompleted}}}}, - {{Key: "$project", Value: bson.M{"id": 1, "stages": firstHumanStagesMongoExpression(), "supplierPool": supplierPoolMongoExpression()}}}, - {{Key: "$unwind", Value: "$stages"}}, - {{Key: "$match", Value: bson.M{"stages.kind": domain.HumanStageAnnotation, "$expr": annotatorStageMongoExpression("$stages")}}}, - {{Key: "$lookup", Value: bson.M{"from": "humanItems", "let": bson.M{"batchId": "$id", "stageId": "$stages.id", "supplierPool": "$supplierPool"}, "pipeline": mongo.Pipeline{{{Key: "$match", Value: itemFilter}}, {{Key: "$count", Value: "count"}}}, "as": "counts"}}}, - {{Key: "$group", Value: bson.M{"_id": nil, "count": bson.M{"$sum": bson.M{"$ifNull": bson.A{bson.M{"$arrayElemAt": bson.A{"$counts.count", 0}}, 0}}}}}}, - } + return bson.M{"organizationId": normalizedOrganizationID(org), "projectId": q.ProjectID, "pluginId": "annotation-qa", "legacyTaskId": bson.M{"$type": "string", "$ne": ""}, "assignedUserId": bson.M{"$in": bson.A{"", nil}}, "status": bson.M{"$in": bson.A{domain.HumanItemReady, domain.HumanItemRework, domain.HumanItemSubmitted}}, "supplierId": bson.M{"$in": suppliers}, "$nor": bson.A{bson.M{"processingScope": "internal", "supplierId": bson.M{"$in": bson.A{"", nil}}}}} } + func (s *MongoStore) CountAnnotationClaimCandidates(ctx context.Context, org string, q AnnotationClaimCountQuery) (int, error) { - cursor, err := s.humanBatches.Aggregate(ctx, annotationClaimCountPipeline(org, q)) + if len(q.Memberships) == 0 { + return 0, nil + } + cursor, err := s.humanItems.Aggregate(ctx, annotationClaimCandidateCountPipeline(org, q)) if err != nil { return 0, err } - var result []struct { - Count int `bson:"count"` - } - if err := cursor.All(ctx, &result); err != nil { + var groups []annotationClaimCandidateCount + if err := cursor.All(ctx, &groups); err != nil { return 0, err } - if len(result) == 0 { - return 0, nil + batchIDs := make([]string, 0, len(groups)) + seen := map[string]bool{} + for _, group := range groups { + if !seen[group.ID.BatchID] { + seen[group.ID.BatchID] = true + batchIDs = append(batchIDs, group.ID.BatchID) + } } - return result[0].Count, nil + // Reuse the established access projection, then apply the exact domain stage + // and pool classifiers instead of evaluating metadata for all project history. + batches, err := s.ListHumanBatchAccessRecords(ctx, org, batchIDs) + if err != nil { + return 0, err + } + byID := make(map[string]domain.HumanBatchRecord, len(batches)) + for _, batch := range batches { + byID[batch.ID] = batch + } + count := 0 + for _, group := range groups { + batch, ok := byID[group.ID.BatchID] + if !ok || !annotationClaimCountStageMatches(batch, org, q.ProjectID, group.ID.StageID) { + continue + } + if batch.ProcessingProfile == domain.ProcessingProfileRawHuman && domain.IsFanyuV58ThreeRoleHumanLoop(batch.Stages) { + count += group.PoolCount + } else { + count += group.LegacyCount + } + } + return count, nil } diff --git a/internal/store/human_loop.go b/internal/store/human_loop.go index 5c3df4d7..447e5370 100644 --- a/internal/store/human_loop.go +++ b/internal/store/human_loop.go @@ -721,7 +721,7 @@ func (store *MongoStore) ListHumanItems(ctx context.Context, organizationID stri } filter := humanItemMongoFilter(organizationID, query) if query.BlindAnnotationActorID != "" { - visibility, err := store.blindAnnotationMongoVisibility(ctx, organizationID, query) + visibility, err := store.blindAnnotationMongoVisibility(ctx, organizationID, query, filter) if err != nil { return nil, err } diff --git a/internal/store/pool_count_performance_test.go b/internal/store/pool_count_performance_test.go index 6b52f4b4..e6da0339 100644 --- a/internal/store/pool_count_performance_test.go +++ b/internal/store/pool_count_performance_test.go @@ -208,7 +208,7 @@ func TestPoolPerformanceMongoBacklog(t *testing.T) { } indexed := explain(humanItemMongoFilter(batch.OrganizationID, q)) q.BlindAnnotationActorID = "actor" - visibility, err := s.blindAnnotationMongoVisibility(ctx, batch.OrganizationID, q) + visibility, err := s.blindAnnotationMongoVisibility(ctx, batch.OrganizationID, q, humanItemMongoFilter(batch.OrganizationID, q)) if err != nil { t.Fatal(err) } @@ -230,35 +230,41 @@ func TestPoolPerformanceMongoBacklog(t *testing.T) { } } -func TestPoolStageRoleExpressionMatchesDomainMongo(t *testing.T) { +func TestPoolStageRoleCountAndVisibilityMatchesDomainMongo(t *testing.T) { ctx, s := poolLocalMongo(t) ids := []string{"split_annotation", domain.FanyuV5StageCutAdjustment, "split_quality_review", domain.FanyuV5StageCutReview, domain.FanyuV5StageSemanticAnnotation, "quality_review", domain.FanyuV5StageSemanticReview, "acceptance", "prepackage_acceptance", "custom-stage"} roles := []string{"", "annotator", "cutter", "quality_reviewer", "acceptance_reviewer", "historical-unknown"} - docs := []interface{}{} + org := "role-fixture-org" for _, id := range ids { for _, role := range roles { - docs = append(docs, bson.M{"stage": domain.HumanStageDefinition{ID: id, RequiredRole: role}}) - } - } - collection := s.db.Collection("roleExpressionFixtures") - if _, err := collection.InsertMany(ctx, docs); err != nil { - t.Fatal(err) - } - cursor, err := collection.Aggregate(ctx, mongo.Pipeline{{{Key: "$project", Value: bson.M{"stage": 1, "annotator": annotatorStageMongoExpression("$stage")}}}}) - if err != nil { - t.Fatal(err) - } - var results []struct { - Stage domain.HumanStageDefinition `bson:"stage"` - Annotator bool `bson:"annotator"` - } - if err := cursor.All(ctx, &results); err != nil { - t.Fatal(err) - } - for _, result := range results { - want := domain.AnnotationRoleForHumanStage(result.Stage) == domain.AnnotationRoleAnnotator - if result.Annotator != want { - t.Fatalf("stage=%+v mongo=%t domain=%t", result.Stage, result.Annotator, want) + t.Run(id+"/"+role, func(t *testing.T) { + key := id + "-" + role + stage := domain.HumanStageDefinition{ID: id, Kind: domain.HumanStageAnnotation, RequiredRole: role} + batch := domain.HumanBatchRecord{ID: key, OrganizationID: org, ProjectID: key, Status: domain.HumanBatchActive, Stages: []domain.HumanStageDefinition{stage}} + item := domain.HumanItemRecord{ID: key, BatchID: key, OrganizationID: org, ProjectID: key, PluginID: "annotation-qa", LegacyTaskID: key, CurrentStageID: id, Status: domain.HumanItemReady} + if err := s.UpsertHumanBatch(ctx, batch); err != nil { + t.Fatal(err) + } + if err := s.UpsertHumanItem(ctx, item); err != nil { + t.Fatal(err) + } + want := 0 + if domain.AnnotationRoleForHumanStage(stage) == domain.AnnotationRoleAnnotator { + want = 1 + } + count, err := s.CountAnnotationClaimCandidates(ctx, org, AnnotationClaimCountQuery{ProjectID: key, ActiveSupplierIDs: []string{"vendor"}, Memberships: []AnnotationClaimMembership{{SupplierID: "vendor", Legacy: true}}}) + if err != nil || count != want { + t.Fatalf("stage=%+v count=%d want=%d err=%v", stage, count, want, err) + } + batch.Stages[0].Kind = domain.HumanStageReview + if err := s.UpsertHumanBatch(ctx, batch); err != nil { + t.Fatal(err) + } + items, err := s.ListHumanItems(ctx, org, domain.HumanItemQuery{ProjectID: key, PluginID: "annotation-qa", BlindAnnotationActorID: "actor"}) + if err != nil || len(items) != want { + t.Fatalf("stage=%+v visible=%d want=%d err=%v", batch.Stages[0], len(items), want, err) + } + }) } } }