package store import ( "context" "crypto/sha256" "droboticflow/internal/domain" "errors" "fmt" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/mongo" "go.mongodb.org/mongo-driver/mongo/options" "maps" ) var ErrAnnotationBufferFull = errors.New("annotation work buffer is full") type AnnotationWorkBuffer struct { Capacity int `json:"capacity" bson:"capacity"` Occupied int `json:"occupied" bson:"-"` Items []domain.HumanItemRecord `json:"items" bson:"-"` Version int `json:"-" bson:"version"` Slots map[string]string `json:"-" bson:"slots"` } func bufferKey(org, user, project string) string { return fmt.Sprintf("%x", sha256.Sum256([]byte(org+"\x00"+user+"\x00"+project))) } func occupiesAnnotationBuffer(item domain.HumanItemRecord, org, user, project string) bool { return item.OrganizationID == org && item.ProjectID == project && item.AssignedUserID == user && item.PluginID == "annotation-qa" && item.LegacyTaskID != "" && (item.CurrentStageID == "annotation_combined" || item.CurrentStageID == "semantic_annotation") && (item.Status == domain.HumanItemClaimed || item.Status == domain.HumanItemInProgress || item.Status == domain.HumanItemRework) } func annotationBufferBatchMatches(item domain.HumanItemRecord, batch domain.HumanBatchRecord) bool { return batch.OrganizationID == item.OrganizationID && batch.ProjectID == item.ProjectID && batch.WorkflowID == item.WorkflowID && batch.ProcessingProfile == domain.ProcessingProfileRawHuman && domain.IsFanyuV58ThreeRoleHumanLoop(batch.Stages) } func (s *MemoryStore) AnnotationWorkBuffer(_ context.Context, org, user, project string, capacity int) (AnnotationWorkBuffer, error) { s.mu.Lock() defer s.mu.Unlock() key := bufferKey(org, user, project) if s.annotationBufferCapacities == nil { s.annotationBufferCapacities = map[string]int{} } if capacity > 0 { s.annotationBufferCapacities[key] = capacity } n := s.annotationBufferCapacities[key] if n == 0 { n = 2 } out := AnnotationWorkBuffer{Capacity: n, Items: []domain.HumanItemRecord{}} for _, item := range s.humanItems { if occupiesAnnotationBuffer(item, org, user, project) && annotationBufferBatchMatches(item, s.humanBatches[item.BatchID]) { out.Items = append(out.Items, clone(item)) } } for _, item := range out.Items { if !AnnotationSourceAuditAdvisory() || !AnnotationSourceUnavailable(item) { out.Occupied++ } } return out, nil } // The single-document CAS ledger also protects standalone Mongo. Pending claim // intents retain their slot across process restarts; only a definitive conflict // or handoff frees it. A receipt retry reuses the same slot even after downsizing. func (s *MongoStore) annotationBuffer(ctx context.Context, org, user, project string, capacity int, reserve *HumanClaimReceipt, summary ...bool) (AnnotationWorkBuffer, error) { compact := len(summary) > 0 && summary[0] key := bufferKey(org, user, project) collection := s.queryCollection("annotationWorkBuffers") for { if err := ctx.Err(); err != nil { return AnnotationWorkBuffer{}, err } var old AnnotationWorkBuffer err := collection.FindOne(ctx, bson.M{"_id": key}).Decode(&old) if err == mongo.ErrNoDocuments { _, e := collection.InsertOne(ctx, bson.M{"_id": key, "capacity": 2, "version": 0, "slots": bson.M{}}) if e != nil && !mongo.IsDuplicateKeyError(e) { return old, e } continue } if err != nil { return old, err } next := old next.Items = []domain.HumanItemRecord{} next.Slots = map[string]string{} // The "active" view bounds the scan to the actor's assigned open // annotation items, a superset of occupiesAnnotationBuffer; matching // latestAuthorUserId would read the actor's whole authored history. items, err := s.ListHumanItems(ctx, org, domain.HumanItemQuery{ProjectID: project, PluginID: "annotation-qa", ActorUserID: user, AnnotationView: "active", ListProjection: true, BufferSummaryProjection: compact}) if err != nil { return old, err } occupiedIDs := map[string]bool{} batchIDs := make([]string, 0, len(items)) for _, item := range items { batchIDs = append(batchIDs, item.BatchID) } batches, err := s.annotationBufferBatches(ctx, batchIDs) if err != nil { return old, err } trusted := func(item domain.HumanItemRecord) (bool, error) { batch, ok := batches[item.BatchID] if !ok { // A ledger slot may reference an item outside the active scan. // Resolve its batch with the same projection and request-local cache. missing, e := s.annotationBufferBatches(ctx, []string{item.BatchID}) if e != nil { return false, e } batch = missing[item.BatchID] batches[item.BatchID] = batch } return batch != nil && annotationBufferBatchMatches(item, *batch), nil } for _, item := range items { scoped, e := trusted(item) if e != nil { return old, e } if !scoped { continue } if occupiesAnnotationBuffer(item, org, user, project) { next.Items = append(next.Items, item) if !AnnotationSourceAuditAdvisory() || !AnnotationSourceUnavailable(item) { occupiedIDs[item.ID] = true } } } for request, id := range old.Slots { // An item the active scan already counted keeps its slot whatever // its receipt says; rereading it and the receipt changes nothing. if occupiedIDs[id] { next.Slots[request] = id continue } slotItems, e := s.ListHumanItems(ctx, org, domain.HumanItemQuery{IDs: []string{id}, ListProjection: true, BufferSummaryProjection: compact}) var item *domain.HumanItemRecord if len(slotItems) > 0 { item = &slotItems[0] } if e != nil { return old, e } receipt, e := s.GetHumanClaimReceipt(ctx, request) if e != nil { return old, e } if item != nil { scoped, e := trusted(*item) if e != nil { return old, e } if !scoped { continue } } pending := receipt != nil && receipt.State == "pending" && receipt.Intent != nil && item != nil && item.ClaimRequestID != request && validClaimSnapshot(*item, *receipt.Intent) if item != nil && occupiesAnnotationBuffer(*item, org, user, project) && !occupiedIDs[id] && (!AnnotationSourceAuditAdvisory() || !AnnotationSourceUnavailable(*item)) { occupiedIDs[id] = true next.Items = append(next.Items, *item) } if occupiedIDs[id] || pending { next.Slots[request] = id occupiedIDs[id] = true } } if capacity > 0 { next.Capacity = capacity } full := false if reserve != nil { if _, exists := next.Slots[reserve.ID]; !exists { if len(occupiedIDs) >= next.Capacity { full = true } else { next.Slots[reserve.ID] = reserve.ItemID occupiedIDs[reserve.ItemID] = true } } } next.Occupied = len(occupiedIDs) // A poll that reserves nothing and leaves capacity and slots as stored // has nothing to persist; skipping the CAS write keeps 3s polls read-only. if reserve == nil && next.Capacity == old.Capacity && maps.Equal(next.Slots, old.Slots) { return next, nil } next.Version = old.Version + 1 result, e := collection.ReplaceOne(ctx, bson.M{"_id": key, "version": old.Version}, next, options.Replace()) if e != nil { return old, e } if result.MatchedCount == 0 { continue } if full { return next, ErrAnnotationBufferFull } return next, nil } } func (s *MongoStore) AnnotationWorkBuffer(ctx context.Context, org, user, project string, capacity int) (AnnotationWorkBuffer, error) { return s.annotationBuffer(ctx, org, user, project, capacity, nil) } // annotationBufferBatches reads only fields used by annotationBufferBatchMatches. // Cache missing IDs too, so corrupt historical references do not cause N+1 reads. // This cache lives for one ledger attempt; permission/workflow changes remain live. func (s *MongoStore) annotationBufferBatches(ctx context.Context, ids []string) (map[string]*domain.HumanBatchRecord, error) { result := make(map[string]*domain.HumanBatchRecord, len(ids)) unique := make([]string, 0, len(ids)) for _, id := range ids { if _, exists := result[id]; exists { continue } result[id] = nil unique = append(unique, id) } if len(unique) == 0 { return result, nil } cursor, err := s.humanBatches.Find(ctx, bson.M{"id": bson.M{"$in": unique}}, options.Find().SetProjection(bson.M{ "_id": 0, "id": 1, "organizationId": 1, "projectId": 1, "workflowId": 1, "processingProfile": 1, "stages": 1, })) if err != nil { return nil, err } defer cursor.Close(ctx) for cursor.Next(ctx) { var batch domain.HumanBatchRecord if err := cursor.Decode(&batch); err != nil { return nil, err } result[batch.ID] = &batch } return result, cursor.Err() } func (s *MongoStore) AnnotationWorkBufferSummary(ctx context.Context, org, user, project string, capacity int) (AnnotationWorkBuffer, error) { return s.annotationBuffer(ctx, org, user, project, capacity, nil, true) } func (s *MemoryStore) AnnotationWorkBufferSummary(ctx context.Context, org, user, project string, capacity int) (AnnotationWorkBuffer, error) { out, err := s.AnnotationWorkBuffer(ctx, org, user, project, capacity) for i := range out.Items { out.Items[i] = annotationBufferProjectionClone(out.Items[i]) } return out, err } func annotationBufferProjectionClone(item domain.HumanItemRecord) domain.HumanItemRecord { result := domain.HumanItemRecord{InheritedRework: item.InheritedRework, ID: item.ID, OrganizationID: item.OrganizationID, ProjectID: item.ProjectID, WorkflowID: item.WorkflowID, RunID: item.RunID, BatchID: item.BatchID, PluginID: item.PluginID, LegacyTaskID: item.LegacyTaskID, AssignedUserID: item.AssignedUserID, CurrentStageID: item.CurrentStageID, Status: item.Status, Version: item.Version, ReviewRound: item.ReviewRound, ClaimedAt: item.ClaimedAt, PendingCombinedSubmissionID: item.PendingCombinedSubmissionID} if item.ProcessingJob != nil { result.ProcessingJob = &domain.AnnotationProcessingJob{SourceFingerprint: item.ProcessingJob.SourceFingerprint, SourceIndex: item.ProcessingJob.SourceIndex, Status: item.ProcessingJob.Status, Kind: item.ProcessingJob.Kind, Error: item.ProcessingJob.Error} } phase, ok := item.Subject.Payload["sliceWorkflow"].(map[string]any) if !ok { if value, valid := item.Subject.Payload["sliceWorkflow"].(bson.M); valid { phase = map[string]any(value) } } if phase != nil { selected := map[string]any{} for _, key := range []string{"status", "phase"} { if value, exists := phase[key]; exists { selected[key] = clone(value) } } result.Subject.Payload = map[string]any{"sliceWorkflow": selected} } materialization, ok := item.Subject.Payload["materialization"].(map[string]any) if !ok { if value, valid := item.Subject.Payload["materialization"].(bson.M); valid { materialization = map[string]any(value) } } if materialization != nil { if result.Subject.Payload == nil { result.Subject.Payload = map[string]any{} } result.Subject.Payload["materialization"] = map[string]any{"sourceIndex": materialization["sourceIndex"]} } return result }