droboticflow/internal/store/annotation_work_buffer.go

286 lines
11 KiB
Go

package store
import (
"context"
"crypto/sha256"
"droboticflow/internal/domain"
"errors"
"fmt"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
"maps"
)
var ErrAnnotationBufferFull = errors.New("annotation work buffer is full")
type AnnotationWorkBuffer struct {
Capacity int `json:"capacity" bson:"capacity"`
Occupied int `json:"occupied" bson:"-"`
Items []domain.HumanItemRecord `json:"items" bson:"-"`
Version int `json:"-" bson:"version"`
Slots map[string]string `json:"-" bson:"slots"`
}
func bufferKey(org, user, project string) string {
return fmt.Sprintf("%x", sha256.Sum256([]byte(org+"\x00"+user+"\x00"+project)))
}
func occupiesAnnotationBuffer(item domain.HumanItemRecord, org, user, project string) bool {
return item.OrganizationID == org && item.ProjectID == project && item.AssignedUserID == user && item.PluginID == "annotation-qa" && item.LegacyTaskID != "" && (item.CurrentStageID == "annotation_combined" || item.CurrentStageID == "semantic_annotation") && (item.Status == domain.HumanItemClaimed || item.Status == domain.HumanItemInProgress || item.Status == domain.HumanItemRework)
}
func annotationBufferBatchMatches(item domain.HumanItemRecord, batch domain.HumanBatchRecord) bool {
return batch.OrganizationID == item.OrganizationID && batch.ProjectID == item.ProjectID && batch.WorkflowID == item.WorkflowID && batch.ProcessingProfile == domain.ProcessingProfileRawHuman && domain.IsFanyuV58ThreeRoleHumanLoop(batch.Stages)
}
func (s *MemoryStore) AnnotationWorkBuffer(_ context.Context, org, user, project string, capacity int) (AnnotationWorkBuffer, error) {
s.mu.Lock()
defer s.mu.Unlock()
key := bufferKey(org, user, project)
if s.annotationBufferCapacities == nil {
s.annotationBufferCapacities = map[string]int{}
}
if capacity > 0 {
s.annotationBufferCapacities[key] = capacity
}
n := s.annotationBufferCapacities[key]
if n == 0 {
n = 2
}
out := AnnotationWorkBuffer{Capacity: n, Items: []domain.HumanItemRecord{}}
for _, item := range s.humanItems {
if occupiesAnnotationBuffer(item, org, user, project) && annotationBufferBatchMatches(item, s.humanBatches[item.BatchID]) {
out.Items = append(out.Items, clone(item))
}
}
for _, item := range out.Items {
if !AnnotationSourceAuditAdvisory() || !AnnotationSourceUnavailable(item) {
out.Occupied++
}
}
return out, nil
}
// The single-document CAS ledger also protects standalone Mongo. Pending claim
// intents retain their slot across process restarts; only a definitive conflict
// or handoff frees it. A receipt retry reuses the same slot even after downsizing.
func (s *MongoStore) annotationBuffer(ctx context.Context, org, user, project string, capacity int, reserve *HumanClaimReceipt, summary ...bool) (AnnotationWorkBuffer, error) {
compact := len(summary) > 0 && summary[0]
key := bufferKey(org, user, project)
collection := s.queryCollection("annotationWorkBuffers")
for {
if err := ctx.Err(); err != nil {
return AnnotationWorkBuffer{}, err
}
var old AnnotationWorkBuffer
err := collection.FindOne(ctx, bson.M{"_id": key}).Decode(&old)
if err == mongo.ErrNoDocuments {
_, e := collection.InsertOne(ctx, bson.M{"_id": key, "capacity": 2, "version": 0, "slots": bson.M{}})
if e != nil && !mongo.IsDuplicateKeyError(e) {
return old, e
}
continue
}
if err != nil {
return old, err
}
next := old
next.Items = []domain.HumanItemRecord{}
next.Slots = map[string]string{}
// The "active" view bounds the scan to the actor's assigned open
// annotation items, a superset of occupiesAnnotationBuffer; matching
// latestAuthorUserId would read the actor's whole authored history.
items, err := s.ListHumanItems(ctx, org, domain.HumanItemQuery{ProjectID: project, PluginID: "annotation-qa", ActorUserID: user, AnnotationView: "active", ListProjection: true, BufferSummaryProjection: compact})
if err != nil {
return old, err
}
occupiedIDs := map[string]bool{}
batchIDs := make([]string, 0, len(items))
for _, item := range items {
batchIDs = append(batchIDs, item.BatchID)
}
batches, err := s.annotationBufferBatches(ctx, batchIDs)
if err != nil {
return old, err
}
trusted := func(item domain.HumanItemRecord) (bool, error) {
batch, ok := batches[item.BatchID]
if !ok {
// A ledger slot may reference an item outside the active scan.
// Resolve its batch with the same projection and request-local cache.
missing, e := s.annotationBufferBatches(ctx, []string{item.BatchID})
if e != nil {
return false, e
}
batch = missing[item.BatchID]
batches[item.BatchID] = batch
}
return batch != nil && annotationBufferBatchMatches(item, *batch), nil
}
for _, item := range items {
scoped, e := trusted(item)
if e != nil {
return old, e
}
if !scoped {
continue
}
if occupiesAnnotationBuffer(item, org, user, project) {
next.Items = append(next.Items, item)
if !AnnotationSourceAuditAdvisory() || !AnnotationSourceUnavailable(item) {
occupiedIDs[item.ID] = true
}
}
}
for request, id := range old.Slots {
// An item the active scan already counted keeps its slot whatever
// its receipt says; rereading it and the receipt changes nothing.
if occupiedIDs[id] {
next.Slots[request] = id
continue
}
slotItems, e := s.ListHumanItems(ctx, org, domain.HumanItemQuery{IDs: []string{id}, ListProjection: true, BufferSummaryProjection: compact})
var item *domain.HumanItemRecord
if len(slotItems) > 0 {
item = &slotItems[0]
}
if e != nil {
return old, e
}
receipt, e := s.GetHumanClaimReceipt(ctx, request)
if e != nil {
return old, e
}
if item != nil {
scoped, e := trusted(*item)
if e != nil {
return old, e
}
if !scoped {
continue
}
}
pending := receipt != nil && receipt.State == "pending" && receipt.Intent != nil && item != nil && item.ClaimRequestID != request && validClaimSnapshot(*item, *receipt.Intent)
if item != nil && occupiesAnnotationBuffer(*item, org, user, project) && !occupiedIDs[id] && (!AnnotationSourceAuditAdvisory() || !AnnotationSourceUnavailable(*item)) {
occupiedIDs[id] = true
next.Items = append(next.Items, *item)
}
if occupiedIDs[id] || pending {
next.Slots[request] = id
occupiedIDs[id] = true
}
}
if capacity > 0 {
next.Capacity = capacity
}
full := false
if reserve != nil {
if _, exists := next.Slots[reserve.ID]; !exists {
if len(occupiedIDs) >= next.Capacity {
full = true
} else {
next.Slots[reserve.ID] = reserve.ItemID
occupiedIDs[reserve.ItemID] = true
}
}
}
next.Occupied = len(occupiedIDs)
// A poll that reserves nothing and leaves capacity and slots as stored
// has nothing to persist; skipping the CAS write keeps 3s polls read-only.
if reserve == nil && next.Capacity == old.Capacity && maps.Equal(next.Slots, old.Slots) {
return next, nil
}
next.Version = old.Version + 1
result, e := collection.ReplaceOne(ctx, bson.M{"_id": key, "version": old.Version}, next, options.Replace())
if e != nil {
return old, e
}
if result.MatchedCount == 0 {
continue
}
if full {
return next, ErrAnnotationBufferFull
}
return next, nil
}
}
func (s *MongoStore) AnnotationWorkBuffer(ctx context.Context, org, user, project string, capacity int) (AnnotationWorkBuffer, error) {
return s.annotationBuffer(ctx, org, user, project, capacity, nil)
}
// annotationBufferBatches reads only fields used by annotationBufferBatchMatches.
// Cache missing IDs too, so corrupt historical references do not cause N+1 reads.
// This cache lives for one ledger attempt; permission/workflow changes remain live.
func (s *MongoStore) annotationBufferBatches(ctx context.Context, ids []string) (map[string]*domain.HumanBatchRecord, error) {
result := make(map[string]*domain.HumanBatchRecord, len(ids))
unique := make([]string, 0, len(ids))
for _, id := range ids {
if _, exists := result[id]; exists {
continue
}
result[id] = nil
unique = append(unique, id)
}
if len(unique) == 0 {
return result, nil
}
cursor, err := s.humanBatches.Find(ctx, bson.M{"id": bson.M{"$in": unique}}, options.Find().SetProjection(bson.M{
"_id": 0, "id": 1, "organizationId": 1, "projectId": 1, "workflowId": 1, "processingProfile": 1, "stages": 1,
}))
if err != nil {
return nil, err
}
defer cursor.Close(ctx)
for cursor.Next(ctx) {
var batch domain.HumanBatchRecord
if err := cursor.Decode(&batch); err != nil {
return nil, err
}
result[batch.ID] = &batch
}
return result, cursor.Err()
}
func (s *MongoStore) AnnotationWorkBufferSummary(ctx context.Context, org, user, project string, capacity int) (AnnotationWorkBuffer, error) {
return s.annotationBuffer(ctx, org, user, project, capacity, nil, true)
}
func (s *MemoryStore) AnnotationWorkBufferSummary(ctx context.Context, org, user, project string, capacity int) (AnnotationWorkBuffer, error) {
out, err := s.AnnotationWorkBuffer(ctx, org, user, project, capacity)
for i := range out.Items {
out.Items[i] = annotationBufferProjectionClone(out.Items[i])
}
return out, err
}
func annotationBufferProjectionClone(item domain.HumanItemRecord) domain.HumanItemRecord {
result := domain.HumanItemRecord{InheritedRework: item.InheritedRework, ID: item.ID, OrganizationID: item.OrganizationID, ProjectID: item.ProjectID, WorkflowID: item.WorkflowID, RunID: item.RunID, BatchID: item.BatchID, PluginID: item.PluginID, LegacyTaskID: item.LegacyTaskID, AssignedUserID: item.AssignedUserID, CurrentStageID: item.CurrentStageID, Status: item.Status, Version: item.Version, ReviewRound: item.ReviewRound, ClaimedAt: item.ClaimedAt, PendingCombinedSubmissionID: item.PendingCombinedSubmissionID}
if item.ProcessingJob != nil {
result.ProcessingJob = &domain.AnnotationProcessingJob{SourceFingerprint: item.ProcessingJob.SourceFingerprint, SourceIndex: item.ProcessingJob.SourceIndex, Status: item.ProcessingJob.Status, Kind: item.ProcessingJob.Kind, Error: item.ProcessingJob.Error}
}
phase, ok := item.Subject.Payload["sliceWorkflow"].(map[string]any)
if !ok {
if value, valid := item.Subject.Payload["sliceWorkflow"].(bson.M); valid {
phase = map[string]any(value)
}
}
if phase != nil {
selected := map[string]any{}
for _, key := range []string{"status", "phase"} {
if value, exists := phase[key]; exists {
selected[key] = clone(value)
}
}
result.Subject.Payload = map[string]any{"sliceWorkflow": selected}
}
materialization, ok := item.Subject.Payload["materialization"].(map[string]any)
if !ok {
if value, valid := item.Subject.Payload["materialization"].(bson.M); valid {
materialization = map[string]any(value)
}
}
if materialization != nil {
if result.Subject.Payload == nil {
result.Subject.Payload = map[string]any{}
}
result.Subject.Payload["materialization"] = map[string]any{"sourceIndex": materialization["sourceIndex"]}
}
return result
}