forked from longtao.wu/droboticflow
280 lines
11 KiB
Go
280 lines
11 KiB
Go
package store
|
|
|
|
import (
|
|
"context"
|
|
"crypto/sha256"
|
|
"droboticflow/internal/domain"
|
|
"errors"
|
|
"fmt"
|
|
"go.mongodb.org/mongo-driver/bson"
|
|
"go.mongodb.org/mongo-driver/mongo"
|
|
"go.mongodb.org/mongo-driver/mongo/options"
|
|
"maps"
|
|
)
|
|
|
|
var ErrAnnotationBufferFull = errors.New("annotation work buffer is full")
|
|
|
|
type AnnotationWorkBuffer struct {
|
|
Capacity int `json:"capacity" bson:"capacity"`
|
|
Occupied int `json:"occupied" bson:"-"`
|
|
Items []domain.HumanItemRecord `json:"items" bson:"-"`
|
|
Version int `json:"-" bson:"version"`
|
|
Slots map[string]string `json:"-" bson:"slots"`
|
|
}
|
|
|
|
func bufferKey(org, user, project string) string {
|
|
return fmt.Sprintf("%x", sha256.Sum256([]byte(org+"\x00"+user+"\x00"+project)))
|
|
}
|
|
func occupiesAnnotationBuffer(item domain.HumanItemRecord, org, user, project string) bool {
|
|
return item.OrganizationID == org && item.ProjectID == project && item.AssignedUserID == user && item.PluginID == "annotation-qa" && item.LegacyTaskID != "" && (item.CurrentStageID == "annotation_combined" || item.CurrentStageID == "semantic_annotation") && (item.Status == domain.HumanItemClaimed || item.Status == domain.HumanItemInProgress || item.Status == domain.HumanItemRework)
|
|
}
|
|
func annotationBufferBatchMatches(item domain.HumanItemRecord, batch domain.HumanBatchRecord) bool {
|
|
return batch.OrganizationID == item.OrganizationID && batch.ProjectID == item.ProjectID && batch.WorkflowID == item.WorkflowID && batch.ProcessingProfile == domain.ProcessingProfileRawHuman && domain.IsFanyuV58ThreeRoleHumanLoop(batch.Stages)
|
|
}
|
|
func (s *MemoryStore) AnnotationWorkBuffer(_ context.Context, org, user, project string, capacity int) (AnnotationWorkBuffer, error) {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
key := bufferKey(org, user, project)
|
|
if s.annotationBufferCapacities == nil {
|
|
s.annotationBufferCapacities = map[string]int{}
|
|
}
|
|
if capacity > 0 {
|
|
s.annotationBufferCapacities[key] = capacity
|
|
}
|
|
n := s.annotationBufferCapacities[key]
|
|
if n == 0 {
|
|
n = 2
|
|
}
|
|
out := AnnotationWorkBuffer{Capacity: n, Items: []domain.HumanItemRecord{}}
|
|
for _, item := range s.humanItems {
|
|
if occupiesAnnotationBuffer(item, org, user, project) && annotationBufferBatchMatches(item, s.humanBatches[item.BatchID]) {
|
|
out.Items = append(out.Items, clone(item))
|
|
}
|
|
}
|
|
out.Occupied = len(out.Items)
|
|
return out, nil
|
|
}
|
|
|
|
// The single-document CAS ledger also protects standalone Mongo. Pending claim
|
|
// intents retain their slot across process restarts; only a definitive conflict
|
|
// or handoff frees it. A receipt retry reuses the same slot even after downsizing.
|
|
func (s *MongoStore) annotationBuffer(ctx context.Context, org, user, project string, capacity int, reserve *HumanClaimReceipt, summary ...bool) (AnnotationWorkBuffer, error) {
|
|
compact := len(summary) > 0 && summary[0]
|
|
key := bufferKey(org, user, project)
|
|
collection := s.db.Collection("annotationWorkBuffers")
|
|
for {
|
|
if err := ctx.Err(); err != nil {
|
|
return AnnotationWorkBuffer{}, err
|
|
}
|
|
var old AnnotationWorkBuffer
|
|
err := collection.FindOne(ctx, bson.M{"_id": key}).Decode(&old)
|
|
if err == mongo.ErrNoDocuments {
|
|
_, e := collection.InsertOne(ctx, bson.M{"_id": key, "capacity": 2, "version": 0, "slots": bson.M{}})
|
|
if e != nil && !mongo.IsDuplicateKeyError(e) {
|
|
return old, e
|
|
}
|
|
continue
|
|
}
|
|
if err != nil {
|
|
return old, err
|
|
}
|
|
next := old
|
|
next.Items = []domain.HumanItemRecord{}
|
|
next.Slots = map[string]string{}
|
|
// The "active" view bounds the scan to the actor's assigned open
|
|
// annotation items, a superset of occupiesAnnotationBuffer; matching
|
|
// latestAuthorUserId would read the actor's whole authored history.
|
|
items, err := s.ListHumanItems(ctx, org, domain.HumanItemQuery{ProjectID: project, PluginID: "annotation-qa", ActorUserID: user, AnnotationView: "active", ListProjection: true, BufferSummaryProjection: compact})
|
|
if err != nil {
|
|
return old, err
|
|
}
|
|
occupiedIDs := map[string]bool{}
|
|
batchIDs := make([]string, 0, len(items))
|
|
for _, item := range items {
|
|
batchIDs = append(batchIDs, item.BatchID)
|
|
}
|
|
batches, err := s.annotationBufferBatches(ctx, batchIDs)
|
|
if err != nil {
|
|
return old, err
|
|
}
|
|
trusted := func(item domain.HumanItemRecord) (bool, error) {
|
|
batch, ok := batches[item.BatchID]
|
|
if !ok {
|
|
// A ledger slot may reference an item outside the active scan.
|
|
// Resolve its batch with the same projection and request-local cache.
|
|
missing, e := s.annotationBufferBatches(ctx, []string{item.BatchID})
|
|
if e != nil {
|
|
return false, e
|
|
}
|
|
batch = missing[item.BatchID]
|
|
batches[item.BatchID] = batch
|
|
}
|
|
return batch != nil && annotationBufferBatchMatches(item, *batch), nil
|
|
}
|
|
for _, item := range items {
|
|
scoped, e := trusted(item)
|
|
if e != nil {
|
|
return old, e
|
|
}
|
|
if !scoped {
|
|
continue
|
|
}
|
|
if occupiesAnnotationBuffer(item, org, user, project) {
|
|
next.Items = append(next.Items, item)
|
|
occupiedIDs[item.ID] = true
|
|
}
|
|
}
|
|
for request, id := range old.Slots {
|
|
// An item the active scan already counted keeps its slot whatever
|
|
// its receipt says; rereading it and the receipt changes nothing.
|
|
if occupiedIDs[id] {
|
|
next.Slots[request] = id
|
|
continue
|
|
}
|
|
slotItems, e := s.ListHumanItems(ctx, org, domain.HumanItemQuery{IDs: []string{id}, ListProjection: true, BufferSummaryProjection: compact})
|
|
var item *domain.HumanItemRecord
|
|
if len(slotItems) > 0 {
|
|
item = &slotItems[0]
|
|
}
|
|
if e != nil {
|
|
return old, e
|
|
}
|
|
receipt, e := s.GetHumanClaimReceipt(ctx, request)
|
|
if e != nil {
|
|
return old, e
|
|
}
|
|
if item != nil {
|
|
scoped, e := trusted(*item)
|
|
if e != nil {
|
|
return old, e
|
|
}
|
|
if !scoped {
|
|
continue
|
|
}
|
|
}
|
|
pending := receipt != nil && receipt.State == "pending" && receipt.Intent != nil && item != nil && item.ClaimRequestID != request && validClaimSnapshot(*item, *receipt.Intent)
|
|
if item != nil && occupiesAnnotationBuffer(*item, org, user, project) && !occupiedIDs[id] {
|
|
occupiedIDs[id] = true
|
|
next.Items = append(next.Items, *item)
|
|
}
|
|
if occupiedIDs[id] || pending {
|
|
next.Slots[request] = id
|
|
occupiedIDs[id] = true
|
|
}
|
|
}
|
|
if capacity > 0 {
|
|
next.Capacity = capacity
|
|
}
|
|
full := false
|
|
if reserve != nil {
|
|
if _, exists := next.Slots[reserve.ID]; !exists {
|
|
if len(occupiedIDs) >= next.Capacity {
|
|
full = true
|
|
} else {
|
|
next.Slots[reserve.ID] = reserve.ItemID
|
|
occupiedIDs[reserve.ItemID] = true
|
|
}
|
|
}
|
|
}
|
|
next.Occupied = len(occupiedIDs)
|
|
// A poll that reserves nothing and leaves capacity and slots as stored
|
|
// has nothing to persist; skipping the CAS write keeps 3s polls read-only.
|
|
if reserve == nil && next.Capacity == old.Capacity && maps.Equal(next.Slots, old.Slots) {
|
|
return next, nil
|
|
}
|
|
next.Version = old.Version + 1
|
|
result, e := collection.ReplaceOne(ctx, bson.M{"_id": key, "version": old.Version}, next, options.Replace())
|
|
if e != nil {
|
|
return old, e
|
|
}
|
|
if result.MatchedCount == 0 {
|
|
continue
|
|
}
|
|
if full {
|
|
return next, ErrAnnotationBufferFull
|
|
}
|
|
return next, nil
|
|
}
|
|
}
|
|
func (s *MongoStore) AnnotationWorkBuffer(ctx context.Context, org, user, project string, capacity int) (AnnotationWorkBuffer, error) {
|
|
return s.annotationBuffer(ctx, org, user, project, capacity, nil)
|
|
}
|
|
|
|
// annotationBufferBatches reads only fields used by annotationBufferBatchMatches.
|
|
// Cache missing IDs too, so corrupt historical references do not cause N+1 reads.
|
|
// This cache lives for one ledger attempt; permission/workflow changes remain live.
|
|
func (s *MongoStore) annotationBufferBatches(ctx context.Context, ids []string) (map[string]*domain.HumanBatchRecord, error) {
|
|
result := make(map[string]*domain.HumanBatchRecord, len(ids))
|
|
unique := make([]string, 0, len(ids))
|
|
for _, id := range ids {
|
|
if _, exists := result[id]; exists {
|
|
continue
|
|
}
|
|
result[id] = nil
|
|
unique = append(unique, id)
|
|
}
|
|
if len(unique) == 0 {
|
|
return result, nil
|
|
}
|
|
cursor, err := s.humanBatches.Find(ctx, bson.M{"id": bson.M{"$in": unique}}, options.Find().SetProjection(bson.M{
|
|
"_id": 0, "id": 1, "organizationId": 1, "projectId": 1, "workflowId": 1, "processingProfile": 1, "stages": 1,
|
|
}))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer cursor.Close(ctx)
|
|
for cursor.Next(ctx) {
|
|
var batch domain.HumanBatchRecord
|
|
if err := cursor.Decode(&batch); err != nil {
|
|
return nil, err
|
|
}
|
|
result[batch.ID] = &batch
|
|
}
|
|
return result, cursor.Err()
|
|
}
|
|
|
|
func (s *MongoStore) AnnotationWorkBufferSummary(ctx context.Context, org, user, project string, capacity int) (AnnotationWorkBuffer, error) {
|
|
return s.annotationBuffer(ctx, org, user, project, capacity, nil, true)
|
|
}
|
|
func (s *MemoryStore) AnnotationWorkBufferSummary(ctx context.Context, org, user, project string, capacity int) (AnnotationWorkBuffer, error) {
|
|
out, err := s.AnnotationWorkBuffer(ctx, org, user, project, capacity)
|
|
for i := range out.Items {
|
|
out.Items[i] = annotationBufferProjectionClone(out.Items[i])
|
|
}
|
|
return out, err
|
|
}
|
|
func annotationBufferProjectionClone(item domain.HumanItemRecord) domain.HumanItemRecord {
|
|
result := domain.HumanItemRecord{ID: item.ID, OrganizationID: item.OrganizationID, ProjectID: item.ProjectID, WorkflowID: item.WorkflowID, RunID: item.RunID, BatchID: item.BatchID, PluginID: item.PluginID, LegacyTaskID: item.LegacyTaskID, AssignedUserID: item.AssignedUserID, CurrentStageID: item.CurrentStageID, Status: item.Status, Version: item.Version, ReviewRound: item.ReviewRound, ClaimedAt: item.ClaimedAt, PendingCombinedSubmissionID: item.PendingCombinedSubmissionID}
|
|
if item.ProcessingJob != nil {
|
|
result.ProcessingJob = &domain.AnnotationProcessingJob{SourceFingerprint: item.ProcessingJob.SourceFingerprint, SourceIndex: item.ProcessingJob.SourceIndex, Status: item.ProcessingJob.Status, Kind: item.ProcessingJob.Kind, Error: item.ProcessingJob.Error}
|
|
}
|
|
phase, ok := item.Subject.Payload["sliceWorkflow"].(map[string]any)
|
|
if !ok {
|
|
if value, valid := item.Subject.Payload["sliceWorkflow"].(bson.M); valid {
|
|
phase = map[string]any(value)
|
|
}
|
|
}
|
|
if phase != nil {
|
|
selected := map[string]any{}
|
|
for _, key := range []string{"status", "phase"} {
|
|
if value, exists := phase[key]; exists {
|
|
selected[key] = clone(value)
|
|
}
|
|
}
|
|
result.Subject.Payload = map[string]any{"sliceWorkflow": selected}
|
|
}
|
|
materialization, ok := item.Subject.Payload["materialization"].(map[string]any)
|
|
if !ok {
|
|
if value, valid := item.Subject.Payload["materialization"].(bson.M); valid {
|
|
materialization = map[string]any(value)
|
|
}
|
|
}
|
|
if materialization != nil {
|
|
if result.Subject.Payload == nil {
|
|
result.Subject.Payload = map[string]any{}
|
|
}
|
|
result.Subject.Payload["materialization"] = map[string]any{"sourceIndex": materialization["sourceIndex"]}
|
|
}
|
|
return result
|
|
}
|