droboticflow/internal/store/annotation_processing.go

223 lines
9.8 KiB
Go

package store
import (
"context"
"droboticflow/internal/domain"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
"sort"
"strings"
)
type AnnotationProcessingStore interface {
AppendAnnotationArtifact(context.Context, string, string, domain.ExecutionArtifact) error
GetAnnotationTimelineIndex(context.Context, string) (*domain.AnnotationTimelineIndex, error)
PutAnnotationTimelineIndex(context.Context, domain.AnnotationTimelineIndex) error
EnqueueAnnotationIndex(context.Context, domain.AnnotationTaskRecord) error
EnqueueAnnotationSourceIndex(context.Context, domain.AnnotationTaskRecord, string, bool) error
GetAnnotationIndexJob(context.Context, string) (*domain.AnnotationIndexJob, error)
RetryAnnotationSourceValidation(context.Context, domain.AnnotationTaskRecord, bool, string) error
FailAnnotationSourceValidation(context.Context, domain.AnnotationTaskRecord, string, string) (bool, error)
TouchAnnotationIndex(context.Context, string, string, string) (bool, error)
ClaimAnnotationIndex(context.Context, string, string, string) (*domain.AnnotationIndexJob, error)
FinishAnnotationIndex(context.Context, domain.AnnotationIndexJob) (bool, error)
ClaimAnnotationProcessing(context.Context, string, string, string) (*domain.HumanItemRecord, error)
SaveAnnotationProcessing(context.Context, string, domain.AnnotationProcessingJob) (bool, error)
TouchAnnotationProcessing(context.Context, string, string, string, string, string, int, int) (bool, error)
}
func (s *MemoryStore) TouchAnnotationProcessing(_ context.Context, itemID, id, owner, until, stage string, completed, total int) (bool, error) {
s.mu.Lock()
defer s.mu.Unlock()
h, ok := s.humanItems[itemID]
if !ok || h.ProcessingJob == nil || h.ProcessingJob.ID != id || h.ProcessingJob.Owner != owner || h.ProcessingJob.Status != "running" {
return false, nil
}
h.ProcessingJob.LeaseUntil = until
h.ProcessingJob.Stage = stage
h.ProcessingJob.CompletedSlices = completed
h.ProcessingJob.TotalSlices = total
h.ProcessingJob.UpdatedAt = nowISO()
s.humanItems[itemID] = h
return true, nil
}
func (s *MongoStore) TouchAnnotationProcessing(ctx context.Context, itemID, id, owner, until, stage string, completed, total int) (bool, error) {
r, e := s.humanItems.UpdateOne(ctx, bson.M{"id": itemID, "processingJob.id": id, "processingJob.owner": owner, "processingJob.status": "running"}, bson.M{"$set": bson.M{"processingJob.leaseUntil": until, "processingJob.stage": stage, "processingJob.completedSlices": completed, "processingJob.totalSlices": total, "processingJob.updatedAt": nowISO()}})
if e != nil {
return false, e
}
return r.MatchedCount == 1, nil
}
func (s *MemoryStore) GetAnnotationTimelineIndex(_ context.Context, id string) (*domain.AnnotationTimelineIndex, error) {
s.mu.Lock()
defer s.mu.Unlock()
v, ok := s.annotationTimelineIndexes[id]
if !ok {
return nil, nil
}
v.Body = append([]byte(nil), v.Body...)
return &v, nil
}
func (s *MemoryStore) PutAnnotationTimelineIndex(_ context.Context, v domain.AnnotationTimelineIndex) error {
s.mu.Lock()
defer s.mu.Unlock()
if s.annotationTimelineIndexes == nil {
s.annotationTimelineIndexes = map[string]domain.AnnotationTimelineIndex{}
}
v.Body = append([]byte(nil), v.Body...)
s.annotationTimelineIndexes[v.ID] = v
return nil
}
func (s *MongoStore) GetAnnotationTimelineIndex(ctx context.Context, id string) (*domain.AnnotationTimelineIndex, error) {
return mongoGetByID[domain.AnnotationTimelineIndex](ctx, s.db.Collection("annotationTimelineIndexes"), id)
}
func (s *MongoStore) PutAnnotationTimelineIndex(ctx context.Context, v domain.AnnotationTimelineIndex) error {
return mongoUpsert(ctx, s.db.Collection("annotationTimelineIndexes"), v.ID, v)
}
func annotationIndexJob(t domain.AnnotationTaskRecord) domain.AnnotationIndexJob {
return domain.AnnotationIndexJob{ID: t.ID, OrganizationID: t.OrganizationID, ProjectID: t.ProjectID, WorkflowID: t.WorkflowID, RunID: t.RunID, Status: "queued", CreatedAt: nowISO(), NextAttemptAt: ""}
}
func (s *MemoryStore) EnqueueAnnotationIndex(_ context.Context, t domain.AnnotationTaskRecord) error {
s.mu.Lock()
defer s.mu.Unlock()
if s.annotationIndexJobs == nil {
s.annotationIndexJobs = map[string]domain.AnnotationIndexJob{}
}
if _, ok := s.annotationIndexJobs[t.ID]; !ok {
s.annotationIndexJobs[t.ID] = annotationIndexJob(t)
}
return nil
}
func (s *MongoStore) EnqueueAnnotationIndex(ctx context.Context, t domain.AnnotationTaskRecord) error {
return supplierGuardedMongoWrite(ctx, s.db.Collection("annotationIndexJobs"), t.ID, annotationIndexJob(t), func(ctx context.Context, d bson.M) error {
_, e := s.db.Collection("annotationIndexJobs").UpdateOne(ctx, bson.M{"id": t.ID}, bson.M{"$setOnInsert": d}, options.Update().SetUpsert(true))
return e
})
}
func claimableJob(status, lease, next, now string) bool {
return status == "queued" && next <= now || status == "running" && lease < now
}
func (s *MemoryStore) ClaimAnnotationIndex(_ context.Context, owner, now, until string) (*domain.AnnotationIndexJob, error) {
s.mu.Lock()
defer s.mu.Unlock()
ids := []string{}
for id, j := range s.annotationIndexJobs {
if claimableJob(j.Status, j.LeaseUntil, j.NextAttemptAt, now) {
ids = append(ids, id)
}
}
sort.Slice(ids, func(i, j int) bool {
a, b := strings.HasPrefix(ids[i], "source-v1:"), strings.HasPrefix(ids[j], "source-v1:")
if a != b {
return a
}
return ids[i] < ids[j]
})
if len(ids) == 0 {
return nil, nil
}
j := s.annotationIndexJobs[ids[0]]
j.Status = "running"
j.Owner = owner
j.LeaseUntil = until
j.Attempt++
s.annotationIndexJobs[j.ID] = j
return &j, nil
}
func (s *MongoStore) ClaimAnnotationIndex(ctx context.Context, owner, now, until string) (*domain.AnnotationIndexJob, error) {
var j domain.AnnotationIndexJob
filter := bson.M{"$or": bson.A{bson.M{"status": "queued", "nextAttemptAt": bson.M{"$lte": now}}, bson.M{"status": "running", "leaseUntil": bson.M{"$lt": now}}}}
update := bson.M{"$set": bson.M{"status": "running", "owner": owner, "leaseUntil": until}, "$inc": bson.M{"attempt": 1}}
opts := options.FindOneAndUpdate().SetSort(bson.D{{Key: "createdAt", Value: 1}}).SetReturnDocument(options.After)
// Current explicit source validation outranks historical timeline work.
// The bounded prefix range also works with the existing unique id index;
// no schema migration or historical record rewrite is required.
filter["id"] = bson.M{"$gte": "source-v1:", "$lt": "source-v1;"}
e := s.db.Collection("annotationIndexJobs").FindOneAndUpdate(ctx, filter, update, opts).Decode(&j)
if e != mongo.ErrNoDocuments {
return &j, e
}
delete(filter, "id")
e = s.db.Collection("annotationIndexJobs").FindOneAndUpdate(ctx, filter, update, opts).Decode(&j)
if e == mongo.ErrNoDocuments {
return nil, nil
}
return &j, e
}
func (s *MemoryStore) FinishAnnotationIndex(_ context.Context, j domain.AnnotationIndexJob) (bool, error) {
s.mu.Lock()
defer s.mu.Unlock()
old, ok := s.annotationIndexJobs[j.ID]
if !ok || old.Owner != j.Owner || old.Status != "running" {
return false, nil
}
s.annotationIndexJobs[j.ID] = j
return true, nil
}
func (s *MongoStore) FinishAnnotationIndex(ctx context.Context, j domain.AnnotationIndexJob) (bool, error) {
result, e := s.withSupplierProjectLocks(ctx, []string{j.ProjectID}, func(ctx context.Context) (any, error) {
r, e := s.db.Collection("annotationIndexJobs").ReplaceOne(ctx, bson.M{"id": j.ID, "owner": j.Owner, "status": "running"}, j)
if e != nil {
return false, e
}
return r.MatchedCount == 1, nil
})
if e != nil {
return false, e
}
return result.(bool), nil
}
func (s *MemoryStore) ClaimAnnotationProcessing(_ context.Context, owner, now, until string) (*domain.HumanItemRecord, error) {
s.mu.Lock()
defer s.mu.Unlock()
ids := []string{}
for id, h := range s.humanItems {
if j := h.ProcessingJob; j != nil && claimableJob(j.Status, j.LeaseUntil, j.NextAttemptAt, now) {
ids = append(ids, id)
}
}
sort.Strings(ids)
if len(ids) == 0 {
return nil, nil
}
h := clone(s.humanItems[ids[0]])
h.ProcessingJob.Status = "running"
h.ProcessingJob.Owner = owner
h.ProcessingJob.LeaseUntil = until
h.ProcessingJob.Attempt++
s.humanItems[h.ID] = clone(h)
return &h, nil
}
func (s *MongoStore) ClaimAnnotationProcessing(ctx context.Context, owner, now, until string) (*domain.HumanItemRecord, error) {
var h domain.HumanItemRecord
queued := bson.M{"processingJob.status": "queued", "$or": bson.A{bson.M{"processingJob.nextAttemptAt": bson.M{"$exists": false}}, bson.M{"processingJob.nextAttemptAt": bson.M{"$lte": now}}}}
e := s.humanItems.FindOneAndUpdate(ctx, bson.M{"$or": bson.A{queued, bson.M{"processingJob.status": "running", "processingJob.leaseUntil": bson.M{"$lt": now}}}}, bson.M{"$set": bson.M{"processingJob.status": "running", "processingJob.owner": owner, "processingJob.leaseUntil": until}, "$inc": bson.M{"processingJob.attempt": 1}}, options.FindOneAndUpdate().SetSort(bson.D{{Key: "processingJob.createdAt", Value: 1}}).SetReturnDocument(options.After)).Decode(&h)
if e == mongo.ErrNoDocuments {
return nil, nil
}
return &h, e
}
func (s *MemoryStore) SaveAnnotationProcessing(_ context.Context, itemID string, j domain.AnnotationProcessingJob) (bool, error) {
s.mu.Lock()
defer s.mu.Unlock()
h, ok := s.humanItems[itemID]
if !ok || h.ProcessingJob == nil || h.ProcessingJob.ID != j.ID || h.ProcessingJob.Owner != j.Owner || h.ProcessingJob.Status != "running" {
return false, nil
}
h.ProcessingJob = &j
s.humanItems[itemID] = clone(h)
return true, nil
}
func (s *MongoStore) SaveAnnotationProcessing(ctx context.Context, itemID string, j domain.AnnotationProcessingJob) (bool, error) {
r, e := s.humanItems.UpdateOne(ctx, bson.M{"id": itemID, "processingJob.id": j.ID, "processingJob.owner": j.Owner, "processingJob.status": "running"}, bson.M{"$set": bson.M{"processingJob": j}})
if e != nil {
return false, e
}
return r.MatchedCount == 1, nil
}