forked from longtao.wu/droboticflow
223 lines
9.8 KiB
Go
223 lines
9.8 KiB
Go
package store
|
|
|
|
import (
|
|
"context"
|
|
"droboticflow/internal/domain"
|
|
"go.mongodb.org/mongo-driver/bson"
|
|
"go.mongodb.org/mongo-driver/mongo"
|
|
"go.mongodb.org/mongo-driver/mongo/options"
|
|
"sort"
|
|
"strings"
|
|
)
|
|
|
|
type AnnotationProcessingStore interface {
|
|
AppendAnnotationArtifact(context.Context, string, string, domain.ExecutionArtifact) error
|
|
GetAnnotationTimelineIndex(context.Context, string) (*domain.AnnotationTimelineIndex, error)
|
|
PutAnnotationTimelineIndex(context.Context, domain.AnnotationTimelineIndex) error
|
|
EnqueueAnnotationIndex(context.Context, domain.AnnotationTaskRecord) error
|
|
EnqueueAnnotationSourceIndex(context.Context, domain.AnnotationTaskRecord, string, bool) error
|
|
GetAnnotationIndexJob(context.Context, string) (*domain.AnnotationIndexJob, error)
|
|
RetryAnnotationSourceValidation(context.Context, domain.AnnotationTaskRecord, bool, string) error
|
|
FailAnnotationSourceValidation(context.Context, domain.AnnotationTaskRecord, string, string) (bool, error)
|
|
TouchAnnotationIndex(context.Context, string, string, string) (bool, error)
|
|
ClaimAnnotationIndex(context.Context, string, string, string) (*domain.AnnotationIndexJob, error)
|
|
FinishAnnotationIndex(context.Context, domain.AnnotationIndexJob) (bool, error)
|
|
ClaimAnnotationProcessing(context.Context, string, string, string) (*domain.HumanItemRecord, error)
|
|
SaveAnnotationProcessing(context.Context, string, domain.AnnotationProcessingJob) (bool, error)
|
|
TouchAnnotationProcessing(context.Context, string, string, string, string, string, int, int) (bool, error)
|
|
}
|
|
|
|
func (s *MemoryStore) TouchAnnotationProcessing(_ context.Context, itemID, id, owner, until, stage string, completed, total int) (bool, error) {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
h, ok := s.humanItems[itemID]
|
|
if !ok || h.ProcessingJob == nil || h.ProcessingJob.ID != id || h.ProcessingJob.Owner != owner || h.ProcessingJob.Status != "running" {
|
|
return false, nil
|
|
}
|
|
h.ProcessingJob.LeaseUntil = until
|
|
h.ProcessingJob.Stage = stage
|
|
h.ProcessingJob.CompletedSlices = completed
|
|
h.ProcessingJob.TotalSlices = total
|
|
h.ProcessingJob.UpdatedAt = nowISO()
|
|
s.humanItems[itemID] = h
|
|
return true, nil
|
|
}
|
|
func (s *MongoStore) TouchAnnotationProcessing(ctx context.Context, itemID, id, owner, until, stage string, completed, total int) (bool, error) {
|
|
r, e := s.humanItems.UpdateOne(ctx, bson.M{"id": itemID, "processingJob.id": id, "processingJob.owner": owner, "processingJob.status": "running"}, bson.M{"$set": bson.M{"processingJob.leaseUntil": until, "processingJob.stage": stage, "processingJob.completedSlices": completed, "processingJob.totalSlices": total, "processingJob.updatedAt": nowISO()}})
|
|
if e != nil {
|
|
return false, e
|
|
}
|
|
return r.MatchedCount == 1, nil
|
|
}
|
|
|
|
func (s *MemoryStore) GetAnnotationTimelineIndex(_ context.Context, id string) (*domain.AnnotationTimelineIndex, error) {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
v, ok := s.annotationTimelineIndexes[id]
|
|
if !ok {
|
|
return nil, nil
|
|
}
|
|
v.Body = append([]byte(nil), v.Body...)
|
|
return &v, nil
|
|
}
|
|
func (s *MemoryStore) PutAnnotationTimelineIndex(_ context.Context, v domain.AnnotationTimelineIndex) error {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
if s.annotationTimelineIndexes == nil {
|
|
s.annotationTimelineIndexes = map[string]domain.AnnotationTimelineIndex{}
|
|
}
|
|
v.Body = append([]byte(nil), v.Body...)
|
|
s.annotationTimelineIndexes[v.ID] = v
|
|
return nil
|
|
}
|
|
func (s *MongoStore) GetAnnotationTimelineIndex(ctx context.Context, id string) (*domain.AnnotationTimelineIndex, error) {
|
|
return mongoGetByID[domain.AnnotationTimelineIndex](ctx, s.queryCollection("annotationTimelineIndexes"), id)
|
|
}
|
|
func (s *MongoStore) PutAnnotationTimelineIndex(ctx context.Context, v domain.AnnotationTimelineIndex) error {
|
|
return mongoUpsert(ctx, s.queryCollection("annotationTimelineIndexes"), v.ID, v)
|
|
}
|
|
|
|
func annotationIndexJob(t domain.AnnotationTaskRecord) domain.AnnotationIndexJob {
|
|
return domain.AnnotationIndexJob{ID: t.ID, OrganizationID: t.OrganizationID, ProjectID: t.ProjectID, WorkflowID: t.WorkflowID, RunID: t.RunID, RevalidateSource: true, Status: "queued", CreatedAt: nowISO(), NextAttemptAt: ""}
|
|
}
|
|
func (s *MemoryStore) EnqueueAnnotationIndex(_ context.Context, t domain.AnnotationTaskRecord) error {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
if s.annotationIndexJobs == nil {
|
|
s.annotationIndexJobs = map[string]domain.AnnotationIndexJob{}
|
|
}
|
|
if _, ok := s.annotationIndexJobs[t.ID]; !ok {
|
|
s.annotationIndexJobs[t.ID] = annotationIndexJob(t)
|
|
}
|
|
return nil
|
|
}
|
|
func (s *MongoStore) EnqueueAnnotationIndex(ctx context.Context, t domain.AnnotationTaskRecord) error {
|
|
return supplierGuardedMongoWrite(ctx, s.queryCollection("annotationIndexJobs"), t.ID, annotationIndexJob(t), func(ctx context.Context, d bson.M) error {
|
|
_, e := s.queryCollection("annotationIndexJobs").UpdateOne(ctx, bson.M{"id": t.ID}, bson.M{"$setOnInsert": d}, options.Update().SetUpsert(true))
|
|
return e
|
|
})
|
|
}
|
|
|
|
func claimableJob(status, lease, next, now string) bool {
|
|
return status == "queued" && next <= now || status == "running" && lease < now
|
|
}
|
|
func (s *MemoryStore) ClaimAnnotationIndex(_ context.Context, owner, now, until string) (*domain.AnnotationIndexJob, error) {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
ids := []string{}
|
|
for id, j := range s.annotationIndexJobs {
|
|
if claimableJob(j.Status, j.LeaseUntil, j.NextAttemptAt, now) {
|
|
ids = append(ids, id)
|
|
}
|
|
}
|
|
sort.Slice(ids, func(i, j int) bool {
|
|
a, b := strings.HasPrefix(ids[i], "source-v1:"), strings.HasPrefix(ids[j], "source-v1:")
|
|
if a != b {
|
|
return a
|
|
}
|
|
return ids[i] < ids[j]
|
|
})
|
|
if len(ids) == 0 {
|
|
return nil, nil
|
|
}
|
|
j := s.annotationIndexJobs[ids[0]]
|
|
j.Status = "running"
|
|
j.Owner = owner
|
|
j.LeaseUntil = until
|
|
j.Attempt++
|
|
s.annotationIndexJobs[j.ID] = j
|
|
return &j, nil
|
|
}
|
|
func (s *MongoStore) ClaimAnnotationIndex(ctx context.Context, owner, now, until string) (*domain.AnnotationIndexJob, error) {
|
|
var j domain.AnnotationIndexJob
|
|
filter := bson.M{"$or": bson.A{bson.M{"status": "queued", "nextAttemptAt": bson.M{"$lte": now}}, bson.M{"status": "running", "leaseUntil": bson.M{"$lt": now}}}}
|
|
update := bson.M{"$set": bson.M{"status": "running", "owner": owner, "leaseUntil": until}, "$inc": bson.M{"attempt": 1}}
|
|
opts := options.FindOneAndUpdate().SetSort(bson.D{{Key: "createdAt", Value: 1}}).SetReturnDocument(options.After)
|
|
// Current explicit source validation outranks historical timeline work.
|
|
// The bounded prefix range also works with the existing unique id index;
|
|
// no schema migration or historical record rewrite is required.
|
|
filter["id"] = bson.M{"$gte": "source-v1:", "$lt": "source-v1;"}
|
|
e := s.queryCollection("annotationIndexJobs").FindOneAndUpdate(ctx, filter, update, opts).Decode(&j)
|
|
if e != mongo.ErrNoDocuments {
|
|
return &j, e
|
|
}
|
|
delete(filter, "id")
|
|
e = s.queryCollection("annotationIndexJobs").FindOneAndUpdate(ctx, filter, update, opts).Decode(&j)
|
|
if e == mongo.ErrNoDocuments {
|
|
return nil, nil
|
|
}
|
|
return &j, e
|
|
}
|
|
func (s *MemoryStore) FinishAnnotationIndex(_ context.Context, j domain.AnnotationIndexJob) (bool, error) {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
old, ok := s.annotationIndexJobs[j.ID]
|
|
if !ok || old.Owner != j.Owner || old.Status != "running" {
|
|
return false, nil
|
|
}
|
|
s.annotationIndexJobs[j.ID] = j
|
|
return true, nil
|
|
}
|
|
func (s *MongoStore) FinishAnnotationIndex(ctx context.Context, j domain.AnnotationIndexJob) (bool, error) {
|
|
result, e := s.withSupplierProjectLocks(ctx, []string{j.ProjectID}, func(ctx context.Context) (any, error) {
|
|
r, e := s.queryCollection("annotationIndexJobs").ReplaceOne(ctx, bson.M{"id": j.ID, "owner": j.Owner, "status": "running"}, j)
|
|
if e != nil {
|
|
return false, e
|
|
}
|
|
return r.MatchedCount == 1, nil
|
|
})
|
|
if e != nil {
|
|
return false, e
|
|
}
|
|
return result.(bool), nil
|
|
}
|
|
|
|
func (s *MemoryStore) ClaimAnnotationProcessing(_ context.Context, owner, now, until string) (*domain.HumanItemRecord, error) {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
ids := []string{}
|
|
for id, h := range s.humanItems {
|
|
if j := h.ProcessingJob; j != nil && claimableJob(j.Status, j.LeaseUntil, j.NextAttemptAt, now) {
|
|
ids = append(ids, id)
|
|
}
|
|
}
|
|
sort.Strings(ids)
|
|
if len(ids) == 0 {
|
|
return nil, nil
|
|
}
|
|
h := clone(s.humanItems[ids[0]])
|
|
h.ProcessingJob.Status = "running"
|
|
h.ProcessingJob.Owner = owner
|
|
h.ProcessingJob.LeaseUntil = until
|
|
h.ProcessingJob.Attempt++
|
|
s.humanItems[h.ID] = clone(h)
|
|
return &h, nil
|
|
}
|
|
func (s *MongoStore) ClaimAnnotationProcessing(ctx context.Context, owner, now, until string) (*domain.HumanItemRecord, error) {
|
|
var h domain.HumanItemRecord
|
|
queued := bson.M{"processingJob.status": "queued", "$or": bson.A{bson.M{"processingJob.nextAttemptAt": bson.M{"$exists": false}}, bson.M{"processingJob.nextAttemptAt": bson.M{"$lte": now}}}}
|
|
e := s.humanItems.FindOneAndUpdate(ctx, bson.M{"$or": bson.A{queued, bson.M{"processingJob.status": "running", "processingJob.leaseUntil": bson.M{"$lt": now}}}}, bson.M{"$set": bson.M{"processingJob.status": "running", "processingJob.owner": owner, "processingJob.leaseUntil": until}, "$inc": bson.M{"processingJob.attempt": 1}}, options.FindOneAndUpdate().SetSort(bson.D{{Key: "processingJob.createdAt", Value: 1}}).SetReturnDocument(options.After)).Decode(&h)
|
|
if e == mongo.ErrNoDocuments {
|
|
return nil, nil
|
|
}
|
|
return &h, e
|
|
}
|
|
func (s *MemoryStore) SaveAnnotationProcessing(_ context.Context, itemID string, j domain.AnnotationProcessingJob) (bool, error) {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
h, ok := s.humanItems[itemID]
|
|
if !ok || h.ProcessingJob == nil || h.ProcessingJob.ID != j.ID || h.ProcessingJob.Owner != j.Owner || h.ProcessingJob.Status != "running" {
|
|
return false, nil
|
|
}
|
|
h.ProcessingJob = &j
|
|
s.humanItems[itemID] = clone(h)
|
|
return true, nil
|
|
}
|
|
func (s *MongoStore) SaveAnnotationProcessing(ctx context.Context, itemID string, j domain.AnnotationProcessingJob) (bool, error) {
|
|
r, e := s.humanItems.UpdateOne(ctx, bson.M{"id": itemID, "processingJob.id": j.ID, "processingJob.owner": j.Owner, "processingJob.status": "running"}, bson.M{"$set": bson.M{"processingJob": j}})
|
|
if e != nil {
|
|
return false, e
|
|
}
|
|
return r.MatchedCount == 1, nil
|
|
}
|