package api import ( "context" "droboticflow/internal/config" "droboticflow/internal/domain" "droboticflow/internal/store" "encoding/json" "fmt" "log" "net/http" "sync" "time" ) type annotationJobLease struct{ ItemID, JobID, Owner string } type annotationJobLeaseKey struct{} func (s *server) newAnnotationProcessingJob(ctx context.Context, t domain.AnnotationTaskRecord, actor, id, kind string, input any, revision int) (*domain.AnnotationProcessingJob, error) { run, e := s.store.GetRun(ctx, t.RunID) if e != nil { return nil, e } if run == nil { return nil, fmt.Errorf("annotation source run unavailable") } fingerprint, e := s.annotationSourceFingerprint(ctx, t, *run) if e != nil { return nil, e } b, e := json.Marshal(input) if e != nil { return nil, e } var values map[string]any if e = json.Unmarshal(b, &values); e != nil { return nil, e } return &domain.AnnotationProcessingJob{ID: id, TaskID: t.ID, Kind: kind, ActorID: actor, Input: values, SourceFingerprint: fingerprint, Revision: revision, Status: "queued", Stage: "queued", TotalSlices: len(listValue(values["sliceSegments"])), CreatedAt: nowISO(), UpdatedAt: nowISO()}, nil } func durableAnnotationActive(item *domain.HumanItemRecord) bool { if item == nil || item.ProcessingJob == nil { return false } j := item.ProcessingJob return (j.Status == "queued" || j.Status == "running") && ((j.Kind == "submit" && j.ID == item.PendingCombinedSubmissionID) || (j.Kind == "slice" && j.ID == stringValue(mapValue(item.Subject.Payload["sliceWorkflow"])["requestId"]))) } func (s *server) assertAnnotationLease(ctx context.Context) error { lease, ok := ctx.Value(annotationJobLeaseKey{}).(annotationJobLease) if !ok { return ctx.Err() } h, e := s.store.GetHumanItem(ctx, lease.ItemID) if e != nil { return e } if h == nil || h.ProcessingJob == nil || h.ProcessingJob.ID != lease.JobID || h.ProcessingJob.Owner != lease.Owner || h.ProcessingJob.Status != "running" { return fmt.Errorf("annotation processing lease lost") } if h.Status == domain.HumanItemCancelled { return fmt.Errorf("annotation task cancelled") } project, e := s.store.GetProject(ctx, h.ProjectID) if e != nil { return e } if project == nil || project.DeletionTaskID != "" { return fmt.Errorf("annotation project removed or deleting") } expires, e := time.Parse(time.RFC3339Nano, h.ProcessingJob.LeaseUntil) if e != nil || time.Now().After(expires) { return fmt.Errorf("annotation processing lease expired") } return ctx.Err() } // RunAnnotationProcessor has no public mutation endpoints. Authenticated API // requests create fenced outboxes; this independent process performs the work. func RunAnnotationProcessor(ctx context.Context, db store.Store, cfg config.AppConfig, parallel int, backfill bool) error { queue, ok := db.(store.AnnotationProcessingStore) if !ok { return fmt.Errorf("annotation processing store unavailable") } if parallel < 1 || parallel > 4 { return fmt.Errorf("annotation processor concurrency must be 1..4") } s := &server{store: db, workspaceMountPath: cfg.WorkspaceMountPath, globalBOSEndpoint: cfg.GlobalBOSEndpoint, externalAnnotationProcessor: true} s.materializationSlotsOnce.Do(func() { s.materializationSlots = make(chan struct{}, parallel) }) if backfill { orgs, e := db.ListOrganizations(ctx) if e != nil { return e } for _, org := range orgs { tasks, e := db.ListAnnotationTasks(ctx, org.ID, "") if e != nil { return e } for _, t := range tasks { if t.SourceIsFullVideo && t.TemplateVersion >= 5 && t.Status == domain.AnnotationTaskStatusPendingAnnotation { if e = queue.EnqueueAnnotationIndex(ctx, t); e != nil { return e } } } } } var wg sync.WaitGroup for n := 0; n < parallel; n++ { wg.Add(1) go func() { defer wg.Done() for ctx.Err() == nil { now := time.Now().UTC() item, e := queue.ClaimAnnotationProcessing(ctx, domain.NewID("annotation-lease"), now.Format(time.RFC3339Nano), now.Add(2*time.Minute).Format(time.RFC3339Nano)) if e != nil { log.Printf("[annotation-processor] claim: %v", e) } else if item != nil { s.executeAnnotationJob(ctx, *item) continue } select { case <-ctx.Done(): return case <-time.After(time.Second): } } }() } wg.Add(1) go func() { defer wg.Done() for ctx.Err() == nil { now := time.Now().UTC() j, e := queue.ClaimAnnotationIndex(ctx, domain.NewID("index-lease"), now.Format(time.RFC3339Nano), now.Add(20*time.Minute).Format(time.RFC3339Nano)) if e != nil { log.Printf("[annotation-processor] index claim: %v", e) } else if j != nil { indexCtx, cancel := context.WithTimeout(ctx, 15*time.Minute) task, readErr := db.GetAnnotationTask(indexCtx, j.ID) if readErr == nil && task != nil { _, readErr = s.annotationFrameTimelineForTask(indexCtx, *task, "left") } else if task == nil { readErr = fmt.Errorf("annotation task no longer exists") } cancel() j.Status = "completed" j.Error = "" if readErr != nil { j.Error = readErr.Error() j.Status = "queued" j.NextAttemptAt = time.Now().Add(time.Minute).UTC().Format(time.RFC3339Nano) if j.Attempt >= 5 { j.Status = "failed" } } _, _ = queue.FinishAnnotationIndex(context.WithoutCancel(ctx), *j) continue } select { case <-ctx.Done(): return case <-time.After(2 * time.Second): } } }() wg.Wait() return ctx.Err() } func (s *server) executeAnnotationJob(parent context.Context, claimed domain.HumanItemRecord) { queue := s.store.(store.AnnotationProcessingStore) j := *claimed.ProcessingJob ctx, cancel := context.WithTimeout(parent, 24*time.Hour) defer cancel() ctx = context.WithValue(ctx, annotationJobLeaseKey{}, annotationJobLease{claimed.ID, j.ID, j.Owner}) done, stopped := make(chan struct{}), make(chan struct{}) go func() { defer close(stopped) tick := time.NewTicker(10 * time.Second) defer tick.Stop() for { select { case <-done: return case <-ctx.Done(): return case <-tick.C: stage, completed, total := "validating", 0, j.TotalSlices if raw, ok := s.materializationProgress.Load(claimed.OrganizationID + ":" + j.TaskID); ok { p := mapValue(raw) stage = stringValue(p["stage"]) completed = intValue(p["completedSlices"]) total = intValue(p["totalSlices"]) } ok, e := queue.TouchAnnotationProcessing(ctx, claimed.ID, j.ID, j.Owner, time.Now().Add(2*time.Minute).UTC().Format(time.RFC3339Nano), stage, completed, total) if e != nil || !ok { cancel() return } } } }() e := s.processAnnotationJob(ctx, claimed) close(done) <-stopped finishCtx, stop := context.WithTimeout(context.WithoutCancel(parent), 15*time.Second) defer stop() if parent.Err() != nil || ctx.Err() != nil { j.Status = "queued" j.Stage = "queued" j.Error = "processing interrupted; resuming" _, _ = queue.SaveAnnotationProcessing(finishCtx, claimed.ID, j) return } j.Status = "completed" j.Stage = "completed" j.Input = nil j.CompletedSlices = j.TotalSlices j.UpdatedAt = nowISO() if e != nil { j.Status = "failed" j.Stage = "failed" j.Error = e.Error() log.Printf("[annotation-processor] task=%s job=%s failed: %v", j.TaskID, j.ID, e) } _, _ = queue.SaveAnnotationProcessing(finishCtx, claimed.ID, j) if e != nil { s.failAnnotationJob(finishCtx, claimed.ID, j) } } func (s *server) processAnnotationJob(ctx context.Context, h domain.HumanItemRecord) error { j := h.ProcessingJob if j.Attempt > 3 { return fmt.Errorf("后台处理连续中断,已保留编辑内容,请重试") } t, e := s.store.GetAnnotationTask(ctx, j.TaskID) if e != nil { return e } if t == nil || t.OrganizationID != h.OrganizationID || t.RunID != h.RunID { return fmt.Errorf("annotation job task scope changed") } // Repair a crash after authoritative handoff but before legacy task publication. if j.Kind == "submit" && h.PendingCombinedSubmissionID == "" && h.CurrentStageID == "quality_review" && h.CurrentRevision > 0 { t.CurrentRevision = h.CurrentRevision t.Status = domain.AnnotationTaskStatusPendingQuality t.AssignedUserID = "" return s.store.UpsertAnnotationTask(ctx, *t) } actor, e := s.store.GetUser(ctx, j.ActorID) if e != nil { return e } if actor == nil || actor.Status != domain.UserStatusActive || actor.OrganizationID != h.OrganizationID || h.AssignedUserID != actor.ID || h.CurrentStageID != "annotation_combined" { return fmt.Errorf("annotation processing ownership or user status changed") } org, e := s.store.GetOrganization(ctx, h.OrganizationID) if e != nil { return e } if org == nil || org.Status != domain.OrganizationStatusActive { return fmt.Errorf("annotation organization unavailable") } principal := &authenticatedPrincipal{User: *actor, Organization: *org} r, _ := http.NewRequestWithContext(context.WithValue(ctx, authContextKey{}, principal), http.MethodPost, "http://annotation-processor/internal", nil) w := &asynchronousAnnotationResponseWriter{} if _, _, ok := s.ensureAnnotationTaskAccess(w, r, *t); !ok { return fmt.Errorf("annotation job no longer authorized") } if !annotationManager(principal) && !s.canAccessCurrentHumanStage(ctx, r, h) { return fmt.Errorf("annotation job role changed") } run, e := s.store.GetRun(ctx, t.RunID) if e != nil { return e } if run == nil { return fmt.Errorf("annotation run unavailable") } key, e := s.annotationSourceFingerprint(ctx, *t, *run) if e != nil { return e } if key != j.SourceFingerprint { return fmt.Errorf("源文件已改变,已保留编辑内容,请重新确认") } b, e := json.Marshal(normalizeStructuredValue(j.Input)) if e != nil { return e } switch j.Kind { case "slice": var p sliceConfirmationInput if e = json.Unmarshal(b, &p); e != nil { return e } s.finishSliceConfirmation(r.Context(), t.ID, h.ID, j.ID, p, j.Revision, j.ActorID) fresh, e := s.store.GetHumanItem(ctx, h.ID) if e != nil { return e } if fresh == nil { return fmt.Errorf("annotation task removed") } flow := mapValue(fresh.Subject.Payload["sliceWorkflow"]) if stringValue(flow["status"]) != "completed" { return fmt.Errorf("%s", firstNonEmptyText(stringValue(flow["error"]), "slice processing did not complete")) } case "submit": if h.PendingCombinedSubmissionID != j.ID { return fmt.Errorf("annotation submission superseded") } var p combinedAnnotationSubmissionInput if e = json.Unmarshal(b, &p); e != nil { return e } s.completeCombinedAnnotationSubmission(w, r, t, &h, p, principal, j.ID, j.Attempt > 1) if w.status >= 400 { return fmt.Errorf("annotation submission failed: %s", w.body.String()) } default: return fmt.Errorf("unknown annotation job kind") } return nil } func (s *server) failAnnotationJob(ctx context.Context, id string, j domain.AnnotationProcessingJob) { h, e := s.store.GetHumanItem(ctx, id) if e != nil || h == nil || h.ProcessingJob == nil || h.ProcessingJob.ID != j.ID || h.ProcessingJob.Owner != j.Owner { return } version := h.Version if j.Kind == "submit" && h.PendingCombinedSubmissionID == j.ID { h.PendingCombinedSubmissionID = "" h.PendingCombinedPayloadSHA = "" } if j.Kind == "slice" { w := mapValue(h.Subject.Payload["sliceWorkflow"]) if stringValue(w["requestId"]) == j.ID { w["status"] = "failed" w["error"] = j.Error h.Subject.Payload["sliceWorkflow"] = w } } h.Version++ h.UpdatedAt = nowISO() _, _ = s.store.UpdateHumanItemVersioned(ctx, *h, version) }