package api import ( "context" "droboticflow/internal/config" "droboticflow/internal/domain" "droboticflow/internal/store" "encoding/json" "errors" "fmt" "log" "net/http" "os" "strconv" "sync" "time" ) type annotationJobLease struct{ ItemID, JobID, Owner string } type annotationJobLeaseKey struct{} func (s *server) newAnnotationProcessingJob(ctx context.Context, t domain.AnnotationTaskRecord, actor, id, kind string, input any, revision int) (*domain.AnnotationProcessingJob, error) { run, e := s.store.GetRun(ctx, t.RunID) if e != nil { return nil, e } if run == nil { return nil, fmt.Errorf("annotation source run unavailable") } fingerprint, e := s.annotationSourceFingerprint(ctx, t, *run) if e != nil { return nil, e } b, e := json.Marshal(input) if e != nil { return nil, e } var values map[string]any if e = json.Unmarshal(b, &values); e != nil { return nil, e } return &domain.AnnotationProcessingJob{ID: id, TaskID: t.ID, Kind: kind, ActorID: actor, Input: values, SourceFingerprint: fingerprint, SourceIndex: s.annotationSourceIndex, Revision: revision, Status: "queued", Stage: "queued", TotalSlices: len(listValue(values["sliceSegments"])), CreatedAt: nowISO(), UpdatedAt: nowISO()}, nil } func durableAnnotationActive(item *domain.HumanItemRecord) bool { if item == nil || item.ProcessingJob == nil { return false } return domain.HasActiveAnnotationJob(item, "submit", item.PendingCombinedSubmissionID) || domain.HasActiveAnnotationJob(item, "slice", stringValue(mapValue(item.Subject.Payload["sliceWorkflow"])["requestId"])) } func (s *server) assertAnnotationLease(ctx context.Context) error { lease, ok := ctx.Value(annotationJobLeaseKey{}).(annotationJobLease) if !ok { return ctx.Err() } h, e := s.store.GetHumanItem(ctx, lease.ItemID) if e != nil { return e } if h == nil || h.ProcessingJob == nil || h.ProcessingJob.ID != lease.JobID || h.ProcessingJob.Owner != lease.Owner || h.ProcessingJob.Status != "running" { return fmt.Errorf("annotation processing lease lost") } if h.Status == domain.HumanItemCancelled { return fmt.Errorf("annotation task cancelled") } project, e := s.store.GetProject(ctx, h.ProjectID) if e != nil { return e } if project == nil || project.DeletionTaskID != "" { return fmt.Errorf("annotation project removed or deleting") } expires, e := time.Parse(time.RFC3339Nano, h.ProcessingJob.LeaseUntil) if e != nil || time.Now().After(expires) { return fmt.Errorf("annotation processing lease expired") } return ctx.Err() } // RunAnnotationProcessor has no public mutation endpoints. Authenticated API // requests create fenced outboxes; this independent process performs the work. func RunAnnotationProcessor(ctx context.Context, db store.Store, cfg config.AppConfig, parallel int, backfill bool) error { queue, ok := db.(store.AnnotationProcessingStore) if !ok { return fmt.Errorf("annotation processing store unavailable") } sourceIndex := os.Getenv("ANNOTATION_SOURCE_INDEX") == "true" role := os.Getenv("ANNOTATION_PROCESSOR_ROLE") if role == "" { role = "all" } if role != "all" && role != "revision" && role != "validation" { return fmt.Errorf("invalid annotation processor role") } maximum := 4 if sourceIndex { maximum = 32 } if parallel < 1 || parallel > maximum { return fmt.Errorf("annotation processor concurrency must be 1..%d", maximum) } indexParallel := 1 if sourceIndex { indexParallel = 2 } if raw := os.Getenv("ANNOTATION_INDEX_CONCURRENCY"); raw != "" { n, err := strconv.Atoi(raw) if err != nil || n < 1 || n > 4 { return fmt.Errorf("annotation index concurrency must be 1..4") } indexParallel = n } if role == "revision" { indexParallel = 0 } s := &server{store: db, workspaceMountPath: cfg.WorkspaceMountPath, globalBOSEndpoint: cfg.GlobalBOSEndpoint, externalAnnotationProcessor: true, annotationSourceIndex: sourceIndex} s.materializationSlotsOnce.Do(func() { s.materializationSlots = make(chan struct{}, parallel) }) if backfill { orgs, e := db.ListOrganizations(ctx) if e != nil { return e } for _, org := range orgs { tasks, e := db.ListAnnotationTasks(ctx, org.ID, "") if e != nil { return e } for _, t := range tasks { if t.SourceIsFullVideo && t.TemplateVersion >= 5 && t.Status == domain.AnnotationTaskStatusPendingAnnotation { if e = queue.EnqueueAnnotationIndex(ctx, t); e != nil { return e } } } } } var wg sync.WaitGroup jobParallel := parallel if role == "validation" { jobParallel = 0 } for n := 0; n < jobParallel; n++ { wg.Add(1) go func() { defer wg.Done() for ctx.Err() == nil { now := time.Now().UTC() item, e := queue.ClaimAnnotationProcessing(ctx, domain.NewID("annotation-lease"), now.Format(time.RFC3339Nano), now.Add(2*time.Minute).Format(time.RFC3339Nano)) if e != nil { log.Printf("[annotation-processor] claim: %v", e) } else if item != nil { s.executeAnnotationJob(ctx, *item) continue } select { case <-ctx.Done(): return case <-time.After(time.Second): } } }() } for indexWorker := 0; indexWorker < indexParallel; indexWorker++ { wg.Add(1) go func() { defer wg.Done() for ctx.Err() == nil { now := time.Now().UTC() j, e := queue.ClaimAnnotationIndex(ctx, domain.NewID("index-lease"), now.Format(time.RFC3339Nano), now.Add(90*time.Second).Format(time.RFC3339Nano)) if e != nil { log.Printf("[annotation-processor] index claim: %v", e) } else if j != nil { started := time.Now() indexCtx, cancel := context.WithTimeout(ctx, 15*time.Minute) stopHeartbeat := keepAnnotationIndexLease(indexCtx, queue, *j, cancel) taskID := j.TaskID if taskID == "" { taskID = j.ID } task, readErr := db.GetAnnotationTask(indexCtx, taskID) var stopWatch func() if task != nil { stopWatch = s.watchSupplierDeletion(indexCtx, task.SupplierID, cancel) } if readErr == nil && task != nil { _, readErr = s.annotationFrameTimelineForTask(indexCtx, *task, "left") // Legacy timeline jobs may predate this release. They must // not silently trigger full-media validation/source copies. if readErr == nil && (s.annotationSourceIndex && j.TaskID != "" || j.RevalidateSource) { run, err := db.GetRun(indexCtx, task.RunID) if err != nil { readErr = err } else if run == nil { readErr = fmt.Errorf("source run unavailable") } else { key, err := s.annotationSourceFingerprint(indexCtx, *task, *run) if err != nil { readErr = err } else { _, readErr = s.annotationSourceCertificate(indexCtx, *task, *run, key) if readErr == nil { j.SourceFingerprint = key } } } } } else if task == nil { readErr = fmt.Errorf("annotation task no longer exists") } stopHeartbeat() if readErr == nil { readErr = indexCtx.Err() } if stopWatch != nil { stopWatch() } cancel() j.Status = "completed" j.Error = "" if readErr != nil { j.Error = readErr.Error() j.Status = "queued" j.NextAttemptAt = time.Now().Add(time.Minute).UTC().Format(time.RFC3339Nano) if j.Attempt >= 5 || (task != nil && s.annotationSupplierDeleting(context.WithoutCancel(ctx), task.SupplierID)) { j.Status = "failed" } } finishCtx, finishCancel := context.WithTimeout(context.WithoutCancel(ctx), 15*time.Second) saved, finishErr := queue.FinishAnnotationIndex(finishCtx, *j) finishCancel() log.Printf("[annotation-validator] job=%s status=%s attempt=%d elapsed_ms=%d saved=%t error=%v finish_error=%v", j.ID, j.Status, j.Attempt, time.Since(started).Milliseconds(), saved, readErr, finishErr) continue } select { case <-ctx.Done(): return case <-time.After(2 * time.Second): } } }() } wg.Wait() return ctx.Err() } func (s *server) executeAnnotationJob(parent context.Context, claimed domain.HumanItemRecord) { queue := s.store.(store.AnnotationProcessingStore) j := *claimed.ProcessingJob ctx, cancel := context.WithTimeout(parent, 24*time.Hour) defer cancel() ctx = context.WithValue(ctx, annotationJobLeaseKey{}, annotationJobLease{claimed.ID, j.ID, j.Owner}) done, stopped := make(chan struct{}), make(chan struct{}) go func() { defer close(stopped) tick := time.NewTicker(10 * time.Second) defer tick.Stop() for { select { case <-done: return case <-ctx.Done(): return case <-tick.C: if s.annotationSupplierDeleting(ctx, claimed.SupplierID) { cancel() return } stage, completed, total := "validating", 0, j.TotalSlices if raw, ok := s.materializationProgress.Load(claimed.OrganizationID + ":" + j.TaskID); ok { p := mapValue(raw) stage = stringValue(p["stage"]) completed = intValue(p["completedSlices"]) total = intValue(p["totalSlices"]) } ok, e := queue.TouchAnnotationProcessing(ctx, claimed.ID, j.ID, j.Owner, time.Now().Add(2*time.Minute).UTC().Format(time.RFC3339Nano), stage, completed, total) if e != nil || !ok { cancel() return } } } }() var e error if s.annotationSupplierDeleting(ctx, claimed.SupplierID) { e = fmt.Errorf("供应商正在删除") } else { e = s.processAnnotationJob(ctx, claimed) } close(done) <-stopped finishCtx, stop := context.WithTimeout(context.WithoutCancel(parent), 15*time.Second) defer stop() if s.annotationSupplierDeleting(finishCtx, claimed.SupplierID) { j.Status = "failed" j.Stage = "cancelled" j.Error = "供应商正在级联删除,已停止处理" j.UpdatedAt = nowISO() _, _ = queue.SaveAnnotationProcessing(finishCtx, claimed.ID, j) return } if parent.Err() != nil || ctx.Err() != nil { j.Status = "queued" j.Stage = "queued" j.Error = "processing interrupted; resuming" _, _ = queue.SaveAnnotationProcessing(finishCtx, claimed.ID, j) return } if errors.Is(e, errSourceValidationPending) { if deferForSourceValidation(&j, time.Now()) { _, _ = queue.SaveAnnotationProcessing(finishCtx, claimed.ID, j) return } e = fmt.Errorf("源文件预校验等待超过 %s,编辑内容已保留;请确认校验进程在运行后重试", sourceValidationWaitLimit) } j.Status = "completed" j.Stage = "completed" j.CompletedSlices = j.TotalSlices j.UpdatedAt = nowISO() if e != nil { j.Status = "failed" j.Stage = "failed" j.Error = e.Error() log.Printf("[annotation-processor] task=%s job=%s failed: %v", j.TaskID, j.ID, e) } else { j.Input = nil } _, _ = queue.SaveAnnotationProcessing(finishCtx, claimed.ID, j) if e != nil { s.failAnnotationJob(finishCtx, claimed.ID, j) } } const ( sourceValidationWaitLimit = time.Hour sourceValidationMaxBackoff = time.Minute ) // deferForSourceValidation requeues a revision that waits on source // validation with exponential backoff (2s doubling to 1m). Waiting does not // consume a crash retry, but it is bounded: past the limit the job fails with // its edits kept, so a missing validation process cannot spin it forever. func deferForSourceValidation(j *domain.AnnotationProcessingJob, now time.Time) bool { since, err := time.Parse(time.RFC3339Nano, j.ValidationSince) if err != nil { since = now j.ValidationSince = now.UTC().Format(time.RFC3339Nano) } if now.Sub(since) >= sourceValidationWaitLimit { return false } j.ValidationWaits++ delay := sourceValidationMaxBackoff if j.ValidationWaits <= 5 { delay = min(2*time.Second<<(j.ValidationWaits-1), sourceValidationMaxBackoff) } j.Status, j.Stage, j.Error = "queued", "validating", "" j.NextAttemptAt = now.Add(delay).UTC().Format(time.RFC3339Nano) j.Attempt = max(0, j.Attempt-1) return true } func (s *server) processAnnotationJob(ctx context.Context, h domain.HumanItemRecord) error { j := h.ProcessingJob if j.Attempt > 3 { return fmt.Errorf("后台处理连续中断,已保留编辑内容,请重试") } t, e := s.store.GetAnnotationTask(ctx, j.TaskID) if e != nil { return e } if t == nil || t.OrganizationID != h.OrganizationID || t.RunID != h.RunID { return fmt.Errorf("annotation job task scope changed") } // Repair a crash after authoritative handoff but before legacy task publication. if j.Kind == "submit" && h.PendingCombinedSubmissionID == "" && h.CurrentStageID == "quality_review" && h.CurrentRevision > 0 { t.CurrentRevision = h.CurrentRevision t.Status = domain.AnnotationTaskStatusPendingQuality t.AssignedUserID = "" return s.store.UpsertAnnotationTask(ctx, *t) } actor, e := s.store.GetUser(ctx, j.ActorID) if e != nil { return e } if actor == nil || actor.Status != domain.UserStatusActive || actor.OrganizationID != h.OrganizationID || h.AssignedUserID != actor.ID || h.CurrentStageID != "annotation_combined" { return fmt.Errorf("annotation processing ownership or user status changed") } org, e := s.store.GetOrganization(ctx, h.OrganizationID) if e != nil { return e } if org == nil || org.Status != domain.OrganizationStatusActive { return fmt.Errorf("annotation organization unavailable") } principal := &authenticatedPrincipal{User: *actor, Organization: *org} r, _ := http.NewRequestWithContext(context.WithValue(ctx, authContextKey{}, principal), http.MethodPost, "http://annotation-processor/internal", nil) w := &asynchronousAnnotationResponseWriter{} if _, _, ok := s.ensureAnnotationTaskAccess(w, r, *t); !ok { return fmt.Errorf("annotation job no longer authorized") } if !annotationManager(principal) && !s.canAccessCurrentHumanStage(ctx, r, h) { return fmt.Errorf("annotation job role changed") } run, e := s.store.GetRun(ctx, t.RunID) if e != nil { return e } if run == nil { return fmt.Errorf("annotation run unavailable") } key, e := s.annotationSourceFingerprint(ctx, *t, *run) if e != nil { return e } if key != j.SourceFingerprint { return fmt.Errorf("源文件已改变,已保留编辑内容,请重新确认") } if s.annotationSourceIndex { if e = s.requirePreparedAnnotationSource(ctx, *t, key, j.Stage == "queued"); e != nil { return e } } b, e := json.Marshal(normalizeStructuredValue(j.Input)) if e != nil { return e } switch j.Kind { case "slice": var p sliceConfirmationInput if e = json.Unmarshal(b, &p); e != nil { return e } s.finishSliceConfirmation(r.Context(), t.ID, h.ID, j.ID, p, j.Revision, j.ActorID) fresh, e := s.store.GetHumanItem(ctx, h.ID) if e != nil { return e } if fresh == nil { return fmt.Errorf("annotation task removed") } flow := mapValue(fresh.Subject.Payload["sliceWorkflow"]) if stringValue(flow["status"]) != "completed" { return fmt.Errorf("%s", firstNonEmptyText(stringValue(flow["error"]), "slice processing did not complete")) } case "submit": if h.PendingCombinedSubmissionID != j.ID { return fmt.Errorf("annotation submission superseded") } var p combinedAnnotationSubmissionInput if e = json.Unmarshal(b, &p); e != nil { return e } s.completeCombinedAnnotationSubmission(w, r, t, &h, p, principal, j.ID, j.Attempt > 1) if w.status >= 400 { return fmt.Errorf("annotation submission failed: %s", w.body.String()) } default: return fmt.Errorf("unknown annotation job kind") } return nil } func (s *server) failAnnotationJob(ctx context.Context, id string, j domain.AnnotationProcessingJob) { h, e := s.store.GetHumanItem(ctx, id) if e != nil || h == nil || h.ProcessingJob == nil || h.ProcessingJob.ID != j.ID || h.ProcessingJob.Owner != j.Owner { return } version := h.Version if j.Kind == "submit" && h.PendingCombinedSubmissionID == j.ID { h.PendingCombinedSubmissionID = "" h.PendingCombinedPayloadSHA = "" } if j.Kind == "slice" { w := mapValue(h.Subject.Payload["sliceWorkflow"]) if stringValue(w["requestId"]) == j.ID { w["status"] = "failed" w["error"] = j.Error h.Subject.Payload["sliceWorkflow"] = w } } h.Version++ h.UpdatedAt = nowISO() _, _ = s.store.UpdateHumanItemVersioned(ctx, *h, version) } func (s *server) annotationSupplierDeleting(ctx context.Context, id string) bool { if id == "" { return false } supplier, e := s.store.GetSupplier(ctx, id) return e == nil && supplier != nil && (supplier.Status == "deleting" || supplier.Status == "deleted") } func (s *server) watchSupplierDeletion(ctx context.Context, id string, cancel context.CancelFunc) func() { done := make(chan struct{}) stopped := make(chan struct{}) go func() { defer close(stopped) tick := time.NewTicker(time.Second) defer tick.Stop() for { select { case <-done: return case <-ctx.Done(): return case <-tick.C: if s.annotationSupplierDeleting(ctx, id) { cancel() return } } } }() return func() { close(done); <-stopped } }