droboticflow/internal/api/annotation_processor.go

546 lines
18 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

package api
import (
"context"
"droboticflow/internal/config"
"droboticflow/internal/domain"
"droboticflow/internal/store"
"encoding/json"
"errors"
"fmt"
"log"
"net/http"
"os"
"strconv"
"sync"
"time"
)
type annotationJobLease struct{ ItemID, JobID, Owner string }
type annotationJobLeaseKey struct{}
func (s *server) newAnnotationProcessingJob(ctx context.Context, t domain.AnnotationTaskRecord, actor, id, kind string, input any, revision int) (*domain.AnnotationProcessingJob, error) {
run, e := s.store.GetRun(ctx, t.RunID)
if e != nil {
return nil, e
}
if run == nil {
return nil, fmt.Errorf("annotation source run unavailable")
}
fingerprint, e := s.annotationSourceFingerprint(ctx, t, *run)
if e != nil {
return nil, e
}
b, e := json.Marshal(input)
if e != nil {
return nil, e
}
var values map[string]any
if e = json.Unmarshal(b, &values); e != nil {
return nil, e
}
return &domain.AnnotationProcessingJob{ID: id, TaskID: t.ID, Kind: kind, ActorID: actor, Input: values, SourceFingerprint: fingerprint, SourceIndex: s.annotationSourceIndex, Revision: revision, Status: "queued", Stage: "queued", TotalSlices: len(listValue(values["sliceSegments"])), CreatedAt: nowISO(), UpdatedAt: nowISO()}, nil
}
func durableAnnotationActive(item *domain.HumanItemRecord) bool {
if item == nil || item.ProcessingJob == nil {
return false
}
return domain.HasActiveAnnotationJob(item, "submit", item.PendingCombinedSubmissionID) ||
domain.HasActiveAnnotationJob(item, "slice", stringValue(mapValue(item.Subject.Payload["sliceWorkflow"])["requestId"]))
}
func (s *server) assertAnnotationLease(ctx context.Context) error {
lease, ok := ctx.Value(annotationJobLeaseKey{}).(annotationJobLease)
if !ok {
return ctx.Err()
}
h, e := s.store.GetHumanItem(ctx, lease.ItemID)
if e != nil {
return e
}
if h == nil || h.ProcessingJob == nil || h.ProcessingJob.ID != lease.JobID || h.ProcessingJob.Owner != lease.Owner || h.ProcessingJob.Status != "running" {
return fmt.Errorf("annotation processing lease lost")
}
if h.Status == domain.HumanItemCancelled {
return fmt.Errorf("annotation task cancelled")
}
project, e := s.store.GetProject(ctx, h.ProjectID)
if e != nil {
return e
}
if project == nil || project.DeletionTaskID != "" {
return fmt.Errorf("annotation project removed or deleting")
}
expires, e := time.Parse(time.RFC3339Nano, h.ProcessingJob.LeaseUntil)
if e != nil || time.Now().After(expires) {
return fmt.Errorf("annotation processing lease expired")
}
return ctx.Err()
}
// RunAnnotationProcessor has no public mutation endpoints. Authenticated API
// requests create fenced outboxes; this independent process performs the work.
func RunAnnotationProcessor(ctx context.Context, db store.Store, cfg config.AppConfig, parallel int, backfill bool) error {
queue, ok := db.(store.AnnotationProcessingStore)
if !ok {
return fmt.Errorf("annotation processing store unavailable")
}
sourceIndex := os.Getenv("ANNOTATION_SOURCE_INDEX") == "true"
role := os.Getenv("ANNOTATION_PROCESSOR_ROLE")
if role == "" {
role = "all"
}
if role != "all" && role != "revision" && role != "validation" {
return fmt.Errorf("invalid annotation processor role")
}
maximum := 4
if sourceIndex {
maximum = 32
}
if parallel < 1 || parallel > maximum {
return fmt.Errorf("annotation processor concurrency must be 1..%d", maximum)
}
indexParallel := 1
if sourceIndex {
indexParallel = 2
}
if raw := os.Getenv("ANNOTATION_INDEX_CONCURRENCY"); raw != "" {
n, err := strconv.Atoi(raw)
if err != nil || n < 1 || n > 4 {
return fmt.Errorf("annotation index concurrency must be 1..4")
}
indexParallel = n
}
if role == "revision" {
indexParallel = 0
}
s := &server{store: db, workspaceMountPath: cfg.WorkspaceMountPath, globalBOSEndpoint: cfg.GlobalBOSEndpoint, externalAnnotationProcessor: true, annotationSourceIndex: sourceIndex}
s.materializationSlotsOnce.Do(func() { s.materializationSlots = make(chan struct{}, parallel) })
if backfill {
orgs, e := db.ListOrganizations(ctx)
if e != nil {
return e
}
for _, org := range orgs {
tasks, e := db.ListAnnotationTasks(ctx, org.ID, "")
if e != nil {
return e
}
for _, t := range tasks {
if t.SourceIsFullVideo && t.TemplateVersion >= 5 && t.Status == domain.AnnotationTaskStatusPendingAnnotation {
if e = queue.EnqueueAnnotationIndex(ctx, t); e != nil {
return e
}
}
}
}
}
var wg sync.WaitGroup
jobParallel := parallel
if role == "validation" {
jobParallel = 0
}
for n := 0; n < jobParallel; n++ {
wg.Add(1)
go func() {
defer wg.Done()
for ctx.Err() == nil {
now := time.Now().UTC()
item, e := queue.ClaimAnnotationProcessing(ctx, domain.NewID("annotation-lease"), now.Format(time.RFC3339Nano), now.Add(2*time.Minute).Format(time.RFC3339Nano))
if e != nil {
log.Printf("[annotation-processor] claim: %v", e)
} else if item != nil {
s.executeAnnotationJob(ctx, *item)
continue
}
select {
case <-ctx.Done():
return
case <-time.After(time.Second):
}
}
}()
}
for indexWorker := 0; indexWorker < indexParallel; indexWorker++ {
wg.Add(1)
go func() {
defer wg.Done()
for ctx.Err() == nil {
now := time.Now().UTC()
j, e := queue.ClaimAnnotationIndex(ctx, domain.NewID("index-lease"), now.Format(time.RFC3339Nano), now.Add(90*time.Second).Format(time.RFC3339Nano))
if e != nil {
log.Printf("[annotation-processor] index claim: %v", e)
} else if j != nil {
started := time.Now()
indexCtx, cancel := context.WithTimeout(ctx, 15*time.Minute)
stopHeartbeat := keepAnnotationIndexLease(indexCtx, queue, *j, cancel)
taskID := j.TaskID
if taskID == "" {
taskID = j.ID
}
task, readErr := db.GetAnnotationTask(indexCtx, taskID)
var stopWatch func()
if task != nil {
stopWatch = s.watchSupplierDeletion(indexCtx, task.SupplierID, cancel)
}
if readErr == nil && task != nil {
_, readErr = s.annotationFrameTimelineForTask(indexCtx, *task, "left")
// Queued legacy timeline jobs must also prove the full source,
// not publish a weaker "completed" admission result. Already
// completed records are not silently reset by this worker.
if readErr == nil {
run, err := db.GetRun(indexCtx, task.RunID)
if err != nil {
readErr = err
} else if run == nil {
readErr = fmt.Errorf("source run unavailable")
} else {
key, err := s.annotationSourceFingerprint(indexCtx, *task, *run)
if err != nil {
readErr = err
} else {
_, readErr = s.annotationSourceIndexForMode(indexCtx, *task, *run, key, false)
if readErr == nil {
j.SourceFingerprint, j.RevalidateSource = key, true
}
}
}
}
} else if task == nil {
readErr = fmt.Errorf("annotation task no longer exists")
}
stopHeartbeat()
if readErr == nil {
readErr = indexCtx.Err()
}
if stopWatch != nil {
stopWatch()
}
cancel()
j.Status = "completed"
j.Error = ""
if readErr != nil {
j.Error = readErr.Error()
j.Status = "queued"
j.NextAttemptAt = time.Now().Add(time.Minute).UTC().Format(time.RFC3339Nano)
if j.Attempt >= 5 || (task != nil && s.annotationSupplierDeleting(context.WithoutCancel(ctx), task.SupplierID)) {
j.Status = "failed"
}
}
finishCtx, finishCancel := context.WithTimeout(context.WithoutCancel(ctx), 15*time.Second)
saved, finishErr := queue.FinishAnnotationIndex(finishCtx, *j)
finishCancel()
log.Printf("[annotation-validator] job=%s status=%s attempt=%d elapsed_ms=%d saved=%t error=%v finish_error=%v", j.ID, j.Status, j.Attempt, time.Since(started).Milliseconds(), saved, readErr, finishErr)
continue
}
select {
case <-ctx.Done():
return
case <-time.After(2 * time.Second):
}
}
}()
}
wg.Wait()
return ctx.Err()
}
func (s *server) executeAnnotationJob(parent context.Context, claimed domain.HumanItemRecord) {
queue := s.store.(store.AnnotationProcessingStore)
j := *claimed.ProcessingJob
ctx, cancel := context.WithTimeout(parent, 24*time.Hour)
defer cancel()
ctx = context.WithValue(ctx, annotationJobLeaseKey{}, annotationJobLease{claimed.ID, j.ID, j.Owner})
done, stopped := make(chan struct{}), make(chan struct{})
go func() {
defer close(stopped)
tick := time.NewTicker(10 * time.Second)
defer tick.Stop()
for {
select {
case <-done:
return
case <-ctx.Done():
return
case <-tick.C:
if s.annotationSupplierDeleting(ctx, claimed.SupplierID) {
cancel()
return
}
stage, completed, total := "validating", 0, j.TotalSlices
if raw, ok := s.materializationProgress.Load(claimed.OrganizationID + ":" + j.TaskID); ok {
p := mapValue(raw)
stage = stringValue(p["stage"])
completed = intValue(p["completedSlices"])
total = intValue(p["totalSlices"])
}
ok, e := queue.TouchAnnotationProcessing(ctx, claimed.ID, j.ID, j.Owner, time.Now().Add(2*time.Minute).UTC().Format(time.RFC3339Nano), stage, completed, total)
if e != nil || !ok {
cancel()
return
}
}
}
}()
var e error
if s.annotationSupplierDeleting(ctx, claimed.SupplierID) {
e = fmt.Errorf("供应商正在删除")
} else {
e = s.processAnnotationJob(ctx, claimed)
}
close(done)
<-stopped
finishCtx, stop := context.WithTimeout(context.WithoutCancel(parent), 15*time.Second)
defer stop()
if s.annotationSupplierDeleting(finishCtx, claimed.SupplierID) {
j.Status = "failed"
j.Stage = "cancelled"
j.Error = "供应商正在级联删除,已停止处理"
j.UpdatedAt = nowISO()
_, _ = queue.SaveAnnotationProcessing(finishCtx, claimed.ID, j)
return
}
if parent.Err() != nil || ctx.Err() != nil {
j.Status = "queued"
j.Stage = "queued"
j.Error = "processing interrupted; resuming"
_, _ = queue.SaveAnnotationProcessing(finishCtx, claimed.ID, j)
return
}
if errors.Is(e, errSourceValidationPending) {
if deferForSourceValidation(&j, time.Now()) {
_, _ = queue.SaveAnnotationProcessing(finishCtx, claimed.ID, j)
return
}
e = fmt.Errorf("源文件预校验等待超过 %s,编辑内容已保留;请确认校验进程在运行后重试", sourceValidationWaitLimit)
}
if j.Kind == "submit" && transientAnnotationStoreError(e) {
if deferForAnnotationStore(&j, time.Now()) {
if saved, saveErr := queue.SaveAnnotationProcessing(finishCtx, claimed.ID, j); saveErr != nil || !saved {
log.Printf("[annotation-processor] submission retry persistence unavailable task=%s", j.TaskID)
}
return
}
}
j.Status = "completed"
j.Stage = "completed"
j.CompletedSlices = j.TotalSlices
j.UpdatedAt = nowISO()
if e != nil {
j.Status = "failed"
j.Stage = "failed"
j.Error = e.Error()
log.Printf("[annotation-processor] task=%s job=%s failed: %v", j.TaskID, j.ID, e)
} else {
j.Input = nil
}
_, _ = queue.SaveAnnotationProcessing(finishCtx, claimed.ID, j)
if e != nil {
s.failAnnotationJob(finishCtx, claimed.ID, j)
}
}
const (
sourceValidationWaitLimit = time.Hour
sourceValidationMaxBackoff = time.Minute
)
// deferForSourceValidation requeues a revision that waits on source
// validation with exponential backoff (2s doubling to 1m). Waiting does not
// consume a crash retry, but it is bounded: past the limit the job fails with
// its edits kept, so a missing validation process cannot spin it forever.
func deferForSourceValidation(j *domain.AnnotationProcessingJob, now time.Time) bool {
since, err := time.Parse(time.RFC3339Nano, j.ValidationSince)
if err != nil {
since = now
j.ValidationSince = now.UTC().Format(time.RFC3339Nano)
}
if now.Sub(since) >= sourceValidationWaitLimit {
return false
}
j.ValidationWaits++
delay := sourceValidationMaxBackoff
if j.ValidationWaits <= 5 {
delay = min(2*time.Second<<(j.ValidationWaits-1), sourceValidationMaxBackoff)
}
j.Status, j.Stage, j.Error = "queued", "validating", ""
j.NextAttemptAt = now.Add(delay).UTC().Format(time.RFC3339Nano)
j.Attempt = max(0, j.Attempt-1)
return true
}
func (s *server) processAnnotationJob(ctx context.Context, h domain.HumanItemRecord) error {
j := h.ProcessingJob
if j.Attempt > 3 {
return fmt.Errorf("后台处理连续中断,已保留编辑内容,请重试")
}
t, e := s.store.GetAnnotationTask(ctx, j.TaskID)
if e != nil {
return e
}
if t == nil || t.OrganizationID != h.OrganizationID || t.RunID != h.RunID {
return fmt.Errorf("annotation job task scope changed")
}
// Repair a crash after authoritative handoff but before legacy task publication.
if j.Kind == "submit" && h.PendingCombinedSubmissionID == "" && h.CurrentStageID == "quality_review" && h.CurrentRevision > 0 {
t.CurrentRevision = h.CurrentRevision
t.Status = domain.AnnotationTaskStatusPendingQuality
t.AssignedUserID = ""
return s.store.UpsertAnnotationTask(ctx, *t)
}
actor, e := s.store.GetUser(ctx, j.ActorID)
if e != nil {
return e
}
if actor == nil || actor.Status != domain.UserStatusActive || actor.OrganizationID != h.OrganizationID || h.AssignedUserID != actor.ID || h.CurrentStageID != "annotation_combined" {
return fmt.Errorf("annotation processing ownership or user status changed")
}
org, e := s.store.GetOrganization(ctx, h.OrganizationID)
if e != nil {
return e
}
if org == nil || org.Status != domain.OrganizationStatusActive {
return fmt.Errorf("annotation organization unavailable")
}
principal := &authenticatedPrincipal{User: *actor, Organization: *org}
r, _ := http.NewRequestWithContext(context.WithValue(ctx, authContextKey{}, principal), http.MethodPost, "http://annotation-processor/internal", nil)
w := &asynchronousAnnotationResponseWriter{}
if _, _, ok := s.ensureAnnotationTaskAccess(w, r, *t); !ok {
return fmt.Errorf("annotation job no longer authorized")
}
if !annotationManager(principal) && !s.canAccessCurrentHumanStage(ctx, r, h) {
return fmt.Errorf("annotation job role changed")
}
run, e := s.store.GetRun(ctx, t.RunID)
if e != nil {
return e
}
if run == nil {
return fmt.Errorf("annotation run unavailable")
}
key, e := s.annotationSourceFingerprint(ctx, *t, *run)
if e != nil {
return e
}
if key != j.SourceFingerprint {
return fmt.Errorf("源文件已改变,已保留编辑内容,请重新确认")
}
if s.annotationSourceIndex {
if e = s.requirePreparedAnnotationSource(ctx, *t, key, j.Stage == "queued"); e != nil {
return e
}
}
b, e := json.Marshal(normalizeStructuredValue(j.Input))
if e != nil {
return e
}
switch j.Kind {
case "slice":
var p sliceConfirmationInput
if e = json.Unmarshal(b, &p); e != nil {
return e
}
s.finishSliceConfirmation(r.Context(), t.ID, h.ID, j.ID, p, j.Revision, j.ActorID)
fresh, e := s.store.GetHumanItem(ctx, h.ID)
if e != nil {
return e
}
if fresh == nil {
return fmt.Errorf("annotation task removed")
}
flow := mapValue(fresh.Subject.Payload["sliceWorkflow"])
if stringValue(flow["status"]) != "completed" {
return fmt.Errorf("%s", firstNonEmptyText(stringValue(flow["error"]), "slice processing did not complete"))
}
case "submit":
if h.PendingCombinedSubmissionID != j.ID {
return fmt.Errorf("annotation submission superseded")
}
var p combinedAnnotationSubmissionInput
if e = json.Unmarshal(b, &p); e != nil {
return e
}
s.completeCombinedAnnotationSubmission(w, r, t, &h, p, principal, j.ID, j.Attempt > 1)
if w.status >= 400 {
if w.cause != nil {
return fmt.Errorf("annotation submission failed: %w", w.cause)
}
return fmt.Errorf("annotation submission failed: %s", w.body.String())
}
default:
return fmt.Errorf("unknown annotation job kind")
}
return nil
}
func (s *server) failAnnotationJob(ctx context.Context, id string, j domain.AnnotationProcessingJob) {
h, e := s.store.GetHumanItem(ctx, id)
if e != nil || h == nil || h.ProcessingJob == nil || h.ProcessingJob.ID != j.ID || h.ProcessingJob.Owner != j.Owner {
return
}
if j.Kind == "submit" {
recovery := *h
recovery.ProcessingJob = &j
if err := s.archiveFailedCombinedSubmission(ctx, recovery); err != nil {
// The failed job input remains durable. Any replacement must
// archive it successfully before overwriting it.
log.Printf("[annotation-processor] failed input archive unavailable task=%s", j.TaskID)
}
}
// Persist terminal state in the fenced item CAS even if saving the job
// separately failed; retain the actual current durable input.
h.ProcessingJob.Status = "failed"
h.ProcessingJob.Stage = "failed"
h.ProcessingJob.Error = j.Error
h.ProcessingJob.UpdatedAt = nowISO()
version := h.Version
if j.Kind == "submit" && h.PendingCombinedSubmissionID == j.ID {
h.PendingCombinedSubmissionID = ""
h.PendingCombinedPayloadSHA = ""
}
if j.Kind == "slice" {
w := mapValue(h.Subject.Payload["sliceWorkflow"])
if stringValue(w["requestId"]) == j.ID {
w["status"] = "failed"
w["error"] = j.Error
h.Subject.Payload["sliceWorkflow"] = w
}
}
h.Version++
h.UpdatedAt = nowISO()
_, _ = s.store.UpdateHumanItemVersioned(ctx, *h, version)
}
func (s *server) annotationSupplierDeleting(ctx context.Context, id string) bool {
if id == "" {
return false
}
supplier, e := s.store.GetSupplier(ctx, id)
return e == nil && supplier != nil && (supplier.Status == "deleting" || supplier.Status == "deleted")
}
func (s *server) watchSupplierDeletion(ctx context.Context, id string, cancel context.CancelFunc) func() {
done := make(chan struct{})
stopped := make(chan struct{})
go func() {
defer close(stopped)
tick := time.NewTicker(time.Second)
defer tick.Stop()
for {
select {
case <-done:
return
case <-ctx.Done():
return
case <-tick.C:
if s.annotationSupplierDeleting(ctx, id) {
cancel()
return
}
}
}
}()
return func() { close(done); <-stopped }
}