droboticflow/internal/api/annotation_processor.go

353 lines
11 KiB
Go

package api
import (
"context"
"droboticflow/internal/config"
"droboticflow/internal/domain"
"droboticflow/internal/store"
"encoding/json"
"fmt"
"log"
"net/http"
"sync"
"time"
)
type annotationJobLease struct{ ItemID, JobID, Owner string }
type annotationJobLeaseKey struct{}
func (s *server) newAnnotationProcessingJob(ctx context.Context, t domain.AnnotationTaskRecord, actor, id, kind string, input any, revision int) (*domain.AnnotationProcessingJob, error) {
run, e := s.store.GetRun(ctx, t.RunID)
if e != nil {
return nil, e
}
if run == nil {
return nil, fmt.Errorf("annotation source run unavailable")
}
fingerprint, e := s.annotationSourceFingerprint(ctx, t, *run)
if e != nil {
return nil, e
}
b, e := json.Marshal(input)
if e != nil {
return nil, e
}
var values map[string]any
if e = json.Unmarshal(b, &values); e != nil {
return nil, e
}
return &domain.AnnotationProcessingJob{ID: id, TaskID: t.ID, Kind: kind, ActorID: actor, Input: values, SourceFingerprint: fingerprint, Revision: revision, Status: "queued", Stage: "queued", TotalSlices: len(listValue(values["sliceSegments"])), CreatedAt: nowISO(), UpdatedAt: nowISO()}, nil
}
func durableAnnotationActive(item *domain.HumanItemRecord) bool {
if item == nil || item.ProcessingJob == nil {
return false
}
return domain.HasActiveAnnotationJob(item, "submit", item.PendingCombinedSubmissionID) ||
domain.HasActiveAnnotationJob(item, "slice", stringValue(mapValue(item.Subject.Payload["sliceWorkflow"])["requestId"]))
}
func (s *server) assertAnnotationLease(ctx context.Context) error {
lease, ok := ctx.Value(annotationJobLeaseKey{}).(annotationJobLease)
if !ok {
return ctx.Err()
}
h, e := s.store.GetHumanItem(ctx, lease.ItemID)
if e != nil {
return e
}
if h == nil || h.ProcessingJob == nil || h.ProcessingJob.ID != lease.JobID || h.ProcessingJob.Owner != lease.Owner || h.ProcessingJob.Status != "running" {
return fmt.Errorf("annotation processing lease lost")
}
if h.Status == domain.HumanItemCancelled {
return fmt.Errorf("annotation task cancelled")
}
project, e := s.store.GetProject(ctx, h.ProjectID)
if e != nil {
return e
}
if project == nil || project.DeletionTaskID != "" {
return fmt.Errorf("annotation project removed or deleting")
}
expires, e := time.Parse(time.RFC3339Nano, h.ProcessingJob.LeaseUntil)
if e != nil || time.Now().After(expires) {
return fmt.Errorf("annotation processing lease expired")
}
return ctx.Err()
}
// RunAnnotationProcessor has no public mutation endpoints. Authenticated API
// requests create fenced outboxes; this independent process performs the work.
func RunAnnotationProcessor(ctx context.Context, db store.Store, cfg config.AppConfig, parallel int, backfill bool) error {
queue, ok := db.(store.AnnotationProcessingStore)
if !ok {
return fmt.Errorf("annotation processing store unavailable")
}
if parallel < 1 || parallel > 4 {
return fmt.Errorf("annotation processor concurrency must be 1..4")
}
s := &server{store: db, workspaceMountPath: cfg.WorkspaceMountPath, globalBOSEndpoint: cfg.GlobalBOSEndpoint, externalAnnotationProcessor: true}
s.materializationSlotsOnce.Do(func() { s.materializationSlots = make(chan struct{}, parallel) })
if backfill {
orgs, e := db.ListOrganizations(ctx)
if e != nil {
return e
}
for _, org := range orgs {
tasks, e := db.ListAnnotationTasks(ctx, org.ID, "")
if e != nil {
return e
}
for _, t := range tasks {
if t.SourceIsFullVideo && t.TemplateVersion >= 5 && t.Status == domain.AnnotationTaskStatusPendingAnnotation {
if e = queue.EnqueueAnnotationIndex(ctx, t); e != nil {
return e
}
}
}
}
}
var wg sync.WaitGroup
for n := 0; n < parallel; n++ {
wg.Add(1)
go func() {
defer wg.Done()
for ctx.Err() == nil {
now := time.Now().UTC()
item, e := queue.ClaimAnnotationProcessing(ctx, domain.NewID("annotation-lease"), now.Format(time.RFC3339Nano), now.Add(2*time.Minute).Format(time.RFC3339Nano))
if e != nil {
log.Printf("[annotation-processor] claim: %v", e)
} else if item != nil {
s.executeAnnotationJob(ctx, *item)
continue
}
select {
case <-ctx.Done():
return
case <-time.After(time.Second):
}
}
}()
}
wg.Add(1)
go func() {
defer wg.Done()
for ctx.Err() == nil {
now := time.Now().UTC()
j, e := queue.ClaimAnnotationIndex(ctx, domain.NewID("index-lease"), now.Format(time.RFC3339Nano), now.Add(20*time.Minute).Format(time.RFC3339Nano))
if e != nil {
log.Printf("[annotation-processor] index claim: %v", e)
} else if j != nil {
indexCtx, cancel := context.WithTimeout(ctx, 15*time.Minute)
task, readErr := db.GetAnnotationTask(indexCtx, j.ID)
if readErr == nil && task != nil {
_, readErr = s.annotationFrameTimelineForTask(indexCtx, *task, "left")
} else if task == nil {
readErr = fmt.Errorf("annotation task no longer exists")
}
cancel()
j.Status = "completed"
j.Error = ""
if readErr != nil {
j.Error = readErr.Error()
j.Status = "queued"
j.NextAttemptAt = time.Now().Add(time.Minute).UTC().Format(time.RFC3339Nano)
if j.Attempt >= 5 {
j.Status = "failed"
}
}
_, _ = queue.FinishAnnotationIndex(context.WithoutCancel(ctx), *j)
continue
}
select {
case <-ctx.Done():
return
case <-time.After(2 * time.Second):
}
}
}()
wg.Wait()
return ctx.Err()
}
func (s *server) executeAnnotationJob(parent context.Context, claimed domain.HumanItemRecord) {
queue := s.store.(store.AnnotationProcessingStore)
j := *claimed.ProcessingJob
ctx, cancel := context.WithTimeout(parent, 24*time.Hour)
defer cancel()
ctx = context.WithValue(ctx, annotationJobLeaseKey{}, annotationJobLease{claimed.ID, j.ID, j.Owner})
done, stopped := make(chan struct{}), make(chan struct{})
go func() {
defer close(stopped)
tick := time.NewTicker(10 * time.Second)
defer tick.Stop()
for {
select {
case <-done:
return
case <-ctx.Done():
return
case <-tick.C:
stage, completed, total := "validating", 0, j.TotalSlices
if raw, ok := s.materializationProgress.Load(claimed.OrganizationID + ":" + j.TaskID); ok {
p := mapValue(raw)
stage = stringValue(p["stage"])
completed = intValue(p["completedSlices"])
total = intValue(p["totalSlices"])
}
ok, e := queue.TouchAnnotationProcessing(ctx, claimed.ID, j.ID, j.Owner, time.Now().Add(2*time.Minute).UTC().Format(time.RFC3339Nano), stage, completed, total)
if e != nil || !ok {
cancel()
return
}
}
}
}()
e := s.processAnnotationJob(ctx, claimed)
close(done)
<-stopped
finishCtx, stop := context.WithTimeout(context.WithoutCancel(parent), 15*time.Second)
defer stop()
if parent.Err() != nil || ctx.Err() != nil {
j.Status = "queued"
j.Stage = "queued"
j.Error = "processing interrupted; resuming"
_, _ = queue.SaveAnnotationProcessing(finishCtx, claimed.ID, j)
return
}
j.Status = "completed"
j.Stage = "completed"
j.Input = nil
j.CompletedSlices = j.TotalSlices
j.UpdatedAt = nowISO()
if e != nil {
j.Status = "failed"
j.Stage = "failed"
j.Error = e.Error()
log.Printf("[annotation-processor] task=%s job=%s failed: %v", j.TaskID, j.ID, e)
}
_, _ = queue.SaveAnnotationProcessing(finishCtx, claimed.ID, j)
if e != nil {
s.failAnnotationJob(finishCtx, claimed.ID, j)
}
}
func (s *server) processAnnotationJob(ctx context.Context, h domain.HumanItemRecord) error {
j := h.ProcessingJob
if j.Attempt > 3 {
return fmt.Errorf("后台处理连续中断,已保留编辑内容,请重试")
}
t, e := s.store.GetAnnotationTask(ctx, j.TaskID)
if e != nil {
return e
}
if t == nil || t.OrganizationID != h.OrganizationID || t.RunID != h.RunID {
return fmt.Errorf("annotation job task scope changed")
}
// Repair a crash after authoritative handoff but before legacy task publication.
if j.Kind == "submit" && h.PendingCombinedSubmissionID == "" && h.CurrentStageID == "quality_review" && h.CurrentRevision > 0 {
t.CurrentRevision = h.CurrentRevision
t.Status = domain.AnnotationTaskStatusPendingQuality
t.AssignedUserID = ""
return s.store.UpsertAnnotationTask(ctx, *t)
}
actor, e := s.store.GetUser(ctx, j.ActorID)
if e != nil {
return e
}
if actor == nil || actor.Status != domain.UserStatusActive || actor.OrganizationID != h.OrganizationID || h.AssignedUserID != actor.ID || h.CurrentStageID != "annotation_combined" {
return fmt.Errorf("annotation processing ownership or user status changed")
}
org, e := s.store.GetOrganization(ctx, h.OrganizationID)
if e != nil {
return e
}
if org == nil || org.Status != domain.OrganizationStatusActive {
return fmt.Errorf("annotation organization unavailable")
}
principal := &authenticatedPrincipal{User: *actor, Organization: *org}
r, _ := http.NewRequestWithContext(context.WithValue(ctx, authContextKey{}, principal), http.MethodPost, "http://annotation-processor/internal", nil)
w := &asynchronousAnnotationResponseWriter{}
if _, _, ok := s.ensureAnnotationTaskAccess(w, r, *t); !ok {
return fmt.Errorf("annotation job no longer authorized")
}
if !annotationManager(principal) && !s.canAccessCurrentHumanStage(ctx, r, h) {
return fmt.Errorf("annotation job role changed")
}
run, e := s.store.GetRun(ctx, t.RunID)
if e != nil {
return e
}
if run == nil {
return fmt.Errorf("annotation run unavailable")
}
key, e := s.annotationSourceFingerprint(ctx, *t, *run)
if e != nil {
return e
}
if key != j.SourceFingerprint {
return fmt.Errorf("源文件已改变,已保留编辑内容,请重新确认")
}
b, e := json.Marshal(normalizeStructuredValue(j.Input))
if e != nil {
return e
}
switch j.Kind {
case "slice":
var p sliceConfirmationInput
if e = json.Unmarshal(b, &p); e != nil {
return e
}
s.finishSliceConfirmation(r.Context(), t.ID, h.ID, j.ID, p, j.Revision, j.ActorID)
fresh, e := s.store.GetHumanItem(ctx, h.ID)
if e != nil {
return e
}
if fresh == nil {
return fmt.Errorf("annotation task removed")
}
flow := mapValue(fresh.Subject.Payload["sliceWorkflow"])
if stringValue(flow["status"]) != "completed" {
return fmt.Errorf("%s", firstNonEmptyText(stringValue(flow["error"]), "slice processing did not complete"))
}
case "submit":
if h.PendingCombinedSubmissionID != j.ID {
return fmt.Errorf("annotation submission superseded")
}
var p combinedAnnotationSubmissionInput
if e = json.Unmarshal(b, &p); e != nil {
return e
}
s.completeCombinedAnnotationSubmission(w, r, t, &h, p, principal, j.ID, j.Attempt > 1)
if w.status >= 400 {
return fmt.Errorf("annotation submission failed: %s", w.body.String())
}
default:
return fmt.Errorf("unknown annotation job kind")
}
return nil
}
func (s *server) failAnnotationJob(ctx context.Context, id string, j domain.AnnotationProcessingJob) {
h, e := s.store.GetHumanItem(ctx, id)
if e != nil || h == nil || h.ProcessingJob == nil || h.ProcessingJob.ID != j.ID || h.ProcessingJob.Owner != j.Owner {
return
}
version := h.Version
if j.Kind == "submit" && h.PendingCombinedSubmissionID == j.ID {
h.PendingCombinedSubmissionID = ""
h.PendingCombinedPayloadSHA = ""
}
if j.Kind == "slice" {
w := mapValue(h.Subject.Payload["sliceWorkflow"])
if stringValue(w["requestId"]) == j.ID {
w["status"] = "failed"
w["error"] = j.Error
h.Subject.Payload["sliceWorkflow"] = w
}
}
h.Version++
h.UpdatedAt = nowISO()
_, _ = s.store.UpdateHumanItemVersioned(ctx, *h, version)
}