forked from longtao.wu/droboticflow
353 lines
11 KiB
Go
353 lines
11 KiB
Go
package api
|
|
|
|
import (
|
|
"context"
|
|
"droboticflow/internal/config"
|
|
"droboticflow/internal/domain"
|
|
"droboticflow/internal/store"
|
|
"encoding/json"
|
|
"fmt"
|
|
"log"
|
|
"net/http"
|
|
"sync"
|
|
"time"
|
|
)
|
|
|
|
type annotationJobLease struct{ ItemID, JobID, Owner string }
|
|
type annotationJobLeaseKey struct{}
|
|
|
|
func (s *server) newAnnotationProcessingJob(ctx context.Context, t domain.AnnotationTaskRecord, actor, id, kind string, input any, revision int) (*domain.AnnotationProcessingJob, error) {
|
|
run, e := s.store.GetRun(ctx, t.RunID)
|
|
if e != nil {
|
|
return nil, e
|
|
}
|
|
if run == nil {
|
|
return nil, fmt.Errorf("annotation source run unavailable")
|
|
}
|
|
fingerprint, e := s.annotationSourceFingerprint(ctx, t, *run)
|
|
if e != nil {
|
|
return nil, e
|
|
}
|
|
b, e := json.Marshal(input)
|
|
if e != nil {
|
|
return nil, e
|
|
}
|
|
var values map[string]any
|
|
if e = json.Unmarshal(b, &values); e != nil {
|
|
return nil, e
|
|
}
|
|
return &domain.AnnotationProcessingJob{ID: id, TaskID: t.ID, Kind: kind, ActorID: actor, Input: values, SourceFingerprint: fingerprint, Revision: revision, Status: "queued", Stage: "queued", TotalSlices: len(listValue(values["sliceSegments"])), CreatedAt: nowISO(), UpdatedAt: nowISO()}, nil
|
|
}
|
|
|
|
func durableAnnotationActive(item *domain.HumanItemRecord) bool {
|
|
if item == nil || item.ProcessingJob == nil {
|
|
return false
|
|
}
|
|
return domain.HasActiveAnnotationJob(item, "submit", item.PendingCombinedSubmissionID) ||
|
|
domain.HasActiveAnnotationJob(item, "slice", stringValue(mapValue(item.Subject.Payload["sliceWorkflow"])["requestId"]))
|
|
}
|
|
|
|
func (s *server) assertAnnotationLease(ctx context.Context) error {
|
|
lease, ok := ctx.Value(annotationJobLeaseKey{}).(annotationJobLease)
|
|
if !ok {
|
|
return ctx.Err()
|
|
}
|
|
h, e := s.store.GetHumanItem(ctx, lease.ItemID)
|
|
if e != nil {
|
|
return e
|
|
}
|
|
if h == nil || h.ProcessingJob == nil || h.ProcessingJob.ID != lease.JobID || h.ProcessingJob.Owner != lease.Owner || h.ProcessingJob.Status != "running" {
|
|
return fmt.Errorf("annotation processing lease lost")
|
|
}
|
|
if h.Status == domain.HumanItemCancelled {
|
|
return fmt.Errorf("annotation task cancelled")
|
|
}
|
|
project, e := s.store.GetProject(ctx, h.ProjectID)
|
|
if e != nil {
|
|
return e
|
|
}
|
|
if project == nil || project.DeletionTaskID != "" {
|
|
return fmt.Errorf("annotation project removed or deleting")
|
|
}
|
|
expires, e := time.Parse(time.RFC3339Nano, h.ProcessingJob.LeaseUntil)
|
|
if e != nil || time.Now().After(expires) {
|
|
return fmt.Errorf("annotation processing lease expired")
|
|
}
|
|
return ctx.Err()
|
|
}
|
|
|
|
// RunAnnotationProcessor has no public mutation endpoints. Authenticated API
|
|
// requests create fenced outboxes; this independent process performs the work.
|
|
func RunAnnotationProcessor(ctx context.Context, db store.Store, cfg config.AppConfig, parallel int, backfill bool) error {
|
|
queue, ok := db.(store.AnnotationProcessingStore)
|
|
if !ok {
|
|
return fmt.Errorf("annotation processing store unavailable")
|
|
}
|
|
if parallel < 1 || parallel > 4 {
|
|
return fmt.Errorf("annotation processor concurrency must be 1..4")
|
|
}
|
|
s := &server{store: db, workspaceMountPath: cfg.WorkspaceMountPath, globalBOSEndpoint: cfg.GlobalBOSEndpoint, externalAnnotationProcessor: true}
|
|
s.materializationSlotsOnce.Do(func() { s.materializationSlots = make(chan struct{}, parallel) })
|
|
if backfill {
|
|
orgs, e := db.ListOrganizations(ctx)
|
|
if e != nil {
|
|
return e
|
|
}
|
|
for _, org := range orgs {
|
|
tasks, e := db.ListAnnotationTasks(ctx, org.ID, "")
|
|
if e != nil {
|
|
return e
|
|
}
|
|
for _, t := range tasks {
|
|
if t.SourceIsFullVideo && t.TemplateVersion >= 5 && t.Status == domain.AnnotationTaskStatusPendingAnnotation {
|
|
if e = queue.EnqueueAnnotationIndex(ctx, t); e != nil {
|
|
return e
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
var wg sync.WaitGroup
|
|
for n := 0; n < parallel; n++ {
|
|
wg.Add(1)
|
|
go func() {
|
|
defer wg.Done()
|
|
for ctx.Err() == nil {
|
|
now := time.Now().UTC()
|
|
item, e := queue.ClaimAnnotationProcessing(ctx, domain.NewID("annotation-lease"), now.Format(time.RFC3339Nano), now.Add(2*time.Minute).Format(time.RFC3339Nano))
|
|
if e != nil {
|
|
log.Printf("[annotation-processor] claim: %v", e)
|
|
} else if item != nil {
|
|
s.executeAnnotationJob(ctx, *item)
|
|
continue
|
|
}
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case <-time.After(time.Second):
|
|
}
|
|
}
|
|
}()
|
|
}
|
|
wg.Add(1)
|
|
go func() {
|
|
defer wg.Done()
|
|
for ctx.Err() == nil {
|
|
now := time.Now().UTC()
|
|
j, e := queue.ClaimAnnotationIndex(ctx, domain.NewID("index-lease"), now.Format(time.RFC3339Nano), now.Add(20*time.Minute).Format(time.RFC3339Nano))
|
|
if e != nil {
|
|
log.Printf("[annotation-processor] index claim: %v", e)
|
|
} else if j != nil {
|
|
indexCtx, cancel := context.WithTimeout(ctx, 15*time.Minute)
|
|
task, readErr := db.GetAnnotationTask(indexCtx, j.ID)
|
|
if readErr == nil && task != nil {
|
|
_, readErr = s.annotationFrameTimelineForTask(indexCtx, *task, "left")
|
|
} else if task == nil {
|
|
readErr = fmt.Errorf("annotation task no longer exists")
|
|
}
|
|
cancel()
|
|
j.Status = "completed"
|
|
j.Error = ""
|
|
if readErr != nil {
|
|
j.Error = readErr.Error()
|
|
j.Status = "queued"
|
|
j.NextAttemptAt = time.Now().Add(time.Minute).UTC().Format(time.RFC3339Nano)
|
|
if j.Attempt >= 5 {
|
|
j.Status = "failed"
|
|
}
|
|
}
|
|
_, _ = queue.FinishAnnotationIndex(context.WithoutCancel(ctx), *j)
|
|
continue
|
|
}
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case <-time.After(2 * time.Second):
|
|
}
|
|
}
|
|
}()
|
|
wg.Wait()
|
|
return ctx.Err()
|
|
}
|
|
|
|
func (s *server) executeAnnotationJob(parent context.Context, claimed domain.HumanItemRecord) {
|
|
queue := s.store.(store.AnnotationProcessingStore)
|
|
j := *claimed.ProcessingJob
|
|
ctx, cancel := context.WithTimeout(parent, 24*time.Hour)
|
|
defer cancel()
|
|
ctx = context.WithValue(ctx, annotationJobLeaseKey{}, annotationJobLease{claimed.ID, j.ID, j.Owner})
|
|
done, stopped := make(chan struct{}), make(chan struct{})
|
|
go func() {
|
|
defer close(stopped)
|
|
tick := time.NewTicker(10 * time.Second)
|
|
defer tick.Stop()
|
|
for {
|
|
select {
|
|
case <-done:
|
|
return
|
|
case <-ctx.Done():
|
|
return
|
|
case <-tick.C:
|
|
stage, completed, total := "validating", 0, j.TotalSlices
|
|
if raw, ok := s.materializationProgress.Load(claimed.OrganizationID + ":" + j.TaskID); ok {
|
|
p := mapValue(raw)
|
|
stage = stringValue(p["stage"])
|
|
completed = intValue(p["completedSlices"])
|
|
total = intValue(p["totalSlices"])
|
|
}
|
|
ok, e := queue.TouchAnnotationProcessing(ctx, claimed.ID, j.ID, j.Owner, time.Now().Add(2*time.Minute).UTC().Format(time.RFC3339Nano), stage, completed, total)
|
|
if e != nil || !ok {
|
|
cancel()
|
|
return
|
|
}
|
|
}
|
|
}
|
|
}()
|
|
e := s.processAnnotationJob(ctx, claimed)
|
|
close(done)
|
|
<-stopped
|
|
finishCtx, stop := context.WithTimeout(context.WithoutCancel(parent), 15*time.Second)
|
|
defer stop()
|
|
if parent.Err() != nil || ctx.Err() != nil {
|
|
j.Status = "queued"
|
|
j.Stage = "queued"
|
|
j.Error = "processing interrupted; resuming"
|
|
_, _ = queue.SaveAnnotationProcessing(finishCtx, claimed.ID, j)
|
|
return
|
|
}
|
|
j.Status = "completed"
|
|
j.Stage = "completed"
|
|
j.Input = nil
|
|
j.CompletedSlices = j.TotalSlices
|
|
j.UpdatedAt = nowISO()
|
|
if e != nil {
|
|
j.Status = "failed"
|
|
j.Stage = "failed"
|
|
j.Error = e.Error()
|
|
log.Printf("[annotation-processor] task=%s job=%s failed: %v", j.TaskID, j.ID, e)
|
|
}
|
|
_, _ = queue.SaveAnnotationProcessing(finishCtx, claimed.ID, j)
|
|
if e != nil {
|
|
s.failAnnotationJob(finishCtx, claimed.ID, j)
|
|
}
|
|
}
|
|
|
|
func (s *server) processAnnotationJob(ctx context.Context, h domain.HumanItemRecord) error {
|
|
j := h.ProcessingJob
|
|
if j.Attempt > 3 {
|
|
return fmt.Errorf("后台处理连续中断,已保留编辑内容,请重试")
|
|
}
|
|
t, e := s.store.GetAnnotationTask(ctx, j.TaskID)
|
|
if e != nil {
|
|
return e
|
|
}
|
|
if t == nil || t.OrganizationID != h.OrganizationID || t.RunID != h.RunID {
|
|
return fmt.Errorf("annotation job task scope changed")
|
|
}
|
|
// Repair a crash after authoritative handoff but before legacy task publication.
|
|
if j.Kind == "submit" && h.PendingCombinedSubmissionID == "" && h.CurrentStageID == "quality_review" && h.CurrentRevision > 0 {
|
|
t.CurrentRevision = h.CurrentRevision
|
|
t.Status = domain.AnnotationTaskStatusPendingQuality
|
|
t.AssignedUserID = ""
|
|
return s.store.UpsertAnnotationTask(ctx, *t)
|
|
}
|
|
actor, e := s.store.GetUser(ctx, j.ActorID)
|
|
if e != nil {
|
|
return e
|
|
}
|
|
if actor == nil || actor.Status != domain.UserStatusActive || actor.OrganizationID != h.OrganizationID || h.AssignedUserID != actor.ID || h.CurrentStageID != "annotation_combined" {
|
|
return fmt.Errorf("annotation processing ownership or user status changed")
|
|
}
|
|
org, e := s.store.GetOrganization(ctx, h.OrganizationID)
|
|
if e != nil {
|
|
return e
|
|
}
|
|
if org == nil || org.Status != domain.OrganizationStatusActive {
|
|
return fmt.Errorf("annotation organization unavailable")
|
|
}
|
|
principal := &authenticatedPrincipal{User: *actor, Organization: *org}
|
|
r, _ := http.NewRequestWithContext(context.WithValue(ctx, authContextKey{}, principal), http.MethodPost, "http://annotation-processor/internal", nil)
|
|
w := &asynchronousAnnotationResponseWriter{}
|
|
if _, _, ok := s.ensureAnnotationTaskAccess(w, r, *t); !ok {
|
|
return fmt.Errorf("annotation job no longer authorized")
|
|
}
|
|
if !annotationManager(principal) && !s.canAccessCurrentHumanStage(ctx, r, h) {
|
|
return fmt.Errorf("annotation job role changed")
|
|
}
|
|
run, e := s.store.GetRun(ctx, t.RunID)
|
|
if e != nil {
|
|
return e
|
|
}
|
|
if run == nil {
|
|
return fmt.Errorf("annotation run unavailable")
|
|
}
|
|
key, e := s.annotationSourceFingerprint(ctx, *t, *run)
|
|
if e != nil {
|
|
return e
|
|
}
|
|
if key != j.SourceFingerprint {
|
|
return fmt.Errorf("源文件已改变,已保留编辑内容,请重新确认")
|
|
}
|
|
b, e := json.Marshal(normalizeStructuredValue(j.Input))
|
|
if e != nil {
|
|
return e
|
|
}
|
|
switch j.Kind {
|
|
case "slice":
|
|
var p sliceConfirmationInput
|
|
if e = json.Unmarshal(b, &p); e != nil {
|
|
return e
|
|
}
|
|
s.finishSliceConfirmation(r.Context(), t.ID, h.ID, j.ID, p, j.Revision, j.ActorID)
|
|
fresh, e := s.store.GetHumanItem(ctx, h.ID)
|
|
if e != nil {
|
|
return e
|
|
}
|
|
if fresh == nil {
|
|
return fmt.Errorf("annotation task removed")
|
|
}
|
|
flow := mapValue(fresh.Subject.Payload["sliceWorkflow"])
|
|
if stringValue(flow["status"]) != "completed" {
|
|
return fmt.Errorf("%s", firstNonEmptyText(stringValue(flow["error"]), "slice processing did not complete"))
|
|
}
|
|
case "submit":
|
|
if h.PendingCombinedSubmissionID != j.ID {
|
|
return fmt.Errorf("annotation submission superseded")
|
|
}
|
|
var p combinedAnnotationSubmissionInput
|
|
if e = json.Unmarshal(b, &p); e != nil {
|
|
return e
|
|
}
|
|
s.completeCombinedAnnotationSubmission(w, r, t, &h, p, principal, j.ID, j.Attempt > 1)
|
|
if w.status >= 400 {
|
|
return fmt.Errorf("annotation submission failed: %s", w.body.String())
|
|
}
|
|
default:
|
|
return fmt.Errorf("unknown annotation job kind")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (s *server) failAnnotationJob(ctx context.Context, id string, j domain.AnnotationProcessingJob) {
|
|
h, e := s.store.GetHumanItem(ctx, id)
|
|
if e != nil || h == nil || h.ProcessingJob == nil || h.ProcessingJob.ID != j.ID || h.ProcessingJob.Owner != j.Owner {
|
|
return
|
|
}
|
|
version := h.Version
|
|
if j.Kind == "submit" && h.PendingCombinedSubmissionID == j.ID {
|
|
h.PendingCombinedSubmissionID = ""
|
|
h.PendingCombinedPayloadSHA = ""
|
|
}
|
|
if j.Kind == "slice" {
|
|
w := mapValue(h.Subject.Payload["sliceWorkflow"])
|
|
if stringValue(w["requestId"]) == j.ID {
|
|
w["status"] = "failed"
|
|
w["error"] = j.Error
|
|
h.Subject.Payload["sliceWorkflow"] = w
|
|
}
|
|
}
|
|
h.Version++
|
|
h.UpdatedAt = nowISO()
|
|
_, _ = s.store.UpdateHumanItemVersioned(ctx, *h, version)
|
|
}
|