package api import ( "context" "droboticflow/internal/config" "droboticflow/internal/domain" "encoding/json" "fmt" "net/http" "os" "os/exec" "path/filepath" "strings" "testing" "time" ) func annotationProcessorFixture(t *testing.T) (combinedDraftFixture, string) { t.Helper() if _, e := exec.LookPath("ffmpeg"); e != nil { t.Skip("ffmpeg required") } f := newCombinedDraftFixture(t) if e := f.store.UpsertOrganization(f.ctx, domain.OrganizationRecord{ID: domain.DefaultOrganizationID, Name: "Processor test", Status: domain.OrganizationStatusActive}); e != nil { t.Fatal(e) } root := t.TempDir() seedSliceDictionary(t, f, "测试切片") task := f.task task.NodeID = "human" task.ArtifactID = "left-video" task.RightArtifactID = "right-video" task.ClipEnd = 4 task.SourceIsFullVideo = true task.SourceDurationSeconds = 4 task.CoverageStartSecond = 0 task.CoverageEndSecond = 4 task.CoverageMode = "full_common_range" pts, imu := "timestamp_us\n", "timestamp_us,x,y,z,type\n" for i := 0; i < 32; i++ { stamp := 1000000 + i*125000 pts += fmt.Sprintf("%d\n", stamp) imu += fmt.Sprintf("%d,1,2,3,accel\n%d,4,5,6,gyro\n", stamp, stamp) } bindings := []domain.NodeInputBinding{} for _, side := range []string{"left", "right"} { video := filepath.Join(root, side+".mp4") if out, e := exec.Command("ffmpeg", "-v", "error", "-y", "-f", "lavfi", "-i", "testsrc2=s=64x48:r=8:d=4", "-c:v", "libx264", "-pix_fmt", "yuv420p", video).CombinedOutput(); e != nil { t.Fatalf("%v: %s", e, out) } p := filepath.Join(root, side+"_pts.csv") os.WriteFile(p, []byte(pts), 0600) videoPort := "fullLeftVideos" if side == "right" { videoPort = "fullRightVideos" } bindings = append(bindings, domain.NodeInputBinding{TargetPortID: videoPort, Artifacts: []domain.ExecutionArtifact{{ID: side + "-video", Name: side + ".mp4", Path: video, Data: map[string]any{"collectionId": "capture"}}}}, domain.NodeInputBinding{TargetPortID: side + "Timestamps", Artifacts: []domain.ExecutionArtifact{{ID: side + "-pts", Name: side + "_pts.csv", Path: p, Data: map[string]any{"collectionId": "capture"}}}}) } p := filepath.Join(root, "imu.csv") os.WriteFile(p, []byte(imu), 0600) bindings = append(bindings, domain.NodeInputBinding{TargetPortID: "imu", Artifacts: []domain.ExecutionArtifact{{ID: "imu", Name: "imu.csv", Path: p, Data: map[string]any{"collectionId": "capture"}}}}) f.store.UpsertRun(f.ctx, domain.RunRecord{ID: task.RunID, OrganizationID: task.OrganizationID, ProjectID: task.ProjectID, WorkflowID: task.WorkflowID, NodeExecutions: []domain.NodeExecutionRecord{{NodeID: task.NodeID, InputBindings: bindings}}}) f.store.UpsertAnnotationTask(f.ctx, task) f.task = task if err := f.store.RetryAnnotationSourceValidation(f.ctx, task, true, ""); err != nil { t.Fatal(err) } f.handler = NewHandlerWithConfig(f.store, nil, config.AppConfig{WorkspaceMountPath: root}) return f, root } func TestDurableAnnotationOutboxSurvivesRestartAndFencesOldWorker(t *testing.T) { t.Setenv("ANNOTATION_PROCESSOR_MODE", "external") f, root := annotationProcessorFixture(t) testDurableAnnotationOutbox(t, f, root) } func testDurableAnnotationOutbox(t *testing.T, f combinedDraftFixture, root string, selected ...[]domain.SliceSegment) { t.Helper() slices := []domain.SliceSegment{{ID: "a", Name: "测试切片", Start: 0, End: 2}, {ID: "b", Name: "测试切片", Start: 2, End: 4}} if len(selected) > 0 { slices = selected[0] } r := f.request(http.MethodPost, "/api/plugins/annotation-qa/tasks/"+f.task.ID+"/slice-confirmations", f.annotator, map[string]any{"version": f.item.Version, "sliceSegments": slices}) if r.Code != 202 { t.Fatal(r.Code, r.Body.String()) } h, _ := f.store.GetHumanItem(f.ctx, f.item.ID) if h.ProcessingJob == nil || h.ProcessingJob.Status != "queued" || len(h.ProcessingJob.Input) == 0 { t.Fatal("missing atomic outbox") } if b, _ := json.Marshal(h); stringContains(string(b), "processingJob") || stringContains(string(b), "sourceFingerprint") { t.Fatal("private outbox leaked") } // A new API instance must not clear a durable job when it has no local map. f.handler = NewHandlerWithConfig(f.store, nil, config.AppConfig{WorkspaceMountPath: root}) r = f.request(http.MethodGet, "/api/plugins/annotation-qa/tasks/"+f.task.ID+"?view=status", f.annotator, nil) if r.Code != 200 { t.Fatal(r.Code, r.Body.String()) } now := time.Now().UTC() old, _ := f.store.ClaimAnnotationProcessing(f.ctx, "dead-worker", now.Format(time.RFC3339Nano), now.Add(-time.Second).Format(time.RFC3339Nano)) claimed, _ := f.store.ClaimAnnotationProcessing(f.ctx, "replacement-worker", now.Format(time.RFC3339Nano), now.Add(time.Minute).Format(time.RFC3339Nano)) if old == nil || claimed == nil || claimed.ProcessingJob.Attempt != 2 { t.Fatal("lease not recovered") } if ok, _ := f.store.UpdateHumanItemVersioned(f.ctx, *old, old.Version); ok { t.Fatal("expired worker can publish") } p := &server{store: f.store, workspaceMountPath: root, externalAnnotationProcessor: true, annotationSourceIndex: os.Getenv("ANNOTATION_SOURCE_INDEX") == "true"} p.executeAnnotationJob(context.Background(), *claimed) h, _ = f.store.GetHumanItem(f.ctx, f.item.ID) if h.ProcessingJob.Status != "completed" || len(h.ProcessingJob.Input) != 0 || stringValue(mapValue(h.Subject.Payload["sliceWorkflow"])["status"]) != "completed" { t.Fatalf("slice job failed: %+v", h.ProcessingJob) } actions := []domain.AnnotationSegment{} if len(selected) > 0 { statuses := map[string]string{} for _, slice := range slices { actions = append(actions, domain.AnnotationSegment{ID: "saved-" + slice.ID, Start: slice.Start, End: slice.End, Description: "保留已有标注", Status: "described"}) statuses[slice.ID] = "labeled" } saved := f.request(http.MethodPut, "/api/plugins/annotation-qa/tasks/"+f.task.ID+"/drafts", f.annotator, map[string]any{"version": h.Version, "sliceSegments": slices, "segments": actions, "statuses": statuses}) if saved.Code != 200 { t.Fatalf("save annotations: %d %s", saved.Code, saved.Body.String()) } h, _ = f.store.GetHumanItem(f.ctx, f.item.ID) } baseHandler := f.handler f.handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { r.Header.Set("Prefer", "respond-async") baseHandler.ServeHTTP(w, r) }) r = f.request(http.MethodPost, "/api/plugins/annotation-qa/tasks/"+f.task.ID+"/combined-submissions", f.annotator, map[string]any{"version": h.Version, "sliceSegments": slices, "segments": actions, "requireSliceCompletion": true}) if r.Code != 202 { t.Fatal(r.Code, r.Body.String()) } before, _ := f.store.GetHumanItem(f.ctx, h.ID) if before.PendingCombinedSubmissionID == "" || before.ProcessingJob.Kind != "submit" { t.Fatal("submission not atomically queued") } job, _ := f.store.ClaimAnnotationProcessing(f.ctx, "submit-worker", now.Format(time.RFC3339Nano), now.Add(time.Minute).Format(time.RFC3339Nano)) if job == nil { t.Fatal("submission not claimable") } p.executeAnnotationJob(context.Background(), *job) after, _ := f.store.GetHumanItem(f.ctx, h.ID) if after.CurrentStageID != "quality_review" || after.ProcessingJob.Status != "completed" || after.PendingCombinedSubmissionID != "" { t.Fatalf("submission failed %+v", after.ProcessingJob) } if len(selected) > 0 { revisions, e := f.store.ListAnnotationRevisions(f.ctx, f.task.OrganizationID, f.task.ID) if e != nil || len(revisions) == 0 || len(revisions[0].Segments) != len(actions) { t.Fatal("submitted annotations were not preserved", e) } for _, segment := range revisions[0].Segments { if segment.Description != "保留已有标注" { t.Fatal("description lost during submission") } } } run, _ := f.store.GetRun(f.ctx, f.task.RunID) found := false for _, a := range run.Artifacts { if boolValue(a.Data["metadataOnly"]) { found = true } } if os.Getenv("ANNOTATION_SOURCE_INDEX") == "true" { references := 0 for _, a := range run.Artifacts { if stringValue(a.Data["sourceIndexSchema"]) == sourceIndexSchema { references++ if e := filepath.WalkDir(a.Path, func(path string, d os.DirEntry, e error) error { if e != nil { return e } if !d.IsDir() && !strings.HasSuffix(path, ".json") { t.Errorf("unexpected regenerated media: %s", path) } return nil }); e != nil { t.Fatal(e) } } } if references < 1 { t.Fatal("source revisions not persisted") } } else if !found { t.Fatal("unchanged media was rebuilt") } } func stringContains(s, part string) bool { return len(part) > 0 && len(s) >= len(part) && strings.Contains(s, part) }