package api import ( "bytes" "context" "encoding/json" "fmt" "net/http" "net/http/httptest" "os" "sync" "testing" "time" "droboticflow/internal/domain" "droboticflow/internal/store" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/mongo" "go.mongodb.org/mongo-driver/mongo/options" ) // Opt-in, isolated database on the local Mongo replica. All assertions cross // public HTTP routes; neither receipt documents nor slot ledgers are inspected. func TestAnnotationBufferMongoHTTPConcurrencyAndReceipt(t *testing.T) { if os.Getenv("ANNOTATION_BUFFER_MONGO_HTTP") != "1" { t.Skip("opt-in local Mongo HTTP verification") } ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second) defer cancel() client, err := mongo.Connect(ctx, options.Client().ApplyURI("mongodb://127.0.0.1:27018/?directConnection=true")) if err != nil { t.Fatal(err) } defer client.Disconnect(context.Background()) db := client.Database(fmt.Sprintf("annotation_buffer_http_test_%d", time.Now().UnixNano())) defer func() { cleanup, done := context.WithTimeout(context.Background(), 10*time.Second) defer done() if err := db.Drop(cleanup); err != nil { t.Error(err) } }() ds := store.NewMongoStore(db) for _, name := range []string{"projects", "users", "suppliers", "projectMembers", "annotationTasks", "humanBatches", "humanItems", "humanEvents", "humanStageAttempts", "humanClaimRequests"} { if _, err := db.Collection(name).Indexes().CreateOne(ctx, mongo.IndexModel{Keys: bson.D{{Key: "id", Value: 1}}, Options: options.Index().SetUnique(true)}); err != nil { t.Fatal(err) } } must := func(err error) { t.Helper() if err != nil { t.Fatal(err) } } org := "buffer-http-org" user := domain.UserRecord{ID: "buffer-http-user", OrganizationID: org, SupplierID: "buffer-http-supplier", Role: domain.UserRoleAnnotator, Status: domain.UserStatusActive} must(ds.UpsertUser(ctx, user)) must(ds.UpsertSupplier(ctx, domain.SupplierRecord{ID: "buffer-http-supplier", OrganizationID: org, Status: "active"})) // Separate handlers/stores model requests routed to different API replicas. handlers := []http.Handler{NewHandler(ds, nil), NewHandler(store.NewMongoStore(db), nil)} call := func(handler int, method, path string, body any) *httptest.ResponseRecorder { data, _ := json.Marshal(body) req := httptest.NewRequest(method, path, bytes.NewReader(data)) req.Header.Set("Content-Type", "application/json") req.AddCookie(&http.Cookie{Name: csrfCookieName, Value: "buffer-http-csrf"}) req.Header.Set(csrfHeaderName, "buffer-http-csrf") req = req.WithContext(context.WithValue(ctx, authContextKey{}, &authenticatedPrincipal{User: user, Organization: domain.OrganizationRecord{ID: org, Status: domain.OrganizationStatusActive}})) rec := httptest.NewRecorder() handlers[handler%2].ServeHTTP(rec, req) return rec } for round := 0; round < 5; round++ { project := fmt.Sprintf("buffer-http-project-%d", round) must(ds.UpsertProject(ctx, domain.ProjectRecord{ID: project, OrganizationID: org})) must(ds.UpsertProjectMember(ctx, domain.ProjectMemberRecord{ID: "member-" + project, OrganizationID: org, ProjectID: project, UserID: user.ID, SupplierID: "buffer-http-supplier", Roles: []domain.AnnotationAssignmentRole{domain.AnnotationRoleAnnotator}})) batch := domain.HumanBatchRecord{ID: "batch-" + project, OrganizationID: org, ProjectID: project, WorkflowID: "workflow-" + project, RunID: "run-" + project, GateNodeID: "gate", PluginID: "annotation-qa", SupplierID: "buffer-http-supplier", Status: domain.HumanBatchActive, ProcessingProfile: domain.ProcessingProfileRawHuman, Stages: domain.DefaultFanyuV58HumanStages()} must(ds.UpsertHumanBatch(ctx, batch)) for n := 0; n < 16; n++ { task := domain.AnnotationTaskRecord{ID: fmt.Sprintf("task-%s-%d", project, n), OrganizationID: org, ProjectID: project, WorkflowID: batch.WorkflowID, SupplierID: batch.SupplierID, Status: domain.AnnotationTaskStatusPendingAnnotation, TemplateVersion: 6} must(ds.UpsertAnnotationTask(ctx, task)) must(ds.UpsertHumanItem(ctx, domain.HumanItemRecord{ID: "item-" + task.ID, OrganizationID: org, ProjectID: project, WorkflowID: batch.WorkflowID, PluginID: "annotation-qa", BatchID: batch.ID, LegacyTaskID: task.ID, SupplierID: batch.SupplierID, Subject: domain.HumanSubjectRef{ItemKey: task.ID}, CurrentStageID: domain.FanyuV58StageAnnotationCombined, Status: domain.HumanItemReady, Version: 1})) } bufferPath := "/api/plugins/annotation-qa/work-buffer?projectId=" + project var wg sync.WaitGroup type claim struct{ requestID, id string } successes := make(chan claim, 16) for n := 0; n < 16; n++ { wg.Add(1) go func(n int) { defer wg.Done() requestID := fmt.Sprintf("mongo-buffer-%02d-request-%02d", round, n) // Concurrent buffer polling exercises reconciliation while ownership writes finish. pollPath := bufferPath if n%2 == 0 { pollPath += "&view=summary" } poll := call(n, "GET", pollPath, nil) if poll.Code != 200 { t.Errorf("concurrent poll: %d %s", poll.Code, poll.Body.String()) } r := call(n, "POST", "/api/plugins/annotation-qa/random-claims", map[string]any{"projectId": project, "requestId": requestID}) if r.Code == 200 { var out struct { Item domain.HumanItemRecord `json:"item"` } if err := json.Unmarshal(r.Body.Bytes(), &out); err != nil { t.Error(err) } successes <- claim{requestID, out.Item.ID} } else if r.Code != 409 { t.Errorf("claim: %d %s", r.Code, r.Body.String()) } }(n) } wg.Wait() close(successes) claims := []claim{} for result := range successes { claims = append(claims, result) } if len(claims) != 2 { t.Fatalf("round %d: capacity 2 assigned %d", round, len(claims)) } r := call(0, "GET", bufferPath, nil) var buffer struct { Capacity int `json:"capacity"` Occupied int `json:"occupied"` Items []domain.HumanItemRecord `json:"items"` } must(json.Unmarshal(r.Body.Bytes(), &buffer)) if r.Code != 200 || buffer.Capacity != 2 || buffer.Occupied != 2 || len(buffer.Items) != 2 { t.Fatalf("round %d buffer: %d %s", round, r.Code, r.Body.String()) } checkSummary := func(handler int) { t.Helper() old := call(handler, "GET", bufferPath, nil) compact := call(handler, "GET", bufferPath+"&view=summary", nil) var full struct { Capacity int `json:"capacity"` Occupied int `json:"occupied"` Items []domain.HumanItemRecord `json:"items"` } var summary struct { Capacity int `json:"capacity"` Occupied int `json:"occupied"` Summaries []annotationBufferSummary `json:"summaries"` } must(json.Unmarshal(old.Body.Bytes(), &full)) must(json.Unmarshal(compact.Body.Bytes(), &summary)) if old.Code != 200 || compact.Code != 200 || full.Capacity != summary.Capacity || full.Occupied != summary.Occupied || len(full.Items) != len(summary.Summaries) { t.Fatalf("summary/full mismatch: %s / %s", old.Body.String(), compact.Body.String()) } expected := map[string]annotationBufferSummary{} for _, row := range summarizeAnnotationBuffer(full.Items) { expected[row.ID] = row } for _, row := range summary.Summaries { a, _ := json.Marshal(expected[row.ID]) b, _ := json.Marshal(row) if string(a) != string(b) { t.Fatalf("summary state differs: %s / %s", a, b) } } } checkSummary(0) for _, result := range claims { r = call(1, "POST", "/api/plugins/annotation-qa/random-claims", map[string]any{"projectId": project, "requestId": result.requestID}) var replay struct { Item domain.HumanItemRecord `json:"item"` Replayed bool `json:"replayed"` } must(json.Unmarshal(r.Body.Bytes(), &replay)) if r.Code != 200 || !replay.Replayed || replay.Item.ID != result.id { t.Fatalf("full buffer receipt replay: %d %s", r.Code, r.Body.String()) } } // Fixture fault: ownership succeeded and work reached review, but the // durable receipt finalization was interrupted. A full current buffer // must not prevent recovering that original request on another replica. original := claims[0] item, err := ds.GetHumanItem(ctx, original.id) must(err) item.CurrentStageID = "quality_review" item.Status = domain.HumanItemSubmitted item.LatestAuthorUserID = user.ID item.Version++ must(ds.UpsertHumanItem(ctx, *item)) _, err = db.Collection("humanClaimRequests").UpdateOne(ctx, bson.M{"itemId": original.id}, bson.M{"$set": bson.M{"state": "pending"}}) must(err) r = call(0, "POST", "/api/plugins/annotation-qa/random-claims", map[string]any{"projectId": project, "requestId": fmt.Sprintf("mongo-buffer-%02d-refill-after-review", round)}) if r.Code != 200 { t.Fatalf("refill after handoff: %d %s", r.Code, r.Body.String()) } r = call(1, "POST", "/api/plugins/annotation-qa/random-claims", map[string]any{"projectId": project, "requestId": original.requestID}) var recovered struct { Item domain.HumanItemRecord `json:"item"` Replayed bool `json:"replayed"` } must(json.Unmarshal(r.Body.Bytes(), &recovered)) if r.Code != 200 || !recovered.Replayed || recovered.Item.ID != original.id || recovered.Item.CurrentStageID != "quality_review" { t.Fatalf("interrupted receipt recovery after handoff: %d %s", r.Code, r.Body.String()) } r = call(0, "GET", bufferPath, nil) must(json.Unmarshal(r.Body.Bytes(), &buffer)) if r.Code != 200 || buffer.Occupied != 2 || len(buffer.Items) != 2 { t.Fatalf("receipt replay consumed capacity: %d %s", r.Code, r.Body.String()) } // Fresh process-local caches after recovery must not change summaries. handlers[1] = NewHandler(store.NewMongoStore(db), nil) checkSummary(1) } }