droboticflow/internal/api/annotation_work_buffer_mongo_http_test.go

214 lines
9.7 KiB
Go

package api
import (
"bytes"
"context"
"encoding/json"
"fmt"
"net/http"
"net/http/httptest"
"os"
"sync"
"testing"
"time"
"droboticflow/internal/domain"
"droboticflow/internal/store"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
)
// Opt-in, isolated database on the local Mongo replica. All assertions cross
// public HTTP routes; neither receipt documents nor slot ledgers are inspected.
func TestAnnotationBufferMongoHTTPConcurrencyAndReceipt(t *testing.T) {
if os.Getenv("ANNOTATION_BUFFER_MONGO_HTTP") != "1" {
t.Skip("opt-in local Mongo HTTP verification")
}
ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second)
defer cancel()
client, err := mongo.Connect(ctx, options.Client().ApplyURI("mongodb://127.0.0.1:27018/?directConnection=true"))
if err != nil {
t.Fatal(err)
}
defer client.Disconnect(context.Background())
db := client.Database(fmt.Sprintf("annotation_buffer_http_test_%d", time.Now().UnixNano()))
defer func() {
cleanup, done := context.WithTimeout(context.Background(), 10*time.Second)
defer done()
if err := db.Drop(cleanup); err != nil {
t.Error(err)
}
}()
ds := store.NewMongoStore(db)
for _, name := range []string{"projects", "users", "suppliers", "projectMembers", "annotationTasks", "humanBatches", "humanItems", "humanEvents", "humanStageAttempts", "humanClaimRequests"} {
if _, err := db.Collection(name).Indexes().CreateOne(ctx, mongo.IndexModel{Keys: bson.D{{Key: "id", Value: 1}}, Options: options.Index().SetUnique(true)}); err != nil {
t.Fatal(err)
}
}
must := func(err error) {
t.Helper()
if err != nil {
t.Fatal(err)
}
}
org := "buffer-http-org"
user := domain.UserRecord{ID: "buffer-http-user", OrganizationID: org, SupplierID: "buffer-http-supplier", Role: domain.UserRoleAnnotator, Status: domain.UserStatusActive}
must(ds.UpsertUser(ctx, user))
must(ds.UpsertSupplier(ctx, domain.SupplierRecord{ID: "buffer-http-supplier", OrganizationID: org, Status: "active"}))
// Separate handlers/stores model requests routed to different API replicas.
handlers := []http.Handler{NewHandler(ds, nil), NewHandler(store.NewMongoStore(db), nil)}
call := func(handler int, method, path string, body any) *httptest.ResponseRecorder {
data, _ := json.Marshal(body)
req := httptest.NewRequest(method, path, bytes.NewReader(data))
req.Header.Set("Content-Type", "application/json")
req.AddCookie(&http.Cookie{Name: csrfCookieName, Value: "buffer-http-csrf"})
req.Header.Set(csrfHeaderName, "buffer-http-csrf")
req = req.WithContext(context.WithValue(ctx, authContextKey{}, &authenticatedPrincipal{User: user, Organization: domain.OrganizationRecord{ID: org, Status: domain.OrganizationStatusActive}}))
rec := httptest.NewRecorder()
handlers[handler%2].ServeHTTP(rec, req)
return rec
}
for round := 0; round < 5; round++ {
project := fmt.Sprintf("buffer-http-project-%d", round)
must(ds.UpsertProject(ctx, domain.ProjectRecord{ID: project, OrganizationID: org}))
must(ds.UpsertProjectMember(ctx, domain.ProjectMemberRecord{ID: "member-" + project, OrganizationID: org, ProjectID: project, UserID: user.ID, SupplierID: "buffer-http-supplier", Roles: []domain.AnnotationAssignmentRole{domain.AnnotationRoleAnnotator}}))
batch := domain.HumanBatchRecord{ID: "batch-" + project, OrganizationID: org, ProjectID: project, WorkflowID: "workflow-" + project, RunID: "run-" + project, GateNodeID: "gate", PluginID: "annotation-qa", SupplierID: "buffer-http-supplier", Status: domain.HumanBatchActive, ProcessingProfile: domain.ProcessingProfileRawHuman, Stages: domain.DefaultFanyuV58HumanStages()}
must(ds.UpsertHumanBatch(ctx, batch))
for n := 0; n < 16; n++ {
task := domain.AnnotationTaskRecord{ID: fmt.Sprintf("task-%s-%d", project, n), OrganizationID: org, ProjectID: project, WorkflowID: batch.WorkflowID, SupplierID: batch.SupplierID, Status: domain.AnnotationTaskStatusPendingAnnotation, TemplateVersion: 6}
must(ds.UpsertAnnotationTask(ctx, task))
must(ds.UpsertHumanItem(ctx, domain.HumanItemRecord{ID: "item-" + task.ID, OrganizationID: org, ProjectID: project, WorkflowID: batch.WorkflowID, PluginID: "annotation-qa", BatchID: batch.ID, LegacyTaskID: task.ID, SupplierID: batch.SupplierID, Subject: domain.HumanSubjectRef{ItemKey: task.ID}, CurrentStageID: domain.FanyuV58StageAnnotationCombined, Status: domain.HumanItemReady, Version: 1}))
}
bufferPath := "/api/plugins/annotation-qa/work-buffer?projectId=" + project
var wg sync.WaitGroup
type claim struct{ requestID, id string }
successes := make(chan claim, 16)
for n := 0; n < 16; n++ {
wg.Add(1)
go func(n int) {
defer wg.Done()
requestID := fmt.Sprintf("mongo-buffer-%02d-request-%02d", round, n)
// Concurrent buffer polling exercises reconciliation while ownership writes finish.
pollPath := bufferPath
if n%2 == 0 {
pollPath += "&view=summary"
}
poll := call(n, "GET", pollPath, nil)
if poll.Code != 200 {
t.Errorf("concurrent poll: %d %s", poll.Code, poll.Body.String())
}
r := call(n, "POST", "/api/plugins/annotation-qa/random-claims", map[string]any{"projectId": project, "requestId": requestID})
if r.Code == 200 {
var out struct {
Item domain.HumanItemRecord `json:"item"`
}
if err := json.Unmarshal(r.Body.Bytes(), &out); err != nil {
t.Error(err)
}
successes <- claim{requestID, out.Item.ID}
} else if r.Code != 409 {
t.Errorf("claim: %d %s", r.Code, r.Body.String())
}
}(n)
}
wg.Wait()
close(successes)
claims := []claim{}
for result := range successes {
claims = append(claims, result)
}
if len(claims) != 2 {
t.Fatalf("round %d: capacity 2 assigned %d", round, len(claims))
}
r := call(0, "GET", bufferPath, nil)
var buffer struct {
Capacity int `json:"capacity"`
Occupied int `json:"occupied"`
Items []domain.HumanItemRecord `json:"items"`
}
must(json.Unmarshal(r.Body.Bytes(), &buffer))
if r.Code != 200 || buffer.Capacity != 2 || buffer.Occupied != 2 || len(buffer.Items) != 2 {
t.Fatalf("round %d buffer: %d %s", round, r.Code, r.Body.String())
}
checkSummary := func(handler int) {
t.Helper()
old := call(handler, "GET", bufferPath, nil)
compact := call(handler, "GET", bufferPath+"&view=summary", nil)
var full struct {
Capacity int `json:"capacity"`
Occupied int `json:"occupied"`
Items []domain.HumanItemRecord `json:"items"`
}
var summary struct {
Capacity int `json:"capacity"`
Occupied int `json:"occupied"`
Summaries []annotationBufferSummary `json:"summaries"`
}
must(json.Unmarshal(old.Body.Bytes(), &full))
must(json.Unmarshal(compact.Body.Bytes(), &summary))
if old.Code != 200 || compact.Code != 200 || full.Capacity != summary.Capacity || full.Occupied != summary.Occupied || len(full.Items) != len(summary.Summaries) {
t.Fatalf("summary/full mismatch: %s / %s", old.Body.String(), compact.Body.String())
}
expected := map[string]annotationBufferSummary{}
for _, row := range summarizeAnnotationBuffer(full.Items) {
expected[row.ID] = row
}
for _, row := range summary.Summaries {
a, _ := json.Marshal(expected[row.ID])
b, _ := json.Marshal(row)
if string(a) != string(b) {
t.Fatalf("summary state differs: %s / %s", a, b)
}
}
}
checkSummary(0)
for _, result := range claims {
r = call(1, "POST", "/api/plugins/annotation-qa/random-claims", map[string]any{"projectId": project, "requestId": result.requestID})
var replay struct {
Item domain.HumanItemRecord `json:"item"`
Replayed bool `json:"replayed"`
}
must(json.Unmarshal(r.Body.Bytes(), &replay))
if r.Code != 200 || !replay.Replayed || replay.Item.ID != result.id {
t.Fatalf("full buffer receipt replay: %d %s", r.Code, r.Body.String())
}
}
// Fixture fault: ownership succeeded and work reached review, but the
// durable receipt finalization was interrupted. A full current buffer
// must not prevent recovering that original request on another replica.
original := claims[0]
item, err := ds.GetHumanItem(ctx, original.id)
must(err)
item.CurrentStageID = "quality_review"
item.Status = domain.HumanItemSubmitted
item.LatestAuthorUserID = user.ID
item.Version++
must(ds.UpsertHumanItem(ctx, *item))
_, err = db.Collection("humanClaimRequests").UpdateOne(ctx, bson.M{"itemId": original.id}, bson.M{"$set": bson.M{"state": "pending"}})
must(err)
r = call(0, "POST", "/api/plugins/annotation-qa/random-claims", map[string]any{"projectId": project, "requestId": fmt.Sprintf("mongo-buffer-%02d-refill-after-review", round)})
if r.Code != 200 {
t.Fatalf("refill after handoff: %d %s", r.Code, r.Body.String())
}
r = call(1, "POST", "/api/plugins/annotation-qa/random-claims", map[string]any{"projectId": project, "requestId": original.requestID})
var recovered struct {
Item domain.HumanItemRecord `json:"item"`
Replayed bool `json:"replayed"`
}
must(json.Unmarshal(r.Body.Bytes(), &recovered))
if r.Code != 200 || !recovered.Replayed || recovered.Item.ID != original.id || recovered.Item.CurrentStageID != "quality_review" {
t.Fatalf("interrupted receipt recovery after handoff: %d %s", r.Code, r.Body.String())
}
r = call(0, "GET", bufferPath, nil)
must(json.Unmarshal(r.Body.Bytes(), &buffer))
if r.Code != 200 || buffer.Occupied != 2 || len(buffer.Items) != 2 {
t.Fatalf("receipt replay consumed capacity: %d %s", r.Code, r.Body.String())
}
// Fresh process-local caches after recovery must not change summaries.
handlers[1] = NewHandler(store.NewMongoStore(db), nil)
checkSummary(1)
}
}