forked from longtao.wu/droboticflow
214 lines
9.7 KiB
Go
214 lines
9.7 KiB
Go
package api
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"os"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"droboticflow/internal/domain"
|
|
"droboticflow/internal/store"
|
|
"go.mongodb.org/mongo-driver/bson"
|
|
"go.mongodb.org/mongo-driver/mongo"
|
|
"go.mongodb.org/mongo-driver/mongo/options"
|
|
)
|
|
|
|
// Opt-in, isolated database on the local Mongo replica. All assertions cross
|
|
// public HTTP routes; neither receipt documents nor slot ledgers are inspected.
|
|
func TestAnnotationBufferMongoHTTPConcurrencyAndReceipt(t *testing.T) {
|
|
if os.Getenv("ANNOTATION_BUFFER_MONGO_HTTP") != "1" {
|
|
t.Skip("opt-in local Mongo HTTP verification")
|
|
}
|
|
ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second)
|
|
defer cancel()
|
|
client, err := mongo.Connect(ctx, options.Client().ApplyURI("mongodb://127.0.0.1:27018/?directConnection=true"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer client.Disconnect(context.Background())
|
|
db := client.Database(fmt.Sprintf("annotation_buffer_http_test_%d", time.Now().UnixNano()))
|
|
defer func() {
|
|
cleanup, done := context.WithTimeout(context.Background(), 10*time.Second)
|
|
defer done()
|
|
if err := db.Drop(cleanup); err != nil {
|
|
t.Error(err)
|
|
}
|
|
}()
|
|
ds := store.NewMongoStore(db)
|
|
for _, name := range []string{"projects", "users", "suppliers", "projectMembers", "annotationTasks", "humanBatches", "humanItems", "humanEvents", "humanStageAttempts", "humanClaimRequests"} {
|
|
if _, err := db.Collection(name).Indexes().CreateOne(ctx, mongo.IndexModel{Keys: bson.D{{Key: "id", Value: 1}}, Options: options.Index().SetUnique(true)}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
must := func(err error) {
|
|
t.Helper()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
org := "buffer-http-org"
|
|
user := domain.UserRecord{ID: "buffer-http-user", OrganizationID: org, SupplierID: "buffer-http-supplier", Role: domain.UserRoleAnnotator, Status: domain.UserStatusActive}
|
|
must(ds.UpsertUser(ctx, user))
|
|
must(ds.UpsertSupplier(ctx, domain.SupplierRecord{ID: "buffer-http-supplier", OrganizationID: org, Status: "active"}))
|
|
// Separate handlers/stores model requests routed to different API replicas.
|
|
handlers := []http.Handler{NewHandler(ds, nil), NewHandler(store.NewMongoStore(db), nil)}
|
|
call := func(handler int, method, path string, body any) *httptest.ResponseRecorder {
|
|
data, _ := json.Marshal(body)
|
|
req := httptest.NewRequest(method, path, bytes.NewReader(data))
|
|
req.Header.Set("Content-Type", "application/json")
|
|
req.AddCookie(&http.Cookie{Name: csrfCookieName, Value: "buffer-http-csrf"})
|
|
req.Header.Set(csrfHeaderName, "buffer-http-csrf")
|
|
req = req.WithContext(context.WithValue(ctx, authContextKey{}, &authenticatedPrincipal{User: user, Organization: domain.OrganizationRecord{ID: org, Status: domain.OrganizationStatusActive}}))
|
|
rec := httptest.NewRecorder()
|
|
handlers[handler%2].ServeHTTP(rec, req)
|
|
return rec
|
|
}
|
|
for round := 0; round < 5; round++ {
|
|
project := fmt.Sprintf("buffer-http-project-%d", round)
|
|
must(ds.UpsertProject(ctx, domain.ProjectRecord{ID: project, OrganizationID: org}))
|
|
must(ds.UpsertProjectMember(ctx, domain.ProjectMemberRecord{ID: "member-" + project, OrganizationID: org, ProjectID: project, UserID: user.ID, SupplierID: "buffer-http-supplier", Roles: []domain.AnnotationAssignmentRole{domain.AnnotationRoleAnnotator}}))
|
|
batch := domain.HumanBatchRecord{ID: "batch-" + project, OrganizationID: org, ProjectID: project, WorkflowID: "workflow-" + project, RunID: "run-" + project, GateNodeID: "gate", PluginID: "annotation-qa", SupplierID: "buffer-http-supplier", Status: domain.HumanBatchActive, ProcessingProfile: domain.ProcessingProfileRawHuman, Stages: domain.DefaultFanyuV58HumanStages()}
|
|
must(ds.UpsertHumanBatch(ctx, batch))
|
|
for n := 0; n < 16; n++ {
|
|
task := domain.AnnotationTaskRecord{ID: fmt.Sprintf("task-%s-%d", project, n), OrganizationID: org, ProjectID: project, WorkflowID: batch.WorkflowID, SupplierID: batch.SupplierID, Status: domain.AnnotationTaskStatusPendingAnnotation, TemplateVersion: 6}
|
|
must(ds.UpsertAnnotationTask(ctx, task))
|
|
must(ds.UpsertHumanItem(ctx, domain.HumanItemRecord{ID: "item-" + task.ID, OrganizationID: org, ProjectID: project, WorkflowID: batch.WorkflowID, PluginID: "annotation-qa", BatchID: batch.ID, LegacyTaskID: task.ID, SupplierID: batch.SupplierID, Subject: domain.HumanSubjectRef{ItemKey: task.ID}, CurrentStageID: domain.FanyuV58StageAnnotationCombined, Status: domain.HumanItemReady, Version: 1}))
|
|
}
|
|
bufferPath := "/api/plugins/annotation-qa/work-buffer?projectId=" + project
|
|
var wg sync.WaitGroup
|
|
type claim struct{ requestID, id string }
|
|
successes := make(chan claim, 16)
|
|
for n := 0; n < 16; n++ {
|
|
wg.Add(1)
|
|
go func(n int) {
|
|
defer wg.Done()
|
|
requestID := fmt.Sprintf("mongo-buffer-%02d-request-%02d", round, n)
|
|
// Concurrent buffer polling exercises reconciliation while ownership writes finish.
|
|
pollPath := bufferPath
|
|
if n%2 == 0 {
|
|
pollPath += "&view=summary"
|
|
}
|
|
poll := call(n, "GET", pollPath, nil)
|
|
if poll.Code != 200 {
|
|
t.Errorf("concurrent poll: %d %s", poll.Code, poll.Body.String())
|
|
}
|
|
r := call(n, "POST", "/api/plugins/annotation-qa/random-claims", map[string]any{"projectId": project, "requestId": requestID})
|
|
if r.Code == 200 {
|
|
var out struct {
|
|
Item domain.HumanItemRecord `json:"item"`
|
|
}
|
|
if err := json.Unmarshal(r.Body.Bytes(), &out); err != nil {
|
|
t.Error(err)
|
|
}
|
|
successes <- claim{requestID, out.Item.ID}
|
|
} else if r.Code != 409 {
|
|
t.Errorf("claim: %d %s", r.Code, r.Body.String())
|
|
}
|
|
}(n)
|
|
}
|
|
wg.Wait()
|
|
close(successes)
|
|
claims := []claim{}
|
|
for result := range successes {
|
|
claims = append(claims, result)
|
|
}
|
|
if len(claims) != 2 {
|
|
t.Fatalf("round %d: capacity 2 assigned %d", round, len(claims))
|
|
}
|
|
r := call(0, "GET", bufferPath, nil)
|
|
var buffer struct {
|
|
Capacity int `json:"capacity"`
|
|
Occupied int `json:"occupied"`
|
|
Items []domain.HumanItemRecord `json:"items"`
|
|
}
|
|
must(json.Unmarshal(r.Body.Bytes(), &buffer))
|
|
if r.Code != 200 || buffer.Capacity != 2 || buffer.Occupied != 2 || len(buffer.Items) != 2 {
|
|
t.Fatalf("round %d buffer: %d %s", round, r.Code, r.Body.String())
|
|
}
|
|
checkSummary := func(handler int) {
|
|
t.Helper()
|
|
old := call(handler, "GET", bufferPath, nil)
|
|
compact := call(handler, "GET", bufferPath+"&view=summary", nil)
|
|
var full struct {
|
|
Capacity int `json:"capacity"`
|
|
Occupied int `json:"occupied"`
|
|
Items []domain.HumanItemRecord `json:"items"`
|
|
}
|
|
var summary struct {
|
|
Capacity int `json:"capacity"`
|
|
Occupied int `json:"occupied"`
|
|
Summaries []annotationBufferSummary `json:"summaries"`
|
|
}
|
|
must(json.Unmarshal(old.Body.Bytes(), &full))
|
|
must(json.Unmarshal(compact.Body.Bytes(), &summary))
|
|
if old.Code != 200 || compact.Code != 200 || full.Capacity != summary.Capacity || full.Occupied != summary.Occupied || len(full.Items) != len(summary.Summaries) {
|
|
t.Fatalf("summary/full mismatch: %s / %s", old.Body.String(), compact.Body.String())
|
|
}
|
|
expected := map[string]annotationBufferSummary{}
|
|
for _, row := range summarizeAnnotationBuffer(full.Items) {
|
|
expected[row.ID] = row
|
|
}
|
|
for _, row := range summary.Summaries {
|
|
a, _ := json.Marshal(expected[row.ID])
|
|
b, _ := json.Marshal(row)
|
|
if string(a) != string(b) {
|
|
t.Fatalf("summary state differs: %s / %s", a, b)
|
|
}
|
|
}
|
|
}
|
|
checkSummary(0)
|
|
for _, result := range claims {
|
|
r = call(1, "POST", "/api/plugins/annotation-qa/random-claims", map[string]any{"projectId": project, "requestId": result.requestID})
|
|
var replay struct {
|
|
Item domain.HumanItemRecord `json:"item"`
|
|
Replayed bool `json:"replayed"`
|
|
}
|
|
must(json.Unmarshal(r.Body.Bytes(), &replay))
|
|
if r.Code != 200 || !replay.Replayed || replay.Item.ID != result.id {
|
|
t.Fatalf("full buffer receipt replay: %d %s", r.Code, r.Body.String())
|
|
}
|
|
}
|
|
// Fixture fault: ownership succeeded and work reached review, but the
|
|
// durable receipt finalization was interrupted. A full current buffer
|
|
// must not prevent recovering that original request on another replica.
|
|
original := claims[0]
|
|
item, err := ds.GetHumanItem(ctx, original.id)
|
|
must(err)
|
|
item.CurrentStageID = "quality_review"
|
|
item.Status = domain.HumanItemSubmitted
|
|
item.LatestAuthorUserID = user.ID
|
|
item.Version++
|
|
must(ds.UpsertHumanItem(ctx, *item))
|
|
_, err = db.Collection("humanClaimRequests").UpdateOne(ctx, bson.M{"itemId": original.id}, bson.M{"$set": bson.M{"state": "pending"}})
|
|
must(err)
|
|
r = call(0, "POST", "/api/plugins/annotation-qa/random-claims", map[string]any{"projectId": project, "requestId": fmt.Sprintf("mongo-buffer-%02d-refill-after-review", round)})
|
|
if r.Code != 200 {
|
|
t.Fatalf("refill after handoff: %d %s", r.Code, r.Body.String())
|
|
}
|
|
r = call(1, "POST", "/api/plugins/annotation-qa/random-claims", map[string]any{"projectId": project, "requestId": original.requestID})
|
|
var recovered struct {
|
|
Item domain.HumanItemRecord `json:"item"`
|
|
Replayed bool `json:"replayed"`
|
|
}
|
|
must(json.Unmarshal(r.Body.Bytes(), &recovered))
|
|
if r.Code != 200 || !recovered.Replayed || recovered.Item.ID != original.id || recovered.Item.CurrentStageID != "quality_review" {
|
|
t.Fatalf("interrupted receipt recovery after handoff: %d %s", r.Code, r.Body.String())
|
|
}
|
|
r = call(0, "GET", bufferPath, nil)
|
|
must(json.Unmarshal(r.Body.Bytes(), &buffer))
|
|
if r.Code != 200 || buffer.Occupied != 2 || len(buffer.Items) != 2 {
|
|
t.Fatalf("receipt replay consumed capacity: %d %s", r.Code, r.Body.String())
|
|
}
|
|
// Fresh process-local caches after recovery must not change summaries.
|
|
handlers[1] = NewHandler(store.NewMongoStore(db), nil)
|
|
checkSummary(1)
|
|
|
|
}
|
|
}
|