127 lines
5.6 KiB
Go
127 lines
5.6 KiB
Go
package store
|
|
|
|
import (
|
|
"context"
|
|
"droboticflow/internal/domain"
|
|
"fmt"
|
|
"go.mongodb.org/mongo-driver/bson"
|
|
"go.mongodb.org/mongo-driver/event"
|
|
"go.mongodb.org/mongo-driver/mongo"
|
|
"go.mongodb.org/mongo-driver/mongo/options"
|
|
"os"
|
|
"strings"
|
|
"sync/atomic"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
// Polls must stay read-only and ignore authored history; capacity changes
|
|
// still persist. Opt-in against an isolated database on the local replica.
|
|
func TestMongoAnnotationWorkBufferPollIsReadOnly(t *testing.T) {
|
|
if os.Getenv("ANNOTATION_BUFFER_MONGO_STORE") != "1" {
|
|
t.Skip("opt-in local Mongo verification")
|
|
}
|
|
ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second)
|
|
defer cancel()
|
|
var batchFinds atomic.Int64
|
|
monitor := &event.CommandMonitor{Started: func(_ context.Context, e *event.CommandStartedEvent) {
|
|
if e.CommandName == "find" && e.Command.Lookup("find").StringValue() == "humanBatches" {
|
|
batchFinds.Add(1)
|
|
}
|
|
}}
|
|
c, err := mongo.Connect(ctx, options.Client().ApplyURI("mongodb://127.0.0.1:27018/?directConnection=true").SetMonitor(monitor))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer c.Disconnect(context.Background())
|
|
db := c.Database(fmt.Sprintf("annotation_buffer_store_test_%d", time.Now().UnixNano()))
|
|
defer db.Drop(context.Background())
|
|
s := NewMongoStore(db)
|
|
must := func(err error) {
|
|
t.Helper()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
org, user, project := "buffer-store-org", "buffer-store-user", "buffer-store-project"
|
|
batch := domain.HumanBatchRecord{ID: "buffer-store-batch", OrganizationID: org, ProjectID: project, WorkflowID: "buffer-store-workflow", PluginID: "annotation-qa", Status: domain.HumanBatchActive, ProcessingProfile: domain.ProcessingProfileRawHuman, Stages: domain.DefaultFanyuV58HumanStages()}
|
|
must(s.UpsertHumanBatch(ctx, batch))
|
|
// An unknown large field represents historical metadata: the buffer must not
|
|
// materialize it just to validate the batch's scope and workflow.
|
|
_, err = db.Collection("humanBatches").UpdateOne(ctx, bson.M{"id": batch.ID}, bson.M{"$set": bson.M{"legacyMetadata": strings.Repeat("x", 1024*1024)}})
|
|
must(err)
|
|
other := batch
|
|
other.ID = "other-batch"
|
|
other.OrganizationID = "other-org"
|
|
must(s.UpsertHumanBatch(ctx, other))
|
|
before := batchFinds.Load()
|
|
projected, err := s.annotationBufferBatches(ctx, []string{batch.ID, batch.ID, other.ID, "missing"})
|
|
must(err)
|
|
if batchFinds.Load()-before != 1 || len(projected) != 3 || projected["missing"] != nil {
|
|
t.Fatalf("expected one deduplicated query including missing references: %+v", projected)
|
|
}
|
|
probe := domain.HumanItemRecord{OrganizationID: org, ProjectID: project, WorkflowID: batch.WorkflowID}
|
|
if !annotationBufferBatchMatches(probe, *projected[batch.ID]) || annotationBufferBatchMatches(probe, *projected[other.ID]) {
|
|
t.Fatal("projected batches changed scope decisions")
|
|
}
|
|
if projected[batch.ID].PluginID != "" {
|
|
t.Fatal("batch query returned unneeded fields")
|
|
}
|
|
|
|
item := func(id, stage, assignee, author string, status domain.HumanItemStatus) domain.HumanItemRecord {
|
|
return domain.HumanItemRecord{ID: id, OrganizationID: org, ProjectID: project, WorkflowID: batch.WorkflowID, PluginID: "annotation-qa", BatchID: batch.ID, LegacyTaskID: "task-" + id, Subject: domain.HumanSubjectRef{ItemKey: id}, CurrentStageID: stage, Status: status, AssignedUserID: assignee, LatestAuthorUserID: author, Version: 1}
|
|
}
|
|
must(s.UpsertHumanItem(ctx, item("active", domain.FanyuV58StageAnnotationCombined, user, user, domain.HumanItemClaimed)))
|
|
for n := 0; n < 20; n++ {
|
|
must(s.UpsertHumanItem(ctx, item(fmt.Sprintf("history-%d", n), "quality_review", "qc", user, domain.HumanItemSubmitted)))
|
|
}
|
|
version := func() int {
|
|
t.Helper()
|
|
var doc struct {
|
|
Version int `bson:"version"`
|
|
}
|
|
must(db.Collection("annotationWorkBuffers").FindOne(ctx, bson.M{"_id": bufferKey(org, user, project)}).Decode(&doc))
|
|
return doc.Version
|
|
}
|
|
first, err := s.AnnotationWorkBuffer(ctx, org, user, project, 0)
|
|
must(err)
|
|
if first.Occupied != 1 || len(first.Items) != 1 || first.Items[0].ID != "active" {
|
|
t.Fatalf("buffer = %+v", first)
|
|
}
|
|
settled := version()
|
|
_, err = db.Collection("humanItems").UpdateOne(ctx, bson.M{"id": "active"}, bson.M{"$set": bson.M{"subject.payload.large": strings.Repeat("x", 1024*1024), "subject.payload.sliceWorkflow": bson.M{"status": "completed", "phase": "annotate", "large": strings.Repeat("y", 1024)}, "processingJob": bson.M{"kind": "slice", "status": "failed", "error": "fixture", "input": bson.M{"large": "private"}}}})
|
|
must(err)
|
|
for n := 0; n < 3; n++ {
|
|
summary, err := s.AnnotationWorkBufferSummary(ctx, org, user, project, 0)
|
|
must(err)
|
|
if summary.Occupied != first.Occupied || len(summary.Items) != 1 {
|
|
t.Fatalf("summary membership changed: %+v", summary)
|
|
}
|
|
got := summary.Items[0]
|
|
if got.ID != "active" || got.Subject.Payload["large"] != nil || got.ProcessingJob == nil || got.ProcessingJob.Input != nil || got.ProcessingJob.Error != "fixture" {
|
|
t.Fatalf("summary projection invalid: %+v", got)
|
|
}
|
|
encoded, err := bson.Marshal(got)
|
|
must(err)
|
|
if len(encoded) > 4096 {
|
|
t.Fatalf("projected item retained payload: %d", len(encoded))
|
|
}
|
|
}
|
|
|
|
for n := 0; n < 3; n++ {
|
|
again, err := s.AnnotationWorkBuffer(ctx, org, user, project, 0)
|
|
must(err)
|
|
if again.Occupied != 1 || again.Capacity != first.Capacity {
|
|
t.Fatalf("poll %d buffer = %+v", n, again)
|
|
}
|
|
}
|
|
if got := version(); got != settled {
|
|
t.Fatalf("polls wrote the ledger: version %d -> %d", settled, got)
|
|
}
|
|
resized, err := s.AnnotationWorkBuffer(ctx, org, user, project, 4)
|
|
must(err)
|
|
if resized.Capacity != 4 || version() != settled+1 {
|
|
t.Fatalf("capacity change not persisted: capacity %d version %d", resized.Capacity, version())
|
|
}
|
|
}
|