droboticflow/internal/store/deleted_annotator_test.go

255 lines
13 KiB
Go

package store
import (
"context"
"errors"
"fmt"
"os"
"sync"
"testing"
"time"
"droboticflow/internal/domain"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
)
func deletedAnnotatorStoreContract(t *testing.T, s Store) {
t.Helper()
ctx := t.Context()
org, supplier, actor := "deletion-test-org", "deletion-test-supplier", "recipient"
must := func(err error) {
t.Helper()
if err != nil {
t.Fatal(err)
}
}
must(s.UpsertOrganization(ctx, domain.OrganizationRecord{ID: org, Status: domain.OrganizationStatusActive}))
must(s.UpsertSupplier(ctx, domain.SupplierRecord{ID: supplier, OrganizationID: org, Status: "active"}))
must(s.UpsertUser(ctx, domain.UserRecord{ID: actor, OrganizationID: org, SupplierID: supplier, Role: domain.UserRoleAnnotator, Status: domain.UserStatusActive}))
var commits []HumanClaimCommit
for i := 0; i < 2; i++ {
project := fmt.Sprintf("project-%d", i)
taskID := fmt.Sprintf("task-%d", i)
id := fmt.Sprintf("item-%d", i)
must(s.UpsertProject(ctx, domain.ProjectRecord{ID: project, OrganizationID: org}))
must(s.UpsertProjectMember(ctx, domain.ProjectMemberRecord{ID: "member-" + project, OrganizationID: org, ProjectID: project, UserID: actor, SupplierID: supplier, Roles: []domain.AnnotationAssignmentRole{domain.AnnotationRoleAnnotator}}))
batch := domain.HumanBatchRecord{ID: "batch-" + project, OrganizationID: org, ProjectID: project, PluginID: "annotation-qa", Status: domain.HumanBatchActive, Stages: domain.DefaultFanyuV58HumanStages()}
must(s.UpsertHumanBatch(ctx, batch))
must(s.UpsertAnnotationTask(ctx, domain.AnnotationTaskRecord{ID: taskID, OrganizationID: org, ProjectID: project, SupplierID: supplier, Status: domain.AnnotationTaskStatusRework, CurrentRevision: 1}))
completeSourceAdmissionTestProof(t, s, domain.AnnotationTaskRecord{ID: taskID, OrganizationID: org, ProjectID: project})
item := domain.HumanItemRecord{ID: id, OrganizationID: org, ProjectID: project, SupplierID: supplier, BatchID: batch.ID, LegacyTaskID: taskID, PluginID: "annotation-qa", Status: domain.HumanItemRework, CurrentStageID: domain.FanyuV58StageAnnotationCombined, Version: 1, ReviewRound: 1, CurrentRevision: 1, InheritedRework: true, Subject: domain.HumanSubjectRef{ItemKey: id, Payload: map[string]any{"annotationDraft": map[string]any{"userId": "deleted", "marker": "old private draft"}}}}
must(s.UpsertHumanItem(ctx, item))
must(s.UpsertReworkBaseline(ctx, domain.ReworkBaselineRecord{ID: ReworkBaselineID(id, 1), OrganizationID: org, ProjectID: project, ItemID: id, Round: 1, Slices: []domain.SliceSegment{{ID: "s", Start: 0, End: 10}}, Actions: []domain.AnnotationSegment{{ID: "a", Start: 0, End: 10, Description: "正式内容"}}, Problems: []domain.ReviewProblem{{ID: "opinion", Text: "保留质检意见"}}}))
item.AssignedUserID = actor
item.Status = domain.HumanItemClaimed
item.Version++
commits = append(commits, HumanClaimCommit{Receipt: HumanClaimReceipt{ID: "claim-" + id, ItemID: id, UserID: actor, ProjectID: project, OrganizationID: org}, Item: item, ExpectedVersion: 1, TaskStatus: domain.AnnotationTaskStatusRework, Attempt: domain.HumanStageAttemptRecord{OrganizationID: org, ItemID: id, StageID: item.CurrentStageID, Round: 1, Status: domain.HumanItemClaimed, AssigneeUserID: actor}, Event: domain.HumanEventRecord{ID: "event-" + id, OrganizationID: org, ItemID: id, EventType: "test-handoff"}})
}
start := make(chan struct{})
var wg sync.WaitGroup
errs := make([]error, 2)
for i := range commits {
wg.Add(1)
go func(i int) { defer wg.Done(); <-start; _, errs[i] = s.CommitHumanClaim(ctx, commits[i]) }(i)
}
close(start)
wg.Wait()
winner, loser := -1, -1
for i, err := range errs {
if err == nil {
winner = i
} else if errors.Is(err, ErrAnnotationBufferFull) {
loser = i
} else {
t.Fatal("unexpected concurrent claim", err)
}
}
if winner < 0 || loser < 0 {
t.Fatalf("global cross-project inherited cap failed: %v", errs)
}
item, err := s.GetHumanItem(ctx, commits[winner].Item.ID)
must(err)
if item.Subject.Payload["annotationDraft"] != nil {
t.Fatal("private old draft carried into claim")
}
if _, err := s.CommitHumanClaim(ctx, commits[winner]); err != nil {
t.Fatal("restart/replay allocated or rejected completed assignment", err)
}
item.CurrentStageID = domain.FanyuV58StageQualityReview
item.AssignedUserID = ""
item.Version++
must(s.UpsertHumanItem(ctx, *item))
if _, err := s.CommitHumanClaim(ctx, commits[loser]); err != nil {
t.Fatal("released inherited slot unavailable", err)
}
// Race the publication itself, not merely the final HumanItem CAS. Either
// the old write precedes reset and is superseded, or its old epoch is refused.
for i := 0; i < 4; i++ {
task := domain.AnnotationTaskRecord{ID: fmt.Sprintf("race-task-%d", i), OrganizationID: org, ProjectID: "project-0", SupplierID: supplier}
must(s.UpsertAnnotationTask(ctx, task))
reset := task
reset.AnnotationEpoch = 1
start := make(chan struct{})
var wg sync.WaitGroup
var writeErr, resetErr error
wg.Add(2)
go func() {
defer wg.Done()
<-start
writeErr = s.UpsertAnnotationRevision(ctx, domain.AnnotationRevisionRecord{ID: "late-" + task.ID, OrganizationID: org, TaskID: task.ID, Revision: 99})
}()
go func() { defer wg.Done(); <-start; resetErr = s.ResetDeletedAnnotatorTask(ctx, reset) }()
close(start)
wg.Wait()
must(resetErr)
if writeErr != nil && !errors.Is(writeErr, ErrHumanClaimConflict) {
t.Fatal(writeErr)
}
visible, err := s.ListAnnotationRevisions(ctx, org, task.ID)
must(err)
if len(visible) != 0 {
t.Fatal("racing old write survived reset", visible)
}
current, err := s.GetAnnotationTask(ctx, task.ID)
must(err)
if writeErr == nil && current.AnnotationRevisionFloor < 99 {
t.Fatal("reset reused a published revision number")
}
must(s.UpsertAnnotationRevision(ctx, domain.AnnotationRevisionRecord{ID: "new-" + task.ID, OrganizationID: org, TaskID: task.ID, AnnotationEpoch: 1, Revision: current.AnnotationRevisionFloor + 1}))
}
old := domain.AnnotationTaskRecord{ID: "reset-task", OrganizationID: org, ProjectID: "project-0", SupplierID: supplier}
must(s.UpsertAnnotationTask(ctx, old))
must(s.UpsertSliceRevision(ctx, domain.SliceRevisionRecord{ID: "old-slice", OrganizationID: org, TaskID: old.ID, Revision: 3}))
must(s.UpsertAnnotationRevision(ctx, domain.AnnotationRevisionRecord{ID: "old-annotation", OrganizationID: org, TaskID: old.ID, Revision: 3}))
reset := old
reset.AnnotationEpoch = 1
reset.AnnotationRevisionFloor = 3
must(s.ResetDeletedAnnotatorTask(ctx, reset))
must(s.ResetDeletedAnnotatorTask(ctx, reset))
if err := s.UpsertAnnotationTask(ctx, old); !errors.Is(err, ErrHumanClaimConflict) {
t.Fatal("old task epoch admitted", err)
}
if err := s.UpsertSliceRevision(ctx, domain.SliceRevisionRecord{ID: "late-old", OrganizationID: org, TaskID: old.ID, Revision: 3}); !errors.Is(err, ErrHumanClaimConflict) {
t.Fatal("late old slice admitted", err)
}
if err := s.UpsertAnnotationRevision(ctx, domain.AnnotationRevisionRecord{ID: "old-annotation", OrganizationID: org, TaskID: old.ID, Revision: 3}); !errors.Is(err, ErrHumanClaimConflict) {
t.Fatal("old annotation resurrected", err)
}
if err := s.UpsertSliceRevision(ctx, domain.SliceRevisionRecord{ID: "late-higher-slice", OrganizationID: org, TaskID: old.ID, Revision: 99}); !errors.Is(err, ErrHumanClaimConflict) {
t.Fatal("late higher revision from old epoch admitted", err)
}
if err := s.UpsertAnnotationRevision(ctx, domain.AnnotationRevisionRecord{ID: "late-higher-annotation", OrganizationID: org, TaskID: old.ID, Revision: 99}); !errors.Is(err, ErrHumanClaimConflict) {
t.Fatal("late higher annotation from old epoch admitted", err)
}
slices, err := s.ListSliceRevisions(ctx, org, old.ID)
must(err)
annotations, err := s.ListAnnotationRevisions(ctx, org, old.ID)
must(err)
if len(slices) != 0 || len(annotations) != 0 {
t.Fatal("reset revisions visible to next worker")
}
progressSlices, err := s.ListLatestSliceProgressByTaskIDs(ctx, org, []string{old.ID})
must(err)
progressActions, err := s.ListLatestAnnotationProgressByTaskIDs(ctx, org, []string{old.ID})
must(err)
if len(progressSlices) != 0 || len(progressActions) != 0 {
t.Fatal("reset history leaked into progress summaries")
}
must(s.UpsertSliceRevision(ctx, domain.SliceRevisionRecord{ID: "new-slice", OrganizationID: org, TaskID: old.ID, AnnotationEpoch: 1, Revision: 4}))
slices, err = s.ListSliceRevisions(ctx, org, old.ID)
must(err)
if len(slices) != 1 || slices[0].Revision != 4 {
t.Fatal("new epoch cannot publish")
}
newTask := reset
newTask.AssignedUserID = "next-annotator"
newTask.CurrentRevision = 4
must(s.UpsertAnnotationTask(ctx, newTask))
// Simulate a slow second replica resuming the old reset after release.
must(s.ResetDeletedAnnotatorTask(ctx, reset))
afterReplay, err := s.GetAnnotationTask(ctx, old.ID)
must(err)
if afterReplay.AssignedUserID != newTask.AssignedUserID || afterReplay.CurrentRevision != 4 {
t.Fatal("late recovery rewrote new task projection")
}
slices, err = s.ListSliceRevisions(ctx, org, old.ID)
must(err)
if len(slices) != 1 || slices[0].Revision != 4 {
t.Fatal("late recovery superseded next annotator's revision")
}
}
func TestDeletedAnnotatorStoreMemory(t *testing.T) {
deletedAnnotatorStoreContract(t, NewMemoryStore())
}
func TestDeletedAnnotatorStoreMongo(t *testing.T) {
if os.Getenv("DELETED_ANNOTATOR_LOCAL_MONGO") != "1" {
t.Skip("isolated local Mongo opt-in")
}
ctx, cancel := context.WithTimeout(t.Context(), 90*time.Second)
defer cancel()
client, err := mongo.Connect(ctx, options.Client().ApplyURI("mongodb://127.0.0.1:27018/?directConnection=true"))
if err != nil {
t.Fatal(err)
}
defer client.Disconnect(context.Background())
db := client.Database(fmt.Sprintf("deleted_annotator_test_%d", time.Now().UnixNano()))
defer db.Drop(context.Background())
s := NewMongoStore(db)
for _, name := range []string{"organizations", "projects", "users", "suppliers", "projectMembers", "annotationTasks", "annotationRevisions", "sliceRevisions", "humanItems", "humanBatches", "humanEvents", "humanStageAttempts", "humanClaimRequests"} {
if _, err := db.Collection(name).Indexes().CreateOne(ctx, uniqueIndex("id_unique", bson.D{{Key: "id", Value: 1}})); err != nil {
t.Fatal(err)
}
}
t.Setenv("DROBOTICFLOW_PROJECT_WRITE_LOCK_MODE", "striped")
deletedAnnotatorStoreContract(t, s)
q := domain.HumanItemQuery{NeedsReworkRouting: true}
if _, err := s.ListHumanItems(ctx, "deletion-test-org", q); err == nil {
t.Fatal("background routing scanned without its indexes")
}
for _, index := range []mongo.IndexModel{
{Keys: bson.D{{Key: "organizationId", Value: 1}, {Key: "currentStageId", Value: 1}, {Key: "reworkRoutingPending", Value: 1}, {Key: "status", Value: 1}}, Options: options.Index().SetName("deleted_author_routing_v1")},
{Keys: bson.D{{Key: "organizationId", Value: 1}, {Key: "currentStageId", Value: 1}, {Key: "inheritedRework", Value: 1}, {Key: "assignedUserId", Value: 1}, {Key: "status", Value: 1}}, Options: options.Index().SetName("deleted_author_unassigned_v1")},
} {
if _, err := s.humanItems.Indexes().CreateOne(ctx, index); err != nil {
t.Fatal(err)
}
}
noise := make([]any, 1000)
for i := range noise {
noise[i] = bson.M{"id": fmt.Sprintf("noise-%d", i), "organizationId": "deletion-test-org", "currentStageId": domain.FanyuV58StageAnnotationCombined, "status": domain.HumanItemClaimed, "assignedUserId": "busy-worker", "inheritedRework": false, "reworkRoutingPending": false}
}
if _, err := s.humanItems.InsertMany(ctx, noise); err != nil {
t.Fatal(err)
}
if _, err := s.humanItems.InsertOne(ctx, bson.M{"id": "routing-pending", "organizationId": "deletion-test-org", "currentStageId": domain.FanyuV58StageAnnotationCombined, "status": domain.HumanItemRework, "reworkRoutingPending": true}); err != nil {
t.Fatal(err)
}
rows, err := s.ListHumanItems(ctx, "deletion-test-org", q)
if err != nil || len(rows) != 1 {
t.Fatal("indexed routing inventory changed", err)
}
var explain struct {
ExecutionStats struct {
NReturned int `bson:"nReturned"`
Documents int `bson:"totalDocsExamined"`
Keys int `bson:"totalKeysExamined"`
} `bson:"executionStats"`
}
if err := db.RunCommand(ctx, bson.D{{Key: "explain", Value: bson.D{{Key: "find", Value: "humanItems"}, {Key: "filter", Value: humanItemMongoFilter("deletion-test-org", q)}, {Key: "sort", Value: bson.D{{Key: "createdAt", Value: -1}, {Key: "id", Value: -1}}}}}, {Key: "verbosity", Value: "executionStats"}}).Decode(&explain); err != nil {
t.Fatal(err)
}
if explain.ExecutionStats.NReturned != 1 || explain.ExecutionStats.Documents > 4 || explain.ExecutionStats.Keys > 20 {
t.Fatalf("routing lookup scanned unrelated annotation inventory: %+v", explain.ExecutionStats)
}
if n, err := s.annotationRevisions.CountDocuments(ctx, bson.M{"id": "old-annotation", "supersededByAccountDeletion": true}); err != nil || n != 1 {
t.Fatal("logical reset deleted retained evidence", err)
}
}