forked from longtao.wu/droboticflow
255 lines
13 KiB
Go
255 lines
13 KiB
Go
package store
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"os"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"droboticflow/internal/domain"
|
|
"go.mongodb.org/mongo-driver/bson"
|
|
"go.mongodb.org/mongo-driver/mongo"
|
|
"go.mongodb.org/mongo-driver/mongo/options"
|
|
)
|
|
|
|
func deletedAnnotatorStoreContract(t *testing.T, s Store) {
|
|
t.Helper()
|
|
ctx := t.Context()
|
|
org, supplier, actor := "deletion-test-org", "deletion-test-supplier", "recipient"
|
|
must := func(err error) {
|
|
t.Helper()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
must(s.UpsertOrganization(ctx, domain.OrganizationRecord{ID: org, Status: domain.OrganizationStatusActive}))
|
|
must(s.UpsertSupplier(ctx, domain.SupplierRecord{ID: supplier, OrganizationID: org, Status: "active"}))
|
|
must(s.UpsertUser(ctx, domain.UserRecord{ID: actor, OrganizationID: org, SupplierID: supplier, Role: domain.UserRoleAnnotator, Status: domain.UserStatusActive}))
|
|
var commits []HumanClaimCommit
|
|
for i := 0; i < 2; i++ {
|
|
project := fmt.Sprintf("project-%d", i)
|
|
taskID := fmt.Sprintf("task-%d", i)
|
|
id := fmt.Sprintf("item-%d", i)
|
|
must(s.UpsertProject(ctx, domain.ProjectRecord{ID: project, OrganizationID: org}))
|
|
must(s.UpsertProjectMember(ctx, domain.ProjectMemberRecord{ID: "member-" + project, OrganizationID: org, ProjectID: project, UserID: actor, SupplierID: supplier, Roles: []domain.AnnotationAssignmentRole{domain.AnnotationRoleAnnotator}}))
|
|
batch := domain.HumanBatchRecord{ID: "batch-" + project, OrganizationID: org, ProjectID: project, PluginID: "annotation-qa", Status: domain.HumanBatchActive, Stages: domain.DefaultFanyuV58HumanStages()}
|
|
must(s.UpsertHumanBatch(ctx, batch))
|
|
must(s.UpsertAnnotationTask(ctx, domain.AnnotationTaskRecord{ID: taskID, OrganizationID: org, ProjectID: project, SupplierID: supplier, Status: domain.AnnotationTaskStatusRework, CurrentRevision: 1}))
|
|
completeSourceAdmissionTestProof(t, s, domain.AnnotationTaskRecord{ID: taskID, OrganizationID: org, ProjectID: project})
|
|
item := domain.HumanItemRecord{ID: id, OrganizationID: org, ProjectID: project, SupplierID: supplier, BatchID: batch.ID, LegacyTaskID: taskID, PluginID: "annotation-qa", Status: domain.HumanItemRework, CurrentStageID: domain.FanyuV58StageAnnotationCombined, Version: 1, ReviewRound: 1, CurrentRevision: 1, InheritedRework: true, Subject: domain.HumanSubjectRef{ItemKey: id, Payload: map[string]any{"annotationDraft": map[string]any{"userId": "deleted", "marker": "old private draft"}}}}
|
|
must(s.UpsertHumanItem(ctx, item))
|
|
must(s.UpsertReworkBaseline(ctx, domain.ReworkBaselineRecord{ID: ReworkBaselineID(id, 1), OrganizationID: org, ProjectID: project, ItemID: id, Round: 1, Slices: []domain.SliceSegment{{ID: "s", Start: 0, End: 10}}, Actions: []domain.AnnotationSegment{{ID: "a", Start: 0, End: 10, Description: "正式内容"}}, Problems: []domain.ReviewProblem{{ID: "opinion", Text: "保留质检意见"}}}))
|
|
item.AssignedUserID = actor
|
|
item.Status = domain.HumanItemClaimed
|
|
item.Version++
|
|
commits = append(commits, HumanClaimCommit{Receipt: HumanClaimReceipt{ID: "claim-" + id, ItemID: id, UserID: actor, ProjectID: project, OrganizationID: org}, Item: item, ExpectedVersion: 1, TaskStatus: domain.AnnotationTaskStatusRework, Attempt: domain.HumanStageAttemptRecord{OrganizationID: org, ItemID: id, StageID: item.CurrentStageID, Round: 1, Status: domain.HumanItemClaimed, AssigneeUserID: actor}, Event: domain.HumanEventRecord{ID: "event-" + id, OrganizationID: org, ItemID: id, EventType: "test-handoff"}})
|
|
}
|
|
start := make(chan struct{})
|
|
var wg sync.WaitGroup
|
|
errs := make([]error, 2)
|
|
for i := range commits {
|
|
wg.Add(1)
|
|
go func(i int) { defer wg.Done(); <-start; _, errs[i] = s.CommitHumanClaim(ctx, commits[i]) }(i)
|
|
}
|
|
close(start)
|
|
wg.Wait()
|
|
winner, loser := -1, -1
|
|
for i, err := range errs {
|
|
if err == nil {
|
|
winner = i
|
|
} else if errors.Is(err, ErrAnnotationBufferFull) {
|
|
loser = i
|
|
} else {
|
|
t.Fatal("unexpected concurrent claim", err)
|
|
}
|
|
}
|
|
if winner < 0 || loser < 0 {
|
|
t.Fatalf("global cross-project inherited cap failed: %v", errs)
|
|
}
|
|
item, err := s.GetHumanItem(ctx, commits[winner].Item.ID)
|
|
must(err)
|
|
if item.Subject.Payload["annotationDraft"] != nil {
|
|
t.Fatal("private old draft carried into claim")
|
|
}
|
|
if _, err := s.CommitHumanClaim(ctx, commits[winner]); err != nil {
|
|
t.Fatal("restart/replay allocated or rejected completed assignment", err)
|
|
}
|
|
item.CurrentStageID = domain.FanyuV58StageQualityReview
|
|
item.AssignedUserID = ""
|
|
item.Version++
|
|
must(s.UpsertHumanItem(ctx, *item))
|
|
if _, err := s.CommitHumanClaim(ctx, commits[loser]); err != nil {
|
|
t.Fatal("released inherited slot unavailable", err)
|
|
}
|
|
|
|
// Race the publication itself, not merely the final HumanItem CAS. Either
|
|
// the old write precedes reset and is superseded, or its old epoch is refused.
|
|
for i := 0; i < 4; i++ {
|
|
task := domain.AnnotationTaskRecord{ID: fmt.Sprintf("race-task-%d", i), OrganizationID: org, ProjectID: "project-0", SupplierID: supplier}
|
|
must(s.UpsertAnnotationTask(ctx, task))
|
|
reset := task
|
|
reset.AnnotationEpoch = 1
|
|
start := make(chan struct{})
|
|
var wg sync.WaitGroup
|
|
var writeErr, resetErr error
|
|
wg.Add(2)
|
|
go func() {
|
|
defer wg.Done()
|
|
<-start
|
|
writeErr = s.UpsertAnnotationRevision(ctx, domain.AnnotationRevisionRecord{ID: "late-" + task.ID, OrganizationID: org, TaskID: task.ID, Revision: 99})
|
|
}()
|
|
go func() { defer wg.Done(); <-start; resetErr = s.ResetDeletedAnnotatorTask(ctx, reset) }()
|
|
close(start)
|
|
wg.Wait()
|
|
must(resetErr)
|
|
if writeErr != nil && !errors.Is(writeErr, ErrHumanClaimConflict) {
|
|
t.Fatal(writeErr)
|
|
}
|
|
visible, err := s.ListAnnotationRevisions(ctx, org, task.ID)
|
|
must(err)
|
|
if len(visible) != 0 {
|
|
t.Fatal("racing old write survived reset", visible)
|
|
}
|
|
current, err := s.GetAnnotationTask(ctx, task.ID)
|
|
must(err)
|
|
if writeErr == nil && current.AnnotationRevisionFloor < 99 {
|
|
t.Fatal("reset reused a published revision number")
|
|
}
|
|
must(s.UpsertAnnotationRevision(ctx, domain.AnnotationRevisionRecord{ID: "new-" + task.ID, OrganizationID: org, TaskID: task.ID, AnnotationEpoch: 1, Revision: current.AnnotationRevisionFloor + 1}))
|
|
}
|
|
|
|
old := domain.AnnotationTaskRecord{ID: "reset-task", OrganizationID: org, ProjectID: "project-0", SupplierID: supplier}
|
|
must(s.UpsertAnnotationTask(ctx, old))
|
|
must(s.UpsertSliceRevision(ctx, domain.SliceRevisionRecord{ID: "old-slice", OrganizationID: org, TaskID: old.ID, Revision: 3}))
|
|
must(s.UpsertAnnotationRevision(ctx, domain.AnnotationRevisionRecord{ID: "old-annotation", OrganizationID: org, TaskID: old.ID, Revision: 3}))
|
|
reset := old
|
|
reset.AnnotationEpoch = 1
|
|
reset.AnnotationRevisionFloor = 3
|
|
must(s.ResetDeletedAnnotatorTask(ctx, reset))
|
|
must(s.ResetDeletedAnnotatorTask(ctx, reset))
|
|
if err := s.UpsertAnnotationTask(ctx, old); !errors.Is(err, ErrHumanClaimConflict) {
|
|
t.Fatal("old task epoch admitted", err)
|
|
}
|
|
if err := s.UpsertSliceRevision(ctx, domain.SliceRevisionRecord{ID: "late-old", OrganizationID: org, TaskID: old.ID, Revision: 3}); !errors.Is(err, ErrHumanClaimConflict) {
|
|
t.Fatal("late old slice admitted", err)
|
|
}
|
|
if err := s.UpsertAnnotationRevision(ctx, domain.AnnotationRevisionRecord{ID: "old-annotation", OrganizationID: org, TaskID: old.ID, Revision: 3}); !errors.Is(err, ErrHumanClaimConflict) {
|
|
t.Fatal("old annotation resurrected", err)
|
|
}
|
|
if err := s.UpsertSliceRevision(ctx, domain.SliceRevisionRecord{ID: "late-higher-slice", OrganizationID: org, TaskID: old.ID, Revision: 99}); !errors.Is(err, ErrHumanClaimConflict) {
|
|
t.Fatal("late higher revision from old epoch admitted", err)
|
|
}
|
|
if err := s.UpsertAnnotationRevision(ctx, domain.AnnotationRevisionRecord{ID: "late-higher-annotation", OrganizationID: org, TaskID: old.ID, Revision: 99}); !errors.Is(err, ErrHumanClaimConflict) {
|
|
t.Fatal("late higher annotation from old epoch admitted", err)
|
|
}
|
|
slices, err := s.ListSliceRevisions(ctx, org, old.ID)
|
|
must(err)
|
|
annotations, err := s.ListAnnotationRevisions(ctx, org, old.ID)
|
|
must(err)
|
|
if len(slices) != 0 || len(annotations) != 0 {
|
|
t.Fatal("reset revisions visible to next worker")
|
|
}
|
|
progressSlices, err := s.ListLatestSliceProgressByTaskIDs(ctx, org, []string{old.ID})
|
|
must(err)
|
|
progressActions, err := s.ListLatestAnnotationProgressByTaskIDs(ctx, org, []string{old.ID})
|
|
must(err)
|
|
if len(progressSlices) != 0 || len(progressActions) != 0 {
|
|
t.Fatal("reset history leaked into progress summaries")
|
|
}
|
|
must(s.UpsertSliceRevision(ctx, domain.SliceRevisionRecord{ID: "new-slice", OrganizationID: org, TaskID: old.ID, AnnotationEpoch: 1, Revision: 4}))
|
|
slices, err = s.ListSliceRevisions(ctx, org, old.ID)
|
|
must(err)
|
|
if len(slices) != 1 || slices[0].Revision != 4 {
|
|
t.Fatal("new epoch cannot publish")
|
|
}
|
|
newTask := reset
|
|
newTask.AssignedUserID = "next-annotator"
|
|
newTask.CurrentRevision = 4
|
|
must(s.UpsertAnnotationTask(ctx, newTask))
|
|
// Simulate a slow second replica resuming the old reset after release.
|
|
must(s.ResetDeletedAnnotatorTask(ctx, reset))
|
|
afterReplay, err := s.GetAnnotationTask(ctx, old.ID)
|
|
must(err)
|
|
if afterReplay.AssignedUserID != newTask.AssignedUserID || afterReplay.CurrentRevision != 4 {
|
|
t.Fatal("late recovery rewrote new task projection")
|
|
}
|
|
slices, err = s.ListSliceRevisions(ctx, org, old.ID)
|
|
must(err)
|
|
if len(slices) != 1 || slices[0].Revision != 4 {
|
|
t.Fatal("late recovery superseded next annotator's revision")
|
|
}
|
|
|
|
}
|
|
|
|
func TestDeletedAnnotatorStoreMemory(t *testing.T) {
|
|
deletedAnnotatorStoreContract(t, NewMemoryStore())
|
|
}
|
|
|
|
func TestDeletedAnnotatorStoreMongo(t *testing.T) {
|
|
if os.Getenv("DELETED_ANNOTATOR_LOCAL_MONGO") != "1" {
|
|
t.Skip("isolated local Mongo opt-in")
|
|
}
|
|
ctx, cancel := context.WithTimeout(t.Context(), 90*time.Second)
|
|
defer cancel()
|
|
client, err := mongo.Connect(ctx, options.Client().ApplyURI("mongodb://127.0.0.1:27018/?directConnection=true"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer client.Disconnect(context.Background())
|
|
db := client.Database(fmt.Sprintf("deleted_annotator_test_%d", time.Now().UnixNano()))
|
|
defer db.Drop(context.Background())
|
|
s := NewMongoStore(db)
|
|
for _, name := range []string{"organizations", "projects", "users", "suppliers", "projectMembers", "annotationTasks", "annotationRevisions", "sliceRevisions", "humanItems", "humanBatches", "humanEvents", "humanStageAttempts", "humanClaimRequests"} {
|
|
if _, err := db.Collection(name).Indexes().CreateOne(ctx, uniqueIndex("id_unique", bson.D{{Key: "id", Value: 1}})); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
t.Setenv("DROBOTICFLOW_PROJECT_WRITE_LOCK_MODE", "striped")
|
|
deletedAnnotatorStoreContract(t, s)
|
|
q := domain.HumanItemQuery{NeedsReworkRouting: true}
|
|
if _, err := s.ListHumanItems(ctx, "deletion-test-org", q); err == nil {
|
|
t.Fatal("background routing scanned without its indexes")
|
|
}
|
|
for _, index := range []mongo.IndexModel{
|
|
{Keys: bson.D{{Key: "organizationId", Value: 1}, {Key: "currentStageId", Value: 1}, {Key: "reworkRoutingPending", Value: 1}, {Key: "status", Value: 1}}, Options: options.Index().SetName("deleted_author_routing_v1")},
|
|
{Keys: bson.D{{Key: "organizationId", Value: 1}, {Key: "currentStageId", Value: 1}, {Key: "inheritedRework", Value: 1}, {Key: "assignedUserId", Value: 1}, {Key: "status", Value: 1}}, Options: options.Index().SetName("deleted_author_unassigned_v1")},
|
|
} {
|
|
if _, err := s.humanItems.Indexes().CreateOne(ctx, index); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
noise := make([]any, 1000)
|
|
for i := range noise {
|
|
noise[i] = bson.M{"id": fmt.Sprintf("noise-%d", i), "organizationId": "deletion-test-org", "currentStageId": domain.FanyuV58StageAnnotationCombined, "status": domain.HumanItemClaimed, "assignedUserId": "busy-worker", "inheritedRework": false, "reworkRoutingPending": false}
|
|
}
|
|
if _, err := s.humanItems.InsertMany(ctx, noise); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := s.humanItems.InsertOne(ctx, bson.M{"id": "routing-pending", "organizationId": "deletion-test-org", "currentStageId": domain.FanyuV58StageAnnotationCombined, "status": domain.HumanItemRework, "reworkRoutingPending": true}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
rows, err := s.ListHumanItems(ctx, "deletion-test-org", q)
|
|
if err != nil || len(rows) != 1 {
|
|
t.Fatal("indexed routing inventory changed", err)
|
|
}
|
|
var explain struct {
|
|
ExecutionStats struct {
|
|
NReturned int `bson:"nReturned"`
|
|
Documents int `bson:"totalDocsExamined"`
|
|
Keys int `bson:"totalKeysExamined"`
|
|
} `bson:"executionStats"`
|
|
}
|
|
if err := db.RunCommand(ctx, bson.D{{Key: "explain", Value: bson.D{{Key: "find", Value: "humanItems"}, {Key: "filter", Value: humanItemMongoFilter("deletion-test-org", q)}, {Key: "sort", Value: bson.D{{Key: "createdAt", Value: -1}, {Key: "id", Value: -1}}}}}, {Key: "verbosity", Value: "executionStats"}}).Decode(&explain); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if explain.ExecutionStats.NReturned != 1 || explain.ExecutionStats.Documents > 4 || explain.ExecutionStats.Keys > 20 {
|
|
t.Fatalf("routing lookup scanned unrelated annotation inventory: %+v", explain.ExecutionStats)
|
|
}
|
|
if n, err := s.annotationRevisions.CountDocuments(ctx, bson.M{"id": "old-annotation", "supersededByAccountDeletion": true}); err != nil || n != 1 {
|
|
t.Fatal("logical reset deleted retained evidence", err)
|
|
}
|
|
}
|