forked from longtao.wu/droboticflow
283 lines
10 KiB
Go
283 lines
10 KiB
Go
package store
|
|
|
|
import (
|
|
"context"
|
|
"droboticflow/internal/domain"
|
|
"fmt"
|
|
"go.mongodb.org/mongo-driver/bson"
|
|
"go.mongodb.org/mongo-driver/mongo"
|
|
"go.mongodb.org/mongo-driver/mongo/options"
|
|
"net/url"
|
|
"os"
|
|
"reflect"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
func TestAdminAnnotationPoolMongoParity(t *testing.T) {
|
|
uri := os.Getenv("POOL_LOCAL_MONGO_URI")
|
|
if uri == "" {
|
|
t.Skip("isolated local Mongo opt-in")
|
|
}
|
|
u, err := url.Parse(uri)
|
|
if err != nil || u.Scheme != "mongodb" || u.Host != "127.0.0.1:27018" {
|
|
t.Fatal("local Mongo 27018 only")
|
|
}
|
|
ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second)
|
|
defer cancel()
|
|
client, err := mongo.Connect(ctx, options.Client().ApplyURI(uri))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer client.Disconnect(context.Background())
|
|
db := client.Database(fmt.Sprintf("admin_pool_test_%d", time.Now().UnixNano()))
|
|
defer db.Drop(context.Background())
|
|
s := NewMongoStore(db)
|
|
if err := s.EnsureIndexes(ctx); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
mem := NewMemoryStore()
|
|
const org = "pool-org"
|
|
q := AdminAnnotationPoolQuery{ProjectID: "project", UserID: "admin", Pool: "available", Limit: 201}
|
|
batch := domain.HumanBatchRecord{ID: "batch", OrganizationID: org, ProjectID: q.ProjectID, PluginID: "annotation-qa", Stages: domain.DefaultFanyuV58HumanStages()}
|
|
_ = mem.UpsertHumanBatch(ctx, batch)
|
|
_ = s.UpsertHumanBatch(ctx, batch)
|
|
items, tasks, batchDocs := []any{}, []any{}, []any{}
|
|
count := 1500
|
|
if os.Getenv("ADMIN_POOL_SCALE") == "1" {
|
|
count = 26218
|
|
}
|
|
for n := 0; n < count; n++ {
|
|
i := domain.HumanItemRecord{ID: fmt.Sprintf("i-%04d", n), OrganizationID: org, ProjectID: q.ProjectID, PluginID: "annotation-qa", BatchID: batch.ID, CurrentStageID: "annotation_combined", Status: domain.HumanItemReady, LegacyTaskID: fmt.Sprintf("t-%04d", n), CreatedAt: fmt.Sprintf("2026-10-21T%02d:%02d:%02dZ", n/3600, n/60%60, n%60), Subject: domain.HumanSubjectRef{ItemKey: fmt.Sprintf("CAPTURE_%04d_camera_left_part0001.mp4", n), Payload: map[string]any{"sliceWorkflow": map[string]any{"status": "editing", "phase": "slice"}, "annotationWorkspace": map[string]any{"description": strings.Repeat("x", 8192)}}}}
|
|
b := batch
|
|
b.ID = fmt.Sprintf("batch-%04d", n)
|
|
b.RunID = b.ID
|
|
batchDocs = append(batchDocs, b)
|
|
_ = mem.UpsertHumanBatch(ctx, b)
|
|
i.BatchID = b.ID
|
|
if n%7 == 0 {
|
|
i.AssignedUserID = "admin"
|
|
i.Status = domain.HumanItemClaimed
|
|
}
|
|
if n%11 == 0 {
|
|
i.CurrentStageID = "quality_review"
|
|
}
|
|
if n == 4 {
|
|
i.RunID = "custom-run"
|
|
}
|
|
if n == 1 {
|
|
i.BatchID = "missing"
|
|
}
|
|
if n == 2 {
|
|
i.OrganizationID = "other-org"
|
|
}
|
|
task := domain.AnnotationTaskRecord{ID: i.LegacyTaskID, OrganizationID: i.OrganizationID, ProjectID: i.ProjectID, Status: domain.AnnotationTaskStatusPendingAnnotation, SourceVideo: i.Subject.ItemKey}
|
|
items = append(items, i)
|
|
tasks = append(tasks, task)
|
|
_ = mem.UpsertHumanItem(ctx, i)
|
|
_ = mem.UpsertAnnotationTask(ctx, task)
|
|
}
|
|
if _, err := s.humanBatches.InsertMany(ctx, batchDocs); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := s.humanItems.InsertMany(ctx, items); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := s.annotationTasks.InsertMany(ctx, tasks); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
custom := domain.CaptureRecord{ID: "custom-capture", OrganizationID: org, ProjectID: q.ProjectID, CaptureID: "客户指定Capture"}
|
|
run := domain.RunRecord{ID: "custom-run", OrganizationID: org, ProjectID: q.ProjectID, TaskID: custom.ID}
|
|
if e := s.UpsertCaptureRecord(ctx, custom); e != nil {
|
|
t.Fatal(e)
|
|
}
|
|
_ = mem.UpsertCaptureRecord(ctx, custom)
|
|
if e := s.UpsertRun(ctx, run); e != nil {
|
|
t.Fatal(e)
|
|
}
|
|
_ = mem.UpsertRun(ctx, run)
|
|
|
|
start := time.Now()
|
|
counts, err := s.CountAdminAnnotationPools(ctx, org, q)
|
|
want, _ := mem.CountAdminAnnotationPools(ctx, org, q)
|
|
if err != nil || counts != want {
|
|
t.Fatalf("counts=%+v want=%+v err=%v", counts, want, err)
|
|
}
|
|
t.Logf("%d-item counts: %s %+v", count, time.Since(start), counts)
|
|
for _, scenario := range []struct{ pool, search, category string }{{"available", "", ""}, {"all", "CAPTURE_0003", ""}, {"all", "", "pending_slice"}, {"mine", "", ""}, {"all", "[.*", ""}, {"all", "客户指定Capture", ""}} {
|
|
q.Pool, q.Search, q.Category = scenario.pool, scenario.search, scenario.category
|
|
start = time.Now()
|
|
got, err := s.ListAdminAnnotationPool(ctx, org, q)
|
|
elapsed := time.Since(start)
|
|
expected, _ := mem.ListAdminAnnotationPool(ctx, org, q)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
ids := func(items []domain.HumanItemRecord) []string {
|
|
out := []string{}
|
|
for _, i := range items {
|
|
out = append(out, i.ID)
|
|
if _, ok := i.Subject.Payload["annotationWorkspace"]; ok {
|
|
t.Fatal("heavy workspace leaked")
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
if !reflect.DeepEqual(ids(got), ids(expected)) {
|
|
t.Fatalf("%+v Mongo=%v memory=%v", scenario, ids(got), ids(expected))
|
|
}
|
|
t.Logf("%+v page=%d elapsed=%s", scenario, len(got), elapsed)
|
|
if os.Getenv("ADMIN_POOL_SCALE") == "1" && elapsed > 2*time.Second {
|
|
t.Errorf("administrator pool exceeds 2s at %d rows: %v", count, elapsed)
|
|
}
|
|
if len(got) > 201 {
|
|
t.Fatal("unbounded page")
|
|
}
|
|
if scenario.search == "" && len(got) > 0 {
|
|
last := got[len(got)-1]
|
|
q.Cursor = &domain.HumanItemCursor{CreatedAt: last.CreatedAt, ID: last.ID}
|
|
next, err := s.ListAdminAnnotationPool(ctx, org, q)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for _, i := range next {
|
|
for _, old := range got {
|
|
if i.ID == old.ID {
|
|
t.Fatal("cursor repeated row")
|
|
}
|
|
}
|
|
}
|
|
q.Cursor = nil
|
|
}
|
|
}
|
|
scoped := q
|
|
scoped.Search = ""
|
|
scoped.Category = ""
|
|
scoped.Pool = "all"
|
|
scoped.Capture = "客户指定Capture"
|
|
gotCounts, e := s.CountAdminAnnotationPools(ctx, org, scoped)
|
|
wantCounts, _ := mem.CountAdminAnnotationPools(ctx, org, scoped)
|
|
if e != nil || gotCounts != wantCounts || gotCounts.All != 1 {
|
|
t.Fatalf("capture counts=%+v want=%+v err=%v", gotCounts, wantCounts, e)
|
|
}
|
|
|
|
// Missing routing IDs must not match an arbitrary batch.
|
|
_, err = s.humanItems.InsertOne(ctx, bson.M{"id": "bad", "organizationId": org, "projectId": "project", "pluginId": "annotation-qa", "currentStageId": "annotation_combined", "status": "ready", "createdAt": "z"})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
again, err := s.CountAdminAnnotationPools(ctx, org, q)
|
|
if err != nil || again != counts {
|
|
t.Fatalf("missing batch count=%+v err=%v", again, err)
|
|
}
|
|
}
|
|
|
|
// Batch source joins must preserve failed, missing, foreign and superseded
|
|
// certificates. Compare the existing admission expression on the same records.
|
|
func TestAdminAnnotationPoolMongoAdmissionParity(t *testing.T) {
|
|
uri := os.Getenv("POOL_LOCAL_MONGO_URI")
|
|
if uri == "" {
|
|
t.Skip("isolated local Mongo opt-in")
|
|
}
|
|
u, e := url.Parse(uri)
|
|
if e != nil || u.Host != "127.0.0.1:27018" || u.Scheme != "mongodb" {
|
|
t.Fatal("local Mongo 27018 only")
|
|
}
|
|
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
|
defer cancel()
|
|
client, e := mongo.Connect(ctx, options.Client().ApplyURI(uri))
|
|
if e != nil {
|
|
t.Fatal(e)
|
|
}
|
|
defer client.Disconnect(context.Background())
|
|
db := client.Database(fmt.Sprintf("admin_pool_admission_%d", time.Now().UnixNano()))
|
|
defer db.Drop(context.Background())
|
|
s := NewMongoStore(db)
|
|
if e := s.EnsureIndexes(ctx); e != nil {
|
|
t.Fatal(e)
|
|
}
|
|
const org = "admin-admission"
|
|
batch := domain.HumanBatchRecord{ID: "batch", OrganizationID: org, PluginID: "annotation-qa", Stages: domain.DefaultFanyuV58HumanStages()}
|
|
if e := s.UpsertHumanBatch(ctx, batch); e != nil {
|
|
t.Fatal(e)
|
|
}
|
|
for n := 0; n < 192; n++ {
|
|
id := fmt.Sprintf("case-%03d", n)
|
|
item := domain.HumanItemRecord{ID: id, OrganizationID: org, ProjectID: "project", RunID: "run", PluginID: "annotation-qa", BatchID: batch.ID, CurrentStageID: "annotation_combined", Status: domain.HumanItemReady, LegacyTaskID: id, CreatedAt: id, ProcessingJob: &domain.AnnotationProcessingJob{SourceIndex: n%2 == 0, SourceFingerprint: id}}
|
|
item.Subject.ItemKey = id
|
|
if n%3 == 0 {
|
|
item.Subject.Payload = map[string]any{"materialization": map[string]any{"sourceIndex": true}}
|
|
item.ProcessingJob.SourceFingerprint = ""
|
|
}
|
|
if e := s.UpsertHumanItem(ctx, item); e != nil {
|
|
t.Fatal(e)
|
|
}
|
|
if n%7 != 0 {
|
|
task := domain.AnnotationTaskRecord{ID: id, OrganizationID: org, ProjectID: item.ProjectID, RunID: item.RunID, SourceIsFullVideo: n%2 == 0, TemplateVersion: 6, ArtifactID: "video"}
|
|
if _, e := s.annotationTasks.InsertOne(ctx, task); e != nil {
|
|
t.Fatal(e)
|
|
}
|
|
}
|
|
for k, jobID := range []string{id, "source-v1:" + org + ":" + id} {
|
|
if n%5 == k {
|
|
continue
|
|
}
|
|
job := domain.AnnotationIndexJob{ID: jobID, OrganizationID: org, ProjectID: item.ProjectID, RunID: item.RunID, Status: []string{"completed", "failed", "running"}[n%3]}
|
|
if n%11 == 0 {
|
|
job.OrganizationID = "foreign"
|
|
}
|
|
if k == 0 && n%4 == 0 {
|
|
job.RevalidateSource = true
|
|
job.SourceFingerprint = "repaired"
|
|
job.SupersededSourceFingerprint = id
|
|
}
|
|
if _, e := db.Collection("annotationIndexJobs").InsertOne(ctx, job); e != nil {
|
|
t.Fatal(e)
|
|
}
|
|
}
|
|
}
|
|
legacy := mongo.Pipeline{{{Key: "$match", Value: adminPoolScopeFilter(org, AdminAnnotationPoolQuery{})}}}
|
|
legacy = append(legacy, annotationSourceAdmissionStatusPipeline()...)
|
|
read := func(p mongo.Pipeline) map[string]string {
|
|
t.Helper()
|
|
cur, e := s.humanItems.Aggregate(ctx, p)
|
|
if e != nil {
|
|
t.Fatal(e)
|
|
}
|
|
defer cur.Close(ctx)
|
|
rows, e := decodeMany[adminPoolRow](ctx, cur)
|
|
if e != nil {
|
|
t.Fatal(e)
|
|
}
|
|
out := map[string]string{}
|
|
for _, r := range rows {
|
|
out[r.ID] = r.SourceAdmissionStatus
|
|
}
|
|
return out
|
|
}
|
|
expected := read(legacy)
|
|
got := read(adminPoolPagePipeline(org, AdminAnnotationPoolQuery{Pool: "all"}))
|
|
if len(expected) != 192 || !reflect.DeepEqual(expected, got) {
|
|
t.Fatalf("batch admission changed: expected=%v actual=%v", expected, got)
|
|
}
|
|
available, e := s.ListAdminAnnotationPool(ctx, org, AdminAnnotationPoolQuery{Pool: "available", Limit: 300})
|
|
if e != nil {
|
|
t.Fatal(e)
|
|
}
|
|
ready := 0
|
|
for _, status := range expected {
|
|
if status == "" {
|
|
ready++
|
|
}
|
|
}
|
|
if len(available) != ready {
|
|
t.Fatalf("available=%d expected=%d", len(available), ready)
|
|
}
|
|
for _, item := range available {
|
|
if expected[item.ID] != "" {
|
|
t.Fatalf("unready source exposed: %s", item.ID)
|
|
}
|
|
}
|
|
}
|