droboticflow/internal/store/admin_annotation_pool_test.go
2026-11-10 04:57:09 -08:00

283 lines
10 KiB
Go

package store
import (
"context"
"droboticflow/internal/domain"
"fmt"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
"net/url"
"os"
"reflect"
"strings"
"testing"
"time"
)
func TestAdminAnnotationPoolMongoParity(t *testing.T) {
uri := os.Getenv("POOL_LOCAL_MONGO_URI")
if uri == "" {
t.Skip("isolated local Mongo opt-in")
}
u, err := url.Parse(uri)
if err != nil || u.Scheme != "mongodb" || u.Host != "127.0.0.1:27018" {
t.Fatal("local Mongo 27018 only")
}
ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second)
defer cancel()
client, err := mongo.Connect(ctx, options.Client().ApplyURI(uri))
if err != nil {
t.Fatal(err)
}
defer client.Disconnect(context.Background())
db := client.Database(fmt.Sprintf("admin_pool_test_%d", time.Now().UnixNano()))
defer db.Drop(context.Background())
s := NewMongoStore(db)
if err := s.EnsureIndexes(ctx); err != nil {
t.Fatal(err)
}
mem := NewMemoryStore()
const org = "pool-org"
q := AdminAnnotationPoolQuery{ProjectID: "project", UserID: "admin", Pool: "available", Limit: 201}
batch := domain.HumanBatchRecord{ID: "batch", OrganizationID: org, ProjectID: q.ProjectID, PluginID: "annotation-qa", Stages: domain.DefaultFanyuV58HumanStages()}
_ = mem.UpsertHumanBatch(ctx, batch)
_ = s.UpsertHumanBatch(ctx, batch)
items, tasks, batchDocs := []any{}, []any{}, []any{}
count := 1500
if os.Getenv("ADMIN_POOL_SCALE") == "1" {
count = 26218
}
for n := 0; n < count; n++ {
i := domain.HumanItemRecord{ID: fmt.Sprintf("i-%04d", n), OrganizationID: org, ProjectID: q.ProjectID, PluginID: "annotation-qa", BatchID: batch.ID, CurrentStageID: "annotation_combined", Status: domain.HumanItemReady, LegacyTaskID: fmt.Sprintf("t-%04d", n), CreatedAt: fmt.Sprintf("2026-10-21T%02d:%02d:%02dZ", n/3600, n/60%60, n%60), Subject: domain.HumanSubjectRef{ItemKey: fmt.Sprintf("CAPTURE_%04d_camera_left_part0001.mp4", n), Payload: map[string]any{"sliceWorkflow": map[string]any{"status": "editing", "phase": "slice"}, "annotationWorkspace": map[string]any{"description": strings.Repeat("x", 8192)}}}}
b := batch
b.ID = fmt.Sprintf("batch-%04d", n)
b.RunID = b.ID
batchDocs = append(batchDocs, b)
_ = mem.UpsertHumanBatch(ctx, b)
i.BatchID = b.ID
if n%7 == 0 {
i.AssignedUserID = "admin"
i.Status = domain.HumanItemClaimed
}
if n%11 == 0 {
i.CurrentStageID = "quality_review"
}
if n == 4 {
i.RunID = "custom-run"
}
if n == 1 {
i.BatchID = "missing"
}
if n == 2 {
i.OrganizationID = "other-org"
}
task := domain.AnnotationTaskRecord{ID: i.LegacyTaskID, OrganizationID: i.OrganizationID, ProjectID: i.ProjectID, Status: domain.AnnotationTaskStatusPendingAnnotation, SourceVideo: i.Subject.ItemKey}
items = append(items, i)
tasks = append(tasks, task)
_ = mem.UpsertHumanItem(ctx, i)
_ = mem.UpsertAnnotationTask(ctx, task)
}
if _, err := s.humanBatches.InsertMany(ctx, batchDocs); err != nil {
t.Fatal(err)
}
if _, err := s.humanItems.InsertMany(ctx, items); err != nil {
t.Fatal(err)
}
if _, err := s.annotationTasks.InsertMany(ctx, tasks); err != nil {
t.Fatal(err)
}
custom := domain.CaptureRecord{ID: "custom-capture", OrganizationID: org, ProjectID: q.ProjectID, CaptureID: "客户指定Capture"}
run := domain.RunRecord{ID: "custom-run", OrganizationID: org, ProjectID: q.ProjectID, TaskID: custom.ID}
if e := s.UpsertCaptureRecord(ctx, custom); e != nil {
t.Fatal(e)
}
_ = mem.UpsertCaptureRecord(ctx, custom)
if e := s.UpsertRun(ctx, run); e != nil {
t.Fatal(e)
}
_ = mem.UpsertRun(ctx, run)
start := time.Now()
counts, err := s.CountAdminAnnotationPools(ctx, org, q)
want, _ := mem.CountAdminAnnotationPools(ctx, org, q)
if err != nil || counts != want {
t.Fatalf("counts=%+v want=%+v err=%v", counts, want, err)
}
t.Logf("%d-item counts: %s %+v", count, time.Since(start), counts)
for _, scenario := range []struct{ pool, search, category string }{{"available", "", ""}, {"all", "CAPTURE_0003", ""}, {"all", "", "pending_slice"}, {"mine", "", ""}, {"all", "[.*", ""}, {"all", "客户指定Capture", ""}} {
q.Pool, q.Search, q.Category = scenario.pool, scenario.search, scenario.category
start = time.Now()
got, err := s.ListAdminAnnotationPool(ctx, org, q)
elapsed := time.Since(start)
expected, _ := mem.ListAdminAnnotationPool(ctx, org, q)
if err != nil {
t.Fatal(err)
}
ids := func(items []domain.HumanItemRecord) []string {
out := []string{}
for _, i := range items {
out = append(out, i.ID)
if _, ok := i.Subject.Payload["annotationWorkspace"]; ok {
t.Fatal("heavy workspace leaked")
}
}
return out
}
if !reflect.DeepEqual(ids(got), ids(expected)) {
t.Fatalf("%+v Mongo=%v memory=%v", scenario, ids(got), ids(expected))
}
t.Logf("%+v page=%d elapsed=%s", scenario, len(got), elapsed)
if os.Getenv("ADMIN_POOL_SCALE") == "1" && elapsed > 2*time.Second {
t.Errorf("administrator pool exceeds 2s at %d rows: %v", count, elapsed)
}
if len(got) > 201 {
t.Fatal("unbounded page")
}
if scenario.search == "" && len(got) > 0 {
last := got[len(got)-1]
q.Cursor = &domain.HumanItemCursor{CreatedAt: last.CreatedAt, ID: last.ID}
next, err := s.ListAdminAnnotationPool(ctx, org, q)
if err != nil {
t.Fatal(err)
}
for _, i := range next {
for _, old := range got {
if i.ID == old.ID {
t.Fatal("cursor repeated row")
}
}
}
q.Cursor = nil
}
}
scoped := q
scoped.Search = ""
scoped.Category = ""
scoped.Pool = "all"
scoped.Capture = "客户指定Capture"
gotCounts, e := s.CountAdminAnnotationPools(ctx, org, scoped)
wantCounts, _ := mem.CountAdminAnnotationPools(ctx, org, scoped)
if e != nil || gotCounts != wantCounts || gotCounts.All != 1 {
t.Fatalf("capture counts=%+v want=%+v err=%v", gotCounts, wantCounts, e)
}
// Missing routing IDs must not match an arbitrary batch.
_, err = s.humanItems.InsertOne(ctx, bson.M{"id": "bad", "organizationId": org, "projectId": "project", "pluginId": "annotation-qa", "currentStageId": "annotation_combined", "status": "ready", "createdAt": "z"})
if err != nil {
t.Fatal(err)
}
again, err := s.CountAdminAnnotationPools(ctx, org, q)
if err != nil || again != counts {
t.Fatalf("missing batch count=%+v err=%v", again, err)
}
}
// Batch source joins must preserve failed, missing, foreign and superseded
// certificates. Compare the existing admission expression on the same records.
func TestAdminAnnotationPoolMongoAdmissionParity(t *testing.T) {
uri := os.Getenv("POOL_LOCAL_MONGO_URI")
if uri == "" {
t.Skip("isolated local Mongo opt-in")
}
u, e := url.Parse(uri)
if e != nil || u.Host != "127.0.0.1:27018" || u.Scheme != "mongodb" {
t.Fatal("local Mongo 27018 only")
}
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
defer cancel()
client, e := mongo.Connect(ctx, options.Client().ApplyURI(uri))
if e != nil {
t.Fatal(e)
}
defer client.Disconnect(context.Background())
db := client.Database(fmt.Sprintf("admin_pool_admission_%d", time.Now().UnixNano()))
defer db.Drop(context.Background())
s := NewMongoStore(db)
if e := s.EnsureIndexes(ctx); e != nil {
t.Fatal(e)
}
const org = "admin-admission"
batch := domain.HumanBatchRecord{ID: "batch", OrganizationID: org, PluginID: "annotation-qa", Stages: domain.DefaultFanyuV58HumanStages()}
if e := s.UpsertHumanBatch(ctx, batch); e != nil {
t.Fatal(e)
}
for n := 0; n < 192; n++ {
id := fmt.Sprintf("case-%03d", n)
item := domain.HumanItemRecord{ID: id, OrganizationID: org, ProjectID: "project", RunID: "run", PluginID: "annotation-qa", BatchID: batch.ID, CurrentStageID: "annotation_combined", Status: domain.HumanItemReady, LegacyTaskID: id, CreatedAt: id, ProcessingJob: &domain.AnnotationProcessingJob{SourceIndex: n%2 == 0, SourceFingerprint: id}}
item.Subject.ItemKey = id
if n%3 == 0 {
item.Subject.Payload = map[string]any{"materialization": map[string]any{"sourceIndex": true}}
item.ProcessingJob.SourceFingerprint = ""
}
if e := s.UpsertHumanItem(ctx, item); e != nil {
t.Fatal(e)
}
if n%7 != 0 {
task := domain.AnnotationTaskRecord{ID: id, OrganizationID: org, ProjectID: item.ProjectID, RunID: item.RunID, SourceIsFullVideo: n%2 == 0, TemplateVersion: 6, ArtifactID: "video"}
if _, e := s.annotationTasks.InsertOne(ctx, task); e != nil {
t.Fatal(e)
}
}
for k, jobID := range []string{id, "source-v1:" + org + ":" + id} {
if n%5 == k {
continue
}
job := domain.AnnotationIndexJob{ID: jobID, OrganizationID: org, ProjectID: item.ProjectID, RunID: item.RunID, Status: []string{"completed", "failed", "running"}[n%3]}
if n%11 == 0 {
job.OrganizationID = "foreign"
}
if k == 0 && n%4 == 0 {
job.RevalidateSource = true
job.SourceFingerprint = "repaired"
job.SupersededSourceFingerprint = id
}
if _, e := db.Collection("annotationIndexJobs").InsertOne(ctx, job); e != nil {
t.Fatal(e)
}
}
}
legacy := mongo.Pipeline{{{Key: "$match", Value: adminPoolScopeFilter(org, AdminAnnotationPoolQuery{})}}}
legacy = append(legacy, annotationSourceAdmissionStatusPipeline()...)
read := func(p mongo.Pipeline) map[string]string {
t.Helper()
cur, e := s.humanItems.Aggregate(ctx, p)
if e != nil {
t.Fatal(e)
}
defer cur.Close(ctx)
rows, e := decodeMany[adminPoolRow](ctx, cur)
if e != nil {
t.Fatal(e)
}
out := map[string]string{}
for _, r := range rows {
out[r.ID] = r.SourceAdmissionStatus
}
return out
}
expected := read(legacy)
got := read(adminPoolPagePipeline(org, AdminAnnotationPoolQuery{Pool: "all"}))
if len(expected) != 192 || !reflect.DeepEqual(expected, got) {
t.Fatalf("batch admission changed: expected=%v actual=%v", expected, got)
}
available, e := s.ListAdminAnnotationPool(ctx, org, AdminAnnotationPoolQuery{Pool: "available", Limit: 300})
if e != nil {
t.Fatal(e)
}
ready := 0
for _, status := range expected {
if status == "" {
ready++
}
}
if len(available) != ready {
t.Fatalf("available=%d expected=%d", len(available), ready)
}
for _, item := range available {
if expected[item.ID] != "" {
t.Fatalf("unready source exposed: %s", item.ID)
}
}
}