forked from longtao.wu/droboticflow
283 lines
11 KiB
Go
283 lines
11 KiB
Go
package store
|
|
|
|
import (
|
|
"context"
|
|
"droboticflow/internal/domain"
|
|
"fmt"
|
|
"go.mongodb.org/mongo-driver/bson"
|
|
"go.mongodb.org/mongo-driver/event"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
)
|
|
|
|
func TestPlatformClaimStageReadDoesNotFetchReviewBacklog(t *testing.T) {
|
|
// Exercise the operational/advisory path with real task scope; an orphan
|
|
// remains excluded even when expensive source audits are advisory.
|
|
t.Setenv("ANNOTATION_SOURCE_VALIDATION_MODE", "advisory")
|
|
var mu sync.Mutex
|
|
var query bson.M
|
|
ctx, s := poolLocalMongo(t, &event.CommandMonitor{Started: func(_ context.Context, e *event.CommandStartedEvent) {
|
|
if e.CommandName == "find" && e.Command.Lookup("find").StringValue() == "humanItems" {
|
|
mu.Lock()
|
|
defer mu.Unlock()
|
|
_ = bson.Unmarshal(e.Command, &query)
|
|
}
|
|
}})
|
|
if err := s.EnsureIndexes(ctx); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
const org, project = "closure-org", "closure-project"
|
|
batch := domain.HumanBatchRecord{ID: "batch", OrganizationID: org, ProjectID: project, Stages: domain.DefaultFanyuV58HumanStages(), Status: domain.HumanBatchActive}
|
|
if err := s.UpsertHumanBatch(ctx, batch); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
docs := make([]any, 1042)
|
|
tasks := make([]any, 0, 42)
|
|
for i := range docs {
|
|
stage := domain.FanyuV58StageQualityReview
|
|
if i < 42 {
|
|
stage = domain.FanyuV58StageAnnotationCombined
|
|
tasks = append(tasks, bson.M{"id": fmt.Sprintf("task-%d", i), "organizationId": org, "projectId": project, "runId": "run"})
|
|
}
|
|
docs[i] = bson.M{"id": fmt.Sprintf("item-%d", i), "organizationId": org, "projectId": project, "pluginId": "annotation-qa", "batchId": batch.ID, "runId": "run", "legacyTaskId": fmt.Sprintf("task-%d", i), "currentStageId": stage, "status": "ready", "assignedUserId": "", "supplierId": "supplier", "subject": bson.M{"itemKey": fmt.Sprintf("source-%d", i), "payload": bson.M{"large": strings.Repeat("x", 16<<10)}}}
|
|
}
|
|
docs = append(docs, bson.M{"id": "orphan", "organizationId": org, "projectId": project, "runId": "run", "pluginId": "annotation-qa", "batchId": batch.ID, "legacyTaskId": "missing", "currentStageId": domain.FanyuV58StageAnnotationCombined, "status": "ready", "assignedUserId": "", "supplierId": "supplier"})
|
|
if _, err := s.annotationTasks.InsertMany(ctx, tasks); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := s.humanItems.InsertMany(ctx, docs); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
n, err := s.CountAnnotationClaimCandidates(ctx, org, AnnotationClaimCountQuery{ProjectID: project, ActiveSupplierIDs: []string{"supplier"}, Memberships: []AnnotationClaimMembership{{SupplierID: "supplier", Legacy: true}}})
|
|
if err != nil || n != 42 {
|
|
t.Fatal("wrong inventory", n, err)
|
|
}
|
|
mu.Lock()
|
|
captured := query
|
|
mu.Unlock()
|
|
find := bson.D{{Key: "find", Value: "humanItems"}, {Key: "filter", Value: captured["filter"]}, {Key: "projection", Value: captured["projection"]}}
|
|
if h := captured["hint"]; h != nil {
|
|
find = append(find, bson.E{Key: "hint", Value: h})
|
|
}
|
|
var result bson.M
|
|
if err := s.db.RunCommand(ctx, bson.D{{Key: "explain", Value: find}, {Key: "verbosity", Value: "executionStats"}}).Decode(&result); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
stats := deletionDoc(result["executionStats"])
|
|
examined := supplierNumber(stats["totalDocsExamined"])
|
|
t.Logf("claim candidates=%d examined=%d keys=%v", n, examined, stats["totalKeysExamined"])
|
|
if examined > 50 {
|
|
t.Fatal("annotation inventory fetched unrelated review payloads", examined)
|
|
}
|
|
}
|
|
|
|
func TestPlatformWorkerRunCountsDoNotFetchRunPayloads(t *testing.T) {
|
|
var mu sync.Mutex
|
|
var query bson.M
|
|
ctx, s := poolLocalMongo(t, &event.CommandMonitor{Started: func(_ context.Context, e *event.CommandStartedEvent) {
|
|
if (e.CommandName == "aggregate" && e.Command.Lookup("aggregate").StringValue() == "runs") || (e.CommandName == "find" && e.Command.Lookup("find").StringValue() == "runs") {
|
|
mu.Lock()
|
|
defer mu.Unlock()
|
|
_ = bson.Unmarshal(e.Command, &query)
|
|
}
|
|
}})
|
|
if err := s.EnsureIndexes(ctx); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := s.runs.Indexes().CreateOne(ctx, PlatformReadIndexes()["runs"][0]); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
docs := make([]any, 2300)
|
|
for i := range docs {
|
|
status := "completed"
|
|
if i < 300 {
|
|
status = "queued"
|
|
}
|
|
docs[i] = bson.M{"id": fmt.Sprintf("run-%d", i), "workflowId": fmt.Sprintf("workflow-%d", i%7), "status": status, "logs": strings.Repeat("x", 16<<10)}
|
|
}
|
|
if _, err := s.runs.InsertMany(ctx, docs); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
n, err := s.CountWorkerRuns(ctx)
|
|
if err != nil || n.QueuedTotal != 300 {
|
|
t.Fatal("wrong worker counts", n, err)
|
|
}
|
|
mu.Lock()
|
|
captured := query
|
|
mu.Unlock()
|
|
|
|
command := bson.D{{Key: "aggregate", Value: "runs"}, {Key: "pipeline", Value: captured["pipeline"]}, {Key: "cursor", Value: bson.M{}}}
|
|
if _, ok := captured["find"]; ok {
|
|
command = bson.D{{Key: "find", Value: "runs"}, {Key: "filter", Value: captured["filter"]}, {Key: "projection", Value: captured["projection"]}}
|
|
}
|
|
if h := captured["hint"]; h != nil {
|
|
command = append(command, bson.E{Key: "hint", Value: h})
|
|
}
|
|
var result bson.M
|
|
if err := s.db.RunCommand(ctx, bson.D{{Key: "explain", Value: command}, {Key: "verbosity", Value: "executionStats"}}).Decode(&result); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
stats := deletionDoc(result["executionStats"])
|
|
if stats == nil {
|
|
for _, raw := range deletionArray(result["stages"]) {
|
|
if cursor := deletionDoc(deletionDoc(raw)["$cursor"]); cursor != nil {
|
|
stats = deletionDoc(cursor["executionStats"])
|
|
}
|
|
}
|
|
}
|
|
t.Logf("worker queued=%d examined=%v keys=%v", n.QueuedTotal, stats["totalDocsExamined"], stats["totalKeysExamined"])
|
|
if stats == nil || supplierNumber(stats["totalDocsExamined"]) != 0 {
|
|
t.Fatalf("worker polling fetched large run bodies: docs=%v", stats["totalDocsExamined"])
|
|
}
|
|
}
|
|
|
|
// A broad text match must not build and sort every foreign join before the
|
|
// first page can be returned. The original matcher still decides each row.
|
|
func TestPlatformBroadPoolSearchBoundsFirstPageJoins(t *testing.T) {
|
|
var mu sync.Mutex
|
|
var joined []bson.M
|
|
ctx, s := poolLocalMongo(t, &event.CommandMonitor{Started: func(_ context.Context, e *event.CommandStartedEvent) {
|
|
if e.CommandName != "aggregate" || e.Command.Lookup("aggregate").StringValue() != "humanItems" {
|
|
return
|
|
}
|
|
var q bson.M
|
|
if bson.Unmarshal(e.Command, &q) != nil {
|
|
return
|
|
}
|
|
raw, _ := bson.Marshal(q)
|
|
if strings.Contains(string(raw), "$lookup") {
|
|
mu.Lock()
|
|
joined = append(joined, q)
|
|
mu.Unlock()
|
|
}
|
|
}})
|
|
if err := s.EnsureIndexes(ctx); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
docs := make([]any, 5000)
|
|
for i := range docs {
|
|
docs[i] = bson.M{"id": fmt.Sprintf("pool-%05d", i), "organizationId": "broad-org", "pluginId": "annotation-qa", "currentStageId": "prepackage_acceptance", "status": "ready", "assignedUserId": "", "createdAt": fmt.Sprintf("2026-11-%05d", i), "subject": bson.M{"itemKey": fmt.Sprintf("FYLX_BROAD_%05d/part0001", i)}}
|
|
}
|
|
if _, err := s.humanItems.InsertMany(ctx, docs); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
rows, err := s.ListHumanItems(ctx, "broad-org", domain.HumanItemQuery{PluginID: "annotation-qa", CurrentStageID: "prepackage_acceptance", Search: "FYLX_BROAD", Limit: 25, IdentityProjection: true})
|
|
if err != nil || len(rows) != 25 {
|
|
t.Fatal("first page", len(rows), err)
|
|
}
|
|
if rows[0].ID != "pool-04999" || rows[24].ID != "pool-04975" {
|
|
t.Fatal("order changed")
|
|
}
|
|
mu.Lock()
|
|
queries := joined
|
|
mu.Unlock()
|
|
if len(queries) == 0 {
|
|
t.Fatal("no actual join query observed")
|
|
}
|
|
for _, q := range queries {
|
|
var result bson.M
|
|
cmd := poolTestExplainCommand(q)
|
|
for _, key := range []string{"$db", "lsid", "$clusterTime", "readConcern", "$readPreference"} {
|
|
delete(q, key)
|
|
}
|
|
cmd = poolTestExplainCommand(q)
|
|
if err := s.db.RunCommand(ctx, cmd).Decode(&result); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var examined int64
|
|
if stats := deletionDoc(result["executionStats"]); stats != nil {
|
|
examined = int64(supplierNumber(stats["totalDocsExamined"]))
|
|
}
|
|
for _, raw := range deletionArray(result["stages"]) {
|
|
if c := deletionDoc(deletionDoc(raw)["$cursor"]); c != nil {
|
|
examined = int64(supplierNumber(deletionDoc(c["executionStats"])["totalDocsExamined"]))
|
|
}
|
|
}
|
|
t.Logf("first-page joins=%d root documents=%d", len(queries), examined)
|
|
if examined > 250 {
|
|
t.Fatal("first-page join materialized whole inventory", examined)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestPlatformRecoveryLookupIndexesPreserveLegacyReceipts(t *testing.T) {
|
|
var mu sync.Mutex
|
|
reads := map[string]bson.M{}
|
|
ctx, s := poolLocalMongo(t, &event.CommandMonitor{Started: func(_ context.Context, e *event.CommandStartedEvent) {
|
|
if e.CommandName != "find" {
|
|
return
|
|
}
|
|
name := e.Command.Lookup("find").StringValue()
|
|
if name != "reviewOperations" && name != "reviewSessions" && name != "captureRecords" {
|
|
return
|
|
}
|
|
var q bson.M
|
|
if bson.Unmarshal(e.Command, &q) != nil {
|
|
return
|
|
}
|
|
mu.Lock()
|
|
reads[name] = q
|
|
mu.Unlock()
|
|
}})
|
|
ops, sessions, captures := []any{}, []any{}, []any{}
|
|
for i := 0; i < 3000; i++ {
|
|
ops = append(ops, bson.M{"_id": fmt.Sprintf("legacy-receipt-%d", i), "organizationId": "recovery-org", "userId": "reviewer", "operationId": fmt.Sprintf("op-%d", i), "itemId": "item", "taskId": "task", "status": "completed"})
|
|
sessions = append(sessions, bson.M{"_id": fmt.Sprintf("session-%d", i), "revision": 1})
|
|
captures = append(captures, bson.M{"id": fmt.Sprintf("capture-%d", i), "runId": fmt.Sprintf("run-%d", i)})
|
|
}
|
|
for name, docs := range map[string][]any{"reviewOperations": ops, "reviewSessions": sessions, "captureRecords": captures} {
|
|
if _, err := s.db.Collection(name).InsertMany(ctx, docs); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
explainLast := func(name string) int64 {
|
|
mu.Lock()
|
|
q := reads[name]
|
|
mu.Unlock()
|
|
for _, k := range []string{"$db", "lsid", "$clusterTime", "readConcern", "$readPreference"} {
|
|
delete(q, k)
|
|
}
|
|
var result bson.M
|
|
if err := s.db.RunCommand(ctx, poolTestExplainCommand(q)).Decode(&result); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return supplierNumber(deletionDoc(result["executionStats"])["totalDocsExamined"])
|
|
}
|
|
for _, indexed := range []bool{false, true} {
|
|
if indexed {
|
|
for name, models := range PlatformReadIndexes() {
|
|
if _, err := s.db.Collection(name).Indexes().CreateMany(ctx, models); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
}
|
|
op, err := s.FindReviewOperation(ctx, "recovery-org", "reviewer", "item", "op-2999")
|
|
if err != nil || op == nil || op.ID != "legacy-receipt-2999" {
|
|
t.Fatal("legacy item receipt lookup changed", err)
|
|
}
|
|
itemDocs := explainLast("reviewOperations")
|
|
alias, err := s.FindReviewOperation(ctx, "recovery-org", "reviewer", "task", "op-2999")
|
|
if err != nil || alias == nil || alias.ID != op.ID {
|
|
t.Fatal("task alias receipt lookup changed", err)
|
|
}
|
|
if denied, err := s.FindReviewOperation(ctx, "foreign-org", "reviewer", "item", "op-2999"); err != nil || denied != nil {
|
|
t.Fatal("receipt tenant scope widened", err)
|
|
}
|
|
if q, err := s.GetReviewContinuation(ctx, "queue-with-no-pending-admission"); err != nil || q != nil {
|
|
t.Fatal("missing queue recovery changed", err)
|
|
}
|
|
sessionDocs := explainLast("reviewSessions")
|
|
capture, err := s.GetCaptureRecordByRunID(ctx, "run-2999")
|
|
if err != nil || capture == nil || capture.ID != "capture-2999" {
|
|
t.Fatal("capture run relation changed", err)
|
|
}
|
|
captureDocs := explainLast("captureRecords")
|
|
t.Logf("indexed=%t receiptDocs=%d sessionDocs=%d captureDocs=%d", indexed, itemDocs, sessionDocs, captureDocs)
|
|
if indexed && (itemDocs > 1 || sessionDocs > 0 || captureDocs > 1) {
|
|
t.Fatal("recovery lookup still scans unrelated records")
|
|
}
|
|
}
|
|
reviewContinuationAdmissionContract(t, s)
|
|
}
|