droboticflow/internal/store/human_pool_explain_test.go

176 lines
5.9 KiB
Go

package store
// Opt-in diagnostics: loopback Mongo, unique synthetic database, no production
// configuration or customer documents. Reports counts/plans, never payloads.
import (
"context"
"encoding/json"
"fmt"
"os"
"path/filepath"
"sort"
"strings"
"testing"
"time"
"droboticflow/internal/domain"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
)
type poolExplainResult struct {
Name string `json:"name"`
Returned int64 `json:"returned"`
Keys int64 `json:"keys_examined"`
Documents int64 `json:"documents_examined"`
Milliseconds int64 `json:"execution_ms"`
Stages []string `json:"stages"`
Indexes []string `json:"indexes"`
}
func TestHumanPoolExplainLocal(t *testing.T) {
if os.Getenv("POLL_POOL_EXPLAIN") != "1" {
t.Skip("local synthetic explain opt-in")
}
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute)
defer cancel()
client, err := mongo.Connect(ctx, options.Client().ApplyURI("mongodb://127.0.0.1:27018/?directConnection=true"))
if err != nil {
t.Fatal(err)
}
defer client.Disconnect(context.Background())
db := client.Database(fmt.Sprintf("pool_explain_%d", time.Now().UnixNano()))
defer db.Drop(context.Background())
ds := NewMongoStore(db)
if err := ds.EnsureIndexes(ctx); err != nil {
t.Fatal(err)
}
org := domain.DefaultOrganizationID
stages := []string{"quality_review", "semantic_review", "acceptance", "prepackage_acceptance"}
// Many historical rows authored by the actor; only a small active subset.
for offset := 0; offset < 40000; offset += 1000 {
docs := make([]any, 0, 1000)
for i := offset; i < offset+1000; i++ {
stage := stages[i%4]
status := "accepted"
assigned := ""
author := "other"
if i%10 == 0 {
author = "actor"
}
if i%100 < 5 {
status = "ready"
}
if i%200 == 0 {
assigned = "actor"
status = "claimed"
}
id := fmt.Sprintf("item-%05d", i)
docs = append(docs, bson.M{"id": id, "organizationId": org, "pluginId": "annotation-qa", "projectId": fmt.Sprintf("project-%02d", (i/4)%20), "batchId": "batch", "subject": bson.M{"itemKey": id, "payload": bson.M{"history": strings.Repeat("x", 1024)}}, "currentStageId": stage, "status": status, "assignedUserId": assigned, "latestAuthorUserId": author, "createdAt": fmt.Sprintf("2026-10-%02dT%02d:%02d:%02dZ", 1+i/86400, (i/3600)%24, (i/60)%60, i%60)})
}
if _, err := ds.humanItems.InsertMany(ctx, docs); err != nil {
t.Fatal(err)
}
}
results := []poolExplainResult{}
explain := func(name string, q domain.HumanItemQuery, filter bson.M) {
var raw struct {
QueryPlanner struct {
WinningPlan bson.M `bson:"winningPlan"`
} `bson:"queryPlanner"`
ExecutionStats struct {
Returned int64 `bson:"nReturned"`
Keys int64 `bson:"totalKeysExamined"`
Docs int64 `bson:"totalDocsExamined"`
MS int64 `bson:"executionTimeMillis"`
} `bson:"executionStats"`
}
cmd := bson.D{{Key: "explain", Value: bson.D{{Key: "find", Value: "humanItems"}, {Key: "filter", Value: filter}, {Key: "sort", Value: bson.D{{Key: "createdAt", Value: -1}, {Key: "id", Value: -1}}}, {Key: "projection", Value: humanItemListProjection(q)}, {Key: "limit", Value: 500}}}, {Key: "verbosity", Value: "executionStats"}}
if err := db.RunCommand(ctx, cmd).Decode(&raw); err != nil {
t.Fatal(err)
}
stageSet, indexSet := map[string]bool{}, map[string]bool{}
var visit func(any)
visit = func(v any) {
switch x := v.(type) {
case bson.M:
for k, v := range x {
if s, ok := v.(string); ok {
if k == "stage" {
stageSet[s] = true
}
if k == "indexName" {
indexSet[s] = true
}
}
visit(v)
}
case bson.A:
for _, v := range x {
visit(v)
}
}
}
visit(raw.QueryPlanner.WinningPlan)
r := poolExplainResult{Name: name, Returned: raw.ExecutionStats.Returned, Keys: raw.ExecutionStats.Keys, Documents: raw.ExecutionStats.Docs, Milliseconds: raw.ExecutionStats.MS}
for k := range stageSet {
r.Stages = append(r.Stages, k)
}
for k := range indexSet {
r.Indexes = append(r.Indexes, k)
}
sort.Strings(r.Stages)
sort.Strings(r.Indexes)
results = append(results, r)
t.Logf("%+v", r)
}
for _, scope := range []string{"", "project-00"} {
for _, stage := range []string{"quality", "acceptance", "prepackage_acceptance"} {
stageIDs := []string{stage}
if stage == "quality" {
stageIDs = []string{"quality_review", "semantic_review"}
}
q := domain.HumanItemQuery{PluginID: "annotation-qa", ProjectID: scope, StageIDs: stageIDs, PoolCandidateProjection: true, Limit: 500}
for _, pool := range []string{"available", "mine", "reviewed"} {
current := q
if pool == "available" {
current.UnassignedOnly = true
current.Statuses = []domain.HumanItemStatus{domain.HumanItemReady, domain.HumanItemRework, domain.HumanItemSubmitted}
}
if pool == "mine" {
current.ActorUserID = "actor"
}
if pool == "reviewed" {
current.IDs = []string{}
for i := 0; i < 40000; i += 100 {
current.IDs = append(current.IDs, fmt.Sprintf("item-%05d", i))
}
}
name := stage + "/" + scope + "/" + pool
filter := humanItemMongoFilter(org, current)
explain(name, current, filter)
if pool == "mine" {
current.MinePoolOnly = true
filter = humanItemMongoFilter(org, current)
explain(name+"/eligible-author", current, filter)
}
}
}
}
out := os.Getenv("POLL_POOL_EXPLAIN_OUTPUT")
if out == "" {
out = filepath.Join("..", "..", "local-data", "test-results", "pool-explain", "report.json")
}
if err := os.MkdirAll(filepath.Dir(out), 0700); err != nil {
t.Fatal(err)
}
data, err := json.MarshalIndent(map[string]any{"rows": 40000, "local_synthetic": true, "results": results}, "", " ")
if err != nil {
t.Fatal(err)
}
if err := os.WriteFile(out, data, 0600); err != nil {
t.Fatal(err)
}
}