droboticflow/internal/api/poll_pool_production_fixture_test.go
2026-11-10 04:57:09 -08:00

147 lines
5.2 KiB
Go

package api
import (
"context"
"fmt"
"strings"
"droboticflow/internal/store"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
"time"
)
// Synthetic only. Counts and BSON size classes follow the read-only 2026-11-09
// inventory; no customer names, payloads, credentials or documents are copied.
func seedProductionPoolNoise(ctx context.Context, db *mongo.Database, org string, blocks []string) (map[string]int64, error) {
counts := map[string]int64{}
for n := 0; n < 4; n++ {
id := fmt.Sprintf("noise-project-%d", n)
if _, err := db.Collection("projects").InsertOne(ctx, bson.M{"id": id, "organizationId": org, "name": id}); err != nil {
return nil, err
}
}
targets := map[string]int64{"humanItems": 26218, "runs": 26490, "captureRecords": 29767, "annotationTasks": 23611}
random := strings.Join(blocks, "")
for _, name := range []string{"humanItems", "runs", "captureRecords", "annotationTasks"} {
c := db.Collection(name)
existing, err := c.CountDocuments(ctx, bson.M{"organizationId": org})
if err != nil {
return nil, err
}
if existing > targets[name] {
return nil, fmt.Errorf("fixture %s already exceeds production target: %d", name, existing)
}
batch := []any{}
for n := existing; n < targets[name]; n++ {
id := fmt.Sprintf("noise-%s-%06d", name, n)
row := bson.M{"id": id, "organizationId": org, "projectId": fmt.Sprintf("noise-project-%d", n%4), "createdAt": "2026-08-01T00:00:00Z", "updatedAt": "2026-08-01T00:00:00Z"}
switch name {
case "humanItems":
row["pluginId"] = "annotation-qa"
row["currentStageId"] = "quality_review"
row["status"] = "ready"
row["assignedUserId"] = "noise-owner"
row["supplierId"] = "noise-supplier"
row["version"] = 1
if n%2 == 0 {
row["currentStageId"] = "annotation_combined"
}
size := 8 * 1024
if n%150 == 0 {
size = 128 * 1024
}
if n%1800 == 0 {
size = 2 * 1024 * 1024
}
blob := strings.Repeat(random, size/len(random)+1)[:size]
row["subject"] = bson.M{"itemKey": id, "payload": bson.M{"annotationWorkspace": bson.M{"syntheticBlob": blob}}}
case "runs":
row["status"] = "waiting_for_human"
row["workflowId"] = "noise-workflow"
row["captureDirectories"] = []string{id}
row["taskId"] = "noise-captureRecords-000000"
size := 80 * 1024
if n%800 == 0 {
size = 384 * 1024
}
blob := strings.Repeat(random, size/len(random)+1)[:size]
row["logs"] = []bson.M{{"message": blob}}
row["artifacts"] = bson.A{}
row["nodeExecutions"] = bson.A{}
case "captureRecords":
row["workflowId"] = "noise-workflow"
row["prefix"] = "noise/" + id
row["captureId"] = id
row["sourceKeys"] = []string{random[:2048]}
case "annotationTasks":
row["status"] = "pending_annotation"
row["sourceVideo"] = "noise/" + id + ".mp4"
row["artifactId"] = id
row["syntheticPadding"] = random[:2048]
}
batch = append(batch, row)
if len(batch) == 32 || n+1 == targets[name] {
if _, err = c.InsertMany(ctx, batch); err != nil {
return nil, err
}
batch = nil
}
}
counts[name] = targets[name]
}
// Match the event index working set too: trace=1 must not be benchmarked
// against an almost-empty audit collection. Only synthetic IDs/text.
cursor, err := db.Collection("humanItems").Find(ctx, bson.M{"organizationId": org}, options.Find().SetProjection(bson.M{"_id": 0, "id": 1}).SetSort(bson.D{{Key: "id", Value: 1}}))
if err != nil {
return nil, err
}
var roots []struct {
ID string `bson:"id"`
}
if err = cursor.All(ctx, &roots); err != nil {
return nil, err
}
events := db.Collection("humanEvents")
existing, err := events.CountDocuments(ctx, bson.M{"organizationId": org})
if err != nil {
return nil, err
}
batch := []any{}
for n := existing; n < 342694; n++ {
kind := "approved"
if n%17 == 0 {
kind = "rejected"
}
batch = append(batch, bson.M{"id": fmt.Sprintf("noise-event-%09d", n), "organizationId": org, "itemId": roots[n%int64(len(roots))].ID, "eventType": kind, "createdAt": "2026-10-02T00:00:00Z"})
if len(batch) == 512 || n+1 == 342694 {
if _, err = events.InsertMany(ctx, batch); err != nil {
return nil, err
}
batch = nil
}
}
counts["humanEvents"] = 342694
// Both baseline and candidate receive identical scalar summaries, even when
// the old binary's Go struct does not know these optional BSON fields.
join := bson.M{"$reduce": bson.M{"input": "$poolSearch.text", "initialValue": "", "in": bson.M{"$cond": bson.A{bson.M{"$eq": bson.A{"$$value", ""}}, "$$this", bson.M{"$concat": bson.A{"$$value", "\n", "$$this"}}}}}}
_, err = db.Collection("humanItems").UpdateMany(ctx, bson.M{"organizationId": org, "poolSearch.schema": 1}, mongo.Pipeline{{{Key: "$set", Value: bson.M{"poolSearch.searchText": join, "poolSearch.textVersion": 1}}}})
if err != nil {
return nil, err
}
return counts, nil
}
func installPoolReadIndexes(ctx context.Context, db *mongo.Database, all bool) error {
for name, models := range store.PoolSearchReadIndexes() {
if !all && name != "captureRecords" {
continue
}
if _, err := db.Collection(name).Indexes().CreateMany(ctx, models, options.CreateIndexes().SetMaxTime(2*time.Minute)); err != nil {
return err
}
}
return nil
}