forked from longtao.wu/droboticflow
147 lines
5.2 KiB
Go
147 lines
5.2 KiB
Go
package api
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"strings"
|
|
|
|
"droboticflow/internal/store"
|
|
"go.mongodb.org/mongo-driver/bson"
|
|
"go.mongodb.org/mongo-driver/mongo"
|
|
"go.mongodb.org/mongo-driver/mongo/options"
|
|
"time"
|
|
)
|
|
|
|
// Synthetic only. Counts and BSON size classes follow the read-only 2026-11-09
|
|
// inventory; no customer names, payloads, credentials or documents are copied.
|
|
func seedProductionPoolNoise(ctx context.Context, db *mongo.Database, org string, blocks []string) (map[string]int64, error) {
|
|
counts := map[string]int64{}
|
|
for n := 0; n < 4; n++ {
|
|
id := fmt.Sprintf("noise-project-%d", n)
|
|
if _, err := db.Collection("projects").InsertOne(ctx, bson.M{"id": id, "organizationId": org, "name": id}); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
targets := map[string]int64{"humanItems": 26218, "runs": 26490, "captureRecords": 29767, "annotationTasks": 23611}
|
|
random := strings.Join(blocks, "")
|
|
for _, name := range []string{"humanItems", "runs", "captureRecords", "annotationTasks"} {
|
|
c := db.Collection(name)
|
|
existing, err := c.CountDocuments(ctx, bson.M{"organizationId": org})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if existing > targets[name] {
|
|
return nil, fmt.Errorf("fixture %s already exceeds production target: %d", name, existing)
|
|
}
|
|
batch := []any{}
|
|
for n := existing; n < targets[name]; n++ {
|
|
id := fmt.Sprintf("noise-%s-%06d", name, n)
|
|
row := bson.M{"id": id, "organizationId": org, "projectId": fmt.Sprintf("noise-project-%d", n%4), "createdAt": "2026-08-01T00:00:00Z", "updatedAt": "2026-08-01T00:00:00Z"}
|
|
switch name {
|
|
case "humanItems":
|
|
row["pluginId"] = "annotation-qa"
|
|
row["currentStageId"] = "quality_review"
|
|
row["status"] = "ready"
|
|
row["assignedUserId"] = "noise-owner"
|
|
row["supplierId"] = "noise-supplier"
|
|
row["version"] = 1
|
|
if n%2 == 0 {
|
|
row["currentStageId"] = "annotation_combined"
|
|
}
|
|
size := 8 * 1024
|
|
if n%150 == 0 {
|
|
size = 128 * 1024
|
|
}
|
|
if n%1800 == 0 {
|
|
size = 2 * 1024 * 1024
|
|
}
|
|
blob := strings.Repeat(random, size/len(random)+1)[:size]
|
|
row["subject"] = bson.M{"itemKey": id, "payload": bson.M{"annotationWorkspace": bson.M{"syntheticBlob": blob}}}
|
|
case "runs":
|
|
row["status"] = "waiting_for_human"
|
|
row["workflowId"] = "noise-workflow"
|
|
row["captureDirectories"] = []string{id}
|
|
row["taskId"] = "noise-captureRecords-000000"
|
|
size := 80 * 1024
|
|
if n%800 == 0 {
|
|
size = 384 * 1024
|
|
}
|
|
blob := strings.Repeat(random, size/len(random)+1)[:size]
|
|
row["logs"] = []bson.M{{"message": blob}}
|
|
row["artifacts"] = bson.A{}
|
|
row["nodeExecutions"] = bson.A{}
|
|
case "captureRecords":
|
|
row["workflowId"] = "noise-workflow"
|
|
row["prefix"] = "noise/" + id
|
|
row["captureId"] = id
|
|
row["sourceKeys"] = []string{random[:2048]}
|
|
case "annotationTasks":
|
|
row["status"] = "pending_annotation"
|
|
row["sourceVideo"] = "noise/" + id + ".mp4"
|
|
row["artifactId"] = id
|
|
row["syntheticPadding"] = random[:2048]
|
|
}
|
|
batch = append(batch, row)
|
|
if len(batch) == 32 || n+1 == targets[name] {
|
|
if _, err = c.InsertMany(ctx, batch); err != nil {
|
|
return nil, err
|
|
}
|
|
batch = nil
|
|
}
|
|
}
|
|
counts[name] = targets[name]
|
|
}
|
|
// Match the event index working set too: trace=1 must not be benchmarked
|
|
// against an almost-empty audit collection. Only synthetic IDs/text.
|
|
cursor, err := db.Collection("humanItems").Find(ctx, bson.M{"organizationId": org}, options.Find().SetProjection(bson.M{"_id": 0, "id": 1}).SetSort(bson.D{{Key: "id", Value: 1}}))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
var roots []struct {
|
|
ID string `bson:"id"`
|
|
}
|
|
if err = cursor.All(ctx, &roots); err != nil {
|
|
return nil, err
|
|
}
|
|
events := db.Collection("humanEvents")
|
|
existing, err := events.CountDocuments(ctx, bson.M{"organizationId": org})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
batch := []any{}
|
|
for n := existing; n < 342694; n++ {
|
|
kind := "approved"
|
|
if n%17 == 0 {
|
|
kind = "rejected"
|
|
}
|
|
batch = append(batch, bson.M{"id": fmt.Sprintf("noise-event-%09d", n), "organizationId": org, "itemId": roots[n%int64(len(roots))].ID, "eventType": kind, "createdAt": "2026-10-02T00:00:00Z"})
|
|
if len(batch) == 512 || n+1 == 342694 {
|
|
if _, err = events.InsertMany(ctx, batch); err != nil {
|
|
return nil, err
|
|
}
|
|
batch = nil
|
|
}
|
|
}
|
|
counts["humanEvents"] = 342694
|
|
// Both baseline and candidate receive identical scalar summaries, even when
|
|
// the old binary's Go struct does not know these optional BSON fields.
|
|
join := bson.M{"$reduce": bson.M{"input": "$poolSearch.text", "initialValue": "", "in": bson.M{"$cond": bson.A{bson.M{"$eq": bson.A{"$$value", ""}}, "$$this", bson.M{"$concat": bson.A{"$$value", "\n", "$$this"}}}}}}
|
|
_, err = db.Collection("humanItems").UpdateMany(ctx, bson.M{"organizationId": org, "poolSearch.schema": 1}, mongo.Pipeline{{{Key: "$set", Value: bson.M{"poolSearch.searchText": join, "poolSearch.textVersion": 1}}}})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return counts, nil
|
|
}
|
|
|
|
func installPoolReadIndexes(ctx context.Context, db *mongo.Database, all bool) error {
|
|
for name, models := range store.PoolSearchReadIndexes() {
|
|
if !all && name != "captureRecords" {
|
|
continue
|
|
}
|
|
if _, err := db.Collection(name).Indexes().CreateMany(ctx, models, options.CreateIndexes().SetMaxTime(2*time.Minute)); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|