package api // Local-only read-mix harness. This is NOT full concurrent-user capacity acceptance: // it excludes media, editing/submission, continuation writes and ingress shaping. import ( "bytes" "compress/gzip" "context" "crypto/sha256" "encoding/json" "fmt" "net/http" "net/http/httptest" "net/url" "os" "path/filepath" "runtime" "runtime/pprof" "sort" "strconv" "strings" "sync" "sync/atomic" "syscall" "testing" "time" "droboticflow/internal/domain" "droboticflow/internal/productionstats" "droboticflow/internal/queryserving" "droboticflow/internal/store" "go.mongodb.org/mongo-driver/event" "go.mongodb.org/mongo-driver/mongo" "go.mongodb.org/mongo-driver/mongo/options" ) type pollLoadActor struct { User domain.UserRecord Project string Role string } type pollLoadSample struct { Actor string Key string MS float64 Raw, Gzip, Rows int Code int Hash string } type pollLoadEndpointKey struct{} type pollMongoObservation struct { Endpoint, Collection, Command string Count, Bytes, Nanoseconds, Failed int64 FailedNanoseconds, MaxTimeMS int64 } // Rusage covers this test process (handler plus harness compression/sampling), // not Mongo, another process, or Kubernetes Pod throttling/RSS. func pollLoadProcessCPU() (float64, error) { var usage syscall.Rusage if err := syscall.Getrusage(syscall.RUSAGE_SELF, &usage); err != nil { return 0, err } return float64(usage.Utime.Sec) + float64(usage.Utime.Usec)/1e6 + float64(usage.Stime.Sec) + float64(usage.Stime.Usec)/1e6, nil } func TestPollPoolReadMixLocal(t *testing.T) { if os.Getenv("POLL_POOL_LOAD") != "1" { t.Skip("explicit isolated local read-mix verification") } seconds := func(key string, fallback int) int { v := os.Getenv(key) if v == "" { return fallback } n, e := strconv.Atoi(v) if e != nil || n < 1 || n > 7200 { t.Fatalf("invalid %s", key) } return n } adminPeriod := seconds("POLL_POOL_ADMIN_PERIOD_SECONDS", 12) warmup, duration := seconds("POLL_POOL_WARMUP_SECONDS", 5), seconds("POLL_POOL_DURATION_SECONDS", 30) annotators := seconds("POLL_POOL_ANNOTATORS", 60) quality := seconds("POLL_POOL_QUALITY_REVIEWERS", 12) acceptance := seconds("POLL_POOL_ACCEPTANCE_REVIEWERS", 8) totalActors := annotators + quality + acceptance + 1 if totalActors > 250 { t.Fatal("local read mix supports at most 250 actors") } availablePerStage := seconds("POLL_POOL_AVAILABLE_PER_STAGE", 8) if availablePerStage > 250 { t.Fatal("available fixtures limited to 250 per stage and supplier") } claimCounts := os.Getenv("POLL_POOL_CLAIM_COUNTS") == "1" productionShape := os.Getenv("POLL_POOL_PRODUCTION_SHAPE") == "1" pageShape := os.Getenv("POLL_POOL_PAGE_SHAPE") == "1" allowBaselineFailure := os.Getenv("POLL_POOL_ALLOW_BASELINE_FAILURE") == "1" readModels := os.Getenv("POLL_POOL_READMODELS") == "1" queryServing := os.Getenv("POLL_POOL_QUERY_SERVING") == "1" queryPanels := os.Getenv("POLL_POOL_QUERY_PANELS") == "1" mode := os.Getenv("POLL_POOL_MODE") if mode == "" { mode = "summary" } if mode != "summary" && mode != "legacy" { t.Fatal("mode must be summary or legacy") } ctx, cancel := context.WithTimeout(context.Background(), time.Duration(warmup+duration+600)*time.Second) defer cancel() var measured atomic.Bool var commands, mongoBytes, mongoNS, writes atomic.Int64 var derivedWrites atomic.Int64 var probeMu sync.Mutex pending := map[int64]struct { key string maxMS int64 }{} observations := map[string]*pollMongoObservation{} monitor := &event.CommandMonitor{ Started: func(commandCtx context.Context, e *event.CommandStartedEvent) { if !measured.Load() { return } endpoint, _ := commandCtx.Value(pollLoadEndpointKey{}).(string) collection, _ := e.Command.Lookup(e.CommandName).StringValueOK() if collection == "" { collection, _ = e.Command.Lookup("collection").StringValueOK() } key := endpoint + "/" + collection + "/" + e.CommandName probeMu.Lock() defer probeMu.Unlock() maxMS, _ := e.Command.Lookup("maxTimeMS").Int64OK() if short, ok := e.Command.Lookup("maxTimeMS").Int32OK(); ok { maxMS = int64(short) } pending[e.RequestID] = struct { key string maxMS int64 }{key, maxMS} if observations[key] == nil { observations[key] = &pollMongoObservation{Endpoint: endpoint, Collection: collection, Command: e.CommandName} } }, Failed: func(_ context.Context, e *event.CommandFailedEvent) { probeMu.Lock() defer probeMu.Unlock() if entry, ok := pending[e.RequestID]; ok { key := entry.key observations[key].Failed++ observations[key].FailedNanoseconds += e.Duration.Nanoseconds() observations[key].MaxTimeMS = entry.maxMS if productionShape { t.Logf("synthetic mongo failure key=%s duration_ms=%d maxTimeMS=%d", key, e.Duration.Milliseconds(), entry.maxMS) } delete(pending, e.RequestID) } }, Succeeded: func(_ context.Context, e *event.CommandSucceededEvent) { derived := false probeMu.Lock() if entry, ok := pending[e.RequestID]; ok { key := entry.key row := observations[key] derived = strings.HasPrefix(row.Collection, "queryReadModel") || strings.HasPrefix(row.Collection, "annotationOutputSnapshot") row.Count++ row.Bytes += int64(len(e.Reply)) row.Nanoseconds += e.Duration.Nanoseconds() delete(pending, e.RequestID) } probeMu.Unlock() if measured.Load() { commands.Add(1) mongoBytes.Add(int64(len(e.Reply))) mongoNS.Add(e.Duration.Nanoseconds()) switch e.CommandName { case "insert", "update", "delete", "findAndModify": if queryServing && derived { derivedWrites.Add(1) } else { writes.Add(1) } } } }} uri := os.Getenv("POLL_POOL_LOCAL_MONGO_URI") if uri == "" { uri = "mongodb://127.0.0.1:27018/?directConnection=true" } parsed, parseErr := url.Parse(uri) if parseErr != nil || parsed.Scheme != "mongodb" || (parsed.Host != "127.0.0.1:27017" && parsed.Host != "127.0.0.1:27018" && parsed.Host != "127.0.0.1:27019") { t.Fatal("isolated loopback Mongo only") } client, err := mongo.Connect(ctx, options.Client().ApplyURI(uri).SetMonitor(monitor)) if err != nil { t.Fatal(err) } defer client.Disconnect(context.Background()) clockCtx, stopClock := context.WithCancel(ctx) clockDone := make(chan struct{}) var clockJumpMS atomic.Int64 var clockSamples atomic.Int64 go func() { defer close(clockDone) var previousServer, previousLocal time.Time ticker := time.NewTicker(100 * time.Millisecond) defer ticker.Stop() for { select { case <-clockCtx.Done(): return case <-ticker.C: started := time.Now() var reply struct { LocalTime time.Time `bson:"localTime"` } probe, cancelProbe := context.WithTimeout(clockCtx, time.Second) err := client.Database("admin").RunCommand(probe, map[string]any{"hello": 1}).Decode(&reply) cancelProbe() finished := time.Now() if err != nil || finished.Sub(started) > 200*time.Millisecond { continue } local := started.Add(finished.Sub(started) / 2) if !previousServer.IsZero() { drift := reply.LocalTime.Sub(previousServer) - local.Sub(previousLocal) if drift < 0 { drift = -drift } for old := clockJumpMS.Load(); drift.Milliseconds() > old; old = clockJumpMS.Load() { if clockJumpMS.CompareAndSwap(old, drift.Milliseconds()) { break } } } previousServer, previousLocal = reply.LocalTime, local clockSamples.Add(1) } } }() defer func() { stopClock(); <-clockDone }() reuseFixture := os.Getenv("POLL_POOL_REUSE_FIXTURE") == "1" fixtureName := os.Getenv("POLL_POOL_FIXTURE_DATABASE") if fixtureName == "" { if reuseFixture { t.Fatal("reuse requires an explicit synthetic fixture database") } fixtureName = fmt.Sprintf("poll_pool_read_mix_%d", time.Now().UnixNano()) } if !strings.HasPrefix(fixtureName, "poll_pool_read_mix_") || len(fixtureName) > 63 || strings.Trim(fixtureName, "abcdefghijklmnopqrstuvwxyz0123456789_") != "" { t.Fatal("invalid synthetic fixture database") } db := client.Database(fixtureName) if !reuseFixture { collections, err := db.ListCollectionNames(ctx, map[string]any{}) if err != nil { t.Fatal(err) } if len(collections) != 0 { t.Fatal("preparation requires a new synthetic database") } } cleanupFixture := !reuseFixture defer func() { if !cleanupFixture { return } if os.Getenv("POLL_POOL_KEEP_FIXTURE") == "1" { t.Logf("retained synthetic fixture: %s", db.Name()) return } cleanup, done := context.WithTimeout(context.Background(), 15*time.Second) defer done() if e := db.Drop(cleanup); e != nil { t.Error(e) } }() ds := store.NewMongoStore(db) must := func(e error) { t.Helper() if e != nil { t.Fatal(e) } } if !reuseFixture { must(ds.EnsureIndexes(ctx)) } if os.Getenv("POLL_POOL_QUERY_SERVING_PREPARE") == "1" { // Both variants use the already-documented platform read indexes as // well as the pool-search indexes. These are isolated fixtures only. for collection, models := range store.PlatformReadIndexes() { _, e := db.Collection(collection).Indexes().CreateMany(ctx, models) must(e) } } const org = "load-org" projects := []string{"load-project-a", "load-project-b"} if os.Getenv("POLL_POOL_QUERY_SERVING_PREPARE") == "1" && !reuseFixture { must(ds.UpsertOrganization(ctx, domain.OrganizationRecord{ID: org, Status: domain.OrganizationStatusActive})) _, err := ds.SaveProductionStatsScope(ctx, org, productionstats.Scope{ProjectIDs: projects, Revision: 1}, 0) must(err) must(ds.EnsureQueryReadModelIndexes(ctx)) } if !reuseFixture { for _, p := range projects { must(ds.UpsertProject(ctx, domain.ProjectRecord{ID: p, OrganizationID: org, Name: p})) must(ds.UpsertWorkflow(ctx, domain.WorkflowRecord{ID: "workflow-" + p, OrganizationID: org, ProjectID: p, Name: "load workflow"})) } } suppliers := []string{"load-supplier-a", "load-supplier-b", "load-supplier-c"} if !reuseFixture { for _, s := range suppliers { must(ds.UpsertSupplier(ctx, domain.SupplierRecord{ID: s, OrganizationID: org, Status: "active", Name: s})) _, e := ds.SetSupplierAcceptanceAuthorization(ctx, org, s, true, 0, "load-admin") must(e) } } actors := []pollLoadActor{} for i := 0; i < totalActors; i++ { role := domain.UserRoleAnnotator label := "annotation" if i >= annotators && i < annotators+quality { role = domain.UserRoleQualityReviewer label = "quality" } if i >= annotators+quality && i < totalActors-1 { role = domain.UserRoleAcceptanceReviewer label = "acceptance" } if i == totalActors-1 { role = domain.UserRoleAdmin label = "admin" } u := domain.UserRecord{ID: fmt.Sprintf("load-user-%02d", i), Phone: fmt.Sprintf("synthetic-load-%02d", i), OrganizationID: org, SupplierID: suppliers[i%3], Role: role, Status: domain.UserStatusActive} if !reuseFixture { must(ds.UpsertUser(ctx, u)) for _, p := range projects { must(ds.UpsertProjectMember(ctx, domain.ProjectMemberRecord{ID: p + u.ID, OrganizationID: org, ProjectID: p, UserID: u.ID, SupplierID: u.SupplierID, Roles: domain.AccountProductionRoles(u)})) } } actors = append(actors, pollLoadActor{u, projects[i%2], label}) } payload := make([]string, 512) for i := range payload { payload[i] = fmt.Sprintf("%x", sha256.Sum256([]byte(fmt.Sprintf("fixed-fixture-%d", i)))) } runPayloadKiB := 0 if raw := os.Getenv("POLL_POOL_RUN_PAYLOAD_KIB"); raw != "" { runPayloadKiB, err = strconv.Atoi(raw) must(err) if runPayloadKiB < 0 || runPayloadKiB > 2048 { t.Fatal("run payload must be 0..2048 KiB") } } taskPayloadKiB := 0 if raw := os.Getenv("POLL_POOL_TASK_PAYLOAD_KIB"); raw != "" { taskPayloadKiB, err = strconv.Atoi(raw) must(err) if taskPayloadKiB < 0 || taskPayloadKiB > 2048 { t.Fatal("task payload must be 0..2048 KiB") } } seededRuns := map[string]bool{} seed := func(id, p, s, stage, owner string, status domain.HumanItemStatus) { batchID := "batch-" + p + s stages := domain.DefaultFanyuV58HumanStages() if stage == "semantic_review" { batchID += "-semantic" stages = []domain.HumanStageDefinition{{ID: stage, Kind: domain.HumanStageReview, RequiredRole: "quality_reviewer"}} } if stage == "acceptance" || stage == "prepackage_acceptance" { batchID += "-" + stage stages = []domain.HumanStageDefinition{{ID: stage, Kind: domain.HumanStageAcceptance, RequiredRole: "acceptance_reviewer"}} } must(ds.UpsertHumanBatch(ctx, domain.HumanBatchRecord{ID: batchID, RunID: batchID, GateNodeID: "gate", OrganizationID: org, ProjectID: p, WorkflowID: "workflow-" + p, PluginID: "annotation-qa", SupplierID: s, Status: domain.HumanBatchActive, ProcessingProfile: domain.ProcessingProfileRawHuman, Stages: stages})) runID := "" if runPayloadKiB > 0 { runID = "run-" + batchID if productionShape { runID = "run-" + id } if !seededRuns[runID] { seededRuns[runID] = true if productionShape { must(ds.UpsertCaptureRecord(ctx, domain.CaptureRecord{ID: "capture-" + id, OrganizationID: org, ProjectID: p, WorkflowID: "workflow-" + p, CaptureID: "synthetic-" + id, Prefix: "synthetic/" + id})) } dirs := []string(nil) captureID := "" blob := strings.Repeat("x", runPayloadKiB*1024) if productionShape { dirs = []string{"synthetic-" + id} captureID = "capture-" + id block := strings.Join(payload, "") blob = strings.Repeat(block, runPayloadKiB/32+1)[:runPayloadKiB*1024] } must(ds.UpsertRun(ctx, domain.RunRecord{ID: runID, CaptureDirectories: dirs, TaskID: captureID, OrganizationID: org, ProjectID: p, WorkflowID: "workflow-" + p, ProjectName: p, WorkflowName: "workflow-" + p, NodeExecutions: []domain.NodeExecutionRecord{{NodeID: "synthetic-node", Logs: []domain.LogEntryRecord{{Message: blob}}}}})) } } taskID := "task-" + id must(ds.UpsertAnnotationTask(ctx, domain.AnnotationTaskRecord{ID: taskID, OrganizationID: org, ProjectID: p, WorkflowID: "workflow-" + p, SupplierID: s, TemplateVersion: 6, Status: domain.AnnotationTaskStatusPendingAnnotation, SourceVideo: "synthetic/camera_left.mp4"})) itemPayload := map[string]any{"captureDirectory": "synthetic-" + id, "sliceMergeEvidence": payload, "sliceWorkflow": map[string]any{"status": "completed", "phase": "annotate"}} if taskPayloadKiB > 0 { content := strings.Repeat(strings.Join(payload, ""), taskPayloadKiB/32+1)[:taskPayloadKiB*1024] itemPayload["items"] = []map[string]any{{"annotation": map[string]any{"syntheticPayload": content}, "materialization": map[string]any{"coverage": map[string]any{"durationSeconds": 900}}}} } item := domain.HumanItemRecord{ID: id, RunID: runID, OrganizationID: org, ProjectID: p, WorkflowID: "workflow-" + p, PluginID: "annotation-qa", SupplierID: s, BatchID: batchID, LegacyTaskID: taskID, AssignedUserID: owner, LatestAuthorUserID: "synthetic-author", CurrentStageID: stage, Status: status, Version: 1, CreatedAt: "2026-10-01T00:00:00Z", Subject: domain.HumanSubjectRef{ItemKey: "synthetic/" + id + "/camera_left.mp4", Payload: itemPayload}} if productionShape && stage == "prepackage_acceptance" { sourceID := "source-" + id batchID = "aggregate-" + id must(ds.UpsertHumanBatch(ctx, domain.HumanBatchRecord{ID: sourceID, RunID: runID, GateNodeID: "source-gate", OrganizationID: org, ProjectID: p, WorkflowID: "workflow-" + p, PluginID: "annotation-qa", SupplierID: s, ProcessingProfile: domain.ProcessingProfileRawHuman, Status: domain.HumanBatchActive, Stages: domain.DefaultFanyuV58HumanStages(), RequirePrePackageAcceptance: true})) must(ds.UpsertHumanBatch(ctx, domain.HumanBatchRecord{ID: batchID, SourceBatchID: sourceID, RunID: runID, GateNodeID: "acceptance-gate", OrganizationID: org, ProjectID: p, WorkflowID: "workflow-" + p, PluginID: "annotation-qa", ProcessingProfile: domain.ProcessingProfileRawHuman, Status: domain.HumanBatchActive, Stages: stages})) item.BatchID, item.LegacyTaskID, item.LatestAuthorUserID = batchID, "", "" item.SupplierID = "" if strings.HasSuffix(id, "0") { item.SupplierID = s } item.Subject.ItemKey = sourceID itemPayload["sourceBatchId"], itemPayload["strictThreeRole"] = sourceID, true } must(ds.UpsertHumanItem(ctx, item)) if productionShape { must(ds.AppendHumanEvent(ctx, domain.HumanEventRecord{ID: "created-" + id, OrganizationID: org, ItemID: id, StageID: stage, EventType: "created", CreatedAt: "2026-10-01T00:00:00Z"})) } } if !reuseFixture { for _, a := range actors { if a.Role == "admin" { continue } stage := "annotation_combined" if a.Role == "quality" { stage = "quality_review" } if a.Role == "acceptance" { stage = "prepackage_acceptance" } for n := 0; n < 2; n++ { seed(fmt.Sprintf("owned-%s-%d", a.User.ID, n), a.Project, a.User.SupplierID, stage, a.User.ID, domain.HumanItemClaimed) } if a.Role == "annotation" { taskID := "task-owned-" + a.User.ID + "-0" must(ds.UpsertAnnotationRevision(ctx, domain.AnnotationRevisionRecord{ID: "revision-" + a.User.ID, OrganizationID: org, TaskID: taskID, Revision: 1, CountsAsOutput: true, AuthorUserID: a.User.ID})) must(ds.UpsertAnnotationWorklogEvent(ctx, domain.AnnotationWorklogEventRecord{ID: "worklog-" + a.User.ID, OrganizationID: org, ProjectID: a.Project, TaskID: taskID, UserID: a.User.ID, SupplierID: a.User.SupplierID, EventType: "combined_submitted", Revision: 1, OutputDelta: 1, CreatedAt: "2026-10-02T00:00:00Z"})) } if a.Role != "annotation" { must(ds.AppendHumanEvent(ctx, domain.HumanEventRecord{ID: "history-" + a.User.ID, OrganizationID: org, ItemID: "owned-" + a.User.ID + "-0", ActorUserID: a.User.ID, StageID: stage, EventType: "approved", CreatedAt: "2026-10-02T00:00:00Z"})) } } for _, p := range projects { for _, s := range suppliers { stages := []string{"quality_review", "semantic_review", "acceptance", "prepackage_acceptance"} if claimCounts { stages = append(stages, "annotation_combined") } for _, stage := range stages { for n := 0; n < availablePerStage; n++ { seed(fmt.Sprintf("available-%s-%s-%s-%d", p, s, stage, n), p, s, stage, "", domain.HumanItemReady) } } } } } authorHistory := 0 if raw := os.Getenv("POLL_POOL_AUTHOR_HISTORY"); raw != "" { authorHistory, err = strconv.Atoi(raw) must(err) if authorHistory < 0 || authorHistory > 5000 { t.Fatal("author history must be 0..5000 per reviewer") } } historyPartitions := map[string]domain.ReleasePartitionRecord{} if !reuseFixture { for _, a := range actors { if authorHistory == 0 || (a.Role != "quality" && a.Role != "acceptance") { continue } rows := make([]any, 0, authorHistory) for i := 0; i < authorHistory; i++ { stage := "quality_review" suffix := "" if a.Role == "quality" && i%2 != 0 { stage = "semantic_review" suffix = "-semantic" } if a.Role == "acceptance" { stage = "acceptance" if i%2 != 0 { stage = "prepackage_acceptance" } suffix = "-" + stage } id := fmt.Sprintf("historical-%s-%05d", a.User.ID, i) batchID := "batch-" + a.Project + a.User.SupplierID + suffix partitionID := "released-history-" + batchID partition := historyPartitions[partitionID] partition.ID = partitionID partition.OrganizationID = org partition.BatchID = batchID partition.Sequence = 1 partition.UniqueKey = partitionID partition.Status = domain.ReleasePartitionDispatched partition.ItemIDs = append(partition.ItemIDs, id) historyPartitions[partitionID] = partition rows = append(rows, domain.HumanItemRecord{ID: id, OrganizationID: org, ProjectID: a.Project, WorkflowID: "workflow-" + a.Project, PluginID: "annotation-qa", SupplierID: a.User.SupplierID, BatchID: batchID, ReleasePartitionID: partitionID, LatestAuthorUserID: a.User.ID, CurrentStageID: stage, Status: domain.HumanItemAccepted, Version: 1, CreatedAt: "2026-09-01T00:00:00Z", Subject: domain.HumanSubjectRef{ItemKey: id}}) } _, e := db.Collection("humanItems").InsertMany(ctx, rows) must(e) } for _, partition := range historyPartitions { must(ds.UpsertReleasePartition(ctx, partition)) } } authorizationState := os.Getenv("POLL_POOL_AUTHORIZATION_STATE") if authorizationState == "" { authorizationState = "legacy" } if authorizationState != "legacy" && authorizationState != "initialized" { t.Fatal("unknown authorization state") } if authorizationState == "initialized" && !reuseFixture { must(ds.InitializeSupplierProjectAuthorizations(ctx, org)) } if readModels && !reuseFixture { result, err := ds.BackfillPoolSearchMetadata(ctx, org, true) must(err) if result.Unsupported > 0 || result.Conflicts > 0 { t.Fatalf("summary preparation incomplete: %+v", result) } } handler := NewHandler(ds, nil) var mu sync.Mutex samples := []pollLoadSample{} coldSamples := []pollLoadSample{} lastPaths := map[string]string{} recordingCold := true failureSamples := []map[string]any{} var completedRequests, completedErrors atomic.Int64 call := func(a pollLoadActor, key, path string, record bool) pollLoadSample { mu.Lock() lastPaths[a.User.ID+"\x00"+key] = path mu.Unlock() req := httptest.NewRequest(http.MethodGet, path, nil) requestCtx, done := context.WithTimeout(ctx, 15*time.Second) defer done() requestCtx = context.WithValue(requestCtx, pollLoadEndpointKey{}, key) req = req.WithContext(context.WithValue(requestCtx, authContextKey{}, &authenticatedPrincipal{User: a.User, Organization: domain.OrganizationRecord{ID: org, Status: domain.OrganizationStatusActive}})) response := httptest.NewRecorder() start := time.Now() handler.ServeHTTP(response, req) elapsed := time.Since(start) completedRequests.Add(1) var body struct { Summary *productionstats.Summary `json:"summary"` Totals json.RawMessage `json:"totals"` Count *int `json:"count"` Counts map[string]int `json:"counts"` Code string `json:"code"` Error string `json:"error"` Items []struct { ID string `json:"id"` CanClaim *bool `json:"canClaim"` TraceSummary *domain.HumanTraceSummary `json:"traceSummary"` } `json:"items"` Summaries []struct { ID string `json:"id"` CanClaim *bool `json:"canClaim"` TraceSummary *domain.HumanTraceSummary `json:"traceSummary"` } `json:"summaries"` } _ = json.Unmarshal(response.Body.Bytes(), &body) ids := []string{} for _, v := range body.Items { ids = append(ids, v.ID) if v.CanClaim != nil { ids = append(ids, fmt.Sprintf("claim:%s:%t", v.ID, *v.CanClaim)) } if v.TraceSummary != nil { raw, _ := json.Marshal(v.TraceSummary) ids = append(ids, "trace:"+v.ID+":"+string(raw)) } if pageShape && strings.Contains(path, "trace=1") && v.TraceSummary == nil { response.Code = 599 } } for _, v := range body.Summaries { ids = append(ids, v.ID) if v.CanClaim != nil { ids = append(ids, fmt.Sprintf("claim:%s:%t", v.ID, *v.CanClaim)) } if v.TraceSummary != nil { raw, _ := json.Marshal(v.TraceSummary) ids = append(ids, "trace:"+v.ID+":"+string(raw)) } if pageShape && strings.Contains(path, "trace=1") && v.TraceSummary == nil { response.Code = 599 } } if response.Code != 200 { completedErrors.Add(1) } rowCount := len(body.Items) + len(body.Summaries) if body.Summary != nil { rowCount = len(body.Summary.Rows) raw, _ := json.Marshal(body.Summary) ids = append(ids, string(raw)) } if strings.HasSuffix(key, ".panel") && len(body.Totals) > 0 { rowCount = 1 ids = append(ids, string(body.Totals)) } if body.Count != nil { rowCount = *body.Count ids = append(ids, fmt.Sprintf("count:%d", *body.Count)) } for pool, total := range body.Counts { ids = append(ids, fmt.Sprintf("pool-count:%s:%d", pool, total)) } sort.Strings(ids) var compressed bytes.Buffer zw := gzip.NewWriter(&compressed) _, _ = zw.Write(response.Body.Bytes()) _ = zw.Close() sample := pollLoadSample{a.User.ID, key, float64(elapsed.Microseconds()) / 1000, response.Body.Len(), compressed.Len(), rowCount, response.Code, fmt.Sprintf("%x", sha256.Sum256([]byte(strings.Join(ids, "\n"))))} if response.Code != 200 { class := "unclassified" for _, known := range []string{"operation exceeded time limit", "context deadline exceeded", "context canceled", "duplicate key", "not found", "conflict", "timeout"} { if strings.Contains(strings.ToLower(body.Error), known) { class = known break } } code := strings.Map(func(r rune) rune { if r >= 'a' && r <= 'z' || r >= 'A' && r <= 'Z' || r >= '0' && r <= '9' || r == '_' || r == '-' { return r } return -1 }, body.Code) if len(code) > 80 { code = code[:80] } mu.Lock() if len(failureSamples) < 20 { failureSamples = append(failureSamples, map[string]any{"at": time.Now().UTC().Format(time.RFC3339Nano), "endpoint": key, "actor": a.User.ID, "status": response.Code, "elapsed_ms": sample.MS, "code": code, "class": class, "measured": record}) if productionShape { t.Logf("synthetic error: %.400s", body.Error) } t.Logf("read-mix HTTP failure endpoint=%s actor=%s status=%d elapsed_ms=%.3f code=%s class=%s request_context=%v", key, a.User.ID, response.Code, sample.MS, code, class, requestCtx.Err()) } mu.Unlock() } if recordingCold { coldSamples = append(coldSamples, sample) } if record { mu.Lock() samples = append(samples, sample) mu.Unlock() } return sample } paths := func(a pollLoadActor, all, countCycle bool, scenario int) map[string]string { scope := "&projectId=" + a.Project scopeName := "project" if all { scope = "" scopeName = "all" } if a.Role == "annotation" { path := "/api/plugins/annotation-qa/work-buffer?projectId=" + a.Project if mode == "summary" { path += "&view=summary" } result := map[string]string{"annotation.buffer": path} if claimCounts && countCycle { result["annotation.claim-count"] = "/api/plugins/annotation-qa/claim-count?projectId=" + a.Project } return result } if a.Role == "admin" { if os.Getenv("POLL_POOL_ADMIN_TASKS") == "1" { base := "/api/plugins/annotation-qa/admin-pool?limit=200" + scope suffix, key := "&pool=available", "available" switch scenario % 5 { case 1: suffix, key = "&pool=all&search=synthetic-available-", "search" case 2: suffix, key = "&pool=all&search=__production_shape_no_match__", "search.nohit" case 3: suffix, key = "&pool=mine", "mine" case 4: suffix, key = "&pool=all&category=pending_slice", "category" } return map[string]string{"admin." + scopeName + "." + key: base + suffix} } return map[string]string{"admin.workforce": "/api/dashboard/workforce?from=2026-10-01T00:00:00Z&to=2026-10-28T00:00:00Z"} } if pageShape { // AnnotationOperations loads one selected tab plus poolCounts. // The old broad stress mode below deliberately requests every tab // and both acceptance stages separately; keep it as a separate test. stages := "quality_review,semantic_review" if a.Role == "acceptance" { stages = "acceptance,prepackage_acceptance" } endpoint := "/api/human-item-summaries" pool, search, kind := "available", "", "available" switch scenario % 5 { case 1: stage := "quality_review" if a.Role == "acceptance" { stage = "prepackage_acceptance" } search = "synthetic-available-" + a.Project + "-" + a.User.SupplierID + "-" + stage + "-0" kind = "search" case 2: search, kind = "__production_shape_no_match__", "search.nohit" case 3: pool, kind = "mine", "mine" case 4: pool, kind = "reviewed", "reviewed" } key := a.Role + "." + scopeName + "." + kind query := "?pluginId=annotation-qa" + scope + "&stageIds=" + stages + "&search=" + url.QueryEscape(search) counts := "/api/human-item-counts" + query if a.Role == "acceptance" { counts += "&reviewedStageIds=" + stages } listQuery := query if a.Role == "quality" && pool == "reviewed" { listQuery = strings.Replace(query, "&stageIds="+stages, "", 1) } return map[string]string{key: endpoint + listQuery + "&pool=" + pool + "&limit=200&trace=1", key + ".counts": counts} } result := map[string]string{} for _, pool := range []string{"available", "mine", "reviewed"} { endpoint := "/api/human-items" if readModels { endpoint = "/api/human-item-summaries" } base := endpoint + "?pluginId=annotation-qa&pool=" + pool + scope if a.Role == "quality" { result["quality."+scopeName+"."+pool] = base + "&stageIds=quality_review%2Csemantic_review" } else { for _, stage := range []string{"acceptance", "prepackage_acceptance"} { result["acceptance."+scopeName+"."+stage+"."+pool] = base + "&stageId=" + stage } } } if readModels { stages := "quality_review,semantic_review" reviewed := "" if a.Role == "acceptance" { stages = "acceptance,prepackage_acceptance" reviewed = "&reviewedStageIds=" + stages } result[a.Role+"."+scopeName+".counts"] = "/api/human-item-counts?pluginId=annotation-qa" + scope + "&stageIds=" + stages + reviewed } if os.Getenv("POLL_POOL_SEARCH") == "1" { endpoint := "/api/human-items" if readModels { endpoint = "/api/human-item-summaries" } stage := "quality_review" if a.Role == "acceptance" { stage = "prepackage_acceptance" } result[a.Role+"."+scopeName+".search"] = endpoint + "?pluginId=annotation-qa&pool=available&limit=200" + scope + "&stageId=" + stage + "&search=synthetic-available-" + a.Project + "-" + a.User.SupplierID + "-" + stage + "-0" if productionShape { stages := "quality_review,semantic_review" if a.Role == "acceptance" { stages = "acceptance,prepackage_acceptance" } base := endpoint + "?pluginId=annotation-qa&pool=available&limit=200" + scope + "&stageIds=" + stages result[a.Role+"."+scopeName+".search"] = base + "&search=synthetic-available-" + a.Project + "-" + a.User.SupplierID + "-" + stage + "-0" result[a.Role+"."+scopeName+".search.nohit"] = base + "&search=__production_shape_no_match__" } } return result } if queryPanels { original := paths paths = func(a pollLoadActor, all, countCycle bool, scenario int) map[string]string { out := original(a, all, countCycle, scenario) out[a.Role+".panel"] = "/api/dashboard/production-quality?from=2026-10-01&to=2026-11-13" if a.Role == "admin" { out["admin.production"] = "/api/dashboard/production?from=2026-10-01&to=2026-11-13" } return out } } fixtureSpec := fmt.Sprintf("isolated-preparation-v5|%d|%d|%d|%d|%d|%d|%d|%t|%t|%t|%s|%s", annotators, quality, acceptance, availablePerStage, runPayloadKiB, taskPayloadKiB, authorHistory, productionShape, readModels, claimCounts, authorizationState, os.Getenv("POLL_POOL_SEARCH_INDEXES")) fixtureSpec += "|admin-tasks=" + os.Getenv("POLL_POOL_ADMIN_TASKS") if os.Getenv("POLL_POOL_QUERY_SERVING_PREPARE") == "1" { fixtureSpec += "|query-serving-preparation-v1" } fixtureDigest := fmt.Sprintf("%x", sha256.Sum256([]byte(fixtureSpec))) if reuseFixture { var manifest struct { Digest string `bson:"digest"` } must(db.Collection("pollFixture").FindOne(ctx, map[string]any{"_id": "identity"}).Decode(&manifest)) if manifest.Digest != fixtureDigest { t.Fatal("fixture configuration differs from preparation") } } else { _, err := db.Collection("pollFixture").InsertOne(ctx, map[string]any{"_id": "identity", "digest": fixtureDigest}) must(err) } fixtureCounts := map[string]int64{} if productionShape { if reuseFixture { for collection, want := range map[string]int64{"humanItems": 26218, "runs": 26490, "captureRecords": 29767, "annotationTasks": 23611, "humanEvents": 342694} { got, err := db.Collection(collection).CountDocuments(ctx, map[string]any{"organizationId": org}) must(err) if got != want { t.Fatalf("fixture %s count=%d want=%d", collection, got, want) } fixtureCounts[collection] = got } } else { var err error fixtureCounts, err = seedProductionPoolNoise(ctx, db, org, payload) must(err) must(installPoolReadIndexes(ctx, db, os.Getenv("POLL_POOL_SEARCH_INDEXES") == "1")) } } cleanupFixture = true if os.Getenv("POLL_POOL_FIXTURE_ONLY") == "1" { return } var preparationCPU float64 if queryServing { startCPU, _ := pollLoadProcessCPU() if os.Getenv("QUERY_SERVING_LOCAL_TEST") != "1" { t.Fatal("query acceleration requires isolated local dependencies") } cache, err := queryserving.NewManagedValkey("valkey://127.0.0.1:16379", db.Name()) must(err) defer cache.Close() search, err := queryserving.NewTypesense("http://127.0.0.1:18108", os.Getenv("QUERY_TEST_TYPESENSE_API_KEY")) must(err) ds.ConfigureQueryServing(cache, search) builder := store.NewQueryReadBuilder(ds) buildCtx, stop := context.WithCancel(ctx) buildDone := make(chan error, 1) go func() { buildDone <- builder.Run(buildCtx) }() defer func() { stop(); <-buildDone; builder.Close() }() prepare, finish := context.WithTimeout(ctx, 4*time.Minute) err = builder.WaitReady(prepare, org, projects) finish() must(err) endCPU, _ := pollLoadProcessCPU() preparationCPU = endCPU - startCPU } // Fail before timing if permissions or fixtures accidentally produce empty work. for _, a := range actors { for key, path := range paths(a, false, true, 0) { result := call(a, key, path, false) if result.Code != 200 { if allowBaselineFailure { continue } t.Fatalf("preflight %s status=%d", key, result.Code) } // This workload seeds assigned annotation work, but no unassigned // annotation pool unless claim-count coverage was requested. An empty // admin available pool is then the correct fixture result, not failure. requireRows := a.Role == "annotation" || a.Role == "admin" && os.Getenv("POLL_POOL_ADMIN_TASKS") != "1" || a.Role != "admin" && (strings.HasSuffix(key, ".available") || strings.HasSuffix(key, ".mine") && !strings.Contains(key, ".acceptance.")) || a.Role == "admin" && claimCounts && strings.HasSuffix(key, ".available") if requireRows && result.Rows == 0 { t.Fatalf("preflight %s unexpectedly empty for %s", key, a.User.ID) } } } recordingCold = false var memBefore, memAfter runtime.MemStats runPhase := func(seconds int, record bool) { phaseCtx, stop := context.WithTimeout(ctx, time.Duration(seconds)*time.Second) defer stop() var wg sync.WaitGroup for _, actor := range actors { wg.Add(1) go func(a pollLoadActor) { defer wg.Done() period := 12 * time.Second if a.Role == "annotation" { period = 3 * time.Second } if a.Role == "admin" && os.Getenv("POLL_POOL_ADMIN_TASKS") == "1" { period = time.Duration(adminPeriod) * time.Second } ticker := time.NewTicker(period) defer ticker.Stop() cycles := (time.Duration(seconds)*time.Second + period - 1) / period for cycle := 0; cycle < int(cycles); cycle++ { select { case <-phaseCtx.Done(): return default: } var requests sync.WaitGroup for key, path := range paths(a, cycle%2 == 1, cycle%4 == 0, cycle/2) { requests.Add(1) go func(key, path string) { defer requests.Done(); call(a, key, path, record) }(key, path) } requests.Wait() if cycle+1 == int(cycles) { return } select { case <-phaseCtx.Done(): return case <-ticker.C: } } }(actor) } wg.Wait() <-phaseCtx.Done() } out := os.Getenv("POLL_POOL_OUTPUT") if out == "" { out = filepath.Join("..", "..", "local-data", "test-results", fmt.Sprintf("poll-pool-%s-%d.json", mode, time.Now().Unix())) } must(os.MkdirAll(filepath.Dir(out), 0700)) resourceFile, err := os.Create(out + ".resources.jsonl") must(err) defer resourceFile.Close() resourceEncoder := json.NewEncoder(resourceFile) resourceRows := []map[string]any{} var phase atomic.Value phase.Store("warmup") resourceStart := time.Now() sampleResource := func() error { var m runtime.MemStats runtime.ReadMemStats(&m) cpuSeconds, cpuErr := pollLoadProcessCPU() if cpuErr != nil { return cpuErr } mu.Lock() requests := len(samples) mu.Unlock() row := map[string]any{"at": time.Now().UTC().Format(time.RFC3339Nano), "phase": phase.Load(), "elapsed_seconds": time.Since(resourceStart).Seconds(), "heap_alloc_bytes": m.HeapAlloc, "heap_inuse_bytes": m.HeapInuse, "total_alloc_bytes": m.TotalAlloc, "gc_cycles": m.NumGC, "goroutines": runtime.NumGoroutine(), "retained_request_samples": requests, "errors": completedErrors.Load(), "completed_requests": completedRequests.Load(), "process_cpu_seconds": cpuSeconds} resourceRows = append(resourceRows, row) return resourceEncoder.Encode(row) } must(sampleResource()) resourceStop := make(chan struct{}) resourceDone := make(chan error, 1) go func() { ticker := time.NewTicker(15 * time.Second) defer ticker.Stop() for { select { case <-resourceStop: resourceDone <- sampleResource() return case <-ticker.C: if e := sampleResource(); e != nil { cancel() resourceDone <- e return } } } }() runPhase(warmup, false) if completedErrors.Load() != 0 && !allowBaselineFailure { close(resourceStop) must(<-resourceDone) t.Fatalf("warmup had %d HTTP errors", completedErrors.Load()) } phase.Store("steady") runtime.ReadMemStats(&memBefore) var cpuFile *os.File if filename := os.Getenv("POLL_POOL_CPU_PROFILE"); filename != "" { must(os.MkdirAll(filepath.Dir(filename), 0700)) cpuFile, err = os.Create(filename) must(err) must(pprof.StartCPUProfile(cpuFile)) } cpuBefore, cpuErr := pollLoadProcessCPU() must(cpuErr) measured.Store(true) start := time.Now() runPhase(duration, true) measured.Store(false) phase.Store("finished") close(resourceStop) must(<-resourceDone) must(resourceFile.Close()) if cpuFile != nil { pprof.StopCPUProfile() must(cpuFile.Close()) } cpuAfter, cpuErr := pollLoadProcessCPU() must(cpuErr) actual := time.Since(start) runtime.ReadMemStats(&memAfter) // Outside the measured interval: distinguish retained objects from GC timing. runtime.GC() var memAfterGC runtime.MemStats runtime.ReadMemStats(&memAfterGC) grouped := map[string][]pollLoadSample{} failures := 0 for _, s := range samples { grouped[s.Key] = append(grouped[s.Key], s) if s.Code != 200 { failures++ } } metrics := map[string]any{} for key, rows := range grouped { lat := []float64{} raw, gz, minRows := 0, 0, rows[0].Rows for _, s := range rows { lat = append(lat, s.MS) raw += s.Raw gz += s.Gzip if s.Rows < minRows { minRows = s.Rows } } sort.Float64s(lat) metrics[key] = map[string]any{"requests": len(rows), "p95_ms": lat[(len(lat)*95-1)/100], "max_ms": lat[len(lat)-1], "raw_bytes": raw, "gzip_bytes": gz, "min_items": minRows} } slowSearch := []string{} if raw := os.Getenv("POLL_POOL_MAX_SEARCH_P95_MS"); raw != "" { budget, err := strconv.ParseFloat(raw, 64) if err != nil || budget <= 0 { t.Fatal("invalid search latency budget") } for key, value := range metrics { if strings.Contains(key, ".search") && value.(map[string]any)["p95_ms"].(float64) > budget { slowSearch = append(slowSearch, key) } } } outputState, stateErr := ds.OutputSnapshotState(ctx) must(stateErr) preparedQueries := 0 for _, head := range outputState.Heads { if head.Current != nil && head.Current.Metrics != nil { preparedQueries += head.Current.Metrics.PreparedQueries } } var usage syscall.Rusage must(syscall.Getrusage(syscall.RUSAGE_SELF, &usage)) maxRSS := usage.Maxrss if runtime.GOOS != "darwin" { maxRSS *= 1024 } // Outside all measured latency, CPU, heap and Mongo counters: a serial // retry supplies correctness evidence for baseline endpoints that never // completed under load. Its latency is NOT mixed into benchmark metrics. references := []pollLoadSample{} if queryPanels { succeeded := map[string]bool{} tried := map[string]bool{} all := append(append([]pollLoadSample{}, coldSamples...), samples...) for _, sample := range all { if sample.Code == 200 { succeeded[sample.Actor+"\x00"+sample.Key] = true } } for _, sample := range all { identity := sample.Actor + "\x00" + sample.Key if succeeded[identity] || tried[identity] || len(references) >= 32 { continue } tried[identity] = true for _, actor := range actors { if actor.User.ID == sample.Actor { references = append(references, call(actor, sample.Key, lastPaths[identity], false)) break } } } } report := map[string]any{"mongo_clock_samples": clockSamples.Load(), "mongo_clock_max_jump_ms": clockJumpMS.Load(), "reference_samples": references, "platform_read_indexes": os.Getenv("POLL_POOL_QUERY_SERVING_PREPARE") == "1", "output_prepared_queries": preparedQueries, "process_max_rss_bytes": maxRSS, "query_serving": queryServing, "query_preparation_cpu_seconds": preparationCPU, "query_serving_statistics": ds.QueryServingStatistics(), "query_panels": queryPanels, "derived_write_commands": derivedWrites.Load(), "reused_prepared_fixture": reuseFixture, "fixture_digest": fixtureDigest, "cold_samples": coldSamples, "slow_search_endpoints": slowSearch, "baseline_failure_collection": allowBaselineFailure, "production_shape": productionShape, "page_shape": pageShape, "fixture_counts": fixtureCounts, "search_indexes": os.Getenv("POLL_POOL_SEARCH_INDEXES") == "1", "read_models": readModels, "search_queries": os.Getenv("POLL_POOL_SEARCH") == "1", "schema": 2, "resource_samples": resourceRows, "process_cpu_seconds": cpuAfter - cpuBefore, "authorization_state": authorizationState, "fixture": "read-mix-v5-isolated-preparation", "available_per_stage": availablePerStage, "task_payload_kib": taskPayloadKiB, "run_payload_kib": runPayloadKiB, "author_history_per_reviewer": authorHistory, "mode": mode, "revision": os.Getenv("POLL_POOL_REVISION"), "actors": totalActors, "actor_counts": map[string]int{"annotation": annotators, "quality": quality, "acceptance": acceptance, "admin": 1}, "claim_counts": claimCounts, "warmup_seconds": warmup, "scheduled_seconds": duration, "actual_seconds": actual.Seconds(), "errors": failures, "failure_samples": failureSamples, "metrics": metrics, "mongo_breakdown": observations, "mongo_commands": commands.Load(), "mongo_reply_bytes": mongoBytes.Load(), "mongo_command_ms": float64(mongoNS.Load()) / 1e6, "mongo_write_commands": writes.Load(), "go_heap_before": memBefore.HeapAlloc, "go_heap_after": memAfter.HeapAlloc, "go_heap_after_gc": memAfterGC.HeapAlloc, "go_gc_cycles": memAfter.NumGC - memBefore.NumGC, "go_allocated_bytes": memAfter.TotalAlloc - memBefore.TotalAlloc, "limitations": []string{"handler timing excludes ingress/network/authentication", "gzip computed separately with fixed Go default compression", "no continuation renew, media or workflow writes", "synthetic fixture; not capacity acceptance", "CPU covers handler and in-process load harness, not Mongo/Pod throttling", "resource heap includes retained benchmark request samples; not standalone service RSS"}, "samples": samples} encoded, e := json.MarshalIndent(report, "", " ") must(e) must(os.WriteFile(out, encoded, 0600)) t.Logf("read-mix report: %s, requests=%d errors=%d", out, len(samples), failures) if len(slowSearch) > 0 { t.Errorf("search latency budget exceeded: %v", slowSearch) } if failures != 0 { t.Fatalf("load returned %d errors", failures) } if writes.Load() != 0 { t.Fatalf("stable read mix wrote %d commands", writes.Load()) } }