droboticflow-new/internal/api/poll_pool_load_test.go

1109 lines
43 KiB
Go

package api
// Local-only read-mix harness. This is NOT full concurrent-user capacity acceptance:
// it excludes media, editing/submission, continuation writes and ingress shaping.
import (
"bytes"
"compress/gzip"
"context"
"crypto/sha256"
"encoding/json"
"fmt"
"net/http"
"net/http/httptest"
"net/url"
"os"
"path/filepath"
"runtime"
"runtime/pprof"
"sort"
"strconv"
"strings"
"sync"
"sync/atomic"
"syscall"
"testing"
"time"
"droboticflow/internal/domain"
"droboticflow/internal/productionstats"
"droboticflow/internal/queryserving"
"droboticflow/internal/store"
"go.mongodb.org/mongo-driver/event"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
)
type pollLoadActor struct {
User domain.UserRecord
Project string
Role string
}
type pollLoadSample struct {
Actor string
Key string
MS float64
Raw, Gzip, Rows int
Code int
Hash string
}
type pollLoadEndpointKey struct{}
type pollMongoObservation struct {
Endpoint, Collection, Command string
Count, Bytes, Nanoseconds, Failed int64
FailedNanoseconds, MaxTimeMS int64
}
// Rusage covers this test process (handler plus harness compression/sampling),
// not Mongo, another process, or Kubernetes Pod throttling/RSS.
func pollLoadProcessCPU() (float64, error) {
var usage syscall.Rusage
if err := syscall.Getrusage(syscall.RUSAGE_SELF, &usage); err != nil {
return 0, err
}
return float64(usage.Utime.Sec) + float64(usage.Utime.Usec)/1e6 + float64(usage.Stime.Sec) + float64(usage.Stime.Usec)/1e6, nil
}
func TestPollPoolReadMixLocal(t *testing.T) {
if os.Getenv("POLL_POOL_LOAD") != "1" {
t.Skip("explicit isolated local read-mix verification")
}
seconds := func(key string, fallback int) int {
v := os.Getenv(key)
if v == "" {
return fallback
}
n, e := strconv.Atoi(v)
if e != nil || n < 1 || n > 7200 {
t.Fatalf("invalid %s", key)
}
return n
}
adminPeriod := seconds("POLL_POOL_ADMIN_PERIOD_SECONDS", 12)
warmup, duration := seconds("POLL_POOL_WARMUP_SECONDS", 5), seconds("POLL_POOL_DURATION_SECONDS", 30)
annotators := seconds("POLL_POOL_ANNOTATORS", 60)
quality := seconds("POLL_POOL_QUALITY_REVIEWERS", 12)
acceptance := seconds("POLL_POOL_ACCEPTANCE_REVIEWERS", 8)
totalActors := annotators + quality + acceptance + 1
if totalActors > 250 {
t.Fatal("local read mix supports at most 250 actors")
}
availablePerStage := seconds("POLL_POOL_AVAILABLE_PER_STAGE", 8)
if availablePerStage > 250 {
t.Fatal("available fixtures limited to 250 per stage and supplier")
}
claimCounts := os.Getenv("POLL_POOL_CLAIM_COUNTS") == "1"
productionShape := os.Getenv("POLL_POOL_PRODUCTION_SHAPE") == "1"
pageShape := os.Getenv("POLL_POOL_PAGE_SHAPE") == "1"
allowBaselineFailure := os.Getenv("POLL_POOL_ALLOW_BASELINE_FAILURE") == "1"
readModels := os.Getenv("POLL_POOL_READMODELS") == "1"
queryServing := os.Getenv("POLL_POOL_QUERY_SERVING") == "1"
queryPanels := os.Getenv("POLL_POOL_QUERY_PANELS") == "1"
mode := os.Getenv("POLL_POOL_MODE")
if mode == "" {
mode = "summary"
}
if mode != "summary" && mode != "legacy" {
t.Fatal("mode must be summary or legacy")
}
ctx, cancel := context.WithTimeout(context.Background(), time.Duration(warmup+duration+600)*time.Second)
defer cancel()
var measured atomic.Bool
var commands, mongoBytes, mongoNS, writes atomic.Int64
var derivedWrites atomic.Int64
var probeMu sync.Mutex
pending := map[int64]struct {
key string
maxMS int64
}{}
observations := map[string]*pollMongoObservation{}
monitor := &event.CommandMonitor{
Started: func(commandCtx context.Context, e *event.CommandStartedEvent) {
if !measured.Load() {
return
}
endpoint, _ := commandCtx.Value(pollLoadEndpointKey{}).(string)
collection, _ := e.Command.Lookup(e.CommandName).StringValueOK()
if collection == "" {
collection, _ = e.Command.Lookup("collection").StringValueOK()
}
key := endpoint + "/" + collection + "/" + e.CommandName
probeMu.Lock()
defer probeMu.Unlock()
maxMS, _ := e.Command.Lookup("maxTimeMS").Int64OK()
if short, ok := e.Command.Lookup("maxTimeMS").Int32OK(); ok {
maxMS = int64(short)
}
pending[e.RequestID] = struct {
key string
maxMS int64
}{key, maxMS}
if observations[key] == nil {
observations[key] = &pollMongoObservation{Endpoint: endpoint, Collection: collection, Command: e.CommandName}
}
},
Failed: func(_ context.Context, e *event.CommandFailedEvent) {
probeMu.Lock()
defer probeMu.Unlock()
if entry, ok := pending[e.RequestID]; ok {
key := entry.key
observations[key].Failed++
observations[key].FailedNanoseconds += e.Duration.Nanoseconds()
observations[key].MaxTimeMS = entry.maxMS
if productionShape {
t.Logf("synthetic mongo failure key=%s duration_ms=%d maxTimeMS=%d", key, e.Duration.Milliseconds(), entry.maxMS)
}
delete(pending, e.RequestID)
}
},
Succeeded: func(_ context.Context, e *event.CommandSucceededEvent) {
derived := false
probeMu.Lock()
if entry, ok := pending[e.RequestID]; ok {
key := entry.key
row := observations[key]
derived = strings.HasPrefix(row.Collection, "queryReadModel") || strings.HasPrefix(row.Collection, "annotationOutputSnapshot")
row.Count++
row.Bytes += int64(len(e.Reply))
row.Nanoseconds += e.Duration.Nanoseconds()
delete(pending, e.RequestID)
}
probeMu.Unlock()
if measured.Load() {
commands.Add(1)
mongoBytes.Add(int64(len(e.Reply)))
mongoNS.Add(e.Duration.Nanoseconds())
switch e.CommandName {
case "insert", "update", "delete", "findAndModify":
if queryServing && derived {
derivedWrites.Add(1)
} else {
writes.Add(1)
}
}
}
}}
uri := os.Getenv("POLL_POOL_LOCAL_MONGO_URI")
if uri == "" {
uri = "mongodb://127.0.0.1:27018/?directConnection=true"
}
parsed, parseErr := url.Parse(uri)
if parseErr != nil || parsed.Scheme != "mongodb" || (parsed.Host != "127.0.0.1:27017" && parsed.Host != "127.0.0.1:27018" && parsed.Host != "127.0.0.1:27019") {
t.Fatal("isolated loopback Mongo only")
}
client, err := mongo.Connect(ctx, options.Client().ApplyURI(uri).SetMonitor(monitor))
if err != nil {
t.Fatal(err)
}
defer client.Disconnect(context.Background())
clockCtx, stopClock := context.WithCancel(ctx)
clockDone := make(chan struct{})
var clockJumpMS atomic.Int64
var clockSamples atomic.Int64
go func() {
defer close(clockDone)
var previousServer, previousLocal time.Time
ticker := time.NewTicker(100 * time.Millisecond)
defer ticker.Stop()
for {
select {
case <-clockCtx.Done():
return
case <-ticker.C:
started := time.Now()
var reply struct {
LocalTime time.Time `bson:"localTime"`
}
probe, cancelProbe := context.WithTimeout(clockCtx, time.Second)
err := client.Database("admin").RunCommand(probe, map[string]any{"hello": 1}).Decode(&reply)
cancelProbe()
finished := time.Now()
if err != nil || finished.Sub(started) > 200*time.Millisecond {
continue
}
local := started.Add(finished.Sub(started) / 2)
if !previousServer.IsZero() {
drift := reply.LocalTime.Sub(previousServer) - local.Sub(previousLocal)
if drift < 0 {
drift = -drift
}
for old := clockJumpMS.Load(); drift.Milliseconds() > old; old = clockJumpMS.Load() {
if clockJumpMS.CompareAndSwap(old, drift.Milliseconds()) {
break
}
}
}
previousServer, previousLocal = reply.LocalTime, local
clockSamples.Add(1)
}
}
}()
defer func() { stopClock(); <-clockDone }()
reuseFixture := os.Getenv("POLL_POOL_REUSE_FIXTURE") == "1"
fixtureName := os.Getenv("POLL_POOL_FIXTURE_DATABASE")
if fixtureName == "" {
if reuseFixture {
t.Fatal("reuse requires an explicit synthetic fixture database")
}
fixtureName = fmt.Sprintf("poll_pool_read_mix_%d", time.Now().UnixNano())
}
if !strings.HasPrefix(fixtureName, "poll_pool_read_mix_") || len(fixtureName) > 63 || strings.Trim(fixtureName, "abcdefghijklmnopqrstuvwxyz0123456789_") != "" {
t.Fatal("invalid synthetic fixture database")
}
db := client.Database(fixtureName)
if !reuseFixture {
collections, err := db.ListCollectionNames(ctx, map[string]any{})
if err != nil {
t.Fatal(err)
}
if len(collections) != 0 {
t.Fatal("preparation requires a new synthetic database")
}
}
cleanupFixture := !reuseFixture
defer func() {
if !cleanupFixture {
return
}
if os.Getenv("POLL_POOL_KEEP_FIXTURE") == "1" {
t.Logf("retained synthetic fixture: %s", db.Name())
return
}
cleanup, done := context.WithTimeout(context.Background(), 15*time.Second)
defer done()
if e := db.Drop(cleanup); e != nil {
t.Error(e)
}
}()
ds := store.NewMongoStore(db)
must := func(e error) {
t.Helper()
if e != nil {
t.Fatal(e)
}
}
if !reuseFixture {
must(ds.EnsureIndexes(ctx))
}
if os.Getenv("POLL_POOL_QUERY_SERVING_PREPARE") == "1" {
// Both variants use the already-documented platform read indexes as
// well as the pool-search indexes. These are isolated fixtures only.
for collection, models := range store.PlatformReadIndexes() {
_, e := db.Collection(collection).Indexes().CreateMany(ctx, models)
must(e)
}
}
const org = "load-org"
projects := []string{"load-project-a", "load-project-b"}
if os.Getenv("POLL_POOL_QUERY_SERVING_PREPARE") == "1" && !reuseFixture {
must(ds.UpsertOrganization(ctx, domain.OrganizationRecord{ID: org, Status: domain.OrganizationStatusActive}))
_, err := ds.SaveProductionStatsScope(ctx, org, productionstats.Scope{ProjectIDs: projects, Revision: 1}, 0)
must(err)
must(ds.EnsureQueryReadModelIndexes(ctx))
}
if !reuseFixture {
for _, p := range projects {
must(ds.UpsertProject(ctx, domain.ProjectRecord{ID: p, OrganizationID: org, Name: p}))
must(ds.UpsertWorkflow(ctx, domain.WorkflowRecord{ID: "workflow-" + p, OrganizationID: org, ProjectID: p, Name: "load workflow"}))
}
}
suppliers := []string{"load-supplier-a", "load-supplier-b", "load-supplier-c"}
if !reuseFixture {
for _, s := range suppliers {
must(ds.UpsertSupplier(ctx, domain.SupplierRecord{ID: s, OrganizationID: org, Status: "active", Name: s}))
_, e := ds.SetSupplierAcceptanceAuthorization(ctx, org, s, true, 0, "load-admin")
must(e)
}
}
actors := []pollLoadActor{}
for i := 0; i < totalActors; i++ {
role := domain.UserRoleAnnotator
label := "annotation"
if i >= annotators && i < annotators+quality {
role = domain.UserRoleQualityReviewer
label = "quality"
}
if i >= annotators+quality && i < totalActors-1 {
role = domain.UserRoleAcceptanceReviewer
label = "acceptance"
}
if i == totalActors-1 {
role = domain.UserRoleAdmin
label = "admin"
}
u := domain.UserRecord{ID: fmt.Sprintf("load-user-%02d", i), Phone: fmt.Sprintf("synthetic-load-%02d", i), OrganizationID: org, SupplierID: suppliers[i%3], Role: role, Status: domain.UserStatusActive}
if !reuseFixture {
must(ds.UpsertUser(ctx, u))
for _, p := range projects {
must(ds.UpsertProjectMember(ctx, domain.ProjectMemberRecord{ID: p + u.ID, OrganizationID: org, ProjectID: p, UserID: u.ID, SupplierID: u.SupplierID, Roles: domain.AccountProductionRoles(u)}))
}
}
actors = append(actors, pollLoadActor{u, projects[i%2], label})
}
payload := make([]string, 512)
for i := range payload {
payload[i] = fmt.Sprintf("%x", sha256.Sum256([]byte(fmt.Sprintf("fixed-fixture-%d", i))))
}
runPayloadKiB := 0
if raw := os.Getenv("POLL_POOL_RUN_PAYLOAD_KIB"); raw != "" {
runPayloadKiB, err = strconv.Atoi(raw)
must(err)
if runPayloadKiB < 0 || runPayloadKiB > 2048 {
t.Fatal("run payload must be 0..2048 KiB")
}
}
taskPayloadKiB := 0
if raw := os.Getenv("POLL_POOL_TASK_PAYLOAD_KIB"); raw != "" {
taskPayloadKiB, err = strconv.Atoi(raw)
must(err)
if taskPayloadKiB < 0 || taskPayloadKiB > 2048 {
t.Fatal("task payload must be 0..2048 KiB")
}
}
seededRuns := map[string]bool{}
seed := func(id, p, s, stage, owner string, status domain.HumanItemStatus) {
batchID := "batch-" + p + s
stages := domain.DefaultFanyuV58HumanStages()
if stage == "semantic_review" {
batchID += "-semantic"
stages = []domain.HumanStageDefinition{{ID: stage, Kind: domain.HumanStageReview, RequiredRole: "quality_reviewer"}}
}
if stage == "acceptance" || stage == "prepackage_acceptance" {
batchID += "-" + stage
stages = []domain.HumanStageDefinition{{ID: stage, Kind: domain.HumanStageAcceptance, RequiredRole: "acceptance_reviewer"}}
}
must(ds.UpsertHumanBatch(ctx, domain.HumanBatchRecord{ID: batchID, RunID: batchID, GateNodeID: "gate", OrganizationID: org, ProjectID: p, WorkflowID: "workflow-" + p, PluginID: "annotation-qa", SupplierID: s, Status: domain.HumanBatchActive, ProcessingProfile: domain.ProcessingProfileRawHuman, Stages: stages}))
runID := ""
if runPayloadKiB > 0 {
runID = "run-" + batchID
if productionShape {
runID = "run-" + id
}
if !seededRuns[runID] {
seededRuns[runID] = true
if productionShape {
must(ds.UpsertCaptureRecord(ctx, domain.CaptureRecord{ID: "capture-" + id, OrganizationID: org, ProjectID: p, WorkflowID: "workflow-" + p, CaptureID: "synthetic-" + id, Prefix: "synthetic/" + id}))
}
dirs := []string(nil)
captureID := ""
blob := strings.Repeat("x", runPayloadKiB*1024)
if productionShape {
dirs = []string{"synthetic-" + id}
captureID = "capture-" + id
block := strings.Join(payload, "")
blob = strings.Repeat(block, runPayloadKiB/32+1)[:runPayloadKiB*1024]
}
must(ds.UpsertRun(ctx, domain.RunRecord{ID: runID, CaptureDirectories: dirs, TaskID: captureID, OrganizationID: org, ProjectID: p, WorkflowID: "workflow-" + p, ProjectName: p, WorkflowName: "workflow-" + p, NodeExecutions: []domain.NodeExecutionRecord{{NodeID: "synthetic-node", Logs: []domain.LogEntryRecord{{Message: blob}}}}}))
}
}
taskID := "task-" + id
must(ds.UpsertAnnotationTask(ctx, domain.AnnotationTaskRecord{ID: taskID, OrganizationID: org, ProjectID: p, WorkflowID: "workflow-" + p, SupplierID: s, TemplateVersion: 6, Status: domain.AnnotationTaskStatusPendingAnnotation, SourceVideo: "synthetic/camera_left.mp4"}))
itemPayload := map[string]any{"captureDirectory": "synthetic-" + id, "sliceMergeEvidence": payload, "sliceWorkflow": map[string]any{"status": "completed", "phase": "annotate"}}
if taskPayloadKiB > 0 {
content := strings.Repeat(strings.Join(payload, ""), taskPayloadKiB/32+1)[:taskPayloadKiB*1024]
itemPayload["items"] = []map[string]any{{"annotation": map[string]any{"syntheticPayload": content}, "materialization": map[string]any{"coverage": map[string]any{"durationSeconds": 900}}}}
}
item := domain.HumanItemRecord{ID: id, RunID: runID, OrganizationID: org, ProjectID: p, WorkflowID: "workflow-" + p, PluginID: "annotation-qa", SupplierID: s, BatchID: batchID, LegacyTaskID: taskID, AssignedUserID: owner, LatestAuthorUserID: "synthetic-author", CurrentStageID: stage, Status: status, Version: 1, CreatedAt: "2026-10-01T00:00:00Z", Subject: domain.HumanSubjectRef{ItemKey: "synthetic/" + id + "/camera_left.mp4", Payload: itemPayload}}
if productionShape && stage == "prepackage_acceptance" {
sourceID := "source-" + id
batchID = "aggregate-" + id
must(ds.UpsertHumanBatch(ctx, domain.HumanBatchRecord{ID: sourceID, RunID: runID, GateNodeID: "source-gate", OrganizationID: org, ProjectID: p, WorkflowID: "workflow-" + p, PluginID: "annotation-qa", SupplierID: s, ProcessingProfile: domain.ProcessingProfileRawHuman, Status: domain.HumanBatchActive, Stages: domain.DefaultFanyuV58HumanStages(), RequirePrePackageAcceptance: true}))
must(ds.UpsertHumanBatch(ctx, domain.HumanBatchRecord{ID: batchID, SourceBatchID: sourceID, RunID: runID, GateNodeID: "acceptance-gate", OrganizationID: org, ProjectID: p, WorkflowID: "workflow-" + p, PluginID: "annotation-qa", ProcessingProfile: domain.ProcessingProfileRawHuman, Status: domain.HumanBatchActive, Stages: stages}))
item.BatchID, item.LegacyTaskID, item.LatestAuthorUserID = batchID, "", ""
item.SupplierID = ""
if strings.HasSuffix(id, "0") {
item.SupplierID = s
}
item.Subject.ItemKey = sourceID
itemPayload["sourceBatchId"], itemPayload["strictThreeRole"] = sourceID, true
}
must(ds.UpsertHumanItem(ctx, item))
if productionShape {
must(ds.AppendHumanEvent(ctx, domain.HumanEventRecord{ID: "created-" + id, OrganizationID: org, ItemID: id, StageID: stage, EventType: "created", CreatedAt: "2026-10-01T00:00:00Z"}))
}
}
if !reuseFixture {
for _, a := range actors {
if a.Role == "admin" {
continue
}
stage := "annotation_combined"
if a.Role == "quality" {
stage = "quality_review"
}
if a.Role == "acceptance" {
stage = "prepackage_acceptance"
}
for n := 0; n < 2; n++ {
seed(fmt.Sprintf("owned-%s-%d", a.User.ID, n), a.Project, a.User.SupplierID, stage, a.User.ID, domain.HumanItemClaimed)
}
if a.Role == "annotation" {
taskID := "task-owned-" + a.User.ID + "-0"
must(ds.UpsertAnnotationRevision(ctx, domain.AnnotationRevisionRecord{ID: "revision-" + a.User.ID, OrganizationID: org, TaskID: taskID, Revision: 1, CountsAsOutput: true, AuthorUserID: a.User.ID}))
must(ds.UpsertAnnotationWorklogEvent(ctx, domain.AnnotationWorklogEventRecord{ID: "worklog-" + a.User.ID, OrganizationID: org, ProjectID: a.Project, TaskID: taskID, UserID: a.User.ID, SupplierID: a.User.SupplierID, EventType: "combined_submitted", Revision: 1, OutputDelta: 1, CreatedAt: "2026-10-02T00:00:00Z"}))
}
if a.Role != "annotation" {
must(ds.AppendHumanEvent(ctx, domain.HumanEventRecord{ID: "history-" + a.User.ID, OrganizationID: org, ItemID: "owned-" + a.User.ID + "-0", ActorUserID: a.User.ID, StageID: stage, EventType: "approved", CreatedAt: "2026-10-02T00:00:00Z"}))
}
}
for _, p := range projects {
for _, s := range suppliers {
stages := []string{"quality_review", "semantic_review", "acceptance", "prepackage_acceptance"}
if claimCounts {
stages = append(stages, "annotation_combined")
}
for _, stage := range stages {
for n := 0; n < availablePerStage; n++ {
seed(fmt.Sprintf("available-%s-%s-%s-%d", p, s, stage, n), p, s, stage, "", domain.HumanItemReady)
}
}
}
}
}
authorHistory := 0
if raw := os.Getenv("POLL_POOL_AUTHOR_HISTORY"); raw != "" {
authorHistory, err = strconv.Atoi(raw)
must(err)
if authorHistory < 0 || authorHistory > 5000 {
t.Fatal("author history must be 0..5000 per reviewer")
}
}
historyPartitions := map[string]domain.ReleasePartitionRecord{}
if !reuseFixture {
for _, a := range actors {
if authorHistory == 0 || (a.Role != "quality" && a.Role != "acceptance") {
continue
}
rows := make([]any, 0, authorHistory)
for i := 0; i < authorHistory; i++ {
stage := "quality_review"
suffix := ""
if a.Role == "quality" && i%2 != 0 {
stage = "semantic_review"
suffix = "-semantic"
}
if a.Role == "acceptance" {
stage = "acceptance"
if i%2 != 0 {
stage = "prepackage_acceptance"
}
suffix = "-" + stage
}
id := fmt.Sprintf("historical-%s-%05d", a.User.ID, i)
batchID := "batch-" + a.Project + a.User.SupplierID + suffix
partitionID := "released-history-" + batchID
partition := historyPartitions[partitionID]
partition.ID = partitionID
partition.OrganizationID = org
partition.BatchID = batchID
partition.Sequence = 1
partition.UniqueKey = partitionID
partition.Status = domain.ReleasePartitionDispatched
partition.ItemIDs = append(partition.ItemIDs, id)
historyPartitions[partitionID] = partition
rows = append(rows, domain.HumanItemRecord{ID: id, OrganizationID: org, ProjectID: a.Project, WorkflowID: "workflow-" + a.Project, PluginID: "annotation-qa", SupplierID: a.User.SupplierID, BatchID: batchID, ReleasePartitionID: partitionID, LatestAuthorUserID: a.User.ID, CurrentStageID: stage, Status: domain.HumanItemAccepted, Version: 1, CreatedAt: "2026-09-01T00:00:00Z", Subject: domain.HumanSubjectRef{ItemKey: id}})
}
_, e := db.Collection("humanItems").InsertMany(ctx, rows)
must(e)
}
for _, partition := range historyPartitions {
must(ds.UpsertReleasePartition(ctx, partition))
}
}
authorizationState := os.Getenv("POLL_POOL_AUTHORIZATION_STATE")
if authorizationState == "" {
authorizationState = "legacy"
}
if authorizationState != "legacy" && authorizationState != "initialized" {
t.Fatal("unknown authorization state")
}
if authorizationState == "initialized" && !reuseFixture {
must(ds.InitializeSupplierProjectAuthorizations(ctx, org))
}
if readModels && !reuseFixture {
result, err := ds.BackfillPoolSearchMetadata(ctx, org, true)
must(err)
if result.Unsupported > 0 || result.Conflicts > 0 {
t.Fatalf("summary preparation incomplete: %+v", result)
}
}
handler := NewHandler(ds, nil)
var mu sync.Mutex
samples := []pollLoadSample{}
coldSamples := []pollLoadSample{}
lastPaths := map[string]string{}
recordingCold := true
failureSamples := []map[string]any{}
var completedRequests, completedErrors atomic.Int64
call := func(a pollLoadActor, key, path string, record bool) pollLoadSample {
mu.Lock()
lastPaths[a.User.ID+"\x00"+key] = path
mu.Unlock()
req := httptest.NewRequest(http.MethodGet, path, nil)
requestCtx, done := context.WithTimeout(ctx, 15*time.Second)
defer done()
requestCtx = context.WithValue(requestCtx, pollLoadEndpointKey{}, key)
req = req.WithContext(context.WithValue(requestCtx, authContextKey{}, &authenticatedPrincipal{User: a.User, Organization: domain.OrganizationRecord{ID: org, Status: domain.OrganizationStatusActive}}))
response := httptest.NewRecorder()
start := time.Now()
handler.ServeHTTP(response, req)
elapsed := time.Since(start)
completedRequests.Add(1)
var body struct {
Summary *productionstats.Summary `json:"summary"`
Totals json.RawMessage `json:"totals"`
Count *int `json:"count"`
Counts map[string]int `json:"counts"`
Code string `json:"code"`
Error string `json:"error"`
Items []struct {
ID string `json:"id"`
CanClaim *bool `json:"canClaim"`
TraceSummary *domain.HumanTraceSummary `json:"traceSummary"`
} `json:"items"`
Summaries []struct {
ID string `json:"id"`
CanClaim *bool `json:"canClaim"`
TraceSummary *domain.HumanTraceSummary `json:"traceSummary"`
} `json:"summaries"`
}
_ = json.Unmarshal(response.Body.Bytes(), &body)
ids := []string{}
for _, v := range body.Items {
ids = append(ids, v.ID)
if v.CanClaim != nil {
ids = append(ids, fmt.Sprintf("claim:%s:%t", v.ID, *v.CanClaim))
}
if v.TraceSummary != nil {
raw, _ := json.Marshal(v.TraceSummary)
ids = append(ids, "trace:"+v.ID+":"+string(raw))
}
if pageShape && strings.Contains(path, "trace=1") && v.TraceSummary == nil {
response.Code = 599
}
}
for _, v := range body.Summaries {
ids = append(ids, v.ID)
if v.CanClaim != nil {
ids = append(ids, fmt.Sprintf("claim:%s:%t", v.ID, *v.CanClaim))
}
if v.TraceSummary != nil {
raw, _ := json.Marshal(v.TraceSummary)
ids = append(ids, "trace:"+v.ID+":"+string(raw))
}
if pageShape && strings.Contains(path, "trace=1") && v.TraceSummary == nil {
response.Code = 599
}
}
if response.Code != 200 {
completedErrors.Add(1)
}
rowCount := len(body.Items) + len(body.Summaries)
if body.Summary != nil {
rowCount = len(body.Summary.Rows)
raw, _ := json.Marshal(body.Summary)
ids = append(ids, string(raw))
}
if strings.HasSuffix(key, ".panel") && len(body.Totals) > 0 {
rowCount = 1
ids = append(ids, string(body.Totals))
}
if body.Count != nil {
rowCount = *body.Count
ids = append(ids, fmt.Sprintf("count:%d", *body.Count))
}
for pool, total := range body.Counts {
ids = append(ids, fmt.Sprintf("pool-count:%s:%d", pool, total))
}
sort.Strings(ids)
var compressed bytes.Buffer
zw := gzip.NewWriter(&compressed)
_, _ = zw.Write(response.Body.Bytes())
_ = zw.Close()
sample := pollLoadSample{a.User.ID, key, float64(elapsed.Microseconds()) / 1000, response.Body.Len(), compressed.Len(), rowCount, response.Code, fmt.Sprintf("%x", sha256.Sum256([]byte(strings.Join(ids, "\n"))))}
if response.Code != 200 {
class := "unclassified"
for _, known := range []string{"operation exceeded time limit", "context deadline exceeded", "context canceled", "duplicate key", "not found", "conflict", "timeout"} {
if strings.Contains(strings.ToLower(body.Error), known) {
class = known
break
}
}
code := strings.Map(func(r rune) rune {
if r >= 'a' && r <= 'z' || r >= 'A' && r <= 'Z' || r >= '0' && r <= '9' || r == '_' || r == '-' {
return r
}
return -1
}, body.Code)
if len(code) > 80 {
code = code[:80]
}
mu.Lock()
if len(failureSamples) < 20 {
failureSamples = append(failureSamples, map[string]any{"at": time.Now().UTC().Format(time.RFC3339Nano), "endpoint": key, "actor": a.User.ID, "status": response.Code, "elapsed_ms": sample.MS, "code": code, "class": class, "measured": record})
if productionShape {
t.Logf("synthetic error: %.400s", body.Error)
}
t.Logf("read-mix HTTP failure endpoint=%s actor=%s status=%d elapsed_ms=%.3f code=%s class=%s request_context=%v", key, a.User.ID, response.Code, sample.MS, code, class, requestCtx.Err())
}
mu.Unlock()
}
if recordingCold {
coldSamples = append(coldSamples, sample)
}
if record {
mu.Lock()
samples = append(samples, sample)
mu.Unlock()
}
return sample
}
paths := func(a pollLoadActor, all, countCycle bool, scenario int) map[string]string {
scope := "&projectId=" + a.Project
scopeName := "project"
if all {
scope = ""
scopeName = "all"
}
if a.Role == "annotation" {
path := "/api/plugins/annotation-qa/work-buffer?projectId=" + a.Project
if mode == "summary" {
path += "&view=summary"
}
result := map[string]string{"annotation.buffer": path}
if claimCounts && countCycle {
result["annotation.claim-count"] = "/api/plugins/annotation-qa/claim-count?projectId=" + a.Project
}
return result
}
if a.Role == "admin" {
if os.Getenv("POLL_POOL_ADMIN_TASKS") == "1" {
base := "/api/plugins/annotation-qa/admin-pool?limit=200" + scope
suffix, key := "&pool=available", "available"
switch scenario % 5 {
case 1:
suffix, key = "&pool=all&search=synthetic-available-", "search"
case 2:
suffix, key = "&pool=all&search=__production_shape_no_match__", "search.nohit"
case 3:
suffix, key = "&pool=mine", "mine"
case 4:
suffix, key = "&pool=all&category=pending_slice", "category"
}
return map[string]string{"admin." + scopeName + "." + key: base + suffix}
}
return map[string]string{"admin.workforce": "/api/dashboard/workforce?from=2026-10-01T00:00:00Z&to=2026-10-28T00:00:00Z"}
}
if pageShape {
// AnnotationOperations loads one selected tab plus poolCounts.
// The old broad stress mode below deliberately requests every tab
// and both acceptance stages separately; keep it as a separate test.
stages := "quality_review,semantic_review"
if a.Role == "acceptance" {
stages = "acceptance,prepackage_acceptance"
}
endpoint := "/api/human-item-summaries"
pool, search, kind := "available", "", "available"
switch scenario % 5 {
case 1:
stage := "quality_review"
if a.Role == "acceptance" {
stage = "prepackage_acceptance"
}
search = "synthetic-available-" + a.Project + "-" + a.User.SupplierID + "-" + stage + "-0"
kind = "search"
case 2:
search, kind = "__production_shape_no_match__", "search.nohit"
case 3:
pool, kind = "mine", "mine"
case 4:
pool, kind = "reviewed", "reviewed"
}
key := a.Role + "." + scopeName + "." + kind
query := "?pluginId=annotation-qa" + scope + "&stageIds=" + stages + "&search=" + url.QueryEscape(search)
counts := "/api/human-item-counts" + query
if a.Role == "acceptance" {
counts += "&reviewedStageIds=" + stages
}
listQuery := query
if a.Role == "quality" && pool == "reviewed" {
listQuery = strings.Replace(query, "&stageIds="+stages, "", 1)
}
return map[string]string{key: endpoint + listQuery + "&pool=" + pool + "&limit=200&trace=1", key + ".counts": counts}
}
result := map[string]string{}
for _, pool := range []string{"available", "mine", "reviewed"} {
endpoint := "/api/human-items"
if readModels {
endpoint = "/api/human-item-summaries"
}
base := endpoint + "?pluginId=annotation-qa&pool=" + pool + scope
if a.Role == "quality" {
result["quality."+scopeName+"."+pool] = base + "&stageIds=quality_review%2Csemantic_review"
} else {
for _, stage := range []string{"acceptance", "prepackage_acceptance"} {
result["acceptance."+scopeName+"."+stage+"."+pool] = base + "&stageId=" + stage
}
}
}
if readModels {
stages := "quality_review,semantic_review"
reviewed := ""
if a.Role == "acceptance" {
stages = "acceptance,prepackage_acceptance"
reviewed = "&reviewedStageIds=" + stages
}
result[a.Role+"."+scopeName+".counts"] = "/api/human-item-counts?pluginId=annotation-qa" + scope + "&stageIds=" + stages + reviewed
}
if os.Getenv("POLL_POOL_SEARCH") == "1" {
endpoint := "/api/human-items"
if readModels {
endpoint = "/api/human-item-summaries"
}
stage := "quality_review"
if a.Role == "acceptance" {
stage = "prepackage_acceptance"
}
result[a.Role+"."+scopeName+".search"] = endpoint + "?pluginId=annotation-qa&pool=available&limit=200" + scope + "&stageId=" + stage + "&search=synthetic-available-" + a.Project + "-" + a.User.SupplierID + "-" + stage + "-0"
if productionShape {
stages := "quality_review,semantic_review"
if a.Role == "acceptance" {
stages = "acceptance,prepackage_acceptance"
}
base := endpoint + "?pluginId=annotation-qa&pool=available&limit=200" + scope + "&stageIds=" + stages
result[a.Role+"."+scopeName+".search"] = base + "&search=synthetic-available-" + a.Project + "-" + a.User.SupplierID + "-" + stage + "-0"
result[a.Role+"."+scopeName+".search.nohit"] = base + "&search=__production_shape_no_match__"
}
}
return result
}
if queryPanels {
original := paths
paths = func(a pollLoadActor, all, countCycle bool, scenario int) map[string]string {
out := original(a, all, countCycle, scenario)
out[a.Role+".panel"] = "/api/dashboard/production-quality?from=2026-10-01&to=2026-11-13"
if a.Role == "admin" {
out["admin.production"] = "/api/dashboard/production?from=2026-10-01&to=2026-11-13"
}
return out
}
}
fixtureSpec := fmt.Sprintf("isolated-preparation-v5|%d|%d|%d|%d|%d|%d|%d|%t|%t|%t|%s|%s", annotators, quality, acceptance, availablePerStage, runPayloadKiB, taskPayloadKiB, authorHistory, productionShape, readModels, claimCounts, authorizationState, os.Getenv("POLL_POOL_SEARCH_INDEXES"))
fixtureSpec += "|admin-tasks=" + os.Getenv("POLL_POOL_ADMIN_TASKS")
if os.Getenv("POLL_POOL_QUERY_SERVING_PREPARE") == "1" {
fixtureSpec += "|query-serving-preparation-v1"
}
fixtureDigest := fmt.Sprintf("%x", sha256.Sum256([]byte(fixtureSpec)))
if reuseFixture {
var manifest struct {
Digest string `bson:"digest"`
}
must(db.Collection("pollFixture").FindOne(ctx, map[string]any{"_id": "identity"}).Decode(&manifest))
if manifest.Digest != fixtureDigest {
t.Fatal("fixture configuration differs from preparation")
}
} else {
_, err := db.Collection("pollFixture").InsertOne(ctx, map[string]any{"_id": "identity", "digest": fixtureDigest})
must(err)
}
fixtureCounts := map[string]int64{}
if productionShape {
if reuseFixture {
for collection, want := range map[string]int64{"humanItems": 26218, "runs": 26490, "captureRecords": 29767, "annotationTasks": 23611, "humanEvents": 342694} {
got, err := db.Collection(collection).CountDocuments(ctx, map[string]any{"organizationId": org})
must(err)
if got != want {
t.Fatalf("fixture %s count=%d want=%d", collection, got, want)
}
fixtureCounts[collection] = got
}
} else {
var err error
fixtureCounts, err = seedProductionPoolNoise(ctx, db, org, payload)
must(err)
must(installPoolReadIndexes(ctx, db, os.Getenv("POLL_POOL_SEARCH_INDEXES") == "1"))
}
}
cleanupFixture = true
if os.Getenv("POLL_POOL_FIXTURE_ONLY") == "1" {
return
}
var preparationCPU float64
if queryServing {
startCPU, _ := pollLoadProcessCPU()
if os.Getenv("QUERY_SERVING_LOCAL_TEST") != "1" {
t.Fatal("query acceleration requires isolated local dependencies")
}
cache, err := queryserving.NewManagedValkey("valkey://127.0.0.1:16379", db.Name())
must(err)
defer cache.Close()
search, err := queryserving.NewTypesense("http://127.0.0.1:18108", os.Getenv("QUERY_TEST_TYPESENSE_API_KEY"))
must(err)
ds.ConfigureQueryServing(cache, search)
builder := store.NewQueryReadBuilder(ds)
buildCtx, stop := context.WithCancel(ctx)
buildDone := make(chan error, 1)
go func() { buildDone <- builder.Run(buildCtx) }()
defer func() { stop(); <-buildDone; builder.Close() }()
prepare, finish := context.WithTimeout(ctx, 4*time.Minute)
err = builder.WaitReady(prepare, org, projects)
finish()
must(err)
endCPU, _ := pollLoadProcessCPU()
preparationCPU = endCPU - startCPU
}
// Fail before timing if permissions or fixtures accidentally produce empty work.
for _, a := range actors {
for key, path := range paths(a, false, true, 0) {
result := call(a, key, path, false)
if result.Code != 200 {
if allowBaselineFailure {
continue
}
t.Fatalf("preflight %s status=%d", key, result.Code)
}
// This workload seeds assigned annotation work, but no unassigned
// annotation pool unless claim-count coverage was requested. An empty
// admin available pool is then the correct fixture result, not failure.
requireRows := a.Role == "annotation" || a.Role == "admin" && os.Getenv("POLL_POOL_ADMIN_TASKS") != "1" || a.Role != "admin" && (strings.HasSuffix(key, ".available") || strings.HasSuffix(key, ".mine") && !strings.Contains(key, ".acceptance.")) || a.Role == "admin" && claimCounts && strings.HasSuffix(key, ".available")
if requireRows && result.Rows == 0 {
t.Fatalf("preflight %s unexpectedly empty for %s", key, a.User.ID)
}
}
}
recordingCold = false
var memBefore, memAfter runtime.MemStats
runPhase := func(seconds int, record bool) {
phaseCtx, stop := context.WithTimeout(ctx, time.Duration(seconds)*time.Second)
defer stop()
var wg sync.WaitGroup
for _, actor := range actors {
wg.Add(1)
go func(a pollLoadActor) {
defer wg.Done()
period := 12 * time.Second
if a.Role == "annotation" {
period = 3 * time.Second
}
if a.Role == "admin" && os.Getenv("POLL_POOL_ADMIN_TASKS") == "1" {
period = time.Duration(adminPeriod) * time.Second
}
ticker := time.NewTicker(period)
defer ticker.Stop()
cycles := (time.Duration(seconds)*time.Second + period - 1) / period
for cycle := 0; cycle < int(cycles); cycle++ {
select {
case <-phaseCtx.Done():
return
default:
}
var requests sync.WaitGroup
for key, path := range paths(a, cycle%2 == 1, cycle%4 == 0, cycle/2) {
requests.Add(1)
go func(key, path string) { defer requests.Done(); call(a, key, path, record) }(key, path)
}
requests.Wait()
if cycle+1 == int(cycles) {
return
}
select {
case <-phaseCtx.Done():
return
case <-ticker.C:
}
}
}(actor)
}
wg.Wait()
<-phaseCtx.Done()
}
out := os.Getenv("POLL_POOL_OUTPUT")
if out == "" {
out = filepath.Join("..", "..", "local-data", "test-results", fmt.Sprintf("poll-pool-%s-%d.json", mode, time.Now().Unix()))
}
must(os.MkdirAll(filepath.Dir(out), 0700))
resourceFile, err := os.Create(out + ".resources.jsonl")
must(err)
defer resourceFile.Close()
resourceEncoder := json.NewEncoder(resourceFile)
resourceRows := []map[string]any{}
var phase atomic.Value
phase.Store("warmup")
resourceStart := time.Now()
sampleResource := func() error {
var m runtime.MemStats
runtime.ReadMemStats(&m)
cpuSeconds, cpuErr := pollLoadProcessCPU()
if cpuErr != nil {
return cpuErr
}
mu.Lock()
requests := len(samples)
mu.Unlock()
row := map[string]any{"at": time.Now().UTC().Format(time.RFC3339Nano), "phase": phase.Load(), "elapsed_seconds": time.Since(resourceStart).Seconds(), "heap_alloc_bytes": m.HeapAlloc, "heap_inuse_bytes": m.HeapInuse, "total_alloc_bytes": m.TotalAlloc, "gc_cycles": m.NumGC, "goroutines": runtime.NumGoroutine(), "retained_request_samples": requests, "errors": completedErrors.Load(), "completed_requests": completedRequests.Load(), "process_cpu_seconds": cpuSeconds}
resourceRows = append(resourceRows, row)
return resourceEncoder.Encode(row)
}
must(sampleResource())
resourceStop := make(chan struct{})
resourceDone := make(chan error, 1)
go func() {
ticker := time.NewTicker(15 * time.Second)
defer ticker.Stop()
for {
select {
case <-resourceStop:
resourceDone <- sampleResource()
return
case <-ticker.C:
if e := sampleResource(); e != nil {
cancel()
resourceDone <- e
return
}
}
}
}()
runPhase(warmup, false)
if completedErrors.Load() != 0 && !allowBaselineFailure {
close(resourceStop)
must(<-resourceDone)
t.Fatalf("warmup had %d HTTP errors", completedErrors.Load())
}
phase.Store("steady")
runtime.ReadMemStats(&memBefore)
var cpuFile *os.File
if filename := os.Getenv("POLL_POOL_CPU_PROFILE"); filename != "" {
must(os.MkdirAll(filepath.Dir(filename), 0700))
cpuFile, err = os.Create(filename)
must(err)
must(pprof.StartCPUProfile(cpuFile))
}
cpuBefore, cpuErr := pollLoadProcessCPU()
must(cpuErr)
measured.Store(true)
start := time.Now()
runPhase(duration, true)
measured.Store(false)
phase.Store("finished")
close(resourceStop)
must(<-resourceDone)
must(resourceFile.Close())
if cpuFile != nil {
pprof.StopCPUProfile()
must(cpuFile.Close())
}
cpuAfter, cpuErr := pollLoadProcessCPU()
must(cpuErr)
actual := time.Since(start)
runtime.ReadMemStats(&memAfter)
// Outside the measured interval: distinguish retained objects from GC timing.
runtime.GC()
var memAfterGC runtime.MemStats
runtime.ReadMemStats(&memAfterGC)
grouped := map[string][]pollLoadSample{}
failures := 0
for _, s := range samples {
grouped[s.Key] = append(grouped[s.Key], s)
if s.Code != 200 {
failures++
}
}
metrics := map[string]any{}
for key, rows := range grouped {
lat := []float64{}
raw, gz, minRows := 0, 0, rows[0].Rows
for _, s := range rows {
lat = append(lat, s.MS)
raw += s.Raw
gz += s.Gzip
if s.Rows < minRows {
minRows = s.Rows
}
}
sort.Float64s(lat)
metrics[key] = map[string]any{"requests": len(rows), "p95_ms": lat[(len(lat)*95-1)/100], "max_ms": lat[len(lat)-1], "raw_bytes": raw, "gzip_bytes": gz, "min_items": minRows}
}
slowSearch := []string{}
if raw := os.Getenv("POLL_POOL_MAX_SEARCH_P95_MS"); raw != "" {
budget, err := strconv.ParseFloat(raw, 64)
if err != nil || budget <= 0 {
t.Fatal("invalid search latency budget")
}
for key, value := range metrics {
if strings.Contains(key, ".search") && value.(map[string]any)["p95_ms"].(float64) > budget {
slowSearch = append(slowSearch, key)
}
}
}
outputState, stateErr := ds.OutputSnapshotState(ctx)
must(stateErr)
preparedQueries := 0
for _, head := range outputState.Heads {
if head.Current != nil && head.Current.Metrics != nil {
preparedQueries += head.Current.Metrics.PreparedQueries
}
}
var usage syscall.Rusage
must(syscall.Getrusage(syscall.RUSAGE_SELF, &usage))
maxRSS := usage.Maxrss
if runtime.GOOS != "darwin" {
maxRSS *= 1024
}
// Outside all measured latency, CPU, heap and Mongo counters: a serial
// retry supplies correctness evidence for baseline endpoints that never
// completed under load. Its latency is NOT mixed into benchmark metrics.
references := []pollLoadSample{}
if queryPanels {
succeeded := map[string]bool{}
tried := map[string]bool{}
all := append(append([]pollLoadSample{}, coldSamples...), samples...)
for _, sample := range all {
if sample.Code == 200 {
succeeded[sample.Actor+"\x00"+sample.Key] = true
}
}
for _, sample := range all {
identity := sample.Actor + "\x00" + sample.Key
if succeeded[identity] || tried[identity] || len(references) >= 32 {
continue
}
tried[identity] = true
for _, actor := range actors {
if actor.User.ID == sample.Actor {
references = append(references, call(actor, sample.Key, lastPaths[identity], false))
break
}
}
}
}
report := map[string]any{"mongo_clock_samples": clockSamples.Load(), "mongo_clock_max_jump_ms": clockJumpMS.Load(), "reference_samples": references, "platform_read_indexes": os.Getenv("POLL_POOL_QUERY_SERVING_PREPARE") == "1", "output_prepared_queries": preparedQueries, "process_max_rss_bytes": maxRSS, "query_serving": queryServing, "query_preparation_cpu_seconds": preparationCPU, "query_serving_statistics": ds.QueryServingStatistics(), "query_panels": queryPanels, "derived_write_commands": derivedWrites.Load(), "reused_prepared_fixture": reuseFixture, "fixture_digest": fixtureDigest, "cold_samples": coldSamples, "slow_search_endpoints": slowSearch, "baseline_failure_collection": allowBaselineFailure, "production_shape": productionShape, "page_shape": pageShape, "fixture_counts": fixtureCounts, "search_indexes": os.Getenv("POLL_POOL_SEARCH_INDEXES") == "1", "read_models": readModels, "search_queries": os.Getenv("POLL_POOL_SEARCH") == "1", "schema": 2, "resource_samples": resourceRows, "process_cpu_seconds": cpuAfter - cpuBefore, "authorization_state": authorizationState, "fixture": "read-mix-v5-isolated-preparation", "available_per_stage": availablePerStage, "task_payload_kib": taskPayloadKiB, "run_payload_kib": runPayloadKiB, "author_history_per_reviewer": authorHistory, "mode": mode, "revision": os.Getenv("POLL_POOL_REVISION"), "actors": totalActors, "actor_counts": map[string]int{"annotation": annotators, "quality": quality, "acceptance": acceptance, "admin": 1}, "claim_counts": claimCounts, "warmup_seconds": warmup, "scheduled_seconds": duration, "actual_seconds": actual.Seconds(), "errors": failures, "failure_samples": failureSamples, "metrics": metrics, "mongo_breakdown": observations, "mongo_commands": commands.Load(), "mongo_reply_bytes": mongoBytes.Load(), "mongo_command_ms": float64(mongoNS.Load()) / 1e6, "mongo_write_commands": writes.Load(), "go_heap_before": memBefore.HeapAlloc, "go_heap_after": memAfter.HeapAlloc, "go_heap_after_gc": memAfterGC.HeapAlloc, "go_gc_cycles": memAfter.NumGC - memBefore.NumGC, "go_allocated_bytes": memAfter.TotalAlloc - memBefore.TotalAlloc, "limitations": []string{"handler timing excludes ingress/network/authentication", "gzip computed separately with fixed Go default compression", "no continuation renew, media or workflow writes", "synthetic fixture; not capacity acceptance", "CPU covers handler and in-process load harness, not Mongo/Pod throttling", "resource heap includes retained benchmark request samples; not standalone service RSS"}, "samples": samples}
encoded, e := json.MarshalIndent(report, "", " ")
must(e)
must(os.WriteFile(out, encoded, 0600))
t.Logf("read-mix report: %s, requests=%d errors=%d", out, len(samples), failures)
if len(slowSearch) > 0 {
t.Errorf("search latency budget exceeded: %v", slowSearch)
}
if failures != 0 {
t.Fatalf("load returned %d errors", failures)
}
if writes.Load() != 0 {
t.Fatalf("stable read mix wrote %d commands", writes.Load())
}
}