droboticflow/internal/api/poll_pool_load_test.go

674 lines
26 KiB
Go

package api
// Local-only read-mix harness. This is NOT full concurrent-user capacity acceptance:
// it excludes media, editing/submission, continuation writes and ingress shaping.
import (
"bytes"
"compress/gzip"
"context"
"crypto/sha256"
"encoding/json"
"fmt"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"runtime"
"runtime/pprof"
"sort"
"strconv"
"strings"
"sync"
"sync/atomic"
"syscall"
"testing"
"time"
"droboticflow/internal/domain"
"droboticflow/internal/store"
"go.mongodb.org/mongo-driver/event"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
)
type pollLoadActor struct {
User domain.UserRecord
Project string
Role string
}
type pollLoadSample struct {
Actor string
Key string
MS float64
Raw, Gzip, Rows int
Code int
Hash string
}
type pollLoadEndpointKey struct{}
type pollMongoObservation struct {
Endpoint, Collection, Command string
Count, Bytes, Nanoseconds, Failed int64
}
// Rusage covers this test process (handler plus harness compression/sampling),
// not Mongo, another process, or Kubernetes Pod throttling/RSS.
func pollLoadProcessCPU() (float64, error) {
var usage syscall.Rusage
if err := syscall.Getrusage(syscall.RUSAGE_SELF, &usage); err != nil {
return 0, err
}
return float64(usage.Utime.Sec) + float64(usage.Utime.Usec)/1e6 + float64(usage.Stime.Sec) + float64(usage.Stime.Usec)/1e6, nil
}
func TestPollPoolReadMixLocal(t *testing.T) {
if os.Getenv("POLL_POOL_LOAD") != "1" {
t.Skip("explicit isolated local read-mix verification")
}
seconds := func(key string, fallback int) int {
v := os.Getenv(key)
if v == "" {
return fallback
}
n, e := strconv.Atoi(v)
if e != nil || n < 1 || n > 7200 {
t.Fatalf("invalid %s", key)
}
return n
}
warmup, duration := seconds("POLL_POOL_WARMUP_SECONDS", 5), seconds("POLL_POOL_DURATION_SECONDS", 30)
annotators := seconds("POLL_POOL_ANNOTATORS", 60)
quality := seconds("POLL_POOL_QUALITY_REVIEWERS", 12)
acceptance := seconds("POLL_POOL_ACCEPTANCE_REVIEWERS", 8)
totalActors := annotators + quality + acceptance + 1
if totalActors > 250 {
t.Fatal("local read mix supports at most 250 actors")
}
availablePerStage := seconds("POLL_POOL_AVAILABLE_PER_STAGE", 8)
if availablePerStage > 250 {
t.Fatal("available fixtures limited to 250 per stage and supplier")
}
claimCounts := os.Getenv("POLL_POOL_CLAIM_COUNTS") == "1"
readModels := os.Getenv("POLL_POOL_READMODELS") == "1"
mode := os.Getenv("POLL_POOL_MODE")
if mode == "" {
mode = "summary"
}
if mode != "summary" && mode != "legacy" {
t.Fatal("mode must be summary or legacy")
}
ctx, cancel := context.WithTimeout(context.Background(), time.Duration(warmup+duration+180)*time.Second)
defer cancel()
var measured atomic.Bool
var commands, mongoBytes, mongoNS, writes atomic.Int64
var probeMu sync.Mutex
pending := map[int64]string{}
observations := map[string]*pollMongoObservation{}
monitor := &event.CommandMonitor{
Started: func(commandCtx context.Context, e *event.CommandStartedEvent) {
if !measured.Load() {
return
}
endpoint, _ := commandCtx.Value(pollLoadEndpointKey{}).(string)
collection, _ := e.Command.Lookup(e.CommandName).StringValueOK()
if collection == "" {
collection, _ = e.Command.Lookup("collection").StringValueOK()
}
key := endpoint + "/" + collection + "/" + e.CommandName
probeMu.Lock()
defer probeMu.Unlock()
pending[e.RequestID] = key
if observations[key] == nil {
observations[key] = &pollMongoObservation{Endpoint: endpoint, Collection: collection, Command: e.CommandName}
}
},
Failed: func(_ context.Context, e *event.CommandFailedEvent) {
probeMu.Lock()
defer probeMu.Unlock()
if key, ok := pending[e.RequestID]; ok {
observations[key].Failed++
delete(pending, e.RequestID)
}
},
Succeeded: func(_ context.Context, e *event.CommandSucceededEvent) {
probeMu.Lock()
if key, ok := pending[e.RequestID]; ok {
row := observations[key]
row.Count++
row.Bytes += int64(len(e.Reply))
row.Nanoseconds += e.Duration.Nanoseconds()
delete(pending, e.RequestID)
}
probeMu.Unlock()
if measured.Load() {
commands.Add(1)
mongoBytes.Add(int64(len(e.Reply)))
mongoNS.Add(e.Duration.Nanoseconds())
switch e.CommandName {
case "insert", "update", "delete", "findAndModify":
writes.Add(1)
}
}
}}
client, err := mongo.Connect(ctx, options.Client().ApplyURI("mongodb://127.0.0.1:27018/?directConnection=true").SetMonitor(monitor))
if err != nil {
t.Fatal(err)
}
defer client.Disconnect(context.Background())
db := client.Database(fmt.Sprintf("poll_pool_read_mix_%d", time.Now().UnixNano()))
defer func() {
cleanup, done := context.WithTimeout(context.Background(), 15*time.Second)
defer done()
if e := db.Drop(cleanup); e != nil {
t.Error(e)
}
}()
ds := store.NewMongoStore(db)
must := func(e error) {
t.Helper()
if e != nil {
t.Fatal(e)
}
}
must(ds.EnsureIndexes(ctx))
const org = "load-org"
projects := []string{"load-project-a", "load-project-b"}
for _, p := range projects {
must(ds.UpsertProject(ctx, domain.ProjectRecord{ID: p, OrganizationID: org, Name: p}))
must(ds.UpsertWorkflow(ctx, domain.WorkflowRecord{ID: "workflow-" + p, OrganizationID: org, ProjectID: p, Name: "load workflow"}))
}
suppliers := []string{"load-supplier-a", "load-supplier-b", "load-supplier-c"}
for _, s := range suppliers {
must(ds.UpsertSupplier(ctx, domain.SupplierRecord{ID: s, OrganizationID: org, Status: "active"}))
_, e := ds.SetSupplierAcceptanceAuthorization(ctx, org, s, true, 0, "load-admin")
must(e)
}
actors := []pollLoadActor{}
for i := 0; i < totalActors; i++ {
role := domain.UserRoleAnnotator
label := "annotation"
if i >= annotators && i < annotators+quality {
role = domain.UserRoleQualityReviewer
label = "quality"
}
if i >= annotators+quality && i < totalActors-1 {
role = domain.UserRoleAcceptanceReviewer
label = "acceptance"
}
if i == totalActors-1 {
role = domain.UserRoleAdmin
label = "admin"
}
u := domain.UserRecord{ID: fmt.Sprintf("load-user-%02d", i), Phone: fmt.Sprintf("synthetic-load-%02d", i), OrganizationID: org, SupplierID: suppliers[i%3], Role: role, Status: domain.UserStatusActive}
must(ds.UpsertUser(ctx, u))
for _, p := range projects {
must(ds.UpsertProjectMember(ctx, domain.ProjectMemberRecord{ID: p + u.ID, OrganizationID: org, ProjectID: p, UserID: u.ID, SupplierID: u.SupplierID, Roles: domain.AccountProductionRoles(u)}))
}
actors = append(actors, pollLoadActor{u, projects[i%2], label})
}
payload := make([]string, 512)
for i := range payload {
payload[i] = fmt.Sprintf("%x", sha256.Sum256([]byte(fmt.Sprintf("fixed-fixture-%d", i))))
}
runPayloadKiB := 0
if raw := os.Getenv("POLL_POOL_RUN_PAYLOAD_KIB"); raw != "" {
runPayloadKiB, err = strconv.Atoi(raw)
must(err)
if runPayloadKiB < 0 || runPayloadKiB > 2048 {
t.Fatal("run payload must be 0..2048 KiB")
}
}
taskPayloadKiB := 0
if raw := os.Getenv("POLL_POOL_TASK_PAYLOAD_KIB"); raw != "" {
taskPayloadKiB, err = strconv.Atoi(raw)
must(err)
if taskPayloadKiB < 0 || taskPayloadKiB > 2048 {
t.Fatal("task payload must be 0..2048 KiB")
}
}
seededRuns := map[string]bool{}
seed := func(id, p, s, stage, owner string, status domain.HumanItemStatus) {
batchID := "batch-" + p + s
stages := domain.DefaultFanyuV58HumanStages()
if stage == "semantic_review" {
batchID += "-semantic"
stages = []domain.HumanStageDefinition{{ID: stage, Kind: domain.HumanStageReview, RequiredRole: "quality_reviewer"}}
}
if stage == "acceptance" || stage == "prepackage_acceptance" {
batchID += "-" + stage
stages = []domain.HumanStageDefinition{{ID: stage, Kind: domain.HumanStageAcceptance, RequiredRole: "acceptance_reviewer"}}
}
must(ds.UpsertHumanBatch(ctx, domain.HumanBatchRecord{ID: batchID, RunID: batchID, GateNodeID: "gate", OrganizationID: org, ProjectID: p, WorkflowID: "workflow-" + p, PluginID: "annotation-qa", SupplierID: s, Status: domain.HumanBatchActive, ProcessingProfile: domain.ProcessingProfileRawHuman, Stages: stages}))
runID := ""
if runPayloadKiB > 0 {
runID = "run-" + batchID
if !seededRuns[runID] {
seededRuns[runID] = true
must(ds.UpsertRun(ctx, domain.RunRecord{ID: runID, OrganizationID: org, ProjectID: p, WorkflowID: "workflow-" + p, ProjectName: p, WorkflowName: "workflow-" + p, NodeExecutions: []domain.NodeExecutionRecord{{NodeID: "synthetic-node", Logs: []domain.LogEntryRecord{{Message: strings.Repeat("x", runPayloadKiB*1024)}}}}}))
}
}
taskID := "task-" + id
must(ds.UpsertAnnotationTask(ctx, domain.AnnotationTaskRecord{ID: taskID, OrganizationID: org, ProjectID: p, WorkflowID: "workflow-" + p, SupplierID: s, TemplateVersion: 6, Status: domain.AnnotationTaskStatusPendingAnnotation, SourceVideo: "synthetic/camera_left.mp4"}))
itemPayload := map[string]any{"captureDirectory": "synthetic-" + id, "sliceMergeEvidence": payload, "sliceWorkflow": map[string]any{"status": "completed", "phase": "annotate"}}
if taskPayloadKiB > 0 {
content := strings.Repeat(strings.Join(payload, ""), taskPayloadKiB/32+1)[:taskPayloadKiB*1024]
itemPayload["items"] = []map[string]any{{"annotation": map[string]any{"syntheticPayload": content}, "materialization": map[string]any{"coverage": map[string]any{"durationSeconds": 900}}}}
}
must(ds.UpsertHumanItem(ctx, domain.HumanItemRecord{ID: id, RunID: runID, OrganizationID: org, ProjectID: p, WorkflowID: "workflow-" + p, PluginID: "annotation-qa", SupplierID: s, BatchID: batchID, LegacyTaskID: taskID, AssignedUserID: owner, LatestAuthorUserID: "synthetic-author", CurrentStageID: stage, Status: status, Version: 1, CreatedAt: "2026-10-01T00:00:00Z", Subject: domain.HumanSubjectRef{ItemKey: "synthetic/" + id + "/camera_left.mp4", Payload: itemPayload}}))
}
for _, a := range actors {
if a.Role == "admin" {
continue
}
stage := "annotation_combined"
if a.Role == "quality" {
stage = "quality_review"
}
if a.Role == "acceptance" {
stage = "prepackage_acceptance"
}
for n := 0; n < 2; n++ {
seed(fmt.Sprintf("owned-%s-%d", a.User.ID, n), a.Project, a.User.SupplierID, stage, a.User.ID, domain.HumanItemClaimed)
}
if a.Role == "annotation" {
taskID := "task-owned-" + a.User.ID + "-0"
must(ds.UpsertAnnotationRevision(ctx, domain.AnnotationRevisionRecord{ID: "revision-" + a.User.ID, OrganizationID: org, TaskID: taskID, Revision: 1, CountsAsOutput: true, AuthorUserID: a.User.ID}))
must(ds.UpsertAnnotationWorklogEvent(ctx, domain.AnnotationWorklogEventRecord{ID: "worklog-" + a.User.ID, OrganizationID: org, ProjectID: a.Project, TaskID: taskID, UserID: a.User.ID, SupplierID: a.User.SupplierID, EventType: "combined_submitted", Revision: 1, OutputDelta: 1, CreatedAt: "2026-10-02T00:00:00Z"}))
}
if a.Role != "annotation" {
must(ds.AppendHumanEvent(ctx, domain.HumanEventRecord{ID: "history-" + a.User.ID, OrganizationID: org, ItemID: "owned-" + a.User.ID + "-0", ActorUserID: a.User.ID, StageID: stage, EventType: "approved", CreatedAt: "2026-10-02T00:00:00Z"}))
}
}
for _, p := range projects {
for _, s := range suppliers {
stages := []string{"quality_review", "semantic_review", "acceptance", "prepackage_acceptance"}
if claimCounts {
stages = append(stages, "annotation_combined")
}
for _, stage := range stages {
for n := 0; n < availablePerStage; n++ {
seed(fmt.Sprintf("available-%s-%s-%s-%d", p, s, stage, n), p, s, stage, "", domain.HumanItemReady)
}
}
}
}
authorHistory := 0
if raw := os.Getenv("POLL_POOL_AUTHOR_HISTORY"); raw != "" {
authorHistory, err = strconv.Atoi(raw)
must(err)
if authorHistory < 0 || authorHistory > 5000 {
t.Fatal("author history must be 0..5000 per reviewer")
}
}
historyPartitions := map[string]domain.ReleasePartitionRecord{}
for _, a := range actors {
if authorHistory == 0 || (a.Role != "quality" && a.Role != "acceptance") {
continue
}
rows := make([]any, 0, authorHistory)
for i := 0; i < authorHistory; i++ {
stage := "quality_review"
suffix := ""
if a.Role == "quality" && i%2 != 0 {
stage = "semantic_review"
suffix = "-semantic"
}
if a.Role == "acceptance" {
stage = "acceptance"
if i%2 != 0 {
stage = "prepackage_acceptance"
}
suffix = "-" + stage
}
id := fmt.Sprintf("historical-%s-%05d", a.User.ID, i)
batchID := "batch-" + a.Project + a.User.SupplierID + suffix
partitionID := "released-history-" + batchID
partition := historyPartitions[partitionID]
partition.ID = partitionID
partition.OrganizationID = org
partition.BatchID = batchID
partition.Sequence = 1
partition.UniqueKey = partitionID
partition.Status = domain.ReleasePartitionDispatched
partition.ItemIDs = append(partition.ItemIDs, id)
historyPartitions[partitionID] = partition
rows = append(rows, domain.HumanItemRecord{ID: id, OrganizationID: org, ProjectID: a.Project, WorkflowID: "workflow-" + a.Project, PluginID: "annotation-qa", SupplierID: a.User.SupplierID, BatchID: batchID, ReleasePartitionID: partitionID, LatestAuthorUserID: a.User.ID, CurrentStageID: stage, Status: domain.HumanItemAccepted, Version: 1, CreatedAt: "2026-09-01T00:00:00Z", Subject: domain.HumanSubjectRef{ItemKey: id}})
}
_, e := db.Collection("humanItems").InsertMany(ctx, rows)
must(e)
}
for _, partition := range historyPartitions {
must(ds.UpsertReleasePartition(ctx, partition))
}
authorizationState := os.Getenv("POLL_POOL_AUTHORIZATION_STATE")
if authorizationState == "" {
authorizationState = "legacy"
}
if authorizationState != "legacy" && authorizationState != "initialized" {
t.Fatal("unknown authorization state")
}
if authorizationState == "initialized" {
must(ds.InitializeSupplierProjectAuthorizations(ctx, org))
}
if readModels {
result, err := ds.BackfillPoolSearchMetadata(ctx, org, true)
must(err)
if result.Unsupported > 0 || result.Conflicts > 0 {
t.Fatalf("summary preparation incomplete: %+v", result)
}
}
handler := NewHandler(ds, nil)
var mu sync.Mutex
samples := []pollLoadSample{}
failureSamples := []map[string]any{}
var completedRequests, completedErrors atomic.Int64
call := func(a pollLoadActor, key, path string, record bool) pollLoadSample {
req := httptest.NewRequest(http.MethodGet, path, nil)
requestCtx, done := context.WithTimeout(ctx, 15*time.Second)
defer done()
requestCtx = context.WithValue(requestCtx, pollLoadEndpointKey{}, key)
req = req.WithContext(context.WithValue(requestCtx, authContextKey{}, &authenticatedPrincipal{User: a.User, Organization: domain.OrganizationRecord{ID: org, Status: domain.OrganizationStatusActive}}))
response := httptest.NewRecorder()
start := time.Now()
handler.ServeHTTP(response, req)
elapsed := time.Since(start)
completedRequests.Add(1)
if response.Code != 200 {
completedErrors.Add(1)
}
var body struct {
Count *int `json:"count"`
Code string `json:"code"`
Error string `json:"error"`
Items []struct {
ID string `json:"id"`
} `json:"items"`
Summaries []struct {
ID string `json:"id"`
} `json:"summaries"`
}
_ = json.Unmarshal(response.Body.Bytes(), &body)
ids := []string{}
for _, v := range body.Items {
ids = append(ids, v.ID)
}
for _, v := range body.Summaries {
ids = append(ids, v.ID)
}
rowCount := len(ids)
if body.Count != nil {
rowCount = *body.Count
ids = append(ids, fmt.Sprintf("count:%d", *body.Count))
}
sort.Strings(ids)
var compressed bytes.Buffer
zw := gzip.NewWriter(&compressed)
_, _ = zw.Write(response.Body.Bytes())
_ = zw.Close()
sample := pollLoadSample{a.User.ID, key, float64(elapsed.Microseconds()) / 1000, response.Body.Len(), compressed.Len(), rowCount, response.Code, fmt.Sprintf("%x", sha256.Sum256([]byte(strings.Join(ids, "\n"))))}
if response.Code != 200 {
class := "unclassified"
for _, known := range []string{"context deadline exceeded", "context canceled", "duplicate key", "not found", "conflict", "timeout"} {
if strings.Contains(strings.ToLower(body.Error), known) {
class = known
break
}
}
code := strings.Map(func(r rune) rune {
if r >= 'a' && r <= 'z' || r >= 'A' && r <= 'Z' || r >= '0' && r <= '9' || r == '_' || r == '-' {
return r
}
return -1
}, body.Code)
if len(code) > 80 {
code = code[:80]
}
mu.Lock()
if len(failureSamples) < 20 {
failureSamples = append(failureSamples, map[string]any{"at": time.Now().UTC().Format(time.RFC3339Nano), "endpoint": key, "actor": a.User.ID, "status": response.Code, "elapsed_ms": sample.MS, "code": code, "class": class, "measured": record})
t.Logf("read-mix HTTP failure endpoint=%s actor=%s status=%d elapsed_ms=%.3f code=%s class=%s request_context=%v", key, a.User.ID, response.Code, sample.MS, code, class, requestCtx.Err())
}
mu.Unlock()
}
if record {
mu.Lock()
samples = append(samples, sample)
mu.Unlock()
}
return sample
}
paths := func(a pollLoadActor, all, countCycle bool) map[string]string {
scope := "&projectId=" + a.Project
scopeName := "project"
if all {
scope = ""
scopeName = "all"
}
if a.Role == "annotation" {
path := "/api/plugins/annotation-qa/work-buffer?projectId=" + a.Project
if mode == "summary" {
path += "&view=summary"
}
result := map[string]string{"annotation.buffer": path}
if claimCounts && countCycle {
result["annotation.claim-count"] = "/api/plugins/annotation-qa/claim-count?projectId=" + a.Project
}
return result
}
if a.Role == "admin" {
return map[string]string{"admin.workforce": "/api/dashboard/workforce?from=2026-10-01T00:00:00Z&to=2026-10-28T00:00:00Z"}
}
result := map[string]string{}
for _, pool := range []string{"available", "mine", "reviewed"} {
endpoint := "/api/human-items"
if readModels {
endpoint = "/api/human-item-summaries"
}
base := endpoint + "?pluginId=annotation-qa&pool=" + pool + scope
if a.Role == "quality" {
result["quality."+scopeName+"."+pool] = base + "&stageIds=quality_review%2Csemantic_review"
} else {
for _, stage := range []string{"acceptance", "prepackage_acceptance"} {
result["acceptance."+scopeName+"."+stage+"."+pool] = base + "&stageId=" + stage
}
}
}
if readModels {
stages := "quality_review,semantic_review"
reviewed := ""
if a.Role == "acceptance" {
stages = "acceptance,prepackage_acceptance"
reviewed = "&reviewedStageIds=" + stages
}
result[a.Role+"."+scopeName+".counts"] = "/api/human-item-counts?pluginId=annotation-qa" + scope + "&stageIds=" + stages + reviewed
}
if os.Getenv("POLL_POOL_SEARCH") == "1" {
endpoint := "/api/human-items"
if readModels {
endpoint = "/api/human-item-summaries"
}
stage := "quality_review"
if a.Role == "acceptance" {
stage = "prepackage_acceptance"
}
result[a.Role+"."+scopeName+".search"] = endpoint + "?pluginId=annotation-qa&pool=available&limit=200" + scope + "&stageId=" + stage + "&search=synthetic-available-" + a.Project + "-" + a.User.SupplierID + "-" + stage + "-0"
}
return result
}
// Fail before timing if permissions or fixtures accidentally produce empty work.
for _, a := range actors {
for key, path := range paths(a, false, true) {
result := call(a, key, path, false)
if result.Code != 200 {
t.Fatalf("preflight %s status=%d", key, result.Code)
}
if (a.Role == "annotation" || a.Role == "admin" || strings.HasSuffix(key, ".available") || strings.HasSuffix(key, ".mine") && !strings.Contains(key, ".acceptance.")) && result.Rows == 0 {
t.Fatalf("preflight %s unexpectedly empty for %s", key, a.User.ID)
}
}
}
var memBefore, memAfter runtime.MemStats
runPhase := func(seconds int, record bool) {
phaseCtx, stop := context.WithTimeout(ctx, time.Duration(seconds)*time.Second)
defer stop()
var wg sync.WaitGroup
for _, actor := range actors {
wg.Add(1)
go func(a pollLoadActor) {
defer wg.Done()
period := 12 * time.Second
if a.Role == "annotation" {
period = 3 * time.Second
}
ticker := time.NewTicker(period)
defer ticker.Stop()
cycles := (time.Duration(seconds)*time.Second + period - 1) / period
for cycle := 0; cycle < int(cycles); cycle++ {
select {
case <-phaseCtx.Done():
return
default:
}
var requests sync.WaitGroup
for key, path := range paths(a, cycle%2 == 1, cycle%4 == 0) {
requests.Add(1)
go func(key, path string) { defer requests.Done(); call(a, key, path, record) }(key, path)
}
requests.Wait()
if cycle+1 == int(cycles) {
return
}
select {
case <-phaseCtx.Done():
return
case <-ticker.C:
}
}
}(actor)
}
wg.Wait()
<-phaseCtx.Done()
}
out := os.Getenv("POLL_POOL_OUTPUT")
if out == "" {
out = filepath.Join("..", "..", "local-data", "test-results", fmt.Sprintf("poll-pool-%s-%d.json", mode, time.Now().Unix()))
}
must(os.MkdirAll(filepath.Dir(out), 0700))
resourceFile, err := os.Create(out + ".resources.jsonl")
must(err)
defer resourceFile.Close()
resourceEncoder := json.NewEncoder(resourceFile)
resourceRows := []map[string]any{}
var phase atomic.Value
phase.Store("warmup")
resourceStart := time.Now()
sampleResource := func() error {
var m runtime.MemStats
runtime.ReadMemStats(&m)
cpuSeconds, cpuErr := pollLoadProcessCPU()
if cpuErr != nil {
return cpuErr
}
mu.Lock()
requests := len(samples)
mu.Unlock()
row := map[string]any{"phase": phase.Load(), "elapsed_seconds": time.Since(resourceStart).Seconds(), "heap_alloc_bytes": m.HeapAlloc, "heap_inuse_bytes": m.HeapInuse, "total_alloc_bytes": m.TotalAlloc, "gc_cycles": m.NumGC, "goroutines": runtime.NumGoroutine(), "retained_request_samples": requests, "errors": completedErrors.Load(), "completed_requests": completedRequests.Load(), "process_cpu_seconds": cpuSeconds}
resourceRows = append(resourceRows, row)
return resourceEncoder.Encode(row)
}
must(sampleResource())
resourceStop := make(chan struct{})
resourceDone := make(chan error, 1)
go func() {
ticker := time.NewTicker(15 * time.Second)
defer ticker.Stop()
for {
select {
case <-resourceStop:
resourceDone <- sampleResource()
return
case <-ticker.C:
if e := sampleResource(); e != nil {
cancel()
resourceDone <- e
return
}
}
}
}()
runPhase(warmup, false)
if completedErrors.Load() != 0 {
close(resourceStop)
must(<-resourceDone)
t.Fatalf("warmup had %d HTTP errors", completedErrors.Load())
}
phase.Store("steady")
runtime.ReadMemStats(&memBefore)
var cpuFile *os.File
if filename := os.Getenv("POLL_POOL_CPU_PROFILE"); filename != "" {
must(os.MkdirAll(filepath.Dir(filename), 0700))
cpuFile, err = os.Create(filename)
must(err)
must(pprof.StartCPUProfile(cpuFile))
}
cpuBefore, cpuErr := pollLoadProcessCPU()
must(cpuErr)
measured.Store(true)
start := time.Now()
runPhase(duration, true)
measured.Store(false)
phase.Store("finished")
close(resourceStop)
must(<-resourceDone)
must(resourceFile.Close())
if cpuFile != nil {
pprof.StopCPUProfile()
must(cpuFile.Close())
}
cpuAfter, cpuErr := pollLoadProcessCPU()
must(cpuErr)
actual := time.Since(start)
runtime.ReadMemStats(&memAfter)
// Outside the measured interval: distinguish retained objects from GC timing.
runtime.GC()
var memAfterGC runtime.MemStats
runtime.ReadMemStats(&memAfterGC)
grouped := map[string][]pollLoadSample{}
failures := 0
for _, s := range samples {
grouped[s.Key] = append(grouped[s.Key], s)
if s.Code != 200 {
failures++
}
}
metrics := map[string]any{}
for key, rows := range grouped {
lat := []float64{}
raw, gz, minRows := 0, 0, rows[0].Rows
for _, s := range rows {
lat = append(lat, s.MS)
raw += s.Raw
gz += s.Gzip
if s.Rows < minRows {
minRows = s.Rows
}
}
sort.Float64s(lat)
metrics[key] = map[string]any{"requests": len(rows), "p95_ms": lat[(len(lat)*95-1)/100], "max_ms": lat[len(lat)-1], "raw_bytes": raw, "gzip_bytes": gz, "min_items": minRows}
}
report := map[string]any{"read_models": readModels, "search_queries": os.Getenv("POLL_POOL_SEARCH") == "1", "schema": 2, "resource_samples": resourceRows, "process_cpu_seconds": cpuAfter - cpuBefore, "authorization_state": authorizationState, "fixture": "read-mix-v3-released-history", "available_per_stage": availablePerStage, "task_payload_kib": taskPayloadKiB, "run_payload_kib": runPayloadKiB, "author_history_per_reviewer": authorHistory, "mode": mode, "revision": os.Getenv("POLL_POOL_REVISION"), "actors": totalActors, "actor_counts": map[string]int{"annotation": annotators, "quality": quality, "acceptance": acceptance, "admin": 1}, "claim_counts": claimCounts, "warmup_seconds": warmup, "scheduled_seconds": duration, "actual_seconds": actual.Seconds(), "errors": failures, "failure_samples": failureSamples, "metrics": metrics, "mongo_breakdown": observations, "mongo_commands": commands.Load(), "mongo_reply_bytes": mongoBytes.Load(), "mongo_command_ms": float64(mongoNS.Load()) / 1e6, "mongo_write_commands": writes.Load(), "go_heap_before": memBefore.HeapAlloc, "go_heap_after": memAfter.HeapAlloc, "go_heap_after_gc": memAfterGC.HeapAlloc, "go_gc_cycles": memAfter.NumGC - memBefore.NumGC, "go_allocated_bytes": memAfter.TotalAlloc - memBefore.TotalAlloc, "limitations": []string{"handler timing excludes ingress/network/authentication", "gzip computed separately with fixed Go default compression", "no continuation renew, media or workflow writes", "synthetic fixture; not capacity acceptance", "CPU covers handler and in-process load harness, not Mongo/Pod throttling", "resource heap includes retained benchmark request samples; not standalone service RSS"}, "samples": samples}
encoded, e := json.MarshalIndent(report, "", " ")
must(e)
must(os.WriteFile(out, encoded, 0600))
t.Logf("read-mix report: %s, requests=%d errors=%d", out, len(samples), failures)
if failures != 0 {
t.Fatalf("load returned %d errors", failures)
}
if writes.Load() != 0 {
t.Fatalf("stable read mix wrote %d commands", writes.Load())
}
}