1109 lines
43 KiB
Go
1109 lines
43 KiB
Go
package api
|
|
|
|
// Local-only read-mix harness. This is NOT full concurrent-user capacity acceptance:
|
|
// it excludes media, editing/submission, continuation writes and ingress shaping.
|
|
import (
|
|
"bytes"
|
|
"compress/gzip"
|
|
"context"
|
|
"crypto/sha256"
|
|
"encoding/json"
|
|
"fmt"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"net/url"
|
|
"os"
|
|
"path/filepath"
|
|
"runtime"
|
|
"runtime/pprof"
|
|
"sort"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
"sync/atomic"
|
|
"syscall"
|
|
"testing"
|
|
"time"
|
|
|
|
"droboticflow/internal/domain"
|
|
"droboticflow/internal/productionstats"
|
|
"droboticflow/internal/queryserving"
|
|
"droboticflow/internal/store"
|
|
"go.mongodb.org/mongo-driver/event"
|
|
"go.mongodb.org/mongo-driver/mongo"
|
|
"go.mongodb.org/mongo-driver/mongo/options"
|
|
)
|
|
|
|
type pollLoadActor struct {
|
|
User domain.UserRecord
|
|
Project string
|
|
Role string
|
|
}
|
|
type pollLoadSample struct {
|
|
Actor string
|
|
Key string
|
|
MS float64
|
|
Raw, Gzip, Rows int
|
|
Code int
|
|
Hash string
|
|
}
|
|
|
|
type pollLoadEndpointKey struct{}
|
|
type pollMongoObservation struct {
|
|
Endpoint, Collection, Command string
|
|
Count, Bytes, Nanoseconds, Failed int64
|
|
FailedNanoseconds, MaxTimeMS int64
|
|
}
|
|
|
|
// Rusage covers this test process (handler plus harness compression/sampling),
|
|
// not Mongo, another process, or Kubernetes Pod throttling/RSS.
|
|
func pollLoadProcessCPU() (float64, error) {
|
|
|
|
var usage syscall.Rusage
|
|
if err := syscall.Getrusage(syscall.RUSAGE_SELF, &usage); err != nil {
|
|
return 0, err
|
|
}
|
|
return float64(usage.Utime.Sec) + float64(usage.Utime.Usec)/1e6 + float64(usage.Stime.Sec) + float64(usage.Stime.Usec)/1e6, nil
|
|
}
|
|
|
|
func TestPollPoolReadMixLocal(t *testing.T) {
|
|
if os.Getenv("POLL_POOL_LOAD") != "1" {
|
|
t.Skip("explicit isolated local read-mix verification")
|
|
}
|
|
seconds := func(key string, fallback int) int {
|
|
v := os.Getenv(key)
|
|
if v == "" {
|
|
return fallback
|
|
}
|
|
n, e := strconv.Atoi(v)
|
|
if e != nil || n < 1 || n > 7200 {
|
|
t.Fatalf("invalid %s", key)
|
|
}
|
|
return n
|
|
}
|
|
adminPeriod := seconds("POLL_POOL_ADMIN_PERIOD_SECONDS", 12)
|
|
warmup, duration := seconds("POLL_POOL_WARMUP_SECONDS", 5), seconds("POLL_POOL_DURATION_SECONDS", 30)
|
|
annotators := seconds("POLL_POOL_ANNOTATORS", 60)
|
|
quality := seconds("POLL_POOL_QUALITY_REVIEWERS", 12)
|
|
acceptance := seconds("POLL_POOL_ACCEPTANCE_REVIEWERS", 8)
|
|
totalActors := annotators + quality + acceptance + 1
|
|
if totalActors > 250 {
|
|
t.Fatal("local read mix supports at most 250 actors")
|
|
}
|
|
availablePerStage := seconds("POLL_POOL_AVAILABLE_PER_STAGE", 8)
|
|
if availablePerStage > 250 {
|
|
t.Fatal("available fixtures limited to 250 per stage and supplier")
|
|
}
|
|
claimCounts := os.Getenv("POLL_POOL_CLAIM_COUNTS") == "1"
|
|
productionShape := os.Getenv("POLL_POOL_PRODUCTION_SHAPE") == "1"
|
|
pageShape := os.Getenv("POLL_POOL_PAGE_SHAPE") == "1"
|
|
allowBaselineFailure := os.Getenv("POLL_POOL_ALLOW_BASELINE_FAILURE") == "1"
|
|
readModels := os.Getenv("POLL_POOL_READMODELS") == "1"
|
|
queryServing := os.Getenv("POLL_POOL_QUERY_SERVING") == "1"
|
|
queryPanels := os.Getenv("POLL_POOL_QUERY_PANELS") == "1"
|
|
|
|
mode := os.Getenv("POLL_POOL_MODE")
|
|
if mode == "" {
|
|
mode = "summary"
|
|
}
|
|
if mode != "summary" && mode != "legacy" {
|
|
t.Fatal("mode must be summary or legacy")
|
|
}
|
|
ctx, cancel := context.WithTimeout(context.Background(), time.Duration(warmup+duration+600)*time.Second)
|
|
defer cancel()
|
|
var measured atomic.Bool
|
|
var commands, mongoBytes, mongoNS, writes atomic.Int64
|
|
var derivedWrites atomic.Int64
|
|
var probeMu sync.Mutex
|
|
pending := map[int64]struct {
|
|
key string
|
|
maxMS int64
|
|
}{}
|
|
observations := map[string]*pollMongoObservation{}
|
|
monitor := &event.CommandMonitor{
|
|
Started: func(commandCtx context.Context, e *event.CommandStartedEvent) {
|
|
if !measured.Load() {
|
|
return
|
|
}
|
|
endpoint, _ := commandCtx.Value(pollLoadEndpointKey{}).(string)
|
|
collection, _ := e.Command.Lookup(e.CommandName).StringValueOK()
|
|
if collection == "" {
|
|
collection, _ = e.Command.Lookup("collection").StringValueOK()
|
|
}
|
|
key := endpoint + "/" + collection + "/" + e.CommandName
|
|
probeMu.Lock()
|
|
defer probeMu.Unlock()
|
|
maxMS, _ := e.Command.Lookup("maxTimeMS").Int64OK()
|
|
if short, ok := e.Command.Lookup("maxTimeMS").Int32OK(); ok {
|
|
maxMS = int64(short)
|
|
}
|
|
pending[e.RequestID] = struct {
|
|
key string
|
|
maxMS int64
|
|
}{key, maxMS}
|
|
if observations[key] == nil {
|
|
observations[key] = &pollMongoObservation{Endpoint: endpoint, Collection: collection, Command: e.CommandName}
|
|
}
|
|
},
|
|
Failed: func(_ context.Context, e *event.CommandFailedEvent) {
|
|
probeMu.Lock()
|
|
defer probeMu.Unlock()
|
|
if entry, ok := pending[e.RequestID]; ok {
|
|
key := entry.key
|
|
observations[key].Failed++
|
|
observations[key].FailedNanoseconds += e.Duration.Nanoseconds()
|
|
observations[key].MaxTimeMS = entry.maxMS
|
|
if productionShape {
|
|
t.Logf("synthetic mongo failure key=%s duration_ms=%d maxTimeMS=%d", key, e.Duration.Milliseconds(), entry.maxMS)
|
|
}
|
|
delete(pending, e.RequestID)
|
|
}
|
|
},
|
|
Succeeded: func(_ context.Context, e *event.CommandSucceededEvent) {
|
|
derived := false
|
|
probeMu.Lock()
|
|
if entry, ok := pending[e.RequestID]; ok {
|
|
key := entry.key
|
|
row := observations[key]
|
|
derived = strings.HasPrefix(row.Collection, "queryReadModel") || strings.HasPrefix(row.Collection, "annotationOutputSnapshot")
|
|
row.Count++
|
|
row.Bytes += int64(len(e.Reply))
|
|
row.Nanoseconds += e.Duration.Nanoseconds()
|
|
delete(pending, e.RequestID)
|
|
}
|
|
probeMu.Unlock()
|
|
if measured.Load() {
|
|
commands.Add(1)
|
|
mongoBytes.Add(int64(len(e.Reply)))
|
|
mongoNS.Add(e.Duration.Nanoseconds())
|
|
switch e.CommandName {
|
|
case "insert", "update", "delete", "findAndModify":
|
|
if queryServing && derived {
|
|
derivedWrites.Add(1)
|
|
} else {
|
|
writes.Add(1)
|
|
}
|
|
}
|
|
}
|
|
}}
|
|
uri := os.Getenv("POLL_POOL_LOCAL_MONGO_URI")
|
|
if uri == "" {
|
|
uri = "mongodb://127.0.0.1:27018/?directConnection=true"
|
|
}
|
|
parsed, parseErr := url.Parse(uri)
|
|
if parseErr != nil || parsed.Scheme != "mongodb" || (parsed.Host != "127.0.0.1:27017" && parsed.Host != "127.0.0.1:27018" && parsed.Host != "127.0.0.1:27019") {
|
|
t.Fatal("isolated loopback Mongo only")
|
|
}
|
|
client, err := mongo.Connect(ctx, options.Client().ApplyURI(uri).SetMonitor(monitor))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer client.Disconnect(context.Background())
|
|
clockCtx, stopClock := context.WithCancel(ctx)
|
|
clockDone := make(chan struct{})
|
|
var clockJumpMS atomic.Int64
|
|
var clockSamples atomic.Int64
|
|
go func() {
|
|
defer close(clockDone)
|
|
var previousServer, previousLocal time.Time
|
|
ticker := time.NewTicker(100 * time.Millisecond)
|
|
defer ticker.Stop()
|
|
for {
|
|
select {
|
|
case <-clockCtx.Done():
|
|
return
|
|
case <-ticker.C:
|
|
started := time.Now()
|
|
var reply struct {
|
|
LocalTime time.Time `bson:"localTime"`
|
|
}
|
|
probe, cancelProbe := context.WithTimeout(clockCtx, time.Second)
|
|
err := client.Database("admin").RunCommand(probe, map[string]any{"hello": 1}).Decode(&reply)
|
|
cancelProbe()
|
|
finished := time.Now()
|
|
if err != nil || finished.Sub(started) > 200*time.Millisecond {
|
|
continue
|
|
}
|
|
local := started.Add(finished.Sub(started) / 2)
|
|
if !previousServer.IsZero() {
|
|
drift := reply.LocalTime.Sub(previousServer) - local.Sub(previousLocal)
|
|
if drift < 0 {
|
|
drift = -drift
|
|
}
|
|
for old := clockJumpMS.Load(); drift.Milliseconds() > old; old = clockJumpMS.Load() {
|
|
if clockJumpMS.CompareAndSwap(old, drift.Milliseconds()) {
|
|
break
|
|
}
|
|
}
|
|
}
|
|
previousServer, previousLocal = reply.LocalTime, local
|
|
clockSamples.Add(1)
|
|
}
|
|
}
|
|
}()
|
|
defer func() { stopClock(); <-clockDone }()
|
|
|
|
reuseFixture := os.Getenv("POLL_POOL_REUSE_FIXTURE") == "1"
|
|
fixtureName := os.Getenv("POLL_POOL_FIXTURE_DATABASE")
|
|
if fixtureName == "" {
|
|
if reuseFixture {
|
|
t.Fatal("reuse requires an explicit synthetic fixture database")
|
|
}
|
|
fixtureName = fmt.Sprintf("poll_pool_read_mix_%d", time.Now().UnixNano())
|
|
}
|
|
if !strings.HasPrefix(fixtureName, "poll_pool_read_mix_") || len(fixtureName) > 63 || strings.Trim(fixtureName, "abcdefghijklmnopqrstuvwxyz0123456789_") != "" {
|
|
t.Fatal("invalid synthetic fixture database")
|
|
}
|
|
db := client.Database(fixtureName)
|
|
if !reuseFixture {
|
|
collections, err := db.ListCollectionNames(ctx, map[string]any{})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(collections) != 0 {
|
|
t.Fatal("preparation requires a new synthetic database")
|
|
}
|
|
}
|
|
cleanupFixture := !reuseFixture
|
|
defer func() {
|
|
if !cleanupFixture {
|
|
return
|
|
}
|
|
if os.Getenv("POLL_POOL_KEEP_FIXTURE") == "1" {
|
|
t.Logf("retained synthetic fixture: %s", db.Name())
|
|
return
|
|
}
|
|
cleanup, done := context.WithTimeout(context.Background(), 15*time.Second)
|
|
defer done()
|
|
if e := db.Drop(cleanup); e != nil {
|
|
t.Error(e)
|
|
}
|
|
}()
|
|
ds := store.NewMongoStore(db)
|
|
must := func(e error) {
|
|
t.Helper()
|
|
if e != nil {
|
|
t.Fatal(e)
|
|
}
|
|
}
|
|
if !reuseFixture {
|
|
must(ds.EnsureIndexes(ctx))
|
|
}
|
|
if os.Getenv("POLL_POOL_QUERY_SERVING_PREPARE") == "1" {
|
|
// Both variants use the already-documented platform read indexes as
|
|
// well as the pool-search indexes. These are isolated fixtures only.
|
|
for collection, models := range store.PlatformReadIndexes() {
|
|
_, e := db.Collection(collection).Indexes().CreateMany(ctx, models)
|
|
must(e)
|
|
}
|
|
}
|
|
const org = "load-org"
|
|
projects := []string{"load-project-a", "load-project-b"}
|
|
if os.Getenv("POLL_POOL_QUERY_SERVING_PREPARE") == "1" && !reuseFixture {
|
|
must(ds.UpsertOrganization(ctx, domain.OrganizationRecord{ID: org, Status: domain.OrganizationStatusActive}))
|
|
_, err := ds.SaveProductionStatsScope(ctx, org, productionstats.Scope{ProjectIDs: projects, Revision: 1}, 0)
|
|
must(err)
|
|
must(ds.EnsureQueryReadModelIndexes(ctx))
|
|
}
|
|
if !reuseFixture {
|
|
for _, p := range projects {
|
|
must(ds.UpsertProject(ctx, domain.ProjectRecord{ID: p, OrganizationID: org, Name: p}))
|
|
must(ds.UpsertWorkflow(ctx, domain.WorkflowRecord{ID: "workflow-" + p, OrganizationID: org, ProjectID: p, Name: "load workflow"}))
|
|
}
|
|
|
|
}
|
|
suppliers := []string{"load-supplier-a", "load-supplier-b", "load-supplier-c"}
|
|
if !reuseFixture {
|
|
for _, s := range suppliers {
|
|
must(ds.UpsertSupplier(ctx, domain.SupplierRecord{ID: s, OrganizationID: org, Status: "active", Name: s}))
|
|
_, e := ds.SetSupplierAcceptanceAuthorization(ctx, org, s, true, 0, "load-admin")
|
|
must(e)
|
|
}
|
|
|
|
}
|
|
actors := []pollLoadActor{}
|
|
for i := 0; i < totalActors; i++ {
|
|
role := domain.UserRoleAnnotator
|
|
label := "annotation"
|
|
if i >= annotators && i < annotators+quality {
|
|
role = domain.UserRoleQualityReviewer
|
|
label = "quality"
|
|
}
|
|
if i >= annotators+quality && i < totalActors-1 {
|
|
role = domain.UserRoleAcceptanceReviewer
|
|
label = "acceptance"
|
|
}
|
|
if i == totalActors-1 {
|
|
role = domain.UserRoleAdmin
|
|
label = "admin"
|
|
}
|
|
u := domain.UserRecord{ID: fmt.Sprintf("load-user-%02d", i), Phone: fmt.Sprintf("synthetic-load-%02d", i), OrganizationID: org, SupplierID: suppliers[i%3], Role: role, Status: domain.UserStatusActive}
|
|
if !reuseFixture {
|
|
must(ds.UpsertUser(ctx, u))
|
|
for _, p := range projects {
|
|
must(ds.UpsertProjectMember(ctx, domain.ProjectMemberRecord{ID: p + u.ID, OrganizationID: org, ProjectID: p, UserID: u.ID, SupplierID: u.SupplierID, Roles: domain.AccountProductionRoles(u)}))
|
|
}
|
|
|
|
}
|
|
actors = append(actors, pollLoadActor{u, projects[i%2], label})
|
|
}
|
|
payload := make([]string, 512)
|
|
for i := range payload {
|
|
payload[i] = fmt.Sprintf("%x", sha256.Sum256([]byte(fmt.Sprintf("fixed-fixture-%d", i))))
|
|
}
|
|
runPayloadKiB := 0
|
|
if raw := os.Getenv("POLL_POOL_RUN_PAYLOAD_KIB"); raw != "" {
|
|
runPayloadKiB, err = strconv.Atoi(raw)
|
|
must(err)
|
|
if runPayloadKiB < 0 || runPayloadKiB > 2048 {
|
|
t.Fatal("run payload must be 0..2048 KiB")
|
|
}
|
|
}
|
|
taskPayloadKiB := 0
|
|
if raw := os.Getenv("POLL_POOL_TASK_PAYLOAD_KIB"); raw != "" {
|
|
taskPayloadKiB, err = strconv.Atoi(raw)
|
|
must(err)
|
|
if taskPayloadKiB < 0 || taskPayloadKiB > 2048 {
|
|
t.Fatal("task payload must be 0..2048 KiB")
|
|
}
|
|
}
|
|
seededRuns := map[string]bool{}
|
|
seed := func(id, p, s, stage, owner string, status domain.HumanItemStatus) {
|
|
batchID := "batch-" + p + s
|
|
stages := domain.DefaultFanyuV58HumanStages()
|
|
if stage == "semantic_review" {
|
|
batchID += "-semantic"
|
|
stages = []domain.HumanStageDefinition{{ID: stage, Kind: domain.HumanStageReview, RequiredRole: "quality_reviewer"}}
|
|
}
|
|
if stage == "acceptance" || stage == "prepackage_acceptance" {
|
|
batchID += "-" + stage
|
|
stages = []domain.HumanStageDefinition{{ID: stage, Kind: domain.HumanStageAcceptance, RequiredRole: "acceptance_reviewer"}}
|
|
}
|
|
must(ds.UpsertHumanBatch(ctx, domain.HumanBatchRecord{ID: batchID, RunID: batchID, GateNodeID: "gate", OrganizationID: org, ProjectID: p, WorkflowID: "workflow-" + p, PluginID: "annotation-qa", SupplierID: s, Status: domain.HumanBatchActive, ProcessingProfile: domain.ProcessingProfileRawHuman, Stages: stages}))
|
|
runID := ""
|
|
if runPayloadKiB > 0 {
|
|
runID = "run-" + batchID
|
|
if productionShape {
|
|
runID = "run-" + id
|
|
}
|
|
if !seededRuns[runID] {
|
|
seededRuns[runID] = true
|
|
if productionShape {
|
|
must(ds.UpsertCaptureRecord(ctx, domain.CaptureRecord{ID: "capture-" + id, OrganizationID: org, ProjectID: p, WorkflowID: "workflow-" + p, CaptureID: "synthetic-" + id, Prefix: "synthetic/" + id}))
|
|
}
|
|
dirs := []string(nil)
|
|
captureID := ""
|
|
blob := strings.Repeat("x", runPayloadKiB*1024)
|
|
if productionShape {
|
|
dirs = []string{"synthetic-" + id}
|
|
captureID = "capture-" + id
|
|
block := strings.Join(payload, "")
|
|
blob = strings.Repeat(block, runPayloadKiB/32+1)[:runPayloadKiB*1024]
|
|
}
|
|
must(ds.UpsertRun(ctx, domain.RunRecord{ID: runID, CaptureDirectories: dirs, TaskID: captureID, OrganizationID: org, ProjectID: p, WorkflowID: "workflow-" + p, ProjectName: p, WorkflowName: "workflow-" + p, NodeExecutions: []domain.NodeExecutionRecord{{NodeID: "synthetic-node", Logs: []domain.LogEntryRecord{{Message: blob}}}}}))
|
|
}
|
|
}
|
|
taskID := "task-" + id
|
|
must(ds.UpsertAnnotationTask(ctx, domain.AnnotationTaskRecord{ID: taskID, OrganizationID: org, ProjectID: p, WorkflowID: "workflow-" + p, SupplierID: s, TemplateVersion: 6, Status: domain.AnnotationTaskStatusPendingAnnotation, SourceVideo: "synthetic/camera_left.mp4"}))
|
|
itemPayload := map[string]any{"captureDirectory": "synthetic-" + id, "sliceMergeEvidence": payload, "sliceWorkflow": map[string]any{"status": "completed", "phase": "annotate"}}
|
|
if taskPayloadKiB > 0 {
|
|
content := strings.Repeat(strings.Join(payload, ""), taskPayloadKiB/32+1)[:taskPayloadKiB*1024]
|
|
itemPayload["items"] = []map[string]any{{"annotation": map[string]any{"syntheticPayload": content}, "materialization": map[string]any{"coverage": map[string]any{"durationSeconds": 900}}}}
|
|
}
|
|
item := domain.HumanItemRecord{ID: id, RunID: runID, OrganizationID: org, ProjectID: p, WorkflowID: "workflow-" + p, PluginID: "annotation-qa", SupplierID: s, BatchID: batchID, LegacyTaskID: taskID, AssignedUserID: owner, LatestAuthorUserID: "synthetic-author", CurrentStageID: stage, Status: status, Version: 1, CreatedAt: "2026-10-01T00:00:00Z", Subject: domain.HumanSubjectRef{ItemKey: "synthetic/" + id + "/camera_left.mp4", Payload: itemPayload}}
|
|
if productionShape && stage == "prepackage_acceptance" {
|
|
sourceID := "source-" + id
|
|
batchID = "aggregate-" + id
|
|
must(ds.UpsertHumanBatch(ctx, domain.HumanBatchRecord{ID: sourceID, RunID: runID, GateNodeID: "source-gate", OrganizationID: org, ProjectID: p, WorkflowID: "workflow-" + p, PluginID: "annotation-qa", SupplierID: s, ProcessingProfile: domain.ProcessingProfileRawHuman, Status: domain.HumanBatchActive, Stages: domain.DefaultFanyuV58HumanStages(), RequirePrePackageAcceptance: true}))
|
|
must(ds.UpsertHumanBatch(ctx, domain.HumanBatchRecord{ID: batchID, SourceBatchID: sourceID, RunID: runID, GateNodeID: "acceptance-gate", OrganizationID: org, ProjectID: p, WorkflowID: "workflow-" + p, PluginID: "annotation-qa", ProcessingProfile: domain.ProcessingProfileRawHuman, Status: domain.HumanBatchActive, Stages: stages}))
|
|
item.BatchID, item.LegacyTaskID, item.LatestAuthorUserID = batchID, "", ""
|
|
item.SupplierID = ""
|
|
if strings.HasSuffix(id, "0") {
|
|
item.SupplierID = s
|
|
}
|
|
item.Subject.ItemKey = sourceID
|
|
itemPayload["sourceBatchId"], itemPayload["strictThreeRole"] = sourceID, true
|
|
}
|
|
must(ds.UpsertHumanItem(ctx, item))
|
|
if productionShape {
|
|
must(ds.AppendHumanEvent(ctx, domain.HumanEventRecord{ID: "created-" + id, OrganizationID: org, ItemID: id, StageID: stage, EventType: "created", CreatedAt: "2026-10-01T00:00:00Z"}))
|
|
}
|
|
}
|
|
if !reuseFixture {
|
|
for _, a := range actors {
|
|
if a.Role == "admin" {
|
|
continue
|
|
}
|
|
stage := "annotation_combined"
|
|
if a.Role == "quality" {
|
|
stage = "quality_review"
|
|
}
|
|
if a.Role == "acceptance" {
|
|
stage = "prepackage_acceptance"
|
|
}
|
|
for n := 0; n < 2; n++ {
|
|
seed(fmt.Sprintf("owned-%s-%d", a.User.ID, n), a.Project, a.User.SupplierID, stage, a.User.ID, domain.HumanItemClaimed)
|
|
}
|
|
if a.Role == "annotation" {
|
|
taskID := "task-owned-" + a.User.ID + "-0"
|
|
must(ds.UpsertAnnotationRevision(ctx, domain.AnnotationRevisionRecord{ID: "revision-" + a.User.ID, OrganizationID: org, TaskID: taskID, Revision: 1, CountsAsOutput: true, AuthorUserID: a.User.ID}))
|
|
must(ds.UpsertAnnotationWorklogEvent(ctx, domain.AnnotationWorklogEventRecord{ID: "worklog-" + a.User.ID, OrganizationID: org, ProjectID: a.Project, TaskID: taskID, UserID: a.User.ID, SupplierID: a.User.SupplierID, EventType: "combined_submitted", Revision: 1, OutputDelta: 1, CreatedAt: "2026-10-02T00:00:00Z"}))
|
|
}
|
|
if a.Role != "annotation" {
|
|
must(ds.AppendHumanEvent(ctx, domain.HumanEventRecord{ID: "history-" + a.User.ID, OrganizationID: org, ItemID: "owned-" + a.User.ID + "-0", ActorUserID: a.User.ID, StageID: stage, EventType: "approved", CreatedAt: "2026-10-02T00:00:00Z"}))
|
|
}
|
|
}
|
|
for _, p := range projects {
|
|
for _, s := range suppliers {
|
|
stages := []string{"quality_review", "semantic_review", "acceptance", "prepackage_acceptance"}
|
|
if claimCounts {
|
|
stages = append(stages, "annotation_combined")
|
|
}
|
|
for _, stage := range stages {
|
|
for n := 0; n < availablePerStage; n++ {
|
|
seed(fmt.Sprintf("available-%s-%s-%s-%d", p, s, stage, n), p, s, stage, "", domain.HumanItemReady)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
}
|
|
authorHistory := 0
|
|
if raw := os.Getenv("POLL_POOL_AUTHOR_HISTORY"); raw != "" {
|
|
authorHistory, err = strconv.Atoi(raw)
|
|
must(err)
|
|
if authorHistory < 0 || authorHistory > 5000 {
|
|
t.Fatal("author history must be 0..5000 per reviewer")
|
|
}
|
|
}
|
|
historyPartitions := map[string]domain.ReleasePartitionRecord{}
|
|
if !reuseFixture {
|
|
for _, a := range actors {
|
|
if authorHistory == 0 || (a.Role != "quality" && a.Role != "acceptance") {
|
|
continue
|
|
}
|
|
rows := make([]any, 0, authorHistory)
|
|
for i := 0; i < authorHistory; i++ {
|
|
stage := "quality_review"
|
|
suffix := ""
|
|
if a.Role == "quality" && i%2 != 0 {
|
|
stage = "semantic_review"
|
|
suffix = "-semantic"
|
|
}
|
|
if a.Role == "acceptance" {
|
|
stage = "acceptance"
|
|
if i%2 != 0 {
|
|
stage = "prepackage_acceptance"
|
|
}
|
|
suffix = "-" + stage
|
|
}
|
|
id := fmt.Sprintf("historical-%s-%05d", a.User.ID, i)
|
|
batchID := "batch-" + a.Project + a.User.SupplierID + suffix
|
|
partitionID := "released-history-" + batchID
|
|
partition := historyPartitions[partitionID]
|
|
partition.ID = partitionID
|
|
partition.OrganizationID = org
|
|
partition.BatchID = batchID
|
|
partition.Sequence = 1
|
|
partition.UniqueKey = partitionID
|
|
partition.Status = domain.ReleasePartitionDispatched
|
|
partition.ItemIDs = append(partition.ItemIDs, id)
|
|
historyPartitions[partitionID] = partition
|
|
|
|
rows = append(rows, domain.HumanItemRecord{ID: id, OrganizationID: org, ProjectID: a.Project, WorkflowID: "workflow-" + a.Project, PluginID: "annotation-qa", SupplierID: a.User.SupplierID, BatchID: batchID, ReleasePartitionID: partitionID, LatestAuthorUserID: a.User.ID, CurrentStageID: stage, Status: domain.HumanItemAccepted, Version: 1, CreatedAt: "2026-09-01T00:00:00Z", Subject: domain.HumanSubjectRef{ItemKey: id}})
|
|
}
|
|
_, e := db.Collection("humanItems").InsertMany(ctx, rows)
|
|
must(e)
|
|
}
|
|
|
|
for _, partition := range historyPartitions {
|
|
must(ds.UpsertReleasePartition(ctx, partition))
|
|
}
|
|
|
|
}
|
|
authorizationState := os.Getenv("POLL_POOL_AUTHORIZATION_STATE")
|
|
if authorizationState == "" {
|
|
authorizationState = "legacy"
|
|
}
|
|
if authorizationState != "legacy" && authorizationState != "initialized" {
|
|
t.Fatal("unknown authorization state")
|
|
}
|
|
if authorizationState == "initialized" && !reuseFixture {
|
|
must(ds.InitializeSupplierProjectAuthorizations(ctx, org))
|
|
}
|
|
if readModels && !reuseFixture {
|
|
result, err := ds.BackfillPoolSearchMetadata(ctx, org, true)
|
|
must(err)
|
|
if result.Unsupported > 0 || result.Conflicts > 0 {
|
|
t.Fatalf("summary preparation incomplete: %+v", result)
|
|
}
|
|
}
|
|
handler := NewHandler(ds, nil)
|
|
var mu sync.Mutex
|
|
samples := []pollLoadSample{}
|
|
coldSamples := []pollLoadSample{}
|
|
lastPaths := map[string]string{}
|
|
recordingCold := true
|
|
failureSamples := []map[string]any{}
|
|
var completedRequests, completedErrors atomic.Int64
|
|
call := func(a pollLoadActor, key, path string, record bool) pollLoadSample {
|
|
mu.Lock()
|
|
lastPaths[a.User.ID+"\x00"+key] = path
|
|
mu.Unlock()
|
|
req := httptest.NewRequest(http.MethodGet, path, nil)
|
|
requestCtx, done := context.WithTimeout(ctx, 15*time.Second)
|
|
defer done()
|
|
requestCtx = context.WithValue(requestCtx, pollLoadEndpointKey{}, key)
|
|
req = req.WithContext(context.WithValue(requestCtx, authContextKey{}, &authenticatedPrincipal{User: a.User, Organization: domain.OrganizationRecord{ID: org, Status: domain.OrganizationStatusActive}}))
|
|
response := httptest.NewRecorder()
|
|
start := time.Now()
|
|
handler.ServeHTTP(response, req)
|
|
elapsed := time.Since(start)
|
|
completedRequests.Add(1)
|
|
var body struct {
|
|
Summary *productionstats.Summary `json:"summary"`
|
|
Totals json.RawMessage `json:"totals"`
|
|
Count *int `json:"count"`
|
|
Counts map[string]int `json:"counts"`
|
|
Code string `json:"code"`
|
|
Error string `json:"error"`
|
|
Items []struct {
|
|
ID string `json:"id"`
|
|
CanClaim *bool `json:"canClaim"`
|
|
TraceSummary *domain.HumanTraceSummary `json:"traceSummary"`
|
|
} `json:"items"`
|
|
Summaries []struct {
|
|
ID string `json:"id"`
|
|
CanClaim *bool `json:"canClaim"`
|
|
TraceSummary *domain.HumanTraceSummary `json:"traceSummary"`
|
|
} `json:"summaries"`
|
|
}
|
|
_ = json.Unmarshal(response.Body.Bytes(), &body)
|
|
ids := []string{}
|
|
for _, v := range body.Items {
|
|
ids = append(ids, v.ID)
|
|
if v.CanClaim != nil {
|
|
ids = append(ids, fmt.Sprintf("claim:%s:%t", v.ID, *v.CanClaim))
|
|
}
|
|
if v.TraceSummary != nil {
|
|
raw, _ := json.Marshal(v.TraceSummary)
|
|
ids = append(ids, "trace:"+v.ID+":"+string(raw))
|
|
}
|
|
if pageShape && strings.Contains(path, "trace=1") && v.TraceSummary == nil {
|
|
response.Code = 599
|
|
}
|
|
}
|
|
for _, v := range body.Summaries {
|
|
ids = append(ids, v.ID)
|
|
if v.CanClaim != nil {
|
|
ids = append(ids, fmt.Sprintf("claim:%s:%t", v.ID, *v.CanClaim))
|
|
}
|
|
if v.TraceSummary != nil {
|
|
raw, _ := json.Marshal(v.TraceSummary)
|
|
ids = append(ids, "trace:"+v.ID+":"+string(raw))
|
|
}
|
|
if pageShape && strings.Contains(path, "trace=1") && v.TraceSummary == nil {
|
|
response.Code = 599
|
|
}
|
|
}
|
|
if response.Code != 200 {
|
|
completedErrors.Add(1)
|
|
}
|
|
rowCount := len(body.Items) + len(body.Summaries)
|
|
if body.Summary != nil {
|
|
rowCount = len(body.Summary.Rows)
|
|
raw, _ := json.Marshal(body.Summary)
|
|
ids = append(ids, string(raw))
|
|
}
|
|
if strings.HasSuffix(key, ".panel") && len(body.Totals) > 0 {
|
|
rowCount = 1
|
|
ids = append(ids, string(body.Totals))
|
|
}
|
|
if body.Count != nil {
|
|
rowCount = *body.Count
|
|
ids = append(ids, fmt.Sprintf("count:%d", *body.Count))
|
|
}
|
|
for pool, total := range body.Counts {
|
|
ids = append(ids, fmt.Sprintf("pool-count:%s:%d", pool, total))
|
|
}
|
|
sort.Strings(ids)
|
|
var compressed bytes.Buffer
|
|
zw := gzip.NewWriter(&compressed)
|
|
_, _ = zw.Write(response.Body.Bytes())
|
|
_ = zw.Close()
|
|
sample := pollLoadSample{a.User.ID, key, float64(elapsed.Microseconds()) / 1000, response.Body.Len(), compressed.Len(), rowCount, response.Code, fmt.Sprintf("%x", sha256.Sum256([]byte(strings.Join(ids, "\n"))))}
|
|
if response.Code != 200 {
|
|
class := "unclassified"
|
|
for _, known := range []string{"operation exceeded time limit", "context deadline exceeded", "context canceled", "duplicate key", "not found", "conflict", "timeout"} {
|
|
if strings.Contains(strings.ToLower(body.Error), known) {
|
|
class = known
|
|
break
|
|
}
|
|
}
|
|
code := strings.Map(func(r rune) rune {
|
|
if r >= 'a' && r <= 'z' || r >= 'A' && r <= 'Z' || r >= '0' && r <= '9' || r == '_' || r == '-' {
|
|
return r
|
|
}
|
|
return -1
|
|
}, body.Code)
|
|
if len(code) > 80 {
|
|
code = code[:80]
|
|
}
|
|
mu.Lock()
|
|
if len(failureSamples) < 20 {
|
|
failureSamples = append(failureSamples, map[string]any{"at": time.Now().UTC().Format(time.RFC3339Nano), "endpoint": key, "actor": a.User.ID, "status": response.Code, "elapsed_ms": sample.MS, "code": code, "class": class, "measured": record})
|
|
if productionShape {
|
|
t.Logf("synthetic error: %.400s", body.Error)
|
|
}
|
|
t.Logf("read-mix HTTP failure endpoint=%s actor=%s status=%d elapsed_ms=%.3f code=%s class=%s request_context=%v", key, a.User.ID, response.Code, sample.MS, code, class, requestCtx.Err())
|
|
}
|
|
mu.Unlock()
|
|
}
|
|
if recordingCold {
|
|
coldSamples = append(coldSamples, sample)
|
|
}
|
|
if record {
|
|
mu.Lock()
|
|
samples = append(samples, sample)
|
|
mu.Unlock()
|
|
}
|
|
return sample
|
|
}
|
|
paths := func(a pollLoadActor, all, countCycle bool, scenario int) map[string]string {
|
|
scope := "&projectId=" + a.Project
|
|
scopeName := "project"
|
|
if all {
|
|
scope = ""
|
|
scopeName = "all"
|
|
}
|
|
if a.Role == "annotation" {
|
|
path := "/api/plugins/annotation-qa/work-buffer?projectId=" + a.Project
|
|
if mode == "summary" {
|
|
path += "&view=summary"
|
|
}
|
|
result := map[string]string{"annotation.buffer": path}
|
|
if claimCounts && countCycle {
|
|
result["annotation.claim-count"] = "/api/plugins/annotation-qa/claim-count?projectId=" + a.Project
|
|
}
|
|
return result
|
|
}
|
|
if a.Role == "admin" {
|
|
if os.Getenv("POLL_POOL_ADMIN_TASKS") == "1" {
|
|
base := "/api/plugins/annotation-qa/admin-pool?limit=200" + scope
|
|
suffix, key := "&pool=available", "available"
|
|
switch scenario % 5 {
|
|
case 1:
|
|
suffix, key = "&pool=all&search=synthetic-available-", "search"
|
|
case 2:
|
|
suffix, key = "&pool=all&search=__production_shape_no_match__", "search.nohit"
|
|
case 3:
|
|
suffix, key = "&pool=mine", "mine"
|
|
case 4:
|
|
suffix, key = "&pool=all&category=pending_slice", "category"
|
|
}
|
|
return map[string]string{"admin." + scopeName + "." + key: base + suffix}
|
|
}
|
|
return map[string]string{"admin.workforce": "/api/dashboard/workforce?from=2026-10-01T00:00:00Z&to=2026-10-28T00:00:00Z"}
|
|
}
|
|
if pageShape {
|
|
// AnnotationOperations loads one selected tab plus poolCounts.
|
|
// The old broad stress mode below deliberately requests every tab
|
|
// and both acceptance stages separately; keep it as a separate test.
|
|
stages := "quality_review,semantic_review"
|
|
if a.Role == "acceptance" {
|
|
stages = "acceptance,prepackage_acceptance"
|
|
}
|
|
endpoint := "/api/human-item-summaries"
|
|
pool, search, kind := "available", "", "available"
|
|
switch scenario % 5 {
|
|
case 1:
|
|
stage := "quality_review"
|
|
if a.Role == "acceptance" {
|
|
stage = "prepackage_acceptance"
|
|
}
|
|
search = "synthetic-available-" + a.Project + "-" + a.User.SupplierID + "-" + stage + "-0"
|
|
kind = "search"
|
|
case 2:
|
|
search, kind = "__production_shape_no_match__", "search.nohit"
|
|
case 3:
|
|
pool, kind = "mine", "mine"
|
|
case 4:
|
|
pool, kind = "reviewed", "reviewed"
|
|
}
|
|
key := a.Role + "." + scopeName + "." + kind
|
|
query := "?pluginId=annotation-qa" + scope + "&stageIds=" + stages + "&search=" + url.QueryEscape(search)
|
|
counts := "/api/human-item-counts" + query
|
|
if a.Role == "acceptance" {
|
|
counts += "&reviewedStageIds=" + stages
|
|
}
|
|
listQuery := query
|
|
if a.Role == "quality" && pool == "reviewed" {
|
|
listQuery = strings.Replace(query, "&stageIds="+stages, "", 1)
|
|
}
|
|
return map[string]string{key: endpoint + listQuery + "&pool=" + pool + "&limit=200&trace=1", key + ".counts": counts}
|
|
}
|
|
|
|
result := map[string]string{}
|
|
for _, pool := range []string{"available", "mine", "reviewed"} {
|
|
endpoint := "/api/human-items"
|
|
if readModels {
|
|
endpoint = "/api/human-item-summaries"
|
|
}
|
|
base := endpoint + "?pluginId=annotation-qa&pool=" + pool + scope
|
|
if a.Role == "quality" {
|
|
result["quality."+scopeName+"."+pool] = base + "&stageIds=quality_review%2Csemantic_review"
|
|
} else {
|
|
for _, stage := range []string{"acceptance", "prepackage_acceptance"} {
|
|
result["acceptance."+scopeName+"."+stage+"."+pool] = base + "&stageId=" + stage
|
|
}
|
|
}
|
|
}
|
|
if readModels {
|
|
stages := "quality_review,semantic_review"
|
|
reviewed := ""
|
|
if a.Role == "acceptance" {
|
|
stages = "acceptance,prepackage_acceptance"
|
|
reviewed = "&reviewedStageIds=" + stages
|
|
}
|
|
result[a.Role+"."+scopeName+".counts"] = "/api/human-item-counts?pluginId=annotation-qa" + scope + "&stageIds=" + stages + reviewed
|
|
}
|
|
if os.Getenv("POLL_POOL_SEARCH") == "1" {
|
|
endpoint := "/api/human-items"
|
|
if readModels {
|
|
endpoint = "/api/human-item-summaries"
|
|
}
|
|
stage := "quality_review"
|
|
if a.Role == "acceptance" {
|
|
stage = "prepackage_acceptance"
|
|
}
|
|
result[a.Role+"."+scopeName+".search"] = endpoint + "?pluginId=annotation-qa&pool=available&limit=200" + scope + "&stageId=" + stage + "&search=synthetic-available-" + a.Project + "-" + a.User.SupplierID + "-" + stage + "-0"
|
|
if productionShape {
|
|
stages := "quality_review,semantic_review"
|
|
if a.Role == "acceptance" {
|
|
stages = "acceptance,prepackage_acceptance"
|
|
}
|
|
base := endpoint + "?pluginId=annotation-qa&pool=available&limit=200" + scope + "&stageIds=" + stages
|
|
result[a.Role+"."+scopeName+".search"] = base + "&search=synthetic-available-" + a.Project + "-" + a.User.SupplierID + "-" + stage + "-0"
|
|
result[a.Role+"."+scopeName+".search.nohit"] = base + "&search=__production_shape_no_match__"
|
|
}
|
|
}
|
|
return result
|
|
}
|
|
if queryPanels {
|
|
original := paths
|
|
paths = func(a pollLoadActor, all, countCycle bool, scenario int) map[string]string {
|
|
out := original(a, all, countCycle, scenario)
|
|
out[a.Role+".panel"] = "/api/dashboard/production-quality?from=2026-10-01&to=2026-11-13"
|
|
if a.Role == "admin" {
|
|
out["admin.production"] = "/api/dashboard/production?from=2026-10-01&to=2026-11-13"
|
|
}
|
|
return out
|
|
}
|
|
}
|
|
|
|
fixtureSpec := fmt.Sprintf("isolated-preparation-v5|%d|%d|%d|%d|%d|%d|%d|%t|%t|%t|%s|%s", annotators, quality, acceptance, availablePerStage, runPayloadKiB, taskPayloadKiB, authorHistory, productionShape, readModels, claimCounts, authorizationState, os.Getenv("POLL_POOL_SEARCH_INDEXES"))
|
|
fixtureSpec += "|admin-tasks=" + os.Getenv("POLL_POOL_ADMIN_TASKS")
|
|
if os.Getenv("POLL_POOL_QUERY_SERVING_PREPARE") == "1" {
|
|
fixtureSpec += "|query-serving-preparation-v1"
|
|
}
|
|
fixtureDigest := fmt.Sprintf("%x", sha256.Sum256([]byte(fixtureSpec)))
|
|
if reuseFixture {
|
|
var manifest struct {
|
|
Digest string `bson:"digest"`
|
|
}
|
|
must(db.Collection("pollFixture").FindOne(ctx, map[string]any{"_id": "identity"}).Decode(&manifest))
|
|
if manifest.Digest != fixtureDigest {
|
|
t.Fatal("fixture configuration differs from preparation")
|
|
}
|
|
} else {
|
|
_, err := db.Collection("pollFixture").InsertOne(ctx, map[string]any{"_id": "identity", "digest": fixtureDigest})
|
|
must(err)
|
|
}
|
|
fixtureCounts := map[string]int64{}
|
|
if productionShape {
|
|
if reuseFixture {
|
|
for collection, want := range map[string]int64{"humanItems": 26218, "runs": 26490, "captureRecords": 29767, "annotationTasks": 23611, "humanEvents": 342694} {
|
|
got, err := db.Collection(collection).CountDocuments(ctx, map[string]any{"organizationId": org})
|
|
must(err)
|
|
if got != want {
|
|
t.Fatalf("fixture %s count=%d want=%d", collection, got, want)
|
|
}
|
|
fixtureCounts[collection] = got
|
|
}
|
|
} else {
|
|
var err error
|
|
fixtureCounts, err = seedProductionPoolNoise(ctx, db, org, payload)
|
|
must(err)
|
|
must(installPoolReadIndexes(ctx, db, os.Getenv("POLL_POOL_SEARCH_INDEXES") == "1"))
|
|
}
|
|
}
|
|
|
|
cleanupFixture = true
|
|
if os.Getenv("POLL_POOL_FIXTURE_ONLY") == "1" {
|
|
return
|
|
}
|
|
var preparationCPU float64
|
|
if queryServing {
|
|
startCPU, _ := pollLoadProcessCPU()
|
|
if os.Getenv("QUERY_SERVING_LOCAL_TEST") != "1" {
|
|
t.Fatal("query acceleration requires isolated local dependencies")
|
|
}
|
|
cache, err := queryserving.NewManagedValkey("valkey://127.0.0.1:16379", db.Name())
|
|
must(err)
|
|
defer cache.Close()
|
|
search, err := queryserving.NewTypesense("http://127.0.0.1:18108", os.Getenv("QUERY_TEST_TYPESENSE_API_KEY"))
|
|
must(err)
|
|
ds.ConfigureQueryServing(cache, search)
|
|
builder := store.NewQueryReadBuilder(ds)
|
|
buildCtx, stop := context.WithCancel(ctx)
|
|
buildDone := make(chan error, 1)
|
|
go func() { buildDone <- builder.Run(buildCtx) }()
|
|
defer func() { stop(); <-buildDone; builder.Close() }()
|
|
prepare, finish := context.WithTimeout(ctx, 4*time.Minute)
|
|
err = builder.WaitReady(prepare, org, projects)
|
|
finish()
|
|
must(err)
|
|
endCPU, _ := pollLoadProcessCPU()
|
|
preparationCPU = endCPU - startCPU
|
|
}
|
|
// Fail before timing if permissions or fixtures accidentally produce empty work.
|
|
for _, a := range actors {
|
|
for key, path := range paths(a, false, true, 0) {
|
|
result := call(a, key, path, false)
|
|
if result.Code != 200 {
|
|
if allowBaselineFailure {
|
|
continue
|
|
}
|
|
t.Fatalf("preflight %s status=%d", key, result.Code)
|
|
}
|
|
// This workload seeds assigned annotation work, but no unassigned
|
|
// annotation pool unless claim-count coverage was requested. An empty
|
|
// admin available pool is then the correct fixture result, not failure.
|
|
requireRows := a.Role == "annotation" || a.Role == "admin" && os.Getenv("POLL_POOL_ADMIN_TASKS") != "1" || a.Role != "admin" && (strings.HasSuffix(key, ".available") || strings.HasSuffix(key, ".mine") && !strings.Contains(key, ".acceptance.")) || a.Role == "admin" && claimCounts && strings.HasSuffix(key, ".available")
|
|
if requireRows && result.Rows == 0 {
|
|
t.Fatalf("preflight %s unexpectedly empty for %s", key, a.User.ID)
|
|
}
|
|
}
|
|
}
|
|
recordingCold = false
|
|
var memBefore, memAfter runtime.MemStats
|
|
runPhase := func(seconds int, record bool) {
|
|
phaseCtx, stop := context.WithTimeout(ctx, time.Duration(seconds)*time.Second)
|
|
defer stop()
|
|
var wg sync.WaitGroup
|
|
for _, actor := range actors {
|
|
wg.Add(1)
|
|
go func(a pollLoadActor) {
|
|
defer wg.Done()
|
|
period := 12 * time.Second
|
|
if a.Role == "annotation" {
|
|
period = 3 * time.Second
|
|
}
|
|
if a.Role == "admin" && os.Getenv("POLL_POOL_ADMIN_TASKS") == "1" {
|
|
period = time.Duration(adminPeriod) * time.Second
|
|
}
|
|
ticker := time.NewTicker(period)
|
|
defer ticker.Stop()
|
|
cycles := (time.Duration(seconds)*time.Second + period - 1) / period
|
|
for cycle := 0; cycle < int(cycles); cycle++ {
|
|
select {
|
|
case <-phaseCtx.Done():
|
|
return
|
|
default:
|
|
}
|
|
var requests sync.WaitGroup
|
|
for key, path := range paths(a, cycle%2 == 1, cycle%4 == 0, cycle/2) {
|
|
requests.Add(1)
|
|
go func(key, path string) { defer requests.Done(); call(a, key, path, record) }(key, path)
|
|
}
|
|
requests.Wait()
|
|
if cycle+1 == int(cycles) {
|
|
return
|
|
}
|
|
select {
|
|
case <-phaseCtx.Done():
|
|
return
|
|
case <-ticker.C:
|
|
}
|
|
}
|
|
}(actor)
|
|
}
|
|
wg.Wait()
|
|
<-phaseCtx.Done()
|
|
}
|
|
out := os.Getenv("POLL_POOL_OUTPUT")
|
|
if out == "" {
|
|
out = filepath.Join("..", "..", "local-data", "test-results", fmt.Sprintf("poll-pool-%s-%d.json", mode, time.Now().Unix()))
|
|
}
|
|
must(os.MkdirAll(filepath.Dir(out), 0700))
|
|
resourceFile, err := os.Create(out + ".resources.jsonl")
|
|
must(err)
|
|
defer resourceFile.Close()
|
|
resourceEncoder := json.NewEncoder(resourceFile)
|
|
resourceRows := []map[string]any{}
|
|
var phase atomic.Value
|
|
phase.Store("warmup")
|
|
resourceStart := time.Now()
|
|
sampleResource := func() error {
|
|
var m runtime.MemStats
|
|
runtime.ReadMemStats(&m)
|
|
cpuSeconds, cpuErr := pollLoadProcessCPU()
|
|
if cpuErr != nil {
|
|
return cpuErr
|
|
}
|
|
mu.Lock()
|
|
requests := len(samples)
|
|
mu.Unlock()
|
|
row := map[string]any{"at": time.Now().UTC().Format(time.RFC3339Nano), "phase": phase.Load(), "elapsed_seconds": time.Since(resourceStart).Seconds(), "heap_alloc_bytes": m.HeapAlloc, "heap_inuse_bytes": m.HeapInuse, "total_alloc_bytes": m.TotalAlloc, "gc_cycles": m.NumGC, "goroutines": runtime.NumGoroutine(), "retained_request_samples": requests, "errors": completedErrors.Load(), "completed_requests": completedRequests.Load(), "process_cpu_seconds": cpuSeconds}
|
|
resourceRows = append(resourceRows, row)
|
|
return resourceEncoder.Encode(row)
|
|
}
|
|
must(sampleResource())
|
|
resourceStop := make(chan struct{})
|
|
resourceDone := make(chan error, 1)
|
|
go func() {
|
|
ticker := time.NewTicker(15 * time.Second)
|
|
defer ticker.Stop()
|
|
for {
|
|
select {
|
|
case <-resourceStop:
|
|
resourceDone <- sampleResource()
|
|
return
|
|
case <-ticker.C:
|
|
if e := sampleResource(); e != nil {
|
|
cancel()
|
|
resourceDone <- e
|
|
return
|
|
}
|
|
}
|
|
}
|
|
}()
|
|
runPhase(warmup, false)
|
|
if completedErrors.Load() != 0 && !allowBaselineFailure {
|
|
close(resourceStop)
|
|
must(<-resourceDone)
|
|
t.Fatalf("warmup had %d HTTP errors", completedErrors.Load())
|
|
}
|
|
phase.Store("steady")
|
|
runtime.ReadMemStats(&memBefore)
|
|
var cpuFile *os.File
|
|
if filename := os.Getenv("POLL_POOL_CPU_PROFILE"); filename != "" {
|
|
must(os.MkdirAll(filepath.Dir(filename), 0700))
|
|
cpuFile, err = os.Create(filename)
|
|
must(err)
|
|
must(pprof.StartCPUProfile(cpuFile))
|
|
}
|
|
cpuBefore, cpuErr := pollLoadProcessCPU()
|
|
must(cpuErr)
|
|
measured.Store(true)
|
|
start := time.Now()
|
|
runPhase(duration, true)
|
|
measured.Store(false)
|
|
phase.Store("finished")
|
|
close(resourceStop)
|
|
must(<-resourceDone)
|
|
must(resourceFile.Close())
|
|
if cpuFile != nil {
|
|
pprof.StopCPUProfile()
|
|
must(cpuFile.Close())
|
|
}
|
|
cpuAfter, cpuErr := pollLoadProcessCPU()
|
|
must(cpuErr)
|
|
actual := time.Since(start)
|
|
runtime.ReadMemStats(&memAfter)
|
|
// Outside the measured interval: distinguish retained objects from GC timing.
|
|
runtime.GC()
|
|
var memAfterGC runtime.MemStats
|
|
runtime.ReadMemStats(&memAfterGC)
|
|
grouped := map[string][]pollLoadSample{}
|
|
failures := 0
|
|
for _, s := range samples {
|
|
grouped[s.Key] = append(grouped[s.Key], s)
|
|
if s.Code != 200 {
|
|
failures++
|
|
}
|
|
}
|
|
metrics := map[string]any{}
|
|
for key, rows := range grouped {
|
|
lat := []float64{}
|
|
raw, gz, minRows := 0, 0, rows[0].Rows
|
|
for _, s := range rows {
|
|
lat = append(lat, s.MS)
|
|
raw += s.Raw
|
|
gz += s.Gzip
|
|
if s.Rows < minRows {
|
|
minRows = s.Rows
|
|
}
|
|
}
|
|
sort.Float64s(lat)
|
|
metrics[key] = map[string]any{"requests": len(rows), "p95_ms": lat[(len(lat)*95-1)/100], "max_ms": lat[len(lat)-1], "raw_bytes": raw, "gzip_bytes": gz, "min_items": minRows}
|
|
}
|
|
slowSearch := []string{}
|
|
if raw := os.Getenv("POLL_POOL_MAX_SEARCH_P95_MS"); raw != "" {
|
|
budget, err := strconv.ParseFloat(raw, 64)
|
|
if err != nil || budget <= 0 {
|
|
t.Fatal("invalid search latency budget")
|
|
}
|
|
for key, value := range metrics {
|
|
if strings.Contains(key, ".search") && value.(map[string]any)["p95_ms"].(float64) > budget {
|
|
slowSearch = append(slowSearch, key)
|
|
}
|
|
}
|
|
}
|
|
outputState, stateErr := ds.OutputSnapshotState(ctx)
|
|
must(stateErr)
|
|
preparedQueries := 0
|
|
for _, head := range outputState.Heads {
|
|
if head.Current != nil && head.Current.Metrics != nil {
|
|
preparedQueries += head.Current.Metrics.PreparedQueries
|
|
}
|
|
}
|
|
var usage syscall.Rusage
|
|
must(syscall.Getrusage(syscall.RUSAGE_SELF, &usage))
|
|
maxRSS := usage.Maxrss
|
|
if runtime.GOOS != "darwin" {
|
|
maxRSS *= 1024
|
|
}
|
|
// Outside all measured latency, CPU, heap and Mongo counters: a serial
|
|
// retry supplies correctness evidence for baseline endpoints that never
|
|
// completed under load. Its latency is NOT mixed into benchmark metrics.
|
|
references := []pollLoadSample{}
|
|
if queryPanels {
|
|
succeeded := map[string]bool{}
|
|
tried := map[string]bool{}
|
|
all := append(append([]pollLoadSample{}, coldSamples...), samples...)
|
|
for _, sample := range all {
|
|
if sample.Code == 200 {
|
|
succeeded[sample.Actor+"\x00"+sample.Key] = true
|
|
}
|
|
}
|
|
for _, sample := range all {
|
|
identity := sample.Actor + "\x00" + sample.Key
|
|
if succeeded[identity] || tried[identity] || len(references) >= 32 {
|
|
continue
|
|
}
|
|
tried[identity] = true
|
|
for _, actor := range actors {
|
|
if actor.User.ID == sample.Actor {
|
|
references = append(references, call(actor, sample.Key, lastPaths[identity], false))
|
|
break
|
|
}
|
|
}
|
|
}
|
|
}
|
|
report := map[string]any{"mongo_clock_samples": clockSamples.Load(), "mongo_clock_max_jump_ms": clockJumpMS.Load(), "reference_samples": references, "platform_read_indexes": os.Getenv("POLL_POOL_QUERY_SERVING_PREPARE") == "1", "output_prepared_queries": preparedQueries, "process_max_rss_bytes": maxRSS, "query_serving": queryServing, "query_preparation_cpu_seconds": preparationCPU, "query_serving_statistics": ds.QueryServingStatistics(), "query_panels": queryPanels, "derived_write_commands": derivedWrites.Load(), "reused_prepared_fixture": reuseFixture, "fixture_digest": fixtureDigest, "cold_samples": coldSamples, "slow_search_endpoints": slowSearch, "baseline_failure_collection": allowBaselineFailure, "production_shape": productionShape, "page_shape": pageShape, "fixture_counts": fixtureCounts, "search_indexes": os.Getenv("POLL_POOL_SEARCH_INDEXES") == "1", "read_models": readModels, "search_queries": os.Getenv("POLL_POOL_SEARCH") == "1", "schema": 2, "resource_samples": resourceRows, "process_cpu_seconds": cpuAfter - cpuBefore, "authorization_state": authorizationState, "fixture": "read-mix-v5-isolated-preparation", "available_per_stage": availablePerStage, "task_payload_kib": taskPayloadKiB, "run_payload_kib": runPayloadKiB, "author_history_per_reviewer": authorHistory, "mode": mode, "revision": os.Getenv("POLL_POOL_REVISION"), "actors": totalActors, "actor_counts": map[string]int{"annotation": annotators, "quality": quality, "acceptance": acceptance, "admin": 1}, "claim_counts": claimCounts, "warmup_seconds": warmup, "scheduled_seconds": duration, "actual_seconds": actual.Seconds(), "errors": failures, "failure_samples": failureSamples, "metrics": metrics, "mongo_breakdown": observations, "mongo_commands": commands.Load(), "mongo_reply_bytes": mongoBytes.Load(), "mongo_command_ms": float64(mongoNS.Load()) / 1e6, "mongo_write_commands": writes.Load(), "go_heap_before": memBefore.HeapAlloc, "go_heap_after": memAfter.HeapAlloc, "go_heap_after_gc": memAfterGC.HeapAlloc, "go_gc_cycles": memAfter.NumGC - memBefore.NumGC, "go_allocated_bytes": memAfter.TotalAlloc - memBefore.TotalAlloc, "limitations": []string{"handler timing excludes ingress/network/authentication", "gzip computed separately with fixed Go default compression", "no continuation renew, media or workflow writes", "synthetic fixture; not capacity acceptance", "CPU covers handler and in-process load harness, not Mongo/Pod throttling", "resource heap includes retained benchmark request samples; not standalone service RSS"}, "samples": samples}
|
|
encoded, e := json.MarshalIndent(report, "", " ")
|
|
must(e)
|
|
must(os.WriteFile(out, encoded, 0600))
|
|
t.Logf("read-mix report: %s, requests=%d errors=%d", out, len(samples), failures)
|
|
if len(slowSearch) > 0 {
|
|
t.Errorf("search latency budget exceeded: %v", slowSearch)
|
|
}
|
|
if failures != 0 {
|
|
t.Fatalf("load returned %d errors", failures)
|
|
}
|
|
if writes.Load() != 0 {
|
|
t.Fatalf("stable read mix wrote %d commands", writes.Load())
|
|
}
|
|
}
|