forked from longtao.wu/droboticflow
674 lines
26 KiB
Go
674 lines
26 KiB
Go
package api
|
|
|
|
// Local-only read-mix harness. This is NOT full concurrent-user capacity acceptance:
|
|
// it excludes media, editing/submission, continuation writes and ingress shaping.
|
|
import (
|
|
"bytes"
|
|
"compress/gzip"
|
|
"context"
|
|
"crypto/sha256"
|
|
"encoding/json"
|
|
"fmt"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"os"
|
|
"path/filepath"
|
|
"runtime"
|
|
"runtime/pprof"
|
|
"sort"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
"sync/atomic"
|
|
"syscall"
|
|
"testing"
|
|
"time"
|
|
|
|
"droboticflow/internal/domain"
|
|
"droboticflow/internal/store"
|
|
"go.mongodb.org/mongo-driver/event"
|
|
"go.mongodb.org/mongo-driver/mongo"
|
|
"go.mongodb.org/mongo-driver/mongo/options"
|
|
)
|
|
|
|
type pollLoadActor struct {
|
|
User domain.UserRecord
|
|
Project string
|
|
Role string
|
|
}
|
|
type pollLoadSample struct {
|
|
Actor string
|
|
Key string
|
|
MS float64
|
|
Raw, Gzip, Rows int
|
|
Code int
|
|
Hash string
|
|
}
|
|
|
|
type pollLoadEndpointKey struct{}
|
|
type pollMongoObservation struct {
|
|
Endpoint, Collection, Command string
|
|
Count, Bytes, Nanoseconds, Failed int64
|
|
}
|
|
|
|
// Rusage covers this test process (handler plus harness compression/sampling),
|
|
// not Mongo, another process, or Kubernetes Pod throttling/RSS.
|
|
func pollLoadProcessCPU() (float64, error) {
|
|
var usage syscall.Rusage
|
|
if err := syscall.Getrusage(syscall.RUSAGE_SELF, &usage); err != nil {
|
|
return 0, err
|
|
}
|
|
return float64(usage.Utime.Sec) + float64(usage.Utime.Usec)/1e6 + float64(usage.Stime.Sec) + float64(usage.Stime.Usec)/1e6, nil
|
|
}
|
|
|
|
func TestPollPoolReadMixLocal(t *testing.T) {
|
|
if os.Getenv("POLL_POOL_LOAD") != "1" {
|
|
t.Skip("explicit isolated local read-mix verification")
|
|
}
|
|
seconds := func(key string, fallback int) int {
|
|
v := os.Getenv(key)
|
|
if v == "" {
|
|
return fallback
|
|
}
|
|
n, e := strconv.Atoi(v)
|
|
if e != nil || n < 1 || n > 7200 {
|
|
t.Fatalf("invalid %s", key)
|
|
}
|
|
return n
|
|
}
|
|
warmup, duration := seconds("POLL_POOL_WARMUP_SECONDS", 5), seconds("POLL_POOL_DURATION_SECONDS", 30)
|
|
annotators := seconds("POLL_POOL_ANNOTATORS", 60)
|
|
quality := seconds("POLL_POOL_QUALITY_REVIEWERS", 12)
|
|
acceptance := seconds("POLL_POOL_ACCEPTANCE_REVIEWERS", 8)
|
|
totalActors := annotators + quality + acceptance + 1
|
|
if totalActors > 250 {
|
|
t.Fatal("local read mix supports at most 250 actors")
|
|
}
|
|
availablePerStage := seconds("POLL_POOL_AVAILABLE_PER_STAGE", 8)
|
|
if availablePerStage > 250 {
|
|
t.Fatal("available fixtures limited to 250 per stage and supplier")
|
|
}
|
|
claimCounts := os.Getenv("POLL_POOL_CLAIM_COUNTS") == "1"
|
|
readModels := os.Getenv("POLL_POOL_READMODELS") == "1"
|
|
|
|
mode := os.Getenv("POLL_POOL_MODE")
|
|
if mode == "" {
|
|
mode = "summary"
|
|
}
|
|
if mode != "summary" && mode != "legacy" {
|
|
t.Fatal("mode must be summary or legacy")
|
|
}
|
|
ctx, cancel := context.WithTimeout(context.Background(), time.Duration(warmup+duration+180)*time.Second)
|
|
defer cancel()
|
|
var measured atomic.Bool
|
|
var commands, mongoBytes, mongoNS, writes atomic.Int64
|
|
var probeMu sync.Mutex
|
|
pending := map[int64]string{}
|
|
observations := map[string]*pollMongoObservation{}
|
|
monitor := &event.CommandMonitor{
|
|
Started: func(commandCtx context.Context, e *event.CommandStartedEvent) {
|
|
if !measured.Load() {
|
|
return
|
|
}
|
|
endpoint, _ := commandCtx.Value(pollLoadEndpointKey{}).(string)
|
|
collection, _ := e.Command.Lookup(e.CommandName).StringValueOK()
|
|
if collection == "" {
|
|
collection, _ = e.Command.Lookup("collection").StringValueOK()
|
|
}
|
|
key := endpoint + "/" + collection + "/" + e.CommandName
|
|
probeMu.Lock()
|
|
defer probeMu.Unlock()
|
|
pending[e.RequestID] = key
|
|
if observations[key] == nil {
|
|
observations[key] = &pollMongoObservation{Endpoint: endpoint, Collection: collection, Command: e.CommandName}
|
|
}
|
|
},
|
|
Failed: func(_ context.Context, e *event.CommandFailedEvent) {
|
|
probeMu.Lock()
|
|
defer probeMu.Unlock()
|
|
if key, ok := pending[e.RequestID]; ok {
|
|
observations[key].Failed++
|
|
delete(pending, e.RequestID)
|
|
}
|
|
},
|
|
Succeeded: func(_ context.Context, e *event.CommandSucceededEvent) {
|
|
probeMu.Lock()
|
|
if key, ok := pending[e.RequestID]; ok {
|
|
row := observations[key]
|
|
row.Count++
|
|
row.Bytes += int64(len(e.Reply))
|
|
row.Nanoseconds += e.Duration.Nanoseconds()
|
|
delete(pending, e.RequestID)
|
|
}
|
|
probeMu.Unlock()
|
|
if measured.Load() {
|
|
commands.Add(1)
|
|
mongoBytes.Add(int64(len(e.Reply)))
|
|
mongoNS.Add(e.Duration.Nanoseconds())
|
|
switch e.CommandName {
|
|
case "insert", "update", "delete", "findAndModify":
|
|
writes.Add(1)
|
|
}
|
|
}
|
|
}}
|
|
client, err := mongo.Connect(ctx, options.Client().ApplyURI("mongodb://127.0.0.1:27018/?directConnection=true").SetMonitor(monitor))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer client.Disconnect(context.Background())
|
|
db := client.Database(fmt.Sprintf("poll_pool_read_mix_%d", time.Now().UnixNano()))
|
|
defer func() {
|
|
cleanup, done := context.WithTimeout(context.Background(), 15*time.Second)
|
|
defer done()
|
|
if e := db.Drop(cleanup); e != nil {
|
|
t.Error(e)
|
|
}
|
|
}()
|
|
ds := store.NewMongoStore(db)
|
|
must := func(e error) {
|
|
t.Helper()
|
|
if e != nil {
|
|
t.Fatal(e)
|
|
}
|
|
}
|
|
must(ds.EnsureIndexes(ctx))
|
|
const org = "load-org"
|
|
projects := []string{"load-project-a", "load-project-b"}
|
|
for _, p := range projects {
|
|
must(ds.UpsertProject(ctx, domain.ProjectRecord{ID: p, OrganizationID: org, Name: p}))
|
|
must(ds.UpsertWorkflow(ctx, domain.WorkflowRecord{ID: "workflow-" + p, OrganizationID: org, ProjectID: p, Name: "load workflow"}))
|
|
}
|
|
suppliers := []string{"load-supplier-a", "load-supplier-b", "load-supplier-c"}
|
|
for _, s := range suppliers {
|
|
must(ds.UpsertSupplier(ctx, domain.SupplierRecord{ID: s, OrganizationID: org, Status: "active"}))
|
|
_, e := ds.SetSupplierAcceptanceAuthorization(ctx, org, s, true, 0, "load-admin")
|
|
must(e)
|
|
}
|
|
actors := []pollLoadActor{}
|
|
for i := 0; i < totalActors; i++ {
|
|
role := domain.UserRoleAnnotator
|
|
label := "annotation"
|
|
if i >= annotators && i < annotators+quality {
|
|
role = domain.UserRoleQualityReviewer
|
|
label = "quality"
|
|
}
|
|
if i >= annotators+quality && i < totalActors-1 {
|
|
role = domain.UserRoleAcceptanceReviewer
|
|
label = "acceptance"
|
|
}
|
|
if i == totalActors-1 {
|
|
role = domain.UserRoleAdmin
|
|
label = "admin"
|
|
}
|
|
u := domain.UserRecord{ID: fmt.Sprintf("load-user-%02d", i), Phone: fmt.Sprintf("synthetic-load-%02d", i), OrganizationID: org, SupplierID: suppliers[i%3], Role: role, Status: domain.UserStatusActive}
|
|
must(ds.UpsertUser(ctx, u))
|
|
for _, p := range projects {
|
|
must(ds.UpsertProjectMember(ctx, domain.ProjectMemberRecord{ID: p + u.ID, OrganizationID: org, ProjectID: p, UserID: u.ID, SupplierID: u.SupplierID, Roles: domain.AccountProductionRoles(u)}))
|
|
}
|
|
actors = append(actors, pollLoadActor{u, projects[i%2], label})
|
|
}
|
|
payload := make([]string, 512)
|
|
for i := range payload {
|
|
payload[i] = fmt.Sprintf("%x", sha256.Sum256([]byte(fmt.Sprintf("fixed-fixture-%d", i))))
|
|
}
|
|
runPayloadKiB := 0
|
|
if raw := os.Getenv("POLL_POOL_RUN_PAYLOAD_KIB"); raw != "" {
|
|
runPayloadKiB, err = strconv.Atoi(raw)
|
|
must(err)
|
|
if runPayloadKiB < 0 || runPayloadKiB > 2048 {
|
|
t.Fatal("run payload must be 0..2048 KiB")
|
|
}
|
|
}
|
|
taskPayloadKiB := 0
|
|
if raw := os.Getenv("POLL_POOL_TASK_PAYLOAD_KIB"); raw != "" {
|
|
taskPayloadKiB, err = strconv.Atoi(raw)
|
|
must(err)
|
|
if taskPayloadKiB < 0 || taskPayloadKiB > 2048 {
|
|
t.Fatal("task payload must be 0..2048 KiB")
|
|
}
|
|
}
|
|
seededRuns := map[string]bool{}
|
|
seed := func(id, p, s, stage, owner string, status domain.HumanItemStatus) {
|
|
batchID := "batch-" + p + s
|
|
stages := domain.DefaultFanyuV58HumanStages()
|
|
if stage == "semantic_review" {
|
|
batchID += "-semantic"
|
|
stages = []domain.HumanStageDefinition{{ID: stage, Kind: domain.HumanStageReview, RequiredRole: "quality_reviewer"}}
|
|
}
|
|
if stage == "acceptance" || stage == "prepackage_acceptance" {
|
|
batchID += "-" + stage
|
|
stages = []domain.HumanStageDefinition{{ID: stage, Kind: domain.HumanStageAcceptance, RequiredRole: "acceptance_reviewer"}}
|
|
}
|
|
must(ds.UpsertHumanBatch(ctx, domain.HumanBatchRecord{ID: batchID, RunID: batchID, GateNodeID: "gate", OrganizationID: org, ProjectID: p, WorkflowID: "workflow-" + p, PluginID: "annotation-qa", SupplierID: s, Status: domain.HumanBatchActive, ProcessingProfile: domain.ProcessingProfileRawHuman, Stages: stages}))
|
|
runID := ""
|
|
if runPayloadKiB > 0 {
|
|
runID = "run-" + batchID
|
|
if !seededRuns[runID] {
|
|
seededRuns[runID] = true
|
|
must(ds.UpsertRun(ctx, domain.RunRecord{ID: runID, OrganizationID: org, ProjectID: p, WorkflowID: "workflow-" + p, ProjectName: p, WorkflowName: "workflow-" + p, NodeExecutions: []domain.NodeExecutionRecord{{NodeID: "synthetic-node", Logs: []domain.LogEntryRecord{{Message: strings.Repeat("x", runPayloadKiB*1024)}}}}}))
|
|
}
|
|
}
|
|
taskID := "task-" + id
|
|
must(ds.UpsertAnnotationTask(ctx, domain.AnnotationTaskRecord{ID: taskID, OrganizationID: org, ProjectID: p, WorkflowID: "workflow-" + p, SupplierID: s, TemplateVersion: 6, Status: domain.AnnotationTaskStatusPendingAnnotation, SourceVideo: "synthetic/camera_left.mp4"}))
|
|
itemPayload := map[string]any{"captureDirectory": "synthetic-" + id, "sliceMergeEvidence": payload, "sliceWorkflow": map[string]any{"status": "completed", "phase": "annotate"}}
|
|
if taskPayloadKiB > 0 {
|
|
content := strings.Repeat(strings.Join(payload, ""), taskPayloadKiB/32+1)[:taskPayloadKiB*1024]
|
|
itemPayload["items"] = []map[string]any{{"annotation": map[string]any{"syntheticPayload": content}, "materialization": map[string]any{"coverage": map[string]any{"durationSeconds": 900}}}}
|
|
}
|
|
must(ds.UpsertHumanItem(ctx, domain.HumanItemRecord{ID: id, RunID: runID, OrganizationID: org, ProjectID: p, WorkflowID: "workflow-" + p, PluginID: "annotation-qa", SupplierID: s, BatchID: batchID, LegacyTaskID: taskID, AssignedUserID: owner, LatestAuthorUserID: "synthetic-author", CurrentStageID: stage, Status: status, Version: 1, CreatedAt: "2026-10-01T00:00:00Z", Subject: domain.HumanSubjectRef{ItemKey: "synthetic/" + id + "/camera_left.mp4", Payload: itemPayload}}))
|
|
}
|
|
for _, a := range actors {
|
|
if a.Role == "admin" {
|
|
continue
|
|
}
|
|
stage := "annotation_combined"
|
|
if a.Role == "quality" {
|
|
stage = "quality_review"
|
|
}
|
|
if a.Role == "acceptance" {
|
|
stage = "prepackage_acceptance"
|
|
}
|
|
for n := 0; n < 2; n++ {
|
|
seed(fmt.Sprintf("owned-%s-%d", a.User.ID, n), a.Project, a.User.SupplierID, stage, a.User.ID, domain.HumanItemClaimed)
|
|
}
|
|
if a.Role == "annotation" {
|
|
taskID := "task-owned-" + a.User.ID + "-0"
|
|
must(ds.UpsertAnnotationRevision(ctx, domain.AnnotationRevisionRecord{ID: "revision-" + a.User.ID, OrganizationID: org, TaskID: taskID, Revision: 1, CountsAsOutput: true, AuthorUserID: a.User.ID}))
|
|
must(ds.UpsertAnnotationWorklogEvent(ctx, domain.AnnotationWorklogEventRecord{ID: "worklog-" + a.User.ID, OrganizationID: org, ProjectID: a.Project, TaskID: taskID, UserID: a.User.ID, SupplierID: a.User.SupplierID, EventType: "combined_submitted", Revision: 1, OutputDelta: 1, CreatedAt: "2026-10-02T00:00:00Z"}))
|
|
}
|
|
if a.Role != "annotation" {
|
|
must(ds.AppendHumanEvent(ctx, domain.HumanEventRecord{ID: "history-" + a.User.ID, OrganizationID: org, ItemID: "owned-" + a.User.ID + "-0", ActorUserID: a.User.ID, StageID: stage, EventType: "approved", CreatedAt: "2026-10-02T00:00:00Z"}))
|
|
}
|
|
}
|
|
for _, p := range projects {
|
|
for _, s := range suppliers {
|
|
stages := []string{"quality_review", "semantic_review", "acceptance", "prepackage_acceptance"}
|
|
if claimCounts {
|
|
stages = append(stages, "annotation_combined")
|
|
}
|
|
for _, stage := range stages {
|
|
for n := 0; n < availablePerStage; n++ {
|
|
seed(fmt.Sprintf("available-%s-%s-%s-%d", p, s, stage, n), p, s, stage, "", domain.HumanItemReady)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
authorHistory := 0
|
|
if raw := os.Getenv("POLL_POOL_AUTHOR_HISTORY"); raw != "" {
|
|
authorHistory, err = strconv.Atoi(raw)
|
|
must(err)
|
|
if authorHistory < 0 || authorHistory > 5000 {
|
|
t.Fatal("author history must be 0..5000 per reviewer")
|
|
}
|
|
}
|
|
historyPartitions := map[string]domain.ReleasePartitionRecord{}
|
|
for _, a := range actors {
|
|
if authorHistory == 0 || (a.Role != "quality" && a.Role != "acceptance") {
|
|
continue
|
|
}
|
|
rows := make([]any, 0, authorHistory)
|
|
for i := 0; i < authorHistory; i++ {
|
|
stage := "quality_review"
|
|
suffix := ""
|
|
if a.Role == "quality" && i%2 != 0 {
|
|
stage = "semantic_review"
|
|
suffix = "-semantic"
|
|
}
|
|
if a.Role == "acceptance" {
|
|
stage = "acceptance"
|
|
if i%2 != 0 {
|
|
stage = "prepackage_acceptance"
|
|
}
|
|
suffix = "-" + stage
|
|
}
|
|
id := fmt.Sprintf("historical-%s-%05d", a.User.ID, i)
|
|
batchID := "batch-" + a.Project + a.User.SupplierID + suffix
|
|
partitionID := "released-history-" + batchID
|
|
partition := historyPartitions[partitionID]
|
|
partition.ID = partitionID
|
|
partition.OrganizationID = org
|
|
partition.BatchID = batchID
|
|
partition.Sequence = 1
|
|
partition.UniqueKey = partitionID
|
|
partition.Status = domain.ReleasePartitionDispatched
|
|
partition.ItemIDs = append(partition.ItemIDs, id)
|
|
historyPartitions[partitionID] = partition
|
|
|
|
rows = append(rows, domain.HumanItemRecord{ID: id, OrganizationID: org, ProjectID: a.Project, WorkflowID: "workflow-" + a.Project, PluginID: "annotation-qa", SupplierID: a.User.SupplierID, BatchID: batchID, ReleasePartitionID: partitionID, LatestAuthorUserID: a.User.ID, CurrentStageID: stage, Status: domain.HumanItemAccepted, Version: 1, CreatedAt: "2026-09-01T00:00:00Z", Subject: domain.HumanSubjectRef{ItemKey: id}})
|
|
}
|
|
_, e := db.Collection("humanItems").InsertMany(ctx, rows)
|
|
must(e)
|
|
}
|
|
|
|
for _, partition := range historyPartitions {
|
|
must(ds.UpsertReleasePartition(ctx, partition))
|
|
}
|
|
authorizationState := os.Getenv("POLL_POOL_AUTHORIZATION_STATE")
|
|
if authorizationState == "" {
|
|
authorizationState = "legacy"
|
|
}
|
|
if authorizationState != "legacy" && authorizationState != "initialized" {
|
|
t.Fatal("unknown authorization state")
|
|
}
|
|
if authorizationState == "initialized" {
|
|
must(ds.InitializeSupplierProjectAuthorizations(ctx, org))
|
|
}
|
|
if readModels {
|
|
result, err := ds.BackfillPoolSearchMetadata(ctx, org, true)
|
|
must(err)
|
|
if result.Unsupported > 0 || result.Conflicts > 0 {
|
|
t.Fatalf("summary preparation incomplete: %+v", result)
|
|
}
|
|
}
|
|
handler := NewHandler(ds, nil)
|
|
var mu sync.Mutex
|
|
samples := []pollLoadSample{}
|
|
failureSamples := []map[string]any{}
|
|
var completedRequests, completedErrors atomic.Int64
|
|
call := func(a pollLoadActor, key, path string, record bool) pollLoadSample {
|
|
req := httptest.NewRequest(http.MethodGet, path, nil)
|
|
requestCtx, done := context.WithTimeout(ctx, 15*time.Second)
|
|
defer done()
|
|
requestCtx = context.WithValue(requestCtx, pollLoadEndpointKey{}, key)
|
|
req = req.WithContext(context.WithValue(requestCtx, authContextKey{}, &authenticatedPrincipal{User: a.User, Organization: domain.OrganizationRecord{ID: org, Status: domain.OrganizationStatusActive}}))
|
|
response := httptest.NewRecorder()
|
|
start := time.Now()
|
|
handler.ServeHTTP(response, req)
|
|
elapsed := time.Since(start)
|
|
completedRequests.Add(1)
|
|
if response.Code != 200 {
|
|
completedErrors.Add(1)
|
|
}
|
|
var body struct {
|
|
Count *int `json:"count"`
|
|
Code string `json:"code"`
|
|
Error string `json:"error"`
|
|
Items []struct {
|
|
ID string `json:"id"`
|
|
} `json:"items"`
|
|
Summaries []struct {
|
|
ID string `json:"id"`
|
|
} `json:"summaries"`
|
|
}
|
|
_ = json.Unmarshal(response.Body.Bytes(), &body)
|
|
ids := []string{}
|
|
for _, v := range body.Items {
|
|
ids = append(ids, v.ID)
|
|
}
|
|
for _, v := range body.Summaries {
|
|
ids = append(ids, v.ID)
|
|
}
|
|
rowCount := len(ids)
|
|
if body.Count != nil {
|
|
rowCount = *body.Count
|
|
ids = append(ids, fmt.Sprintf("count:%d", *body.Count))
|
|
}
|
|
sort.Strings(ids)
|
|
var compressed bytes.Buffer
|
|
zw := gzip.NewWriter(&compressed)
|
|
_, _ = zw.Write(response.Body.Bytes())
|
|
_ = zw.Close()
|
|
sample := pollLoadSample{a.User.ID, key, float64(elapsed.Microseconds()) / 1000, response.Body.Len(), compressed.Len(), rowCount, response.Code, fmt.Sprintf("%x", sha256.Sum256([]byte(strings.Join(ids, "\n"))))}
|
|
if response.Code != 200 {
|
|
class := "unclassified"
|
|
for _, known := range []string{"context deadline exceeded", "context canceled", "duplicate key", "not found", "conflict", "timeout"} {
|
|
if strings.Contains(strings.ToLower(body.Error), known) {
|
|
class = known
|
|
break
|
|
}
|
|
}
|
|
code := strings.Map(func(r rune) rune {
|
|
if r >= 'a' && r <= 'z' || r >= 'A' && r <= 'Z' || r >= '0' && r <= '9' || r == '_' || r == '-' {
|
|
return r
|
|
}
|
|
return -1
|
|
}, body.Code)
|
|
if len(code) > 80 {
|
|
code = code[:80]
|
|
}
|
|
mu.Lock()
|
|
if len(failureSamples) < 20 {
|
|
failureSamples = append(failureSamples, map[string]any{"at": time.Now().UTC().Format(time.RFC3339Nano), "endpoint": key, "actor": a.User.ID, "status": response.Code, "elapsed_ms": sample.MS, "code": code, "class": class, "measured": record})
|
|
t.Logf("read-mix HTTP failure endpoint=%s actor=%s status=%d elapsed_ms=%.3f code=%s class=%s request_context=%v", key, a.User.ID, response.Code, sample.MS, code, class, requestCtx.Err())
|
|
}
|
|
mu.Unlock()
|
|
}
|
|
if record {
|
|
mu.Lock()
|
|
samples = append(samples, sample)
|
|
mu.Unlock()
|
|
}
|
|
return sample
|
|
}
|
|
paths := func(a pollLoadActor, all, countCycle bool) map[string]string {
|
|
scope := "&projectId=" + a.Project
|
|
scopeName := "project"
|
|
if all {
|
|
scope = ""
|
|
scopeName = "all"
|
|
}
|
|
if a.Role == "annotation" {
|
|
path := "/api/plugins/annotation-qa/work-buffer?projectId=" + a.Project
|
|
if mode == "summary" {
|
|
path += "&view=summary"
|
|
}
|
|
result := map[string]string{"annotation.buffer": path}
|
|
if claimCounts && countCycle {
|
|
result["annotation.claim-count"] = "/api/plugins/annotation-qa/claim-count?projectId=" + a.Project
|
|
}
|
|
return result
|
|
}
|
|
if a.Role == "admin" {
|
|
return map[string]string{"admin.workforce": "/api/dashboard/workforce?from=2026-10-01T00:00:00Z&to=2026-10-28T00:00:00Z"}
|
|
}
|
|
result := map[string]string{}
|
|
for _, pool := range []string{"available", "mine", "reviewed"} {
|
|
endpoint := "/api/human-items"
|
|
if readModels {
|
|
endpoint = "/api/human-item-summaries"
|
|
}
|
|
base := endpoint + "?pluginId=annotation-qa&pool=" + pool + scope
|
|
if a.Role == "quality" {
|
|
result["quality."+scopeName+"."+pool] = base + "&stageIds=quality_review%2Csemantic_review"
|
|
} else {
|
|
for _, stage := range []string{"acceptance", "prepackage_acceptance"} {
|
|
result["acceptance."+scopeName+"."+stage+"."+pool] = base + "&stageId=" + stage
|
|
}
|
|
}
|
|
}
|
|
if readModels {
|
|
stages := "quality_review,semantic_review"
|
|
reviewed := ""
|
|
if a.Role == "acceptance" {
|
|
stages = "acceptance,prepackage_acceptance"
|
|
reviewed = "&reviewedStageIds=" + stages
|
|
}
|
|
result[a.Role+"."+scopeName+".counts"] = "/api/human-item-counts?pluginId=annotation-qa" + scope + "&stageIds=" + stages + reviewed
|
|
}
|
|
if os.Getenv("POLL_POOL_SEARCH") == "1" {
|
|
endpoint := "/api/human-items"
|
|
if readModels {
|
|
endpoint = "/api/human-item-summaries"
|
|
}
|
|
stage := "quality_review"
|
|
if a.Role == "acceptance" {
|
|
stage = "prepackage_acceptance"
|
|
}
|
|
result[a.Role+"."+scopeName+".search"] = endpoint + "?pluginId=annotation-qa&pool=available&limit=200" + scope + "&stageId=" + stage + "&search=synthetic-available-" + a.Project + "-" + a.User.SupplierID + "-" + stage + "-0"
|
|
}
|
|
return result
|
|
}
|
|
// Fail before timing if permissions or fixtures accidentally produce empty work.
|
|
for _, a := range actors {
|
|
for key, path := range paths(a, false, true) {
|
|
result := call(a, key, path, false)
|
|
if result.Code != 200 {
|
|
t.Fatalf("preflight %s status=%d", key, result.Code)
|
|
}
|
|
if (a.Role == "annotation" || a.Role == "admin" || strings.HasSuffix(key, ".available") || strings.HasSuffix(key, ".mine") && !strings.Contains(key, ".acceptance.")) && result.Rows == 0 {
|
|
t.Fatalf("preflight %s unexpectedly empty for %s", key, a.User.ID)
|
|
}
|
|
}
|
|
}
|
|
var memBefore, memAfter runtime.MemStats
|
|
runPhase := func(seconds int, record bool) {
|
|
phaseCtx, stop := context.WithTimeout(ctx, time.Duration(seconds)*time.Second)
|
|
defer stop()
|
|
var wg sync.WaitGroup
|
|
for _, actor := range actors {
|
|
wg.Add(1)
|
|
go func(a pollLoadActor) {
|
|
defer wg.Done()
|
|
period := 12 * time.Second
|
|
if a.Role == "annotation" {
|
|
period = 3 * time.Second
|
|
}
|
|
ticker := time.NewTicker(period)
|
|
defer ticker.Stop()
|
|
cycles := (time.Duration(seconds)*time.Second + period - 1) / period
|
|
for cycle := 0; cycle < int(cycles); cycle++ {
|
|
select {
|
|
case <-phaseCtx.Done():
|
|
return
|
|
default:
|
|
}
|
|
var requests sync.WaitGroup
|
|
for key, path := range paths(a, cycle%2 == 1, cycle%4 == 0) {
|
|
requests.Add(1)
|
|
go func(key, path string) { defer requests.Done(); call(a, key, path, record) }(key, path)
|
|
}
|
|
requests.Wait()
|
|
if cycle+1 == int(cycles) {
|
|
return
|
|
}
|
|
select {
|
|
case <-phaseCtx.Done():
|
|
return
|
|
case <-ticker.C:
|
|
}
|
|
}
|
|
}(actor)
|
|
}
|
|
wg.Wait()
|
|
<-phaseCtx.Done()
|
|
}
|
|
out := os.Getenv("POLL_POOL_OUTPUT")
|
|
if out == "" {
|
|
out = filepath.Join("..", "..", "local-data", "test-results", fmt.Sprintf("poll-pool-%s-%d.json", mode, time.Now().Unix()))
|
|
}
|
|
must(os.MkdirAll(filepath.Dir(out), 0700))
|
|
resourceFile, err := os.Create(out + ".resources.jsonl")
|
|
must(err)
|
|
defer resourceFile.Close()
|
|
resourceEncoder := json.NewEncoder(resourceFile)
|
|
resourceRows := []map[string]any{}
|
|
var phase atomic.Value
|
|
phase.Store("warmup")
|
|
resourceStart := time.Now()
|
|
sampleResource := func() error {
|
|
var m runtime.MemStats
|
|
runtime.ReadMemStats(&m)
|
|
cpuSeconds, cpuErr := pollLoadProcessCPU()
|
|
if cpuErr != nil {
|
|
return cpuErr
|
|
}
|
|
mu.Lock()
|
|
requests := len(samples)
|
|
mu.Unlock()
|
|
row := map[string]any{"phase": phase.Load(), "elapsed_seconds": time.Since(resourceStart).Seconds(), "heap_alloc_bytes": m.HeapAlloc, "heap_inuse_bytes": m.HeapInuse, "total_alloc_bytes": m.TotalAlloc, "gc_cycles": m.NumGC, "goroutines": runtime.NumGoroutine(), "retained_request_samples": requests, "errors": completedErrors.Load(), "completed_requests": completedRequests.Load(), "process_cpu_seconds": cpuSeconds}
|
|
resourceRows = append(resourceRows, row)
|
|
return resourceEncoder.Encode(row)
|
|
}
|
|
must(sampleResource())
|
|
resourceStop := make(chan struct{})
|
|
resourceDone := make(chan error, 1)
|
|
go func() {
|
|
ticker := time.NewTicker(15 * time.Second)
|
|
defer ticker.Stop()
|
|
for {
|
|
select {
|
|
case <-resourceStop:
|
|
resourceDone <- sampleResource()
|
|
return
|
|
case <-ticker.C:
|
|
if e := sampleResource(); e != nil {
|
|
cancel()
|
|
resourceDone <- e
|
|
return
|
|
}
|
|
}
|
|
}
|
|
}()
|
|
runPhase(warmup, false)
|
|
if completedErrors.Load() != 0 {
|
|
close(resourceStop)
|
|
must(<-resourceDone)
|
|
t.Fatalf("warmup had %d HTTP errors", completedErrors.Load())
|
|
}
|
|
phase.Store("steady")
|
|
runtime.ReadMemStats(&memBefore)
|
|
var cpuFile *os.File
|
|
if filename := os.Getenv("POLL_POOL_CPU_PROFILE"); filename != "" {
|
|
must(os.MkdirAll(filepath.Dir(filename), 0700))
|
|
cpuFile, err = os.Create(filename)
|
|
must(err)
|
|
must(pprof.StartCPUProfile(cpuFile))
|
|
}
|
|
cpuBefore, cpuErr := pollLoadProcessCPU()
|
|
must(cpuErr)
|
|
measured.Store(true)
|
|
start := time.Now()
|
|
runPhase(duration, true)
|
|
measured.Store(false)
|
|
phase.Store("finished")
|
|
close(resourceStop)
|
|
must(<-resourceDone)
|
|
must(resourceFile.Close())
|
|
if cpuFile != nil {
|
|
pprof.StopCPUProfile()
|
|
must(cpuFile.Close())
|
|
}
|
|
cpuAfter, cpuErr := pollLoadProcessCPU()
|
|
must(cpuErr)
|
|
actual := time.Since(start)
|
|
runtime.ReadMemStats(&memAfter)
|
|
// Outside the measured interval: distinguish retained objects from GC timing.
|
|
runtime.GC()
|
|
var memAfterGC runtime.MemStats
|
|
runtime.ReadMemStats(&memAfterGC)
|
|
grouped := map[string][]pollLoadSample{}
|
|
failures := 0
|
|
for _, s := range samples {
|
|
grouped[s.Key] = append(grouped[s.Key], s)
|
|
if s.Code != 200 {
|
|
failures++
|
|
}
|
|
}
|
|
metrics := map[string]any{}
|
|
for key, rows := range grouped {
|
|
lat := []float64{}
|
|
raw, gz, minRows := 0, 0, rows[0].Rows
|
|
for _, s := range rows {
|
|
lat = append(lat, s.MS)
|
|
raw += s.Raw
|
|
gz += s.Gzip
|
|
if s.Rows < minRows {
|
|
minRows = s.Rows
|
|
}
|
|
}
|
|
sort.Float64s(lat)
|
|
metrics[key] = map[string]any{"requests": len(rows), "p95_ms": lat[(len(lat)*95-1)/100], "max_ms": lat[len(lat)-1], "raw_bytes": raw, "gzip_bytes": gz, "min_items": minRows}
|
|
}
|
|
report := map[string]any{"read_models": readModels, "search_queries": os.Getenv("POLL_POOL_SEARCH") == "1", "schema": 2, "resource_samples": resourceRows, "process_cpu_seconds": cpuAfter - cpuBefore, "authorization_state": authorizationState, "fixture": "read-mix-v3-released-history", "available_per_stage": availablePerStage, "task_payload_kib": taskPayloadKiB, "run_payload_kib": runPayloadKiB, "author_history_per_reviewer": authorHistory, "mode": mode, "revision": os.Getenv("POLL_POOL_REVISION"), "actors": totalActors, "actor_counts": map[string]int{"annotation": annotators, "quality": quality, "acceptance": acceptance, "admin": 1}, "claim_counts": claimCounts, "warmup_seconds": warmup, "scheduled_seconds": duration, "actual_seconds": actual.Seconds(), "errors": failures, "failure_samples": failureSamples, "metrics": metrics, "mongo_breakdown": observations, "mongo_commands": commands.Load(), "mongo_reply_bytes": mongoBytes.Load(), "mongo_command_ms": float64(mongoNS.Load()) / 1e6, "mongo_write_commands": writes.Load(), "go_heap_before": memBefore.HeapAlloc, "go_heap_after": memAfter.HeapAlloc, "go_heap_after_gc": memAfterGC.HeapAlloc, "go_gc_cycles": memAfter.NumGC - memBefore.NumGC, "go_allocated_bytes": memAfter.TotalAlloc - memBefore.TotalAlloc, "limitations": []string{"handler timing excludes ingress/network/authentication", "gzip computed separately with fixed Go default compression", "no continuation renew, media or workflow writes", "synthetic fixture; not capacity acceptance", "CPU covers handler and in-process load harness, not Mongo/Pod throttling", "resource heap includes retained benchmark request samples; not standalone service RSS"}, "samples": samples}
|
|
encoded, e := json.MarshalIndent(report, "", " ")
|
|
must(e)
|
|
must(os.WriteFile(out, encoded, 0600))
|
|
t.Logf("read-mix report: %s, requests=%d errors=%d", out, len(samples), failures)
|
|
if failures != 0 {
|
|
t.Fatalf("load returned %d errors", failures)
|
|
}
|
|
if writes.Load() != 0 {
|
|
t.Fatalf("stable read mix wrote %d commands", writes.Load())
|
|
}
|
|
}
|